diff options
Diffstat (limited to 'src')
187 files changed, 15241 insertions, 4563 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e07117496..9a0e9f08d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -33,18 +33,18 @@ std::string escapeUri(std::string uri) static string currentLoad; -static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) +static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { +static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!settings.systemFeatures.get().count(feature)) return false; + if (!store.systemFeatures.get().count(feature)) return false; return true; } -static int _main(int argc, char * * argv) +static int main_build_remote(int argc, char * * argv) { { logger = makeJSONLogger(*logger); @@ -103,10 +103,10 @@ static int _main(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling + auto canBuildLocally = amWilling && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) - && allSupportedLocally(requiredFeatures); + && allSupportedLocally(*store, requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -119,7 +119,7 @@ static int _main(int argc, char * * argv) bool rightType = false; Machine * bestMachine = nullptr; - unsigned long long bestLoad = 0; + uint64_t bestLoad = 0; for (auto & m : machines) { debug("considering building on remote machine '%s'", m.storeUri); @@ -130,8 +130,8 @@ static int _main(int argc, char * * argv) m.mandatoryMet(requiredFeatures)) { rightType = true; AutoCloseFD free; - unsigned long long load = 0; - for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + uint64_t load = 0; + for (uint64_t slot = 0; slot < m.maxJobs; ++slot) { auto slotLock = openSlotLock(m, slot); if (lockFile(slotLock.get(), ltWrite, false)) { if (!free) { @@ -170,7 +170,45 @@ static int _main(int argc, char * * argv) if (rightType && !canBuildLocally) std::cerr << "# postpone\n"; else + { + // build the hint template. + string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)"; + hintstring += "\n%s available machines:"; + hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; + + for (unsigned int i = 0; i < machines.size(); ++i) { + hintstring += "\n(%s, %s, %s, %s)"; + } + + // add the template values. + string drvstr; + if (drvPath.has_value()) + drvstr = drvPath->to_string(); + else + drvstr = "<unknown>"; + + auto hint = hintformat(hintstring); + hint + % drvstr + % neededSystem + % concatStringsSep<StringSet>(", ", requiredFeatures) + % machines.size(); + + for (auto & m : machines) { + hint % concatStringsSep<vector<string>>(", ", m.systemTypes) + % m.maxJobs + % concatStringsSep<StringSet>(", ", m.supportedFeatures) + % concatStringsSep<StringSet>(", ", m.mandatoryFeatures); + } + + logErrorInfo(lvlInfo, { + .name = "Remote build", + .description = "Failed to find a machine for remote build!", + .hint = hint + }); + std::cerr << "# decline\n"; + } break; } @@ -186,15 +224,7 @@ static int _main(int argc, char * * argv) Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); - Store::Params storeParams; - if (hasPrefix(bestMachine->storeUri, "ssh://")) { - storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; - if (bestMachine->sshKey != "") - storeParams["ssh-key"] = bestMachine->sshKey; - } - - sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore = bestMachine->openStore(); sshStore->connect(); storeUri = bestMachine->storeUri; @@ -267,4 +297,4 @@ connected: } } -static RegisterLegacyCommand s1("build-remote", _main); +static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 44baadd53..10c1a6975 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -4,6 +4,8 @@ #include "util.hh" #include "eval.hh" #include "fetchers.hh" +#include "registry.hh" +#include "flake/flakeref.hh" #include "store-api.hh" namespace nix { @@ -27,10 +29,31 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up <...> file names", + .description = "add a path to the list of locations used to look up `<...>` file names", .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); + + addFlag({ + .longName = "impure", + .description = "allow access to mutable paths and repositories", + .handler = {[&]() { + evalSettings.pureEval = false; + }}, + }); + + addFlag({ + .longName = "override-flake", + .description = "override a flake registry value", + .labels = {"original-ref", "resolved-ref"}, + .handler = {[&](std::string _from, std::string _to) { + auto from = parseFlakeRef(_from, absPath(".")); + auto to = parseFlakeRef(_to, absPath(".")); + fetchers::Attrs extraAttrs; + if (to.subdir != "") extraAttrs["dir"] = to.subdir; + fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }} + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) @@ -53,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc new file mode 100644 index 000000000..381344b40 --- /dev/null +++ b/src/libexpr/eval-cache.cc @@ -0,0 +1,620 @@ +#include "eval-cache.hh" +#include "sqlite.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" + +namespace nix::eval_cache { + +static const char * schema = R"sql( +create table if not exists Attributes ( + parent integer not null, + name text, + type integer not null, + value text, + context text, + primary key (parent, name) +); +)sql"; + +struct AttrDb +{ + std::atomic_bool failed{false}; + + struct State + { + SQLite db; + SQLiteStmt insertAttribute; + SQLiteStmt insertAttributeWithContext; + SQLiteStmt queryAttribute; + SQLiteStmt queryAttributes; + std::unique_ptr<SQLiteTxn> txn; + }; + + std::unique_ptr<Sync<State>> _state; + + AttrDb(const Hash & fingerprint) + : _state(std::make_unique<Sync<State>>()) + { + auto state(_state->lock()); + + Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; + createDirs(cacheDir); + + Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + + state->insertAttributeWithContext.create(state->db, + "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select rowid, type, value, context from Attributes where parent = ? and name = ?"); + + state->queryAttributes.create(state->db, + "select name from Attributes where parent = ?"); + + state->txn = std::make_unique<SQLiteTxn>(state->db); + } + + ~AttrDb() + { + try { + auto state(_state->lock()); + if (!failed) + state->txn->commit(); + state->txn.reset(); + } catch (...) { + ignoreException(); + } + } + + template<typename F> + AttrId doSQLite(F && fun) + { + if (failed) return 0; + try { + return fun(); + } catch (SQLiteError &) { + ignoreException(); + failed = true; + return 0; + } + } + + AttrId setAttrs( + AttrKey key, + const std::vector<Symbol> & attrs) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::FullAttrs) + (0, false).exec(); + + AttrId rowId = state->db.getLastInsertedRowId(); + assert(rowId); + + for (auto & attr : attrs) + state->insertAttribute.use() + (rowId) + (attr) + (AttrType::Placeholder) + (0, false).exec(); + + return rowId; + }); + } + + AttrId setString( + AttrKey key, + std::string_view s, + const char * * context = nullptr) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + if (context) { + std::string ctx; + for (const char * * p = context; *p; ++p) { + if (p != context) ctx.push_back(' '); + ctx.append(*p); + } + state->insertAttributeWithContext.use() + (key.first) + (key.second) + (AttrType::String) + (s) + (ctx).exec(); + } else { + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::String) + (s).exec(); + } + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setBool( + AttrKey key, + bool b) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Bool) + (b ? 1 : 0).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setPlaceholder(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Placeholder) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setMissing(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Missing) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setMisc(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Misc) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setFailed(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Failed) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + std::optional<std::pair<AttrId, AttrValue>> getAttr( + AttrKey key, + SymbolTable & symbols) + { + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + if (!queryAttribute.next()) return {}; + + auto rowId = (AttrType) queryAttribute.getInt(0); + auto type = (AttrType) queryAttribute.getInt(1); + + switch (type) { + case AttrType::Placeholder: + return {{rowId, placeholder_t()}}; + case AttrType::FullAttrs: { + // FIXME: expensive, should separate this out. + std::vector<Symbol> attrs; + auto queryAttributes(state->queryAttributes.use()(rowId)); + while (queryAttributes.next()) + attrs.push_back(symbols.create(queryAttributes.getStr(0))); + return {{rowId, attrs}}; + } + case AttrType::String: { + std::vector<std::pair<Path, std::string>> context; + if (!queryAttribute.isNull(3)) + for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) + context.push_back(decodeContext(s)); + return {{rowId, string_t{queryAttribute.getStr(2), context}}}; + } + case AttrType::Bool: + return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Missing: + return {{rowId, missing_t()}}; + case AttrType::Misc: + return {{rowId, misc_t()}}; + case AttrType::Failed: + return {{rowId, failed_t()}}; + default: + throw Error("unexpected type in evaluation cache"); + } + } +}; + +static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint) +{ + try { + return std::make_shared<AttrDb>(fingerprint); + } catch (SQLiteError &) { + ignoreException(); + return nullptr; + } +} + +EvalCache::EvalCache( + std::optional<std::reference_wrapper<const Hash>> useCache, + EvalState & state, + RootLoader rootLoader) + : db(useCache ? makeAttrDb(*useCache) : nullptr) + , state(state) + , rootLoader(rootLoader) +{ +} + +Value * EvalCache::getRootValue() +{ + if (!value) { + debug("getting root value"); + value = allocRootValue(rootLoader()); + } + return *value; +} + +std::shared_ptr<AttrCursor> EvalCache::getRoot() +{ + return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt); +} + +AttrCursor::AttrCursor( + ref<EvalCache> root, + Parent parent, + Value * value, + std::optional<std::pair<AttrId, AttrValue>> && cachedValue) + : root(root), parent(parent), cachedValue(std::move(cachedValue)) +{ + if (value) + _value = allocRootValue(value); +} + +AttrKey AttrCursor::getKey() +{ + if (!parent) + return {0, root->state.sEpsilon}; + if (!parent->first->cachedValue) { + parent->first->cachedValue = root->db->getAttr( + parent->first->getKey(), root->state.symbols); + assert(parent->first->cachedValue); + } + return {parent->first->cachedValue->first, parent->second}; +} + +Value & AttrCursor::getValue() +{ + if (!_value) { + if (parent) { + auto & vParent = parent->first->getValue(); + root->state.forceAttrs(vParent); + auto attr = vParent.attrs->get(parent->second); + if (!attr) + throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); + _value = allocRootValue(attr->value); + } else + _value = allocRootValue(root->getRootValue()); + } + return **_value; +} + +std::vector<Symbol> AttrCursor::getAttrPath() const +{ + if (parent) { + auto attrPath = parent->first->getAttrPath(); + attrPath.push_back(parent->second); + return attrPath; + } else + return {}; +} + +std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const +{ + auto attrPath = getAttrPath(); + attrPath.push_back(name); + return attrPath; +} + +std::string AttrCursor::getAttrPathStr() const +{ + return concatStringsSep(".", getAttrPath()); +} + +std::string AttrCursor::getAttrPathStr(Symbol name) const +{ + return concatStringsSep(".", getAttrPath(name)); +} + +Value & AttrCursor::forceValue() +{ + debug("evaluating uncached attribute %s", getAttrPathStr()); + + auto & v = getValue(); + + try { + root->state.forceValue(v); + } catch (EvalError &) { + debug("setting '%s' to failed", getAttrPathStr()); + if (root->db) + cachedValue = {root->db->setFailed(getKey()), failed_t()}; + throw; + } + + if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) { + if (v.type == tString) + cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), + string_t{v.string.s, {}}}; + else if (v.type == tPath) + cachedValue = {root->db->setString(getKey(), v.path), v.path}; + else if (v.type == tBool) + cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + else if (v.type == tAttrs) + ; // FIXME: do something? + else + cachedValue = {root->db->setMisc(getKey()), misc_t()}; + } + + return v; +} + +std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + + if (cachedValue) { + if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) { + for (auto & attr : *attrs) + if (attr == name) + return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name)); + return nullptr; + } else if (std::get_if<placeholder_t>(&cachedValue->second)) { + auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); + if (attr) { + if (std::get_if<missing_t>(&attr->second)) + return nullptr; + else if (std::get_if<failed_t>(&attr->second)) { + if (forceErrors) + debug("reevaluating failed cached attribute '%s'"); + else + throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + } else + return std::make_shared<AttrCursor>(root, + std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); + } + // Incomplete attrset, so need to fall thru and + // evaluate to see whether 'name' exists + } else + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tAttrs) + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + auto attr = v.attrs->get(name); + + if (!attr) { + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + root->db->setMissing({cachedValue->first, name}); + } + return nullptr; + } + + std::optional<std::pair<AttrId, AttrValue>> cachedValue2; + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; + } + + return std::make_shared<AttrCursor>( + root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); +} + +std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name) +{ + return maybeGetAttr(root->state.symbols.create(name)); +} + +std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) +{ + auto p = maybeGetAttr(name, forceErrors); + if (!p) + throw Error("attribute '%s' does not exist", getAttrPathStr(name)); + return p; +} + +std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name) +{ + return getAttr(root->state.symbols.create(name)); +} + +std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath) +{ + auto res = shared_from_this(); + for (auto & attr : attrPath) { + res = res->maybeGetAttr(attr); + if (!res) return {}; + } + return res; +} + +std::string AttrCursor::getString() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto s = std::get_if<string_t>(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return s->first; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tString && v.type != tPath) + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + + return v.type == tString ? v.string.s : v.path; +} + +string_t AttrCursor::getStringWithContext() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto s = std::get_if<string_t>(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type == tString) + return {v.string.s, v.getContext()}; + else if (v.type == tPath) + return {v.path, {}}; + else + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); +} + +bool AttrCursor::getBool() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto b = std::get_if<bool>(&cachedValue->second)) { + debug("using cached Boolean attribute '%s'", getAttrPathStr()); + return *b; + } else + throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tBool) + throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + + return v.boolean; +} + +std::vector<Symbol> AttrCursor::getAttrs() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) { + debug("using cached attrset attribute '%s'", getAttrPathStr()); + return *attrs; + } else + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tAttrs) + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + std::vector<Symbol> attrs; + for (auto & attr : *getValue().attrs) + attrs.push_back(attr.name); + std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { + return (const string &) a < (const string &) b; + }); + + if (root->db) + cachedValue = {root->db->setAttrs(getKey(), attrs), attrs}; + + return attrs; +} + +bool AttrCursor::isDerivation() +{ + auto aType = maybeGetAttr("type"); + return aType && aType->getString() == "derivation"; +} + +StorePath AttrCursor::forceDerivation() +{ + auto aDrvPath = getAttr(root->state.sDrvPath, true); + auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); + if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + if (!root->state.store->isValidPath(drvPath)) + throw Error("don't know how to recreate store derivation '%s'!", + root->state.store->printStorePath(drvPath)); + } + return drvPath; +} + +} diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh new file mode 100644 index 000000000..e23e45c94 --- /dev/null +++ b/src/libexpr/eval-cache.hh @@ -0,0 +1,123 @@ +#pragma once + +#include "sync.hh" +#include "hash.hh" +#include "eval.hh" + +#include <functional> +#include <variant> + +namespace nix::eval_cache { + +MakeError(CachedEvalError, EvalError); + +struct AttrDb; +class AttrCursor; + +class EvalCache : public std::enable_shared_from_this<EvalCache> +{ + friend class AttrCursor; + + std::shared_ptr<AttrDb> db; + EvalState & state; + typedef std::function<Value *()> RootLoader; + RootLoader rootLoader; + RootValue value; + + Value * getRootValue(); + +public: + + EvalCache( + std::optional<std::reference_wrapper<const Hash>> useCache, + EvalState & state, + RootLoader rootLoader); + + std::shared_ptr<AttrCursor> getRoot(); +}; + +enum AttrType { + Placeholder = 0, + FullAttrs = 1, + String = 2, + Missing = 3, + Misc = 4, + Failed = 5, + Bool = 6, +}; + +struct placeholder_t {}; +struct missing_t {}; +struct misc_t {}; +struct failed_t {}; +typedef uint64_t AttrId; +typedef std::pair<AttrId, Symbol> AttrKey; +typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t; + +typedef std::variant< + std::vector<Symbol>, + string_t, + placeholder_t, + missing_t, + misc_t, + failed_t, + bool + > AttrValue; + +class AttrCursor : public std::enable_shared_from_this<AttrCursor> +{ + friend class EvalCache; + + ref<EvalCache> root; + typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent; + Parent parent; + RootValue _value; + std::optional<std::pair<AttrId, AttrValue>> cachedValue; + + AttrKey getKey(); + + Value & getValue(); + +public: + + AttrCursor( + ref<EvalCache> root, + Parent parent, + Value * value = nullptr, + std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {}); + + std::vector<Symbol> getAttrPath() const; + + std::vector<Symbol> getAttrPath(Symbol name) const; + + std::string getAttrPathStr() const; + + std::string getAttrPathStr(Symbol name) const; + + std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false); + + std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name); + + std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false); + + std::shared_ptr<AttrCursor> getAttr(std::string_view name); + + std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath); + + std::string getString(); + + string_t getStringWithContext(); + + bool getBool(); + + std::vector<Symbol> getAttrs(); + + bool isDerivation(); + + Value & forceValue(); + + /* Force creation of the .drv file in the Nix store. */ + StorePath forceDerivation(); +}; + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c1a9af9b2..d6366050c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -87,6 +87,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con else if (*i == '\n') str << "\\n"; else if (*i == '\r') str << "\\r"; else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; else str << *i; str << "\""; break; @@ -199,6 +200,18 @@ string showType(const Value & v) } +bool Value::isTrivial() const +{ + return + type != tApp + && type != tPrimOpApp + && (type != tThunk + || (dynamic_cast<ExprAttrs *>(thunk.expr) + && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast<ExprLambda *>(thunk.expr)); +} + + #if HAVE_BOEHMGC /* Called when the Boehm GC runs out of memory. */ static void * oomHandler(size_t requested) @@ -333,12 +346,17 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) + , sContentAddressed(symbols.create("__contentAddressed")) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) , sRecurseForDerivations(symbols.create("recurseForDerivations")) + , sDescription(symbols.create("description")) + , sSelf(symbols.create("self")) + , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -353,7 +371,11 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + try { + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + } catch (Error &) { + } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); @@ -365,10 +387,14 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) auto path = r.second; if (store->isInStore(r.second)) { - StorePathSet closure; - store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure); - for (auto & path : closure) - allowedPaths->insert(store->printStorePath(path)); + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(r.second).first, closure); + for (auto & path : closure) + allowedPaths->insert(store->printStorePath(path)); + } catch (InvalidPath &) { + allowedPaths->insert(r.second); + } } else allowedPaths->insert(r.second); } @@ -493,7 +519,7 @@ Value * EvalState::addPrimOp(const string & name, if (arity == 0) { auto vPrimOp = allocValue(); vPrimOp->type = tPrimOp; - vPrimOp->primOp = new PrimOp(primOp, 1, sym); + vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym }; Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); @@ -501,7 +527,7 @@ Value * EvalState::addPrimOp(const string & name, Value * v = allocValue(); v->type = tPrimOp; - v->primOp = new PrimOp(primOp, arity, sym); + v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym }; staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); @@ -509,12 +535,59 @@ Value * EvalState::addPrimOp(const string & name, } +Value * EvalState::addPrimOp(PrimOp && primOp) +{ + /* Hack to make constants lazy: turn them into a application of + the primop to a dummy value. */ + if (primOp.arity == 0) { + primOp.arity = 1; + auto vPrimOp = allocValue(); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = new PrimOp(std::move(primOp)); + Value v; + mkApp(v, *vPrimOp, *vPrimOp); + return addConstant(primOp.name, v); + } + + Symbol envName = primOp.name; + if (hasPrefix(primOp.name, "__")) + primOp.name = symbols.create(std::string(primOp.name, 2)); + + Value * v = allocValue(); + v->type = tPrimOp; + v->primOp = new PrimOp(std::move(primOp)); + staticBaseEnv.vars[envName] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); + return v; +} + + Value & EvalState::getBuiltin(const string & name) { return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; } +std::optional<EvalState::Doc> EvalState::getDoc(Value & v) +{ + if (v.type == tPrimOp || v.type == tPrimOpApp) { + auto v2 = &v; + while (v2->type == tPrimOpApp) + v2 = v2->primOpApp.left; + if (v2->primOp->doc) + return Doc { + .pos = noPos, + .name = v2->primOp->name, + .arity = v2->primOp->arity, + .args = v2->primOp->args, + .doc = v2->primOp->doc, + }; + } + return {}; +} + + /* Every "format" object (even temporary) takes up a few hundred bytes of stack space, which is a real killer in the recursive evaluator. So here are some helper functions for throwing @@ -782,7 +855,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v) +void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -811,6 +884,11 @@ void EvalState::evalFile(const Path & path_, Value & v) fileParseCache[path2] = e; try { + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast<ExprAttrs *>(e))) + throw Error("file '%s' must be an attribute set", path); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", path2); @@ -1236,10 +1314,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po try { lambda.body->eval(*this, env2, v); } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambdaction")); + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); addErrorTrace(e, pos, "from call site%s", ""); throw; } @@ -1276,14 +1354,25 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } Value * actualArgs = allocValue(); - mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); - - for (auto & i : fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i.name); - if (j != args.end()) - actualArgs->attrs->push_back(*j); - else if (!i.def) - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size())); + + if (fun.lambda.fun->formals->ellipsis) { + // If the formals have an ellipsis (eg the function accepts extra args) pass + // all available automatic arguments (which includes arguments specified on + // the command line via --arg/--argstr) + for (auto& v : args) { + actualArgs->attrs->push_back(v); + } + } else { + // Otherwise, only pass the arguments that the function accepts + for (auto & i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args.find(i.name); + if (j != args.end()) { + actualArgs->attrs->push_back(*j); + } else if (!i.def) { + throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + } + } } actualArgs->attrs->sort(); @@ -1586,6 +1675,18 @@ string EvalState::forceString(Value & v, const Pos & pos) } +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<string, string> decodeContext(std::string_view s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; + } else + return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; +} + + void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -1594,6 +1695,17 @@ void copyContext(const Value & v, PathSet & context) } +std::vector<std::pair<Path, std::string>> Value::getContext() +{ + std::vector<std::pair<Path, std::string>> res; + assert(type == tString); + if (string.context) + for (const char * * p = string.context; *p; ++p) + res.push_back(decodeContext(*p)); + return res; +} + + string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { string s = forceString(v, pos); @@ -1969,7 +2081,7 @@ Strings EvalSettings::getDefaultNixPath() EvalSettings evalSettings; -static GlobalConfig::Register r1(&evalSettings); +static GlobalConfig::Register rEvalSettings(&evalSettings); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0d52a7f63..0e1f61baa 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -4,13 +4,12 @@ #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" -#include "hash.hh" #include "config.hh" -#include <regex> #include <map> #include <optional> #include <unordered_map> +#include <mutex> namespace nix { @@ -30,8 +29,8 @@ struct PrimOp PrimOpFun fun; size_t arity; Symbol name; - PrimOp(PrimOpFun fun, size_t arity, Symbol name) - : fun(fun), arity(arity), name(name) { } + std::vector<std::string> args; + const char * doc = nullptr; }; @@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath; void initGC(); +struct RegexCache; + +std::shared_ptr<RegexCache> makeRegexCache(); + + class EvalState { public: @@ -74,8 +78,10 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sContentAddressed, sOutputHash, sOutputHashAlgo, sOutputHashMode, - sRecurseForDerivations; + sRecurseForDerivations, + sDescription, sSelf, sEpsilon; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -90,6 +96,7 @@ public: const ref<Store> store; + private: SrcToStore srcToStore; @@ -117,7 +124,7 @@ private: std::unordered_map<Path, Path> resolvedPaths; /* Cache used by prim_match(). */ - std::unordered_map<std::string, std::regex> regexCache; + std::shared_ptr<RegexCache> regexCache; public: @@ -152,8 +159,9 @@ public: Expr * parseStdin(); /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); + form. Optionally enforce that the top-level expression is + trivial (i.e. doesn't require arbitrary computation). */ + void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); void resetFileCache(); @@ -238,10 +246,23 @@ private: Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); + Value * addPrimOp(PrimOp && primOp); + public: Value & getBuiltin(const string & name); + struct Doc + { + Pos pos; + std::optional<Symbol> name; + size_t arity; + std::vector<std::string> args; + const char * doc; + }; + + std::optional<Doc> getDoc(Value & v); + private: inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); @@ -330,7 +351,7 @@ string showType(const Value & v); /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<string, string> decodeContext(const string & s); +std::pair<string, string> decodeContext(std::string_view s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); @@ -353,24 +374,60 @@ struct EvalSettings : Config Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "Whether builtin functions that allow executing native code should be enabled."}; - Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for <...> file references."}; + Setting<Strings> nixPath{ + this, getDefaultNixPath(), "nix-path", + "List of directories to be searched for `<...>` file references."}; - Setting<bool> restrictEval{this, false, "restrict-eval", - "Whether to restrict file system access to paths in $NIX_PATH, " - "and network access to the URI prefixes listed in 'allowed-uris'."}; + Setting<bool> restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of the Nix search path (as set via the `NIX_PATH` + environment variable or the `-I` option), or to URIs outside of + `allowed-uri`. The default is `false`. + )"}; Setting<bool> pureEval{this, false, "pure-eval", "Whether to restrict file system and network access to files specified by cryptographic hash."}; - Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", - "Whether the evaluator allows importing the result of a derivation."}; + Setting<bool> enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows you to `import` from a derivation, allowing + building at evaluation time. With this option set to false, Nix will + throw an error when evaluating an expression that uses this feature, + allowing users to ensure their evaluation will not require any + builds to take place. + )"}; Setting<Strings> allowedUris{this, {}, "allowed-uris", - "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + )"}; Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting<bool> useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix new file mode 100644 index 000000000..932ac5e90 --- /dev/null +++ b/src/libexpr/flake/call-flake.nix @@ -0,0 +1,56 @@ +lockFileStr: rootSrc: rootSubdir: + +let + + lockFile = builtins.fromJSON lockFileStr; + + allNodes = + builtins.mapAttrs + (key: node: + let + + sourceInfo = + if key == lockFile.root + then rootSrc + else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); + + subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; + + flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + + inputs = builtins.mapAttrs + (inputName: inputSpec: allNodes.${resolveInput inputSpec}) + (node.inputs or {}); + + # Resolve a input spec into a node name. An input spec is + # either a node name, or a 'follows' path from the root + # node. + resolveInput = inputSpec: + if builtins.isList inputSpec + then getInputByPath lockFile.root inputSpec + else inputSpec; + + # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the + # root node, returning the final node. + getInputByPath = nodeName: path: + if path == [] + then nodeName + else + getInputByPath + # Since this could be a 'follows' input, call resolveInput. + (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) + (builtins.tail path); + + outputs = flake.outputs (inputs // { self = result; }); + + result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }; + in + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo + ) + lockFile.nodes; + +in allNodes.${lockFile.root} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..bae4d65e5 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,608 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "finally.hh" + +namespace nix { + +using namespace flake; + +namespace flake { + +typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake; +typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache; + +static std::optional<FetchedFlake> lookupInFlakeCache( + const FlakeCache & flakeCache, + const FlakeRef & flakeRef) +{ + // FIXME: inefficient. + for (auto & i : flakeCache) { + if (flakeRef == i.first) { + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i.first, i.second.second); + return i.second; + } + } + + return std::nullopt; +} + +static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto fetched = lookupInFlakeCache(flakeCache, originalRef); + FlakeRef resolvedRef = originalRef; + + if (!fetched) { + if (originalRef.input.isDirect()) { + fetched.emplace(originalRef.fetchTree(state.store)); + } else { + if (allowLookup) { + resolvedRef = originalRef.resolve(state.store); + auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); + if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); + flakeCache.push_back({resolvedRef, *fetchedResolved}); + fetched.emplace(*fetchedResolved); + } + else { + throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); + } + } + flakeCache.push_back({originalRef, *fetched}); + } + + auto [tree, lockedRef] = *fetched; + + debug("got tree '%s' from '%s'", + state.store->printStorePath(tree.storePath), lockedRef); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + + return {std::move(tree), resolvedRef, lockedRef}; +} + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + if (value.type == tThunk && value.isTrivial()) + state.forceValue(value, pos); + if (value.type != type) + throw Error("expected %s but got %s at %s", + showType(type), showType(value.type), pos); +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos); + +static FlakeInput parseFlakeInput(EvalState & state, + const std::string & inputName, Value * value, const Pos & pos) +{ + expectType(state, tAttrs, *value, pos); + + FlakeInput input; + + auto sInputs = state.symbols.create("inputs"); + auto sUrl = state.symbols.create("url"); + auto sFlake = state.symbols.create("flake"); + auto sFollows = state.symbols.create("follows"); + + fetchers::Attrs attrs; + std::optional<std::string> url; + + for (nix::Attr attr : *(value->attrs)) { + try { + if (attr.name == sUrl) { + expectType(state, tString, *attr.value, *attr.pos); + url = attr.value->string.s; + attrs.emplace("url", *url); + } else if (attr.name == sFlake) { + expectType(state, tBool, *attr.value, *attr.pos); + input.isFlake = attr.value->boolean; + } else if (attr.name == sInputs) { + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); + } else if (attr.name == sFollows) { + expectType(state, tString, *attr.value, *attr.pos); + input.follows = parseInputPath(attr.value->string.s); + } else { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else + throw TypeError("flake input attribute '%s' is %s while a string is expected", + attr.name, showType(*attr.value)); + } + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); + throw; + } + } + + if (attrs.count("type")) + try { + input.ref = FlakeRef::fromAttrs(attrs); + } catch (Error & e) { + e.addTrace(pos, hintfmt("in flake input")); + throw; + } + else { + attrs.erase("url"); + if (!attrs.empty()) + throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); + if (url) + input.ref = parseFlakeRef(*url, {}, true); + } + + if (!input.follows && !input.ref) + input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + + return input; +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos) +{ + std::map<FlakeId, FlakeInput> inputs; + + expectType(state, tAttrs, *value, pos); + + for (nix::Attr & inputAttr : *(*value).attrs) { + inputs.emplace(inputAttr.name, + parseFlakeInput(state, + inputAttr.name, + inputAttr.value, + *inputAttr.pos)); + } + + return inputs; +} + +static Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, originalRef, allowLookup, flakeCache); + + // Guard against symlink attacks. + auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); + if (!isInDir(flakeFile, sourceInfo.actualPath)) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", + lockedRef, state.store->printStorePath(sourceInfo.storePath)); + + Flake flake { + .originalRef = originalRef, + .resolvedRef = resolvedRef, + .lockedRef = lockedRef, + .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo)) + }; + + if (!pathExists(flakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + + Value vInfo; + state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + + expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + + auto sEdition = state.symbols.create("edition"); // FIXME: remove soon + + if (vInfo.attrs->get(sEdition)) + warn("flake '%s' has deprecated attribute 'edition'", lockedRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) { + expectType(state, tString, *description->value, *description->pos); + flake.description = description->value->string.s; + } + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + expectType(state, tLambda, *outputs->value, *outputs->pos); + + if (outputs->value->lambda.fun->matchAttrs) { + for (auto & formal : outputs->value->lambda.fun->formals->formals) { + if (formal.name != state.sSelf) + flake.inputs.emplace(formal.name, FlakeInput { + .ref = parseFlakeRef(formal.name) + }); + } + } + + } else + throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEdition && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + lockedRef, attr.name, *attr.pos); + } + + return flake; +} + +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +{ + FlakeCache flakeCache; + return getFlake(state, originalRef, allowLookup, flakeCache); +} + +/* Compute an in-memory lock file for the specified top-level flake, + and optionally write it to file, if the flake is writable. */ +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags) +{ + settings.requireExperimentalFeature("flakes"); + + FlakeCache flakeCache; + + auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); + + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + + debug("old lock file: %s", oldLockFile); + + // FIXME: check whether all overrides are used. + std::map<InputPath, FlakeInput> overrides; + std::set<InputPath> overridesUsed, updatesUsed; + + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + + LockFile newLockFile; + + std::vector<FlakeRef> parents; + + std::function<void( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode)> + computeLocks; + + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode) + { + debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + // FIXME: check this + for (auto & [id, input] : flake.inputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + auto inputPathS = printInputPath(inputPath); + debug("computing input '%s'", inputPathS); + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) overridesUsed.insert(inputPath); + auto & input = hasOverride ? i->second : input2; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputPath target; + if (hasOverride || input.absolute) + /* 'follows' from an override is relative to the + root of the graph. */ + target = *input.follows; + else { + /* Otherwise, it's relative to the current flake. */ + target = inputPathPrefix; + for (auto & i : *input.follows) target.push_back(i); + } + debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } + + assert(input.ref); + + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr<LockedNode> oldLock; + + updatesUsed.insert(inputPath); + + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; + + if (oldLock + && oldLock->originalRef == *input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared<LockedNode>( + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; + + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .follows = *follows, + .absolute = true + }); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputPathS); + + if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared<LockedNode>( + inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast<const Node>(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + } + + else { + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, lockFlags.useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared<LockedNode>(lockedRef, *input.ref, false)); + } + } + } + }; + + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + printInputPath(i.first), i.second); + + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + + /* Check 'follows' inputs. */ + newLockFile.check(); + + debug("new lock file: %s", newLockFile); + + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { + + auto diff = LockFile::diff(oldLockFile, newLockFile); + + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input.getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + + auto path = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(path); + + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", path); + else + warn("updating lock file '%s':\n%s", path, s); + } else + warn("creating lock file '%s'", path); + + newLockFile.write(path); + + topRef.input.markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", diff)) + : std::nullopt); + + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); + + if (lockFlags.commitLockFile && + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input.isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); + } else + warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); + } + + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; +} + +void callFlake(EvalState & state, + const LockedFlake & lockedFlake, + Value & vRes) +{ + auto vLocks = state.allocValue(); + auto vRootSrc = state.allocValue(); + auto vRootSubdir = state.allocValue(); + auto vTmp1 = state.allocValue(); + auto vTmp2 = state.allocValue(); + + mkString(*vLocks, lockedFlake.lockFile.to_string()); + + emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); + + mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + + static RootValue vCallFlake = nullptr; + + if (!vCallFlake) { + vCallFlake = allocRootValue(state.allocValue()); + state.eval(state.parseExprFromString( + #include "call-flake.nix.gen.hh" + , "/"), **vCallFlake); + } + + state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); + state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto flakeRefS = state.forceStringNoCtx(*args[0], pos); + auto flakeRef = parseFlakeRef(flakeRefS, {}, true); + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) + throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + + callFlake(state, + lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowMutable = !evalSettings.pureEval, + }), + v); +} + +static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes"); + +} + +Fingerprint LockedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lock file + // and we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%d;%d;%s", + flake.sourceInfo->storePath.to_string(), + flake.lockedRef.input.getRevCount().value_or(0), + flake.lockedRef.input.getLastModified().value_or(0), + lockFile)); +} + +Flake::~Flake() { } + +} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh new file mode 100644 index 000000000..69c779af8 --- /dev/null +++ b/src/libexpr/flake/flake.hh @@ -0,0 +1,110 @@ +#pragma once + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" +#include "value.hh" + +namespace nix { + +class EvalState; + +namespace fetchers { struct Tree; } + +namespace flake { + +struct FlakeInput; + +typedef std::map<FlakeId, FlakeInput> FlakeInputs; + +struct FlakeInput +{ + std::optional<FlakeRef> ref; + bool isFlake = true; + std::optional<InputPath> follows; + bool absolute = false; // whether 'follows' is relative to the flake root + FlakeInputs overrides; +}; + +struct Flake +{ + FlakeRef originalRef; + FlakeRef resolvedRef; + FlakeRef lockedRef; + std::optional<std::string> description; + std::shared_ptr<const fetchers::Tree> sourceInfo; + FlakeInputs inputs; + ~Flake(); +}; + +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); + +/* Fingerprint of a locked flake; used as a cache key. */ +typedef Hash Fingerprint; + +struct LockedFlake +{ + Flake flake; + LockFile lockFile; + + Fingerprint getFingerprint() const; +}; + +struct LockFlags +{ + /* Whether to ignore the existing lock file, creating a new one + from scratch. */ + bool recreateLockFile = false; + + /* Whether to update the lock file at all. If set to false, if any + change to the lock file is needed (e.g. when an input has been + added to flake.nix), you get a fatal error. */ + bool updateLockFile = true; + + /* Whether to write the lock file to disk. If set to true, if the + any changes to the lock file are needed and the flake is not + writable (i.e. is not a local Git working tree or similar), you + get a fatal error. If set to false, Nix will use the modified + lock file in memory only, without writing it to disk. */ + bool writeLockFile = true; + + /* Whether to use the registries to lookup indirect flake + references like 'nixpkgs'. */ + bool useRegistries = true; + + /* Whether mutable flake references (i.e. those without a Git + revision or similar) without a corresponding lock are + allowed. Mutable flake references with a lock are always + allowed. */ + bool allowMutable = true; + + /* Whether to commit changes to flake.lock. */ + bool commitLockFile = false; + + /* Flake inputs to be overriden. */ + std::map<InputPath, FlakeRef> inputOverrides; + + /* Flake inputs to be updated. This means that any existing lock + for those inputs will be ignored. */ + std::set<InputPath> inputUpdates; +}; + +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & flakeRef, + const LockFlags & lockFlags); + +void callFlake( + EvalState & state, + const LockedFlake & lockedFlake, + Value & v); + +} + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + const fetchers::Input & input, + Value & v, bool emptyRevFallback = false); + +} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc new file mode 100644 index 000000000..833e8a776 --- /dev/null +++ b/src/libexpr/flake/flakeref.cc @@ -0,0 +1,206 @@ +#include "flakeref.hh" +#include "store-api.hh" +#include "url.hh" +#include "url-parts.hh" +#include "fetchers.hh" +#include "registry.hh" + +namespace nix { + +#if 0 +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; +#endif + +std::string FlakeRef::to_string() const +{ + std::map<std::string, std::string> extraQuery; + if (subdir != "") + extraQuery.insert_or_assign("dir", subdir); + return input.toURLString(extraQuery); +} + +fetchers::Attrs FlakeRef::toAttrs() const +{ + auto attrs = input.toAttrs(); + if (subdir != "") + attrs.emplace("dir", subdir); + return attrs; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::operator ==(const FlakeRef & other) const +{ + return input == other.input && subdir == other.subdir; +} + +FlakeRef FlakeRef::resolve(ref<Store> store) const +{ + auto [input2, extraAttrs] = lookupInRegistries(store, input); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); +} + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir, bool allowMissing) +{ + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing); + if (fragment != "") + throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); + return flakeRef; +} + +std::optional<FlakeRef> maybeParseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir) +{ + try { + return parseFlakeRef(url, baseDir); + } catch (Error &) { + return {}; + } +} + +std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir, bool allowMissing) +{ + using namespace fetchers; + + static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; + + static std::regex pathUrlRegex( + "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + static std::regex flakeRegex( + "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:<flake-id>?ref=<ref>&rev=<rev>'. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + std::string(match[1]), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), ""), + percentDecode(std::string(match[6]))); + } + + else if (std::regex_match(url, match, pathUrlRegex)) { + std::string path = match[1]; + std::string fragment = percentDecode(std::string(match[3])); + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir, true); + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = decodeQuery(match[2]), + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + auto query = decodeQuery(match[2]); + path = canonPath(path + "/" + get(query, "dir").value_or("")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); + } + + else { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } +} + +std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir) +{ + try { + return parseFlakeRefWithFragment(url, baseDir); + } catch (Error & e) { + return {}; + } +} + +FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("dir"); + return FlakeRef( + fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); +} + +std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const +{ + auto [tree, lockedInput] = input.fetch(store); + return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; +} + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh new file mode 100644 index 000000000..f4eb825a6 --- /dev/null +++ b/src/libexpr/flake/flakeref.hh @@ -0,0 +1,53 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "fetchers.hh" + +#include <variant> + +namespace nix { + +class Store; + +typedef std::string FlakeId; + +struct FlakeRef +{ + fetchers::Input input; + + Path subdir; + + bool operator==(const FlakeRef & other) const; + + FlakeRef(fetchers::Input && input, const Path & subdir) + : input(std::move(input)), subdir(subdir) + { } + + // FIXME: change to operator <<. + std::string to_string() const; + + fetchers::Attrs toAttrs() const; + + FlakeRef resolve(ref<Store> store) const; + + static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + + std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false); + +std::optional<FlakeRef> maybeParseFlake( + const std::string & url, const std::optional<Path> & baseDir = {}); + +std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false); + +std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir = {}); + +} diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc new file mode 100644 index 000000000..bb46e1bb4 --- /dev/null +++ b/src/libexpr/flake/lockfile.cc @@ -0,0 +1,339 @@ +#include "lockfile.hh" +#include "store-api.hh" +#include "url-parts.hh" + +#include <nlohmann/json.hpp> + +namespace nix::flake { + +FlakeRef getFlakeRef( + const nlohmann::json & json, + const char * attr, + const char * info) +{ + auto i = json.find(attr); + if (i != json.end()) { + auto attrs = fetchers::jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : fetchers::jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(attrs); + } + + throw Error("attribute '%s' missing in lock file", attr); +} + +LockedNode::LockedNode(const nlohmann::json & json) + : lockedRef(getFlakeRef(json, "locked", "info")) + , originalRef(getFlakeRef(json, "original", nullptr)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) +{ + if (!lockedRef.input.isImmutable()) + throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); +} + +StorePath LockedNode::computeStorePath(Store & store) const +{ + return lockedRef.input.computeStorePath(store); +} + +std::shared_ptr<Node> LockFile::findInput(const InputPath & path) +{ + auto pos = root; + + if (!pos) return {}; + + for (auto & elem : path) { + if (auto i = get(pos->inputs, elem)) { + if (auto node = std::get_if<0>(&*i)) + pos = *node; + else if (auto follows = std::get_if<1>(&*i)) { + pos = findInput(*follows); + if (!pos) return {}; + } + } else + return {}; + } + + return pos; +} + +LockFile::LockFile(const nlohmann::json & json, const Path & path) +{ + auto version = json.value("version", 0); + if (version < 5 || version > 7) + throw Error("lock file '%s' has unsupported version %d", path, version); + + std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap; + + std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs; + + getInputs = [&](Node & node, const nlohmann::json & jsonNode) + { + if (jsonNode.find("inputs") == jsonNode.end()) return; + for (auto & i : jsonNode["inputs"].items()) { + if (i.value().is_array()) { + InputPath path; + for (auto & j : i.value()) + path.push_back(j); + node.inputs.insert_or_assign(i.key(), path); + } else { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto jsonNode2 = json["nodes"][inputKey]; + auto input = std::make_shared<LockedNode>(jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, jsonNode2); + } + if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second)) + node.inputs.insert_or_assign(i.key(), child); + else + // FIXME: replace by follows node + throw Error("lock file contains cycle to root node"); + } + } + }; + + std::string rootKey = json["root"]; + nodeMap.insert_or_assign(rootKey, root); + getInputs(*root, json["nodes"][rootKey]); + + // FIXME: check that there are no cycles in version >= 7. Cycles + // between inputs are only possible using 'follows' indirections. + // Once we drop support for version <= 6, we can simplify the code + // a bit since we don't need to worry about cycles. +} + +nlohmann::json LockFile::toJson() const +{ + nlohmann::json nodes; + std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys; + std::unordered_set<std::string> keys; + + std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode; + + dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string + { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2; ; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } + } + } + + nodeKeys.insert_or_assign(node, key); + + auto n = nlohmann::json::object(); + + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) { + if (auto child = std::get_if<0>(&i.second)) { + inputs[i.first] = dumpNode(i.first, *child); + } else if (auto follows = std::get_if<1>(&i.second)) { + auto arr = nlohmann::json::array(); + for (auto & x : *follows) + arr.push_back(x); + inputs[i.first] = std::move(arr); + } + } + n["inputs"] = std::move(inputs); + } + + if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) { + n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); + if (!lockedNode->isFlake) n["flake"] = false; + } + + nodes[key] = std::move(n); + + return key; + }; + + nlohmann::json json; + json["version"] = 7; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); + + return json; +} + +std::string LockFile::to_string() const +{ + return toJson().dump(2); +} + +LockFile LockFile::read(const Path & path) +{ + if (!pathExists(path)) return LockFile(); + return LockFile(nlohmann::json::parse(readFile(path)), path); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(2); + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +bool LockFile::isImmutable() const +{ + std::unordered_set<std::shared_ptr<const Node>> nodes; + + std::function<void(std::shared_ptr<const Node> node)> visit; + + visit = [&](std::shared_ptr<const Node> node) + { + if (!nodes.insert(node).second) return; + for (auto & i : node->inputs) + if (auto child = std::get_if<0>(&i.second)) + visit(*child); + }; + + visit(root); + + for (auto & i : nodes) { + if (i == root) continue; + auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); + if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; + } + + return true; +} + +bool LockFile::operator ==(const LockFile & other) const +{ + // FIXME: slow + return toJson() == other.toJson(); +} + +InputPath parseInputPath(std::string_view s) +{ + InputPath path; + + for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) { + if (!std::regex_match(elem, flakeIdRegex)) + throw UsageError("invalid flake input path element '%s'", elem); + path.push_back(elem); + } + + return path; +} + +std::map<InputPath, Node::Edge> LockFile::getAllInputs() const +{ + std::unordered_set<std::shared_ptr<Node>> done; + std::map<InputPath, Node::Edge> res; + + std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse; + + recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node) + { + if (!done.insert(node).second) return; + + for (auto &[id, input] : node->inputs) { + auto inputPath(prefix); + inputPath.push_back(id); + res.emplace(inputPath, input); + if (auto child = std::get_if<0>(&input)) + recurse(inputPath, *child); + } + }; + + recurse({}, root); + + return res; +} + +std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge) +{ + if (auto node = std::get_if<0>(&edge)) + stream << "'" << (*node)->lockedRef << "'"; + else if (auto follows = std::get_if<1>(&edge)) + stream << fmt("follows '%s'", printInputPath(*follows)); + return stream; +} + +static bool equals(const Node::Edge & e1, const Node::Edge & e2) +{ + if (auto n1 = std::get_if<0>(&e1)) + if (auto n2 = std::get_if<0>(&e2)) + return (*n1)->lockedRef == (*n2)->lockedRef; + if (auto f1 = std::get_if<1>(&e1)) + if (auto f2 = std::get_if<1>(&e2)) + return *f1 == *f2; + return false; +} + +std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks) +{ + auto oldFlat = oldLocks.getAllInputs(); + auto newFlat = newLocks.getAllInputs(); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + std::string res; + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("* Removed '%s'\n", printInputPath(i->first)); + ++i; + } else { + if (!equals(i->second, j->second)) { + res += fmt("* Updated '%s': %s -> %s\n", + printInputPath(i->first), + i->second, + j->second); + } + ++i; + ++j; + } + } + + return res; +} + +void LockFile::check() +{ + auto inputs = getAllInputs(); + + for (auto & [inputPath, input] : inputs) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !get(inputs, *follows)) + throw Error("input '%s' follows a non-existent input '%s'", + printInputPath(inputPath), + printInputPath(*follows)); + } + } +} + +void check(); + +std::string printInputPath(const InputPath & path) +{ + return concatStringsSep("/", path); +} + +} diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh new file mode 100644 index 000000000..627794d8c --- /dev/null +++ b/src/libexpr/flake/lockfile.hh @@ -0,0 +1,83 @@ +#pragma once + +#include "flakeref.hh" + +#include <nlohmann/json_fwd.hpp> + +namespace nix { +class Store; +class StorePath; +} + +namespace nix::flake { + +typedef std::vector<FlakeId> InputPath; + +struct LockedNode; + +/* A node in the lock file. It has outgoing edges to other nodes (its + inputs). Only the root node has this type; all other nodes have + type LockedNode. */ +struct Node : std::enable_shared_from_this<Node> +{ + typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge; + + std::map<FlakeId, Edge> inputs; + + virtual ~Node() { } +}; + +/* A non-root node in the lock file. */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + bool isFlake = true; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + bool isFlake = true) + : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) + { } + + LockedNode(const nlohmann::json & json); + + StorePath computeStorePath(Store & store) const; +}; + +struct LockFile +{ + std::shared_ptr<Node> root = std::make_shared<Node>(); + + LockFile() {}; + LockFile(const nlohmann::json & json, const Path & path); + + nlohmann::json toJson() const; + + std::string to_string() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; + + bool isImmutable() const; + + bool operator ==(const LockFile & other) const; + + std::shared_ptr<Node> findInput(const InputPath & path); + + std::map<InputPath, Node::Edge> getAllInputs() const; + + static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); + + /* Check that every 'follows' input target exists. */ + void check(); +}; + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + +InputPath parseInputPath(std::string_view s); + +std::string printInputPath(const InputPath & path); + +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 9055f59a1..91916e8bf 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -38,8 +38,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat auto i = drv.outputs.find(outputName); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); + auto & [outputName, output] = *i; - outPath = store->printStorePath(i->second.path); + auto optStorePath = output.path(*store, drv.name, outputName); + if (optStorePath) + outPath = store->printStorePath(*optStorePath); } @@ -77,12 +80,15 @@ string DrvInfo::queryDrvPath() const string DrvInfo::queryOutPath() const { - if (outPath == "" && attrs) { + if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); PathSet context; - outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + if (i != attrs->end()) + outPath = state->coerceToPath(*i->pos, *i->value, context); } - return outPath; + if (!outPath) + throw UnimplementedError("CA derivations are not yet supported"); + return *outPath; } diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index d7860fc6a..29bb6a660 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -20,7 +20,7 @@ private: mutable string name; mutable string system; mutable string drvPath; - mutable string outPath; + mutable std::optional<string> outPath; mutable string outputName; Outputs outputs; diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 76e1a26bf..9ca5ac86d 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -115,6 +115,14 @@ public: { return handle_value<void(Value&, const char*)>(mkString, val.c_str()); } +#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 + bool binary(binary_t&) + { + // This function ought to be unreachable + assert(false); + return true; + } +#endif bool start_object(std::size_t len) { diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9ed39e745..687a8ccda 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr @@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh +$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) + +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh + +$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 878f06c96..a4c84c526 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -614,8 +614,7 @@ Path resolveExprPath(Path path) // Basic cycle/depth limit to avoid infinite loops. if (++followCount >= maxFollow) throw Error("too many symbolic links encountered while traversing the path '%s'", path); - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%s'", path); + st = lstat(path); if (!S_ISLNK(st.st_mode)) break; path = absPath(readLink(path), dirOf(path)); } @@ -719,7 +718,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ .name = "Entry download", diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dec917b38..2b304aab0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -30,18 +30,6 @@ namespace nix { *************************************************************/ -/* Decode a context string ‘!<name>!<path>’ into a pair <path, - name>. */ -std::pair<string, string> decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), ""); -} - - InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} @@ -56,16 +44,6 @@ void EvalState::realiseContext(const PathSet & context) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); - - /* Add the output of this derivation to the allowed - paths. */ - if (allowedPaths) { - auto drv = store->derivationFromPath(ctx); - DerivationOutputs::iterator i = drv.outputs.find(outputName); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); - allowedPaths->insert(store->printStorePath(i->second.path)); - } } } @@ -77,19 +55,61 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(drvs); + + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + for (auto & [drvPath, outputs] : drvs) { + auto outputPaths = store->queryDerivationOutputMap(drvPath); + for (auto & outputName : outputs) { + if (outputPaths.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store->printStorePath(drvPath), outputName); + allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); + } + } + } } +/* Add and attribute to the given attribute map from the output name to + the output path, or a placeholder. + + Where possible the path is used, but for floating CA derivations we + may not know it. For sake of determinism we always assume we don't + and instead put in a place holder. In either case, however, the + string context will contain the drv path and output name, so + downstream derivations will have the proper dependency, and in + addition, before building, the placeholder will be rewritten to be + the actual path. + + The 'drv' and 'drvPath' outputs must correspond. */ +static void mkOutputString(EvalState & state, Value & v, + const StorePath & drvPath, const BasicDerivation & drv, + std::pair<string, DerivationOutput> o) +{ + auto optOutputPath = o.second.path(*state.store, drv.name, o.first); + mkString( + *state.allocAttr(v, state.symbols.create(o.first)), + optOutputPath + ? state.store->printStorePath(*optOutputPath) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + /* FIXME: we need to depend on the basic derivation, not + derivation */ + : downstreamPlaceholder(*state.store, drvPath, o.first), + {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); +} /* Load and evaluate an expression from path specified by the argument. */ -static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); + Path path = state.coerceToPath(pos, vPath, context); try { state.realiseContext(context); @@ -103,8 +123,18 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { - Derivation drv = readDerivation(*state.store, realPath); + auto isValidDerivationInStore = [&]() -> std::optional<StorePath> { + if (!state.store->isStorePath(path)) + return std::nullopt; + auto storePath = state.store->parseStorePath(path); + if (!(state.store->isValidPath(storePath) && isDerivation(path))) + return std::nullopt; + return storePath; + }; + + if (auto optStorePath = isValidDerivationInStore()) { + auto storePath = *optStorePath; + Derivation drv = state.store->readDerivation(storePath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); @@ -117,8 +147,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args unsigned int outputs_index = 0; for (const auto & o : drv.outputs) { - v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); + mkOutputString(state, w, storePath, drv, o); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } @@ -136,17 +165,18 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args mkApp(v, **fun, w); state.forceAttrs(v, pos); } else { - state.forceAttrs(*args[0]); - if (args[0]->attrs->empty()) + if (!vScope) state.evalFile(realPath, v); else { - Env * env = &state.allocEnv(args[0]->attrs->size()); + state.forceAttrs(*vScope); + + Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; StaticEnv staticEnv(false, &state.staticBaseEnv); unsigned int displ = 0; - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *vScope->attrs) { staticEnv.vars[attr.name] = displ; env->values[displ++] = attr.value; } @@ -159,6 +189,76 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } } +static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { + .name = "scopedImport", + .arity = 2, + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + import(state, pos, *args[1], args[0], v); + } +}); + +static RegisterPrimOp primop_import({ + .name = "import", + .args = {"path"}, + .doc = R"( + Load, parse and return the Nix expression in the file *path*. If + *path* is a directory, the file ` default.nix ` in that directory + is loaded. Evaluation aborts if the file doesn’t exist or contains + an incorrect Nix expression. `import` implements Nix’s module + system: you can put any Nix expression (such as a set or a + function) in a separate file, and use it from Nix expressions in + other files. + + > **Note** + > + > Unlike some languages, `import` is a regular function in Nix. + > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*) + > are [normal path values](language-values.md). + + A Nix expression loaded by `import` must not contain any *free + variables* (identifiers that are not defined in the Nix expression + itself and are not built-in). Therefore, it cannot refer to + variables that are in scope at the call site. For instance, if you + have a calling expression + + ```nix + rec { + x = 123; + y = import ./foo.nix; + } + ``` + + then the following `foo.nix` will give an error: + + ```nix + x + 456 + ``` + + since `x` is not in scope in `foo.nix`. If you want `x` to be + available in `foo.nix`, you should pass it as a function argument: + + ```nix + rec { + x = 123; + y = import ./foo.nix x; + } + ``` + + and + + ```nix + x: x + 456 + ``` + + (The function argument doesn’t have to be called `x` in `foo.nix`; + any name would work.) + )", + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + import(state, pos, *args[0], nullptr, v); + } +}); /* Want reasonable symbol names, so extern C */ /* !!! Should we pass the Pos or the file name too? */ @@ -278,6 +378,16 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, state.symbols.create(t)); } +static RegisterPrimOp primop_typeOf({ + .name = "__typeOf", + .args = {"e"}, + .doc = R"( + Return a string representing the type of the value *e*, namely + `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`, + `"list"`, `"lambda"` or `"float"`. + )", + .fun = prim_typeOf, +}); /* Determine whether the argument is the null value. */ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -286,6 +396,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tNull); } +static RegisterPrimOp primop_isNull({ + .name = "isNull", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to `null`, and `false` otherwise. + + > **Warning** + > + > This function is *deprecated*; just write `e == null` instead. + )", + .fun = prim_isNull, +}); /* Determine whether the argument is a function. */ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -305,6 +427,14 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, mkBool(v, res); } +static RegisterPrimOp primop_isFunction({ + .name = "__isFunction", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a function, and `false` otherwise. + )", + .fun = prim_isFunction, +}); /* Determine whether the argument is an integer. */ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -313,6 +443,15 @@ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value mkBool(v, args[0]->type == tInt); } +static RegisterPrimOp primop_isInt({ + .name = "__isInt", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to an integer, and `false` otherwise. + )", + .fun = prim_isInt, +}); + /* Determine whether the argument is a float. */ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -320,6 +459,15 @@ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Val mkBool(v, args[0]->type == tFloat); } +static RegisterPrimOp primop_isFloat({ + .name = "__isFloat", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a float, and `false` otherwise. + )", + .fun = prim_isFloat, +}); + /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -327,6 +475,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va mkBool(v, args[0]->type == tString); } +static RegisterPrimOp primop_isString({ + .name = "__isString", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a string, and `false` otherwise. + )", + .fun = prim_isString, +}); /* Determine whether the argument is a Boolean. */ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -335,6 +491,15 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tBool); } +static RegisterPrimOp primop_isBool({ + .name = "__isBool", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a bool, and `false` otherwise. + )", + .fun = prim_isBool, +}); + /* Determine whether the argument is a path. */ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -342,6 +507,15 @@ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tPath); } +static RegisterPrimOp primop_isPath({ + .name = "__isPath", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a path, and `false` otherwise. + )", + .fun = prim_isPath, +}); + struct CompareValues { bool operator () (const Value * v1, const Value * v2) const @@ -447,22 +621,43 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar v.listElems()[n++] = i; } +static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { + .name = "__genericClosure", + .arity = 1, + .fun = prim_genericClosure, +}); -static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - throw Abort("evaluation aborted with the following error message: '%1%'", s); -} - - -static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - throw ThrownError(s); -} - +static RegisterPrimOp primop_abort({ + .name = "abort", + .args = {"s"}, + .doc = R"( + Abort Nix expression evaluation and print the error message *s*. + )", + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw Abort("evaluation aborted with the following error message: '%1%'", s); + } +}); + +static RegisterPrimOp primop_throw({ + .name = "throw", + .args = {"s"}, + .doc = R"( + Throw an error message *s*. This usually aborts Nix expression + evaluation, but in `nix-env -qa` and other commands that try to + evaluate a set of derivations to get information about those + derivations, a derivation that throws an error is silently skipped + (which is not the case for `abort`). + )", + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw ThrownError(s); + } +}); static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -476,6 +671,11 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a } } +static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { + .name = "__addErrorContext", + .arity = 2, + .fun = prim_addErrorContext, +}); /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ @@ -493,6 +693,22 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val v.attrs->sort(); } +static RegisterPrimOp primop_tryEval({ + .name = "__tryEval", + .args = {"e"}, + .doc = R"( + Try to shallowly evaluate *e*. Return a set containing the + attributes `success` (`true` if *e* evaluated successfully, + `false` if an error was thrown) and `value`, equalling *e* if + successful and `false` otherwise. Note that this doesn't evaluate + *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval + e).success ` will be `true`. Using ` builtins.deepSeq ` one can + get the expected result: `let e = { x = throw ""; }; in + (builtins.tryEval (builtins.deepSeq e e)).success` will be + `false`. + )", + .fun = prim_tryEval, +}); /* Return an environment variable. Use with care. */ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -501,6 +717,22 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } +static RegisterPrimOp primop_getEnv({ + .name = "__getEnv", + .args = {"s"}, + .doc = R"( + `getEnv` returns the value of the environment variable *s*, or an + empty string if the variable doesn’t exist. This function should be + used with care, as it can introduce all sorts of nasty environment + dependencies in your Nix expression. + + `getEnv` is used in Nix Packages to locate the file + `~/.nixpkgs/config.nix`, which contains user-local settings for Nix + Packages. (That is, it does a `getEnv "HOME"` to locate the user’s + home directory.) + )", + .fun = prim_getEnv, +}); /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -510,6 +742,15 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v = *args[1]; } +static RegisterPrimOp primop_seq({ + .name = "__seq", + .args = {"e1", "e2"}, + .doc = R"( + Evaluate *e1*, then evaluate and return *e2*. This ensures that a + computation is strict in the value of *e1*. + )", + .fun = prim_seq, +}); /* Evaluate the first argument deeply (i.e. recursing into lists and attrsets), then return the second argument. */ @@ -520,6 +761,16 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val v = *args[1]; } +static RegisterPrimOp primop_deepSeq({ + .name = "__deepSeq", + .args = {"e1", "e2"}, + .doc = R"( + This is like `seq e1 e2`, except that *e1* is evaluated *deeply*: + if it’s a list or set, its elements or attributes are also + evaluated recursively. + )", + .fun = prim_deepSeq, +}); /* Evaluate the first expression and print it on standard error. Then return the second expression. Useful for debugging. */ @@ -534,6 +785,17 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value v = *args[1]; } +static RegisterPrimOp primop_trace({ + .name = "__trace", + .args = {"e1", "e2"}, + .doc = R"( + Evaluate *e1* and print its abstract syntax representation on + standard error. Then return *e2*. This function is useful for + debugging. + )", + .fun = prim_trace, +}); + /************************************************************* * Derivations @@ -582,9 +844,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Build the derivation expression by processing the attributes. */ Derivation drv; + drv.name = drvName; PathSet context; + bool contentAddressed = false; std::optional<std::string> outputHash; std::string outputHashAlgo; auto ingestionMethod = FileIngestionMethod::Flat; @@ -641,9 +905,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->value->type == tNull) continue; } + if (i->name == state.sContentAddressed) { + settings.requireExperimentalFeature("ca-derivations"); + contentAddressed = state.forceBool(*i->value, pos); + } + /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (i->name == state.sArgs) { + else if (i->name == state.sArgs) { state.forceList(*i->value, pos); for (unsigned int n = 0; n < i->value->listSize(); ++n) { string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); @@ -696,7 +965,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } catch (Error & e) { - e.addTrace(posDrvName, + e.addTrace(posDrvName, "while evaluating the attribute '%1%' of the derivation '%2%'", key, drvName); throw; @@ -763,7 +1032,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * }); if (outputHash) { - /* Handle fixed-output derivations. */ + /* Handle fixed-output derivations. + + Ignore `__contentAddressed` because fixed output derivations are + already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error({ .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), @@ -774,16 +1046,30 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); - if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); + drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .path = std::move(outPath), - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + .output = DerivationOutputCAFixed { + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, + }, }); } + else if (contentAddressed) { + HashType ht = parseHashType(outputHashAlgo); + for (auto & i : outputs) { + drv.env[i] = hashPlaceholder(i); + drv.outputs.insert_or_assign(i, DerivationOutput { + .output = DerivationOutputCAFloating { + .method = ingestionMethod, + .hashType = std::move(ht), + }, + }); + } + } + else { /* Compute a hash over the "masked" store derivation, which is the final one except that in the list of outputs, the @@ -792,48 +1078,59 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * that changes in the set of output names do get reflected in the hash. */ for (auto & i : outputs) { - if (!jsonObject) drv.env[i] = ""; + drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .path = StorePath::dummy, - .hash = std::optional<FixedOutputHash> {}, + .output = DerivationOutputInputAddressed { + .path = StorePath::dummy, + }, }); } - Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); + // Regular, non-CA derivation should always return a single hash and not + // hash per output. + Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true)); for (auto & i : outputs) { auto outPath = state.store->makeOutputPath(i, h, drvName); - if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); + drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .path = std::move(outPath), - .hash = std::optional<FixedOutputHash>(), + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, }); } } /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPath = writeDerivation(*state.store, drv, state.repair); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't - read them later. */ - drvHashes.insert_or_assign(drvPath, - hashDerivationModulo(*state.store, Derivation(drv), false)); + read them later. + + However, we don't bother doing this for floating CA derivations because + their "hash modulo" is indeterminate until built. */ + if (drv.type() != DerivationType::CAFloating) + drvHashes.insert_or_assign(drvPath, + hashDerivationModulo(*state.store, Derivation(drv), false)); state.mkAttrs(v, 1 + drv.outputs.size()); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); - for (auto & i : drv.outputs) { - mkString(*state.allocAttr(v, state.symbols.create(i.first)), - state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); - } + for (auto & i : drv.outputs) + mkOutputString(state, v, drvPath, drv, i); v.attrs->sort(); } +static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { + .name = "derivationStrict", + .arity = 1, + .fun = prim_derivationStrict, +}); /* Return a placeholder string for the specified output that will be substituted by the corresponding output path at build time. For @@ -847,6 +1144,17 @@ static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); } +static RegisterPrimOp primop_placeholder({ + .name = "placeholder", + .args = {"output"}, + .doc = R"( + Return a placeholder string for the specified *output* that will be + substituted by the corresponding output path at build time. Typical + outputs would be `"out"`, `"bin"` or `"dev"`. + )", + .fun = prim_placeholder, +}); + /************************************************************* * Paths @@ -861,6 +1169,15 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, canonPath(path), context); } +static RegisterPrimOp primop_toPath({ + .name = "__toPath", + .args = {"s"}, + .doc = R"( + **DEPRECATED.** Use `/. + "/path"` to convert a string into an absolute + path. For relative paths, use `./. + "/path"`. + )", + .fun = prim_toPath, +}); /* Allow a valid store path to be used in an expression. This is useful in some generated expressions such as in nix-push, which @@ -872,6 +1189,9 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu corner cases. */ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v) { + if (evalSettings.pureEval) + throw EvalError("builtins.storePath' is not allowed in pure evaluation mode"); + PathSet context; Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink @@ -883,13 +1203,30 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V .hint = hintfmt("path '%1%' is not in the Nix store", path), .errPos = pos }); - Path path2 = state.store->toStorePath(path); + auto path2 = state.store->toStorePath(path).first; if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(path2)); - context.insert(path2); + state.store->ensurePath(path2); + context.insert(state.store->printStorePath(path2)); mkString(v, path, context); } +static RegisterPrimOp primop_storePath({ + .name = "__storePath", + .args = {"path"}, + .doc = R"( + This function allows you to define a dependency on an already + existing store path. For example, the derivation attribute `src + = builtins.storePath /nix/store/f1d18v1y…-source` causes the + derivation to depend on the specified path, which must exist or + be substitutable. Note that this differs from a plain path + (e.g. `src = /nix/store/f1d18v1y…-source`) in that the latter + causes the path to be *copied* again to the Nix store, resulting + in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). + + This function is not available in pure evaluation mode. + )", + .fun = prim_storePath, +}); static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -917,6 +1254,15 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, } } +static RegisterPrimOp primop_pathExists({ + .name = "__pathExists", + .args = {"path"}, + .doc = R"( + Return `true` if the path *path* exists at evaluation time, and + `false` otherwise. + )", + .fun = prim_pathExists, +}); /* Return the base name of the given string, i.e., everything following the last slash. */ @@ -926,6 +1272,16 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); } +static RegisterPrimOp primop_baseNameOf({ + .name = "baseNameOf", + .args = {"s"}, + .doc = R"( + Return the *base name* of the string *s*, that is, everything + following the final slash in the string. This is similar to the GNU + `basename` command. + )", + .fun = prim_baseNameOf, +}); /* Return the directory of the given path, i.e., everything before the last slash. Return either a path or a string depending on the type @@ -937,6 +1293,16 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } +static RegisterPrimOp primop_dirOf({ + .name = "dirOf", + .args = {"s"}, + .doc = R"( + Return the directory part of the string *s*, that is, everything + before the final slash in the string. This is similar to the GNU + `dirname` command. + )", + .fun = prim_dirOf, +}); /* Return the contents of a file as a string. */ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -957,6 +1323,14 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va mkString(v, s.c_str()); } +static RegisterPrimOp primop_readFile({ + .name = "__readFile", + .args = {"path"}, + .doc = R"( + Return the contents of the file *path* as a string. + )", + .fun = prim_readFile, +}); /* Find a file in the Nix search path. Used to implement <x> paths, which are desugared to 'findFile __nixPath "x"'. */ @@ -1002,6 +1376,12 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { + .name = "__findFile", + .arity = 2, + .fun = prim_findFile, +}); + /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1019,6 +1399,17 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context); } +static RegisterPrimOp primop_hashFile({ + .name = "__hashFile", + .args = {"type", "p"}, + .doc = R"( + Return a base-16 representation of the cryptographic hash of the + file at path *p*. The hash algorithm specified by *type* must be one + of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`. + )", + .fun = prim_hashFile, +}); + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1050,6 +1441,25 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val v.attrs->sort(); } +static RegisterPrimOp primop_readDir({ + .name = "__readDir", + .args = {"path"}, + .doc = R"( + Return the contents of the directory *path* as a set mapping + directory entries to the corresponding file type. For instance, if + directory `A` contains a regular file `B` and another directory + `C`, then `builtins.readDir ./A` will return the set + + ```nix + { B = "regular"; C = "directory"; } + ``` + + The possible values for the file type are `"regular"`, + `"directory"`, `"symlink"` and `"unknown"`. + )", + .fun = prim_readDir, +}); + /************************************************************* * Creating files @@ -1067,6 +1477,102 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value mkString(v, out.str(), context); } +static RegisterPrimOp primop_toXML({ + .name = "__toXML", + .args = {"e"}, + .doc = R"( + Return a string containing an XML representation of *e*. The main + application for `toXML` is to communicate information with the + builder in a more structured format than plain environment + variables. + + Here is an example where this is the case: + + ```nix + { stdenv, fetchurl, libxslt, jira, uberwiki }: + + stdenv.mkDerivation (rec { + name = "web-server"; + + buildInputs = [ libxslt ]; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + mkdir $out + echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ① + "; + + stylesheet = builtins.toFile "stylesheet.xsl" ② + "<?xml version='1.0' encoding='UTF-8'?> + <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'> + <xsl:template match='/'> + <Configure> + <xsl:for-each select='/expr/list/attrs'> + <Call name='addWebApplication'> + <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg> + <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg> + </Call> + </xsl:for-each> + </Configure> + </xsl:template> + </xsl:stylesheet> + "; + + servlets = builtins.toXML [ ③ + { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; } + { path = "/wiki"; war = uberwiki + "/uberwiki.war"; } + ]; + }) + ``` + + The builder is supposed to generate the configuration file for a + [Jetty servlet container](http://jetty.mortbay.org/). A servlet + container contains a number of servlets (`*.war` files) each + exported under a specific URI prefix. So the servlet configuration + is a list of sets containing the `path` and `war` of the servlet + (①). This kind of information is difficult to communicate with the + normal method of passing information through an environment + variable, which just concatenates everything together into a + string (which might just work in this case, but wouldn’t work if + fields are optional or contain lists themselves). Instead the Nix + expression is converted to an XML representation with `toXML`, + which is unambiguous and can easily be processed with the + appropriate tools. For instance, in the example an XSLT stylesheet + (at point ②) is applied to it (at point ①) to generate the XML + configuration file for the Jetty server. The XML representation + produced at point ③ by `toXML` is as follows: + + ```xml + <?xml version='1.0' encoding='utf-8'?> + <expr> + <list> + <attrs> + <attr name="path"> + <string value="/bugtracker" /> + </attr> + <attr name="war"> + <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" /> + </attr> + </attrs> + <attrs> + <attr name="path"> + <string value="/wiki" /> + </attr> + <attr name="war"> + <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" /> + </attr> + </attrs> + </list> + </expr> + ``` + + Note that we used the `toFile` built-in to write the builder and + the stylesheet “inline” in the Nix expression. The path of the + stylesheet is spliced into the builder using the syntax `xsltproc + ${stylesheet}`. + )", + .fun = prim_toXML, +}); /* Convert the argument (which can be any Nix expression) to a JSON string. Not all Nix expressions can be sensibly or completely @@ -1079,6 +1585,19 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, out.str(), context); } +static RegisterPrimOp primop_toJSON({ + .name = "__toJSON", + .args = {"e"}, + .doc = R"( + Return a string containing a JSON representation of *e*. Strings, + integers, floats, booleans, nulls and lists are mapped to their JSON + equivalents. Sets (except derivations) are represented as objects. + Derivations are translated to a JSON string containing the + derivation’s output path. Paths are copied to the store and + represented as a JSON string of the resulting store path. + )", + .fun = prim_toJSON, +}); /* Parse a JSON string to a value. */ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1087,6 +1606,20 @@ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Va parseJSON(state, s, v); } +static RegisterPrimOp primop_fromJSON({ + .name = "__fromJSON", + .args = {"e"}, + .doc = R"( + Convert a JSON string to a Nix value. For example, + + ```nix + builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' + ``` + + returns the value `{ x = [ 1 2 3 ]; y = null; }`. + )", + .fun = prim_fromJSON, +}); /* Store a string in the Nix store as a source file that can be used as an input by derivations. */ @@ -1121,9 +1654,86 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, storePath, {storePath}); } +static RegisterPrimOp primop_toFile({ + .name = "__toFile", + .args = {"name", "s"}, + .doc = R"( + Store the string *s* in a file in the Nix store and return its + path. The file has suffix *name*. This file can be used as an + input to derivations. One application is to write builders + “inline”. For instance, the following Nix expression combines the + [Nix expression for GNU Hello](expression-syntax.md) and its + [build script](build-script.md) into one file: + + ```nix + { stdenv, fetchurl, perl }: + + stdenv.mkDerivation { + name = "hello-2.1.1"; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + + PATH=$perl/bin:$PATH + + tar xvfz $src + cd hello-* + ./configure --prefix=$out + make + make install + "; + + src = fetchurl { + url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + }; + inherit perl; + } + ``` + + It is even possible for one file to refer to another, e.g., + + ```nix + builder = let + configFile = builtins.toFile "foo.conf" " + # This is some dummy configuration file. + ... + "; + in builtins.toFile "builder.sh" " + source $stdenv/setup + ... + cp ${configFile} $out/etc/foo.conf + "; + ``` + + Note that `${configFile}` is an + [antiquotation](language-values.md), so the result of the + expression `configFile` + (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be + spliced into the resulting string. + + It is however *not* allowed to have files mutually referring to each + other, like so: + + ```nix + let + foo = builtins.toFile "foo" "...${bar}..."; + bar = builtins.toFile "bar" "...${foo}..."; + in foo + ``` + + This is not allowed because it would cause a cyclic dependency in + the computation of the cryptographic hashes for `foo` and `bar`. + + It is also not possible to reference the result of a derivation. If + you are using Nixpkgs, the `writeTextFile` function is able to do + that. + )", + .fun = prim_toFile, +}); static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1154,7 +1764,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1188,8 +1798,50 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); -} + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); +} + +static RegisterPrimOp primop_filterSource({ + .name = "__filterSource", + .args = {"e1", "e2"}, + .doc = R"( + This function allows you to copy sources into the Nix store while + filtering certain files. For instance, suppose that you want to use + the directory `source-dir` as an input to a Nix expression, e.g. + + ```nix + stdenv.mkDerivation { + ... + src = ./source-dir; + } + ``` + + However, if `source-dir` is a Subversion working copy, then all + those annoying `.svn` subdirectories will also be copied to the + store. Worse, the contents of those directories may change a lot, + causing lots of spurious rebuilds. With `filterSource` you can + filter out the `.svn` directories: + + ```nix + src = builtins.filterSource + (path: type: type != "directory" || baseNameOf path != ".svn") + ./source-dir; + ``` + + Thus, the first argument *e1* must be a predicate function that is + called for each regular file, directory or symlink in the source + tree *e2*. If the function returns `true`, the file is copied to the + Nix store, otherwise it is omitted. The function is called with two + arguments. The first is the full path of the file. The second is a + string that identifies the type of the file, which is either + `"regular"`, `"directory"`, `"symlink"` or `"unknown"` (for other + kinds of files such as device nodes or fifos — but note that those + cannot be copied to the Nix store, so if the predicate returns + `true` for them, the copy will fail). If you exclude a directory, + the entire corresponding subtree of *e2* will be excluded. + )", + .fun = prim_filterSource, +}); static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1198,7 +1850,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + std::optional<Hash> expectedHash; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); @@ -1236,6 +1888,41 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value addPath(state, pos, name, path, filterFun, method, expectedHash, v); } +static RegisterPrimOp primop_path({ + .name = "__path", + .args = {"args"}, + .doc = R"( + An enrichment of the built-in path type, based on the attributes + present in *args*. All are optional except `path`: + + - path + The underlying path. + + - name + The name of the path when added to the store. This can used to + reference paths that have nix-illegal characters in their names, + like `@`. + + - filter + A function of the type expected by `builtins.filterSource`, + with the same semantics. + + - recursive + When `false`, when `path` is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of the + file. Thus, `path` must refer to a regular file, not a + directory. This allows similar behavior to `fetchurl`. Defaults + to `true`. + + - sha256 + When provided, this is the expected hash of the file at the + path. Evaluation will fail if the hash is incorrect, and + providing a hash allows `builtins.path` to be used even when the + `pure-eval` nix config option is on. + )", + .fun = prim_path, +}); + /************************************************************* * Sets @@ -1258,6 +1945,16 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } +static RegisterPrimOp primop_attrNames({ + .name = "__attrNames", + .args = {"set"}, + .doc = R"( + Return the names of the attributes in the set *set* in an + alphabetically sorted list. For instance, `builtins.attrNames { y + = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. + )", + .fun = prim_attrNames, +}); /* Return the values of the attributes in a set as a list, in the same order as attrNames. */ @@ -1278,6 +1975,15 @@ static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, v.listElems()[i] = ((Attr *) v.listElems()[i])->value; } +static RegisterPrimOp primop_attrValues({ + .name = "__attrValues", + .args = {"set"}, + .doc = R"( + Return the values of the attributes in the set *set* in the order + corresponding to the sorted attribute names. + )", + .fun = prim_attrValues, +}); /* Dynamic version of the `.' operator. */ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1297,9 +2003,20 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) v = *i->value; } +static RegisterPrimOp primop_getAttr({ + .name = "__getAttr", + .args = {"s", "set"}, + .doc = R"( + `getAttr` returns the attribute named *s* from *set*. Evaluation + aborts if the attribute doesn’t exist. This is a dynamic version of + the `.` operator, since *s* is an expression rather than an + identifier. + )", + .fun = prim_getAttr, +}); /* Return position information of the specified attribute. */ -void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) { string attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); @@ -1310,6 +2027,11 @@ void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, V state.mkPos(v, i->pos); } +static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { + .name = "__unsafeGetAttrPos", + .arity = 2, + .fun = prim_unsafeGetAttrPos, +}); /* Dynamic version of the `?' operator. */ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1319,6 +2041,16 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); } +static RegisterPrimOp primop_hasAttr({ + .name = "__hasAttr", + .args = {"s", "set"}, + .doc = R"( + `hasAttr` returns `true` if *set* has an attribute named *s*, and + `false` otherwise. This is a dynamic version of the `?` operator, + since *s* is an expression rather than an identifier. + )", + .fun = prim_hasAttr, +}); /* Determine whether the argument is a set. */ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1327,6 +2059,14 @@ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Val mkBool(v, args[0]->type == tAttrs); } +static RegisterPrimOp primop_isAttrs({ + .name = "__isAttrs", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a set, and `false` otherwise. + )", + .fun = prim_isAttrs, +}); static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1350,6 +2090,21 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, } } +static RegisterPrimOp primop_removeAttrs({ + .name = "removeAttrs", + .args = {"set", "list"}, + .doc = R"( + Remove the attributes listed in *list* from *set*. The attributes + don’t have to exist in *set*. For instance, + + ```nix + removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ] + ``` + + evaluates to `{ y = 2; }`. + )", + .fun = prim_removeAttrs, +}); /* Builds a set from a list specifying (name, value) pairs. To be precise, a list [{name = "name1"; value = value1;} ... {name = @@ -1391,10 +2146,31 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, v.attrs->sort(); } +static RegisterPrimOp primop_listToAttrs({ + .name = "__listToAttrs", + .args = {"e"}, + .doc = R"( + Construct a set from a list specifying the names and values of each + attribute. Each element of the list should be a set consisting of a + string-valued attribute `name` specifying the name of the attribute, + and an attribute `value` specifying its value. Example: + + ```nix + builtins.listToAttrs + [ { name = "foo"; value = 123; } + { name = "bar"; value = 456; } + ] + ``` + + evaluates to + + ```nix + { foo = 123; bar = 456; } + ``` + )", + .fun = prim_listToAttrs, +}); -/* Return the right-biased intersection of two sets as1 and as2, - i.e. a set that contains every attribute from as2 that is also a - member of as1. */ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos); @@ -1409,14 +2185,16 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar } } +static RegisterPrimOp primop_intersectAttrs({ + .name = "__intersectAttrs", + .args = {"e1", "e2"}, + .doc = R"( + Return a set consisting of the attributes in the set *e2* that also + exist in the set *e1*. + )", + .fun = prim_intersectAttrs, +}); -/* Collect each attribute named `attr' from a list of attribute sets. - Sets that don't contain the named attribute are ignored. - - Example: - catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] - => [1 2] -*/ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); @@ -1438,23 +2216,30 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va v.listElems()[n] = res[n]; } +static RegisterPrimOp primop_catAttrs({ + .name = "__catAttrs", + .args = {"attr", "list"}, + .doc = R"( + Collect each attribute named *attr* from a list of attribute + sets. Attrsets that don't contain the named attribute are + ignored. For example, -/* Return a set containing the names of the formal arguments expected - by the function `f'. The value of each attribute is a Boolean - denoting whether the corresponding argument has a default value. For instance, + ```nix + builtins.catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + ``` - functionArgs ({ x, y ? 123}: ...) - => { x = false; y = true; } + evaluates to `[1 2]`. + )", + .fun = prim_catAttrs, +}); - "Formal argument" here refers to the attributes pattern-matched by - the function. Plain lambdas are not included, e.g. - - functionArgs (x: ...) - => { } -*/ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); + if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + state.mkAttrs(v, 0); + return; + } if (args[0]->type != tLambda) throw TypeError({ .hint = hintfmt("'functionArgs' requires a function"), @@ -1476,8 +2261,24 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args v.attrs->sort(); } - -/* Apply a function to every element of an attribute set. */ +static RegisterPrimOp primop_functionArgs({ + .name = "__functionArgs", + .args = {"f"}, + .doc = R"( + Return a set containing the names of the formal arguments expected + by the function *f*. The value of each attribute is a Boolean + denoting whether the corresponding argument has a default value. For + instance, `functionArgs ({ x, y ? 123}: ...) = { x = false; y = + true; }`. + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. `functionArgs (x: + ...) = { }`. + )", + .fun = prim_functionArgs, +}); + +/* */ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceAttrs(*args[1], pos); @@ -1493,6 +2294,20 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va } } +static RegisterPrimOp primop_mapAttrs({ + .name = "__mapAttrs", + .args = {"f", "attrset"}, + .doc = R"( + Apply function *f* to every element of *attrset*. For example, + + ```nix + builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; } + ``` + + evaluates to `{ a = 10; b = 20; }`. + )", + .fun = prim_mapAttrs, +}); /************************************************************* @@ -1507,6 +2322,14 @@ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->isList()); } +static RegisterPrimOp primop_isList({ + .name = "__isList", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a list, and `false` otherwise. + )", + .fun = prim_isList, +}); static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) { @@ -1520,13 +2343,21 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu v = *list.listElems()[n]; } - /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const Pos & pos, Value * * args, Value & v) { elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); } +static RegisterPrimOp primop_elemAt({ + .name = "__elemAt", + .args = {"xs", "n"}, + .doc = R"( + Return element *n* from the list *xs*. Elements are counted starting + from 0. A fatal error occurs if the index is out of bounds. + )", + .fun = prim_elemAt, +}); /* Return the first element of a list. */ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1534,6 +2365,16 @@ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value elemAt(state, pos, *args[0], 0, v); } +static RegisterPrimOp primop_head({ + .name = "__head", + .args = {"list"}, + .doc = R"( + Return the first element of a list; abort evaluation if the argument + isn’t a list or is an empty list. You can test whether a list is + empty by comparing it with `[]`. + )", + .fun = prim_head, +}); /* Return a list consisting of everything but the first element of a list. Warning: this function takes O(n) time, so you probably @@ -1552,6 +2393,21 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[0]->listElems()[n + 1]; } +static RegisterPrimOp primop_tail({ + .name = "__tail", + .args = {"list"}, + .doc = R"( + Return the second to last elements of a list; abort evaluation if + the argument isn’t a list or is an empty list. + + > **Warning** + > + > This function should generally be avoided since it's inefficient: + > unlike Haskell's `tail`, it takes O(n) time, so recursing over a + > list by repeatedly calling `tail` takes O(n^2) time. + )", + .fun = prim_tail, +}); /* Apply a function to every element of a list. */ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1565,6 +2421,21 @@ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & *args[0], *args[1]->listElems()[n]); } +static RegisterPrimOp primop_map({ + .name = "map", + .args = {"f", "list"}, + .doc = R"( + Apply the function *f* to each element in the list *list*. For + example, + + ```nix + map (x: "foo" + x) [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + )", + .fun = prim_map, +}); /* Filter a list using a predicate; that is, return a list containing every element from the list for which the predicate function @@ -1596,6 +2467,15 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu } } +static RegisterPrimOp primop_filter({ + .name = "__filter", + .args = {"f", "list"}, + .doc = R"( + Return a list consisting of the elements of *list* for which the + function *f* returns `true`. + )", + .fun = prim_filter, +}); /* Return true if a list contains a given element. */ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1610,6 +2490,15 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value mkBool(v, res); } +static RegisterPrimOp primop_elem({ + .name = "__elem", + .args = {"x", "xs"}, + .doc = R"( + Return `true` if a value equal to *x* occurs in the list *xs*, and + `false` otherwise. + )", + .fun = prim_elem, +}); /* Concatenate a list of lists. */ static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1618,6 +2507,14 @@ static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); } +static RegisterPrimOp primop_concatLists({ + .name = "__concatLists", + .args = {"lists"}, + .doc = R"( + Concatenate a list of lists into a single list. + )", + .fun = prim_concatLists, +}); /* Return the length of a list. This is an O(1) time operation. */ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1626,6 +2523,14 @@ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Valu mkInt(v, args[0]->listSize()); } +static RegisterPrimOp primop_length({ + .name = "__length", + .args = {"e"}, + .doc = R"( + Return the length of the list *e*. + )", + .fun = prim_length, +}); /* Reduce a list by applying a binary operator, from left to right. The operator is applied strictly. */ @@ -1650,6 +2555,18 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, } } +static RegisterPrimOp primop_foldlStrict({ + .name = "__foldl'", + .args = {"op", "nul", "list"}, + .doc = R"( + Reduce a list by applying a binary operator, from left to right, + e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + ...`. The operator is applied strictly, i.e., its arguments are + evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluates to 6. + )", + .fun = prim_foldlStrict, +}); static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1675,12 +2592,30 @@ static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value & anyOrAll(true, state, pos, args, v); } +static RegisterPrimOp primop_any({ + .name = "__any", + .args = {"pred", "list"}, + .doc = R"( + Return `true` if the function *pred* returns `true` for at least one + element of *list*, and `false` otherwise. + )", + .fun = prim_any, +}); static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v) { anyOrAll(false, state, pos, args, v); } +static RegisterPrimOp primop_all({ + .name = "__all", + .args = {"pred", "list"}, + .doc = R"( + Return `true` if the function *pred* returns `true` for all elements + of *list*, and `false` otherwise. + )", + .fun = prim_all, +}); static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1701,6 +2636,21 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val } } +static RegisterPrimOp primop_genList({ + .name = "__genList", + .args = {"generator", "length"}, + .doc = R"( + Generate list of size *length*, with each element *i* equal to the + value returned by *generator* `i`. For example, + + ```nix + builtins.genList (x: x * x) 5 + ``` + + returns the list `[ 0 1 4 9 16 ]`. + )", + .fun = prim_genList, +}); static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -1736,6 +2686,26 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value std::stable_sort(v.listElems(), v.listElems() + len, comparator); } +static RegisterPrimOp primop_sort({ + .name = "__sort", + .args = {"comparator", "list"}, + .doc = R"( + Return *list* in sorted order. It repeatedly calls the function + *comparator* with two elements. The comparator should return `true` + if the first element is less than the second, and `false` otherwise. + For example, + + ```nix + builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] + ``` + + produces the list `[ 42 77 147 249 483 526 ]`. + + This is a stable sort: it preserves the relative order of elements + deemed equal by the comparator. + )", + .fun = prim_sort, +}); static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1774,9 +2744,29 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V v.attrs->sort(); } +static RegisterPrimOp primop_partition({ + .name = "__partition", + .args = {"pred", "list"}, + .doc = R"( + Given a predicate function *pred*, this function returns an + attrset containing a list named `right`, containing the elements + in *list* for which *pred* returned `true`, and a list named + `wrong`, containing the elements for which it returned + `false`. For example, + + ```nix + builtins.partition (x: x > 10) [1 23 9 3 42] + ``` + + evaluates to + + ```nix + { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } + ``` + )", + .fun = prim_partition, +}); -/* concatMap = f: list: concatLists (map f list); */ -/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); @@ -1803,6 +2793,16 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V } } +static RegisterPrimOp primop_concatMap({ + .name = "__concatMap", + .args = {"f", "list"}, + .doc = R"( + This function is equivalent to `builtins.concatLists (map f list)` + but is more efficient. + )", + .fun = prim_concatMap, +}); + /************************************************************* * Integer arithmetic @@ -1819,6 +2819,14 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_add({ + .name = "__add", + .args = {"e1", "e2"}, + .doc = R"( + Return the sum of the numbers *e1* and *e2*. + )", + .fun = prim_add, +}); static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1830,6 +2838,14 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_sub({ + .name = "__sub", + .args = {"e1", "e2"}, + .doc = R"( + Return the difference between the numbers *e1* and *e2*. + )", + .fun = prim_sub, +}); static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1841,6 +2857,14 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_mul({ + .name = "__mul", + .args = {"e1", "e2"}, + .doc = R"( + Return the product of the numbers *e1* and *e2*. + )", + .fun = prim_mul, +}); static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1870,21 +2894,57 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & } } +static RegisterPrimOp primop_div({ + .name = "__div", + .args = {"e1", "e2"}, + .doc = R"( + Return the quotient of the numbers *e1* and *e2*. + )", + .fun = prim_div, +}); + static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) { mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_bitAnd({ + .name = "__bitAnd", + .args = {"e1", "e2"}, + .doc = R"( + Return the bitwise AND of the integers *e1* and *e2*. + )", + .fun = prim_bitAnd, +}); + static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) { mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_bitOr({ + .name = "__bitOr", + .args = {"e1", "e2"}, + .doc = R"( + Return the bitwise OR of the integers *e1* and *e2*. + )", + .fun = prim_bitOr, +}); + static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) { mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_bitXor({ + .name = "__bitXor", + .args = {"e1", "e2"}, + .doc = R"( + Return the bitwise XOR of the integers *e1* and *e2*. + )", + .fun = prim_bitXor, +}); + static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); @@ -1893,6 +2953,17 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va mkBool(v, comp(args[0], args[1])); } +static RegisterPrimOp primop_lessThan({ + .name = "__lessThan", + .args = {"e1", "e2"}, + .doc = R"( + Return `true` if the number *e1* is less than the number *e2*, and + `false` otherwise. Evaluation aborts if either *e1* or *e2* does not + evaluate to a number. + )", + .fun = prim_lessThan, +}); + /************************************************************* * String manipulation @@ -1909,6 +2980,29 @@ static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Va mkString(v, s, context); } +static RegisterPrimOp primop_toString({ + .name = "toString", + .args = {"e"}, + .doc = R"( + Convert the expression *e* to a string. *e* can be: + + - A string (in which case the string is returned unmodified). + + - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`. + + - A set containing `{ __toString = self: ...; }`. + + - An integer. + + - A list, in which case the string representations of its elements + are joined with spaces. + + - A Boolean (`false` yields `""`, `true` yields `"1"`). + + - `null`, which yields the empty string. + )", + .fun = prim_toString, +}); /* `substring start len str' returns the substring of `str' starting at character position `min(start, stringLength str)' inclusive and @@ -1930,6 +3024,25 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } +static RegisterPrimOp primop_substring({ + .name = "__substring", + .args = {"start", "len", "s"}, + .doc = R"( + Return the substring of *s* from character position *start* + (zero-based) up to but not including *start + len*. If *start* is + greater than the length of the string, an empty string is returned, + and if *start + len* lies beyond the end of the string, only the + substring up to the end of the string is returned. *start* must be + non-negative. For example, + + ```nix + builtins.substring 0 3 "nixos" + ``` + + evaluates to `"nix"`. + )", + .fun = prim_substring, +}); static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1938,6 +3051,15 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args mkInt(v, s.size()); } +static RegisterPrimOp primop_stringLength({ + .name = "__stringLength", + .args = {"e"}, + .doc = R"( + Return the length of the string *e*. If *e* is not a string, + evaluation is aborted. + )", + .fun = prim_stringLength, +}); /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1956,18 +3078,36 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, mkString(v, hashString(*ht, s).to_string(Base16, false), context); } +static RegisterPrimOp primop_hashString({ + .name = "__hashString", + .args = {"type", "s"}, + .doc = R"( + Return a base-16 representation of the cryptographic hash of string + *s*. The hash algorithm specified by *type* must be one of `"md5"`, + `"sha1"`, `"sha256"` or `"sha512"`. + )", + .fun = prim_hashString, +}); + +struct RegexCache +{ + std::unordered_map<std::string, std::regex> cache; +}; + +std::shared_ptr<RegexCache> makeRegexCache() +{ + return std::make_shared<RegexCache>(); +} -/* Match a regular expression against a string and return either - ‘null’ or a list containing substring matches. */ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto re = state.forceStringNoCtx(*args[0], pos); try { - auto regex = state.regexCache.find(re); - if (regex == state.regexCache.end()) - regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first; + auto regex = state.regexCache->cache.find(re); + if (regex == state.regexCache->cache.end()) + regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first; PathSet context; const std::string str = state.forceString(*args[1], context, pos); @@ -2004,6 +3144,41 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) } } +static RegisterPrimOp primop_match({ + .name = "__match", + .args = {"regex", "str"}, + .doc = R"s( + Returns a list if the [extended POSIX regular + expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) + *regex* matches *str* precisely, otherwise returns `null`. Each item + in the list is a regex group. + + ```nix + builtins.match "ab" "abc" + ``` + + Evaluates to `null`. + + ```nix + builtins.match "abc" "abc" + ``` + + Evaluates to `[ ]`. + + ```nix + builtins.match "a(b)(c)" "abc" + ``` + + Evaluates to `[ "b" "c" ]`. + + ```nix + builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " + ``` + + Evaluates to `[ "foo" ]`. + )s", + .fun = prim_match, +}); /* Split a string with a regular expression, and return a list of the non-matching parts interleaved by the lists of the matching groups. */ @@ -2077,8 +3252,44 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value } } +static RegisterPrimOp primop_split({ + .name = "__split", + .args = {"regex", "str"}, + .doc = R"s( + Returns a list composed of non matched strings interleaved with the + lists of the [extended POSIX regular + expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) + *regex* matches of *str*. Each item in the lists of matched + sequences is a regex group. + + ```nix + builtins.split "(a)b" "abc" + ``` -static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v) + Evaluates to `[ "" [ "a" ] "c" ]`. + + ```nix + builtins.split "([ac])" "abc" + ``` + + Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`. + + ```nix + builtins.split "(a)|(c)" "abc" + ``` + + Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`. + + ```nix + builtins.split "([[:upper:]]+)" " FOO " + ``` + + Evaluates to `[ " " [ "FOO" ] " " ]`. + )s", + .fun = prim_split, +}); + +static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; @@ -2097,6 +3308,16 @@ static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * a mkString(v, res, context); } +static RegisterPrimOp primop_concatStringsSep({ + .name = "__concatStringsSep", + .args = {"separator", "list"}, + .doc = R"( + Concatenate a list of strings with a separator between each + element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == + "usr/local/bin"`. + )", + .fun = prim_concatStringsSep, +}); static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2156,6 +3377,22 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar mkString(v, res, context); } +static RegisterPrimOp primop_replaceStrings({ + .name = "__replaceStrings", + .args = {"from", "to", "s"}, + .doc = R"( + Given string *s*, replace every occurrence of the strings in *from* + with the corresponding string in *to*. For example, + + ```nix + builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" + ``` + + evaluates to `"fabir"`. + )", + .fun = prim_replaceStrings, +}); + /************************************************************* * Versions @@ -2172,6 +3409,19 @@ static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args v.attrs->sort(); } +static RegisterPrimOp primop_parseDrvName({ + .name = "__parseDrvName", + .args = {"s"}, + .doc = R"( + Split the string *s* into a package name and version. The package + name is everything up to but not including the first dash followed + by a digit, and the version is everything following that dash. The + result is returned in a set `{ name, version }`. Thus, + `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = + "nix"; version = "0.12pre12876"; }`. + )", + .fun = prim_parseDrvName, +}); static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2180,6 +3430,18 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a mkInt(v, compareVersions(version1, version2)); } +static RegisterPrimOp primop_compareVersions({ + .name = "__compareVersions", + .args = {"s1", "s2"}, + .doc = R"( + Compare two strings representing versions and return `-1` if + version *s1* is older than version *s2*, `0` if they are the same, + and `1` if *s1* is newer than *s2*. The version comparison + algorithm is the same as the one used by [`nix-env + -u`](../command-ref/nix-env.md#operation---upgrade). + )", + .fun = prim_compareVersions, +}); static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2200,6 +3462,17 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args } } +static RegisterPrimOp primop_splitVersion({ + .name = "__splitVersion", + .args = {"s"}, + .doc = R"( + Split a string representing a version into its components, by the + same version splitting logic underlying the version comparison in + [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade). + )", + .fun = prim_splitVersion, +}); + /************************************************************* * Primop registration @@ -2213,7 +3486,20 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun, std::optional<std::string> requiredFeature) { if (!primOps) primOps = new PrimOps; - primOps->push_back({name, arity, fun, requiredFeature}); + primOps->push_back({ + .name = name, + .args = {}, + .arity = arity, + .requiredFeature = std::move(requiredFeature), + .fun = fun + }); +} + + +RegisterPrimOp::RegisterPrimOp(Info && info) +{ + if (!primOps) primOps = new PrimOps; + primOps->push_back(std::move(info)); } @@ -2237,15 +3523,6 @@ void EvalState::createBaseEnv() mkNull(v); addConstant("null", v); - auto vThrow = addPrimOp("throw", 1, prim_throw); - - auto addPurityError = [&](const std::string & name) { - Value * v2 = allocValue(); - mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); - mkApp(v, *vThrow, *v2); - addConstant(name, v); - }; - if (!evalSettings.pureEval) { mkInt(v, time(0)); addConstant("__currentTime", v); @@ -2270,132 +3547,16 @@ void EvalState::createBaseEnv() addConstant("__langVersion", v); // Miscellaneous - auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); - Value * v2 = allocValue(); - mkAttrs(*v2, 0); - mkApp(v, *vScopedImport, *v2); - forceValue(v); - addConstant("import", v); if (evalSettings.enableNativeCode) { addPrimOp("__importNative", 2, prim_importNative); addPrimOp("__exec", 1, prim_exec); } - addPrimOp("__typeOf", 1, prim_typeOf); - addPrimOp("isNull", 1, prim_isNull); - addPrimOp("__isFunction", 1, prim_isFunction); - addPrimOp("__isString", 1, prim_isString); - addPrimOp("__isInt", 1, prim_isInt); - addPrimOp("__isFloat", 1, prim_isFloat); - addPrimOp("__isBool", 1, prim_isBool); - addPrimOp("__isPath", 1, prim_isPath); - addPrimOp("__genericClosure", 1, prim_genericClosure); - addPrimOp("abort", 1, prim_abort); - addPrimOp("__addErrorContext", 2, prim_addErrorContext); - addPrimOp("__tryEval", 1, prim_tryEval); - addPrimOp("__getEnv", 1, prim_getEnv); - - // Strictness - addPrimOp("__seq", 2, prim_seq); - addPrimOp("__deepSeq", 2, prim_deepSeq); - - // Debugging - addPrimOp("__trace", 2, prim_trace); - - // Paths - addPrimOp("__toPath", 1, prim_toPath); - if (evalSettings.pureEval) - addPurityError("__storePath"); - else - addPrimOp("__storePath", 1, prim_storePath); - addPrimOp("__pathExists", 1, prim_pathExists); - addPrimOp("baseNameOf", 1, prim_baseNameOf); - addPrimOp("dirOf", 1, prim_dirOf); - addPrimOp("__readFile", 1, prim_readFile); - addPrimOp("__readDir", 1, prim_readDir); - addPrimOp("__findFile", 2, prim_findFile); - addPrimOp("__hashFile", 2, prim_hashFile); - - // Creating files - addPrimOp("__toXML", 1, prim_toXML); - addPrimOp("__toJSON", 1, prim_toJSON); - addPrimOp("__fromJSON", 1, prim_fromJSON); - addPrimOp("__toFile", 2, prim_toFile); - addPrimOp("__filterSource", 2, prim_filterSource); - addPrimOp("__path", 1, prim_path); - - // Sets - addPrimOp("__attrNames", 1, prim_attrNames); - addPrimOp("__attrValues", 1, prim_attrValues); - addPrimOp("__getAttr", 2, prim_getAttr); - addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); - addPrimOp("__hasAttr", 2, prim_hasAttr); - addPrimOp("__isAttrs", 1, prim_isAttrs); - addPrimOp("removeAttrs", 2, prim_removeAttrs); - addPrimOp("__listToAttrs", 1, prim_listToAttrs); - addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); - addPrimOp("__catAttrs", 2, prim_catAttrs); - addPrimOp("__functionArgs", 1, prim_functionArgs); - addPrimOp("__mapAttrs", 2, prim_mapAttrs); - - // Lists - addPrimOp("__isList", 1, prim_isList); - addPrimOp("__elemAt", 2, prim_elemAt); - addPrimOp("__head", 1, prim_head); - addPrimOp("__tail", 1, prim_tail); - addPrimOp("map", 2, prim_map); - addPrimOp("__filter", 2, prim_filter); - addPrimOp("__elem", 2, prim_elem); - addPrimOp("__concatLists", 1, prim_concatLists); - addPrimOp("__length", 1, prim_length); - addPrimOp("__foldl'", 3, prim_foldlStrict); - addPrimOp("__any", 2, prim_any); - addPrimOp("__all", 2, prim_all); - addPrimOp("__genList", 2, prim_genList); - addPrimOp("__sort", 2, prim_sort); - addPrimOp("__partition", 2, prim_partition); - addPrimOp("__concatMap", 2, prim_concatMap); - - // Integer arithmetic - addPrimOp("__add", 2, prim_add); - addPrimOp("__sub", 2, prim_sub); - addPrimOp("__mul", 2, prim_mul); - addPrimOp("__div", 2, prim_div); - addPrimOp("__bitAnd", 2, prim_bitAnd); - addPrimOp("__bitOr", 2, prim_bitOr); - addPrimOp("__bitXor", 2, prim_bitXor); - addPrimOp("__lessThan", 2, prim_lessThan); - - // String manipulation - addPrimOp("toString", 1, prim_toString); - addPrimOp("__substring", 3, prim_substring); - addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hashString", 2, prim_hashString); - addPrimOp("__match", 2, prim_match); - addPrimOp("__split", 2, prim_split); - addPrimOp("__concatStringsSep", 2, prim_concatStringSep); - addPrimOp("__replaceStrings", 3, prim_replaceStrings); - - // Versions - addPrimOp("__parseDrvName", 1, prim_parseDrvName); - addPrimOp("__compareVersions", 2, prim_compareVersions); - addPrimOp("__splitVersion", 1, prim_splitVersion); - - // Derivations - addPrimOp("derivationStrict", 1, prim_derivationStrict); - addPrimOp("placeholder", 1, prim_placeholder); - - /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ - string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); - sDerivationNix = symbols.create(path); - evalFile(path, v); - addConstant("derivation", v); /* Add a value containing the current Nix expression search path. */ mkList(v, searchPath.size()); int n = 0; for (auto & i : searchPath) { - v2 = v.listElems()[n++] = allocValue(); + auto v2 = v.listElems()[n++] = allocValue(); mkAttrs(*v2, 2); mkString(*allocAttr(*v2, symbols.create("path")), i.second); mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); @@ -2406,7 +3567,21 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature)) - addPrimOp(primOp.name, primOp.arity, primOp.primOp); + addPrimOp({ + .fun = primOp.fun, + .arity = std::max(primOp.args.size(), primOp.arity), + .name = symbols.create(primOp.name), + .args = std::move(primOp.args), + .doc = primOp.doc, + }); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + sDerivationNix = symbols.create("//builtin/derivation.nix"); + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), v); + addConstant("derivation", v); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 75c460ecf..9d42d6539 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,3 +1,5 @@ +#pragma once + #include "eval.hh" #include <tuple> @@ -10,9 +12,11 @@ struct RegisterPrimOp struct Info { std::string name; - size_t arity; - PrimOpFun primOp; + std::vector<std::string> args; + size_t arity = 0; + const char * doc; std::optional<std::string> requiredFeature; + PrimOpFun fun; }; typedef std::vector<Info> PrimOps; @@ -26,6 +30,8 @@ struct RegisterPrimOp size_t arity, PrimOpFun fun, std::optional<std::string> requiredFeature = {}); + + RegisterPrimOp(Info && info); }; /* These primops are disabled without enableNativeCode, but plugins diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index dbb93bae6..b570fca31 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, mkString(v, s, PathSet()); } -static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, mkBool(v, !context.empty()); } -static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po mkString(v, s, context2); } -static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); /* Extract the context of a string as a structured Nix value. @@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, v.attrs->sort(); } -static RegisterPrimOp r4("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); /* Append the given context to a given string. @@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg mkString(v, orig, context); } -static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); } diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix new file mode 100644 index 000000000..c0fbe8082 --- /dev/null +++ b/src/libexpr/primops/derivation.nix @@ -0,0 +1,27 @@ +/* This is the implementation of the ‘derivation’ builtin function. + It's actually a wrapper around the ‘derivationStrict’ primop. */ + +drvAttrs @ { outputs ? [ "out" ], ... }: + +let + + strict = derivationStrict drvAttrs; + + commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvAttrs; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName strict; + drvPath = strict.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc deleted file mode 100644 index 36b0db2bd..000000000 --- a/src/libexpr/primops/fetchGit.cc +++ /dev/null @@ -1,91 +0,0 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "store-api.hh" -#include "hash.hh" -#include "fetchers.hh" -#include "url.hh" - -namespace nix { - -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::optional<std::string> ref; - std::optional<Hash> rev; - std::string name = "source"; - bool fetchSubmodules = false; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "ref") - ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "rev") - rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "submodules") - fetchSubmodules = state.forceBool(*attr.value, *attr.pos); - else - throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), - .errPos = *attr.pos - }); - } - - if (url.empty()) - throw EvalError({ - .hint = hintfmt("'url' argument required"), - .errPos = pos - }); - - } else - url = state.coerceToString(pos, *args[0], context, false, false); - - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); - - if (evalSettings.pureEval && !rev) - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); - if (ref) attrs.insert_or_assign("ref", *ref); - if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", true); - auto input = fetchers::inputFromAttrs(attrs); - - // FIXME: use name? - auto [tree, input2] = input->fetchTree(state.store); - - state.mkAttrs(v, 8); - auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - // Backward compatibility: set 'rev' to - // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); - // Backward compatibility: set 'revCount' to 0 for a dirty tree. - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - tree.info.revCount.value_or(0)); - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); - v.attrs->sort(); - - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); -} - -static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); - -} diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 59166b777..a77035c16 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -3,8 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "url.hh" - -#include <regex> +#include "url-parts.hh" namespace nix { @@ -31,7 +30,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, *attr.pos); if (std::regex_match(value, revRegex)) - rev = Hash(value, htSHA1); + rev = Hash::parseAny(value, htSHA1); else ref = value; } @@ -65,29 +64,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::inputFromAttrs(attrs); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - if (input2->getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + if (input2.getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input2.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); } -static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 01d6ad8b0..7cd4d0fbf 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" +#include "registry.hh" #include <ctime> #include <iomanip> @@ -12,39 +13,73 @@ namespace nix { void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr<const fetchers::Input> input, - Value & v) + const fetchers::Input & input, + Value & v, + bool emptyRevFallback) { + assert(input.isImmutable()); + state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - assert(tree.info.narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI, true)); + // FIXME: support arbitrary input attributes. - if (input->getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + auto narHash = input.getNarHash(); + assert(narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + narHash->to_string(SRI, true)); + + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + } else if (emptyRevFallback) { + // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev + auto emptyHash = Hash(htSHA1); + mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); } - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (input.getType() == "git") + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); - if (tree.info.lastModified) - mkString(*state.allocAttr(v, state.symbols.create("lastModified")), - fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + else if (emptyRevFallback) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); + + if (auto lastModified = input.getLastModified()) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); + mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), + fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); + } v.attrs->sort(); } -static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +std::string fixURI(std::string uri, EvalState &state) { - settings.requireExperimentalFeature("flakes"); + state.checkURI(uri); + return uri.find("://") != std::string::npos ? uri : "file://" + uri; +} + +void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) +{ + string n(name); + attrs.emplace(name, n == "url" ? fixURI(v, state) : v); +} - std::shared_ptr<const fetchers::Input> input; +static void fetchTree( + EvalState &state, + const Pos &pos, + Value **args, + Value &v, + const std::optional<std::string> type, + bool emptyRevFallback = false +) { + fetchers::Input input; PathSet context; state.forceValue(*args[0]); @@ -56,38 +91,68 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tString) - attrs.emplace(attr.name, attr.value->string.s); + if (attr.value->type == tPath || attr.value->type == tString) + addURI( + state, + attrs, + attr.name, + state.coerceToString(*attr.pos, *attr.value, context, false, false) + ); + else if (attr.value->type == tString) + addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, attr.value->boolean); + attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean}); + else if (attr.value->type == tInt) + attrs.emplace(attr.name, attr.value->integer); else - throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); } + if (type) + attrs.emplace("type", type.value()); + if (!attrs.count("type")) throw Error({ .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), .errPos = pos }); - input = fetchers::inputFromAttrs(attrs); - } else - input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); + + if (type == "git") { + fetchers::Attrs attrs; + attrs.emplace("type", "git"); + attrs.emplace("url", url); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + input = fetchers::Input::fromURL(url); + } + } + + if (!evalSettings.pureEval && !input.isDirect()) + input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input->isImmutable()) - throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + if (evalSettings.pureEval && !input.isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); - // FIXME: use fetchOrSubstituteTree - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v); + emitTreeAttrs(state, tree, input2, v, emptyRevFallback); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + fetchTree(state, pos, args, v, std::nullopt); } -static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, std::string name) @@ -136,7 +201,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); @@ -161,12 +226,187 @@ static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Va fetch(state, pos, args, v, "fetchurl", false, ""); } +static RegisterPrimOp primop_fetchurl({ + .name = "__fetchurl", + .args = {"url"}, + .doc = R"( + Download the specified URL and return the path of the downloaded + file. This function is not available if [restricted evaluation + mode](../command-ref/conf-file.md) is enabled. + )", + .fun = prim_fetchurl, +}); + static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) { fetch(state, pos, args, v, "fetchTarball", true, "source"); } -static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); -static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); +static RegisterPrimOp primop_fetchTarball({ + .name = "fetchTarball", + .args = {"args"}, + .doc = R"( + Download the specified URL, unpack it and return the path of the + unpacked tree. The file must be a tape archive (`.tar`) compressed + with `gzip`, `bzip2` or `xz`. The top-level path component of the + files in the tarball is removed, so it is best if the tarball + contains a single directory at top level. The typical use of the + function is to obtain external Nix expression dependencies, such as + a particular version of Nixpkgs, e.g. + + ```nix + with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; + + stdenv.mkDerivation { … } + ``` + + The fetched tarball is cached for a certain amount of time (1 hour + by default) in `~/.cache/nix/tarballs/`. You can change the cache + timeout either on the command line with `--option tarball-ttl number + of seconds` or in the Nix configuration file with this option: ` + number of seconds to cache `. + + Note that when obtaining the hash with ` nix-prefetch-url ` the + option `--unpack` is required. + + This function can also verify the contents against a hash. In that + case, the function takes a set instead of a URL. The set requires + the attribute `url` and the attribute `sha256`, e.g. + + ```nix + with import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz"; + sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; + }) {}; + + stdenv.mkDerivation { … } + ``` + + This function is not available if [restricted evaluation + mode](../command-ref/conf-file.md) is enabled. + )", + .fun = prim_fetchTarball, +}); + +static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +{ + fetchTree(state, pos, args, v, "git", true); +} + +static RegisterPrimOp primop_fetchGit({ + .name = "fetchGit", + .args = {"args"}, + .doc = R"( + Fetch a path from git. *args* can be a URL, in which case the HEAD + of the repo at that URL is fetched. Otherwise, it can be an + attribute with the following attributes (all except `url` optional): + + - url + The URL of the repo. + + - name + The name of the directory the repo should be exported to in the + store. Defaults to the basename of the URL. + + - rev + The git revision to fetch. Defaults to the tip of `ref`. + + - ref + The git ref to look for the requested revision under. This is + often a branch or tag name. Defaults to `HEAD`. + + By default, the `ref` value is prefixed with `refs/heads/`. As + of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts + with `refs/`. + + - submodules + A Boolean parameter that specifies whether submodules should be + checked out. Defaults to `false`. + + Here are some examples of how to use `fetchGit`. + + - To fetch a private repository over SSH: + + ```nix + builtins.fetchGit { + url = "git@github.com:my-secret/repository.git"; + ref = "master"; + rev = "adab8b916a45068c044658c4158d81878f9ed1c3"; + } + ``` + + - To fetch an arbitrary reference: + + ```nix + builtins.fetchGit { + url = "https://github.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; + } + ``` + + - If the revision you're looking for is in the default branch of + the git repository you don't strictly need to specify the branch + name in the `ref` attribute. + + However, if the revision you're looking for is in a future + branch for the non-default branch you will need to specify the + the `ref` attribute as well. + + ```nix + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + ref = "1.11-maintenance"; + } + ``` + + > **Note** + > + > It is nice to always specify the branch which a revision + > belongs to. Without the branch being specified, the fetcher + > might fail if the default branch changes. Additionally, it can + > be confusing to try a commit from a non-default branch and see + > the fetch fail. If the branch is specified the fault is much + > more obvious. + + - If the revision you're looking for is in the default branch of + the git repository you may omit the `ref` attribute. + + ```nix + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + } + ``` + + - To fetch a specific tag: + + ```nix + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + ref = "refs/tags/1.9"; + } + ``` + + - To fetch the latest version of a remote branch: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + ref = "master"; + } + ``` + + > **Note** + > + > Nix will refetch the branch in accordance with + > the option `tarball-ttl`. + + > **Note** + > + > This behavior is disabled in *Pure evaluation mode*. + )", + .fun = prim_fetchGit, +}); } diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index b00827a4b..77bff44ae 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -88,6 +88,6 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va } } -static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 7ba5e1c14..4eb6dac81 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -28,6 +28,12 @@ public: return s == s2.s; } + // FIXME: remove + bool operator == (std::string_view s2) const + { + return s->compare(s2) == 0; + } + bool operator != (const Symbol & s2) const { return s != s2.s; @@ -68,9 +74,10 @@ private: Symbols symbols; public: - Symbol create(const string & s) + Symbol create(std::string_view s) { - std::pair<Symbols::iterator, bool> res = symbols.insert(s); + // FIXME: avoid allocation if 's' already exists in the symbol table. + std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s)); return Symbol(&*res.first); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 71025824e..fe11bb2ed 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -166,6 +166,13 @@ struct Value { return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; } + + /* Check whether forcing this value requires a trivial amount of + computation. In particular, function applications are + non-trivial. */ + bool isTrivial() const; + + std::vector<std::pair<Path, std::string>> getContext(); }; diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index feb0a6085..1e59faa73 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if<int64_t>(&attr.second)) { + if (auto v = std::get_if<uint64_t>(&attr.second)) { json[attr.first] = *v; } else if (auto v = std::get_if<std::string>(&attr.second)) { json[attr.first] = *v; @@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name) return *s; } -std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) +std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if<int64_t>(&i->second)) + if (auto v = std::get_if<uint64_t>(&i->second)) return *v; throw Error("input attribute '%s' is not an integer", name); } -int64_t getIntAttr(const Attrs & attrs, const std::string & name) +uint64_t getIntAttr(const Attrs & attrs, const std::string & name) { auto s = maybeGetIntAttr(attrs, name); if (!s) @@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if<int64_t>(&i->second)) - return *v; + if (auto v = std::get_if<Explicit<bool>>(&i->second)) + return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs) { std::map<std::string, std::string> query; for (auto & attr : attrs) { - if (auto v = std::get_if<int64_t>(&attr.second)) { + if (auto v = std::get_if<uint64_t>(&attr.second)) { query.insert_or_assign(attr.first, fmt("%d", *v)); } else if (auto v = std::get_if<std::string>(&attr.second)) { query.insert_or_assign(attr.first, *v); diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index d6e0ae000..4b4630c80 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,9 +13,14 @@ namespace nix::fetchers { template<typename T> struct Explicit { T t; + + bool operator ==(const Explicit<T> & other) const + { + return t == other.t; + } }; -typedef std::variant<std::string, int64_t, Explicit<bool>> Attr; +typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr; typedef std::map<std::string, Attr> Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin std::string getStrAttr(const Attrs & attrs, const std::string & name); -std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); +std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); -int64_t getIntAttr(const Attrs & attrs, const std::string & name); +uint64_t getIntAttr(const Attrs & attrs, const std::string & name); std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9174c3de4..49851f7bc 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,71 +5,276 @@ namespace nix::fetchers { -std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr; +std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr; -void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme) +void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>(); + if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>(); inputSchemes->push_back(std::move(inputScheme)); } -std::unique_ptr<Input> inputFromURL(const ParsedURL & url) +Input Input::fromURL(const std::string & url) +{ + return fromURL(parseURL(url)); +} + +static void fixupInput(Input & input) +{ + // Check common attributes. + input.getType(); + input.getRef(); + if (input.getRev()) + input.immutable = true; + input.getRevCount(); + input.getLastModified(); + if (input.getNarHash()) + input.immutable = true; +} + +Input Input::fromURL(const ParsedURL & url) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url); - if (res) return res; + if (res) { + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); + } } - throw Error("input '%s' is unsupported", url.url); -} -std::unique_ptr<Input> inputFromURL(const std::string & url) -{ - return inputFromURL(parseURL(url)); + throw Error("input '%s' is unsupported", url.url); } -std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) +Input Input::fromAttrs(Attrs && attrs) { - auto attrs2(attrs); - attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs2); + auto res = inputScheme->inputFromAttrs(attrs); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = newHashAllowEmpty(*narHash, {}); - return res; + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); + + Input input; + input.attrs = attrs; + fixupInput(input); + return input; +} + +ParsedURL Input::toURL() const +{ + if (!scheme) + throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + return scheme->toURL(*this); +} + +std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const +{ + auto url = toURL(); + for (auto & attr : extraQuery) + url.query.insert(attr); + return url.to_string(); +} + +std::string Input::to_string() const +{ + return toURL().to_string(); } Attrs Input::toAttrs() const { - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI, true)); - attrs.emplace("type", type()); return attrs; } -std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const +bool Input::hasAllInfo() const { - auto [tree, input] = fetchTreeInternal(store); + return getNarHash() && scheme && scheme->hasAllInfo(*this); +} + +bool Input::operator ==(const Input & other) const +{ + return attrs == other.attrs; +} + +bool Input::contains(const Input & other) const +{ + if (*this == other) return true; + auto other2(other); + other2.attrs.erase("ref"); + other2.attrs.erase("rev"); + if (*this == other2) return true; + return false; +} + +std::pair<Tree, Input> Input::fetch(ref<Store> store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (hasAllInfo()) { + try { + auto storePath = computeStorePath(*store); + + store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + to_string(), store->printStorePath(storePath)); + + auto actualPath = store->toRealPath(storePath); + + return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this}; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", to_string(), e.what()); + } + } + + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(tree.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); - if (input->narHash) - assert(input->narHash == tree.info.narHash); + if (auto prevNarHash = getNarHash()) { + if (narHash != *prevNarHash) + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + } - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); + if (auto prevLastModified = getLastModified()) { + if (input.getLastModified() != prevLastModified) + throw Error("'lastModified' attribute mismatch in input '%s', expected %d", + input.to_string(), *prevLastModified); + } + + if (auto prevRevCount = getRevCount()) { + if (input.getRevCount() != prevRevCount) + throw Error("'revCount' attribute mismatch in input '%s', expected %d", + input.to_string(), *prevRevCount); + } + + input.immutable = true; + + assert(input.hasAllInfo()); return {std::move(tree), input}; } +Input Input::applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const +{ + if (!scheme) return *this; + return scheme->applyOverrides(*this, ref, rev); +} + +void Input::clone(const Path & destDir) const +{ + assert(scheme); + scheme->clone(*this, destDir); +} + +std::optional<Path> Input::getSourcePath() const +{ + assert(scheme); + return scheme->getSourcePath(*this); +} + +void Input::markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const +{ + assert(scheme); + return scheme->markChangedFile(*this, file, commitMsg); +} + +StorePath Input::computeStorePath(Store & store) const +{ + auto narHash = getNarHash(); + if (!narHash) + throw Error("cannot compute store path for mutable input '%s'", to_string()); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); +} + +std::string Input::getType() const +{ + return getStrAttr(attrs, "type"); +} + +std::optional<Hash> Input::getNarHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "narHash")) { + auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); + if (hash.type != htSHA256) + throw UsageError("narHash must use SHA-256"); + return hash; + } + return {}; +} + +std::optional<std::string> Input::getRef() const +{ + if (auto s = maybeGetStrAttr(attrs, "ref")) + return *s; + return {}; +} + +std::optional<Hash> Input::getRev() const +{ + if (auto s = maybeGetStrAttr(attrs, "rev")) + return Hash::parseAny(*s, htSHA1); + return {}; +} + +std::optional<uint64_t> Input::getRevCount() const +{ + if (auto n = maybeGetIntAttr(attrs, "revCount")) + return *n; + return {}; +} + +std::optional<time_t> Input::getLastModified() const +{ + if (auto n = maybeGetIntAttr(attrs, "lastModified")) + return *n; + return {}; +} + +ParsedURL InputScheme::toURL(const Input & input) +{ + throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); +} + +Input InputScheme::applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) +{ + if (ref) + throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); + if (rev) + throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev()); + return input; +} + +std::optional<Path> InputScheme::getSourcePath(const Input & input) +{ + return {}; +} + +void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) +{ + assert(false); +} + +void InputScheme::clone(const Input & input, const Path & destDir) +{ + throw Error("do not know how to clone input '%s'", input.to_string()); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae67..e8ae59143 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "hash.hh" #include "path.hh" -#include "tree-info.hh" #include "attrs.hh" #include "url.hh" @@ -13,73 +12,106 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Tree { Path actualPath; StorePath storePath; - TreeInfo info; + Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {} }; -struct Input : std::enable_shared_from_this<Input> +struct InputScheme; + +struct Input { - std::optional<Hash> narHash; // FIXME: implement + friend struct InputScheme; + + std::shared_ptr<InputScheme> scheme; // note: can be null + Attrs attrs; + bool immutable = false; + bool direct = true; + +public: + static Input fromURL(const std::string & url); - virtual std::string type() const = 0; + static Input fromURL(const ParsedURL & url); - virtual ~Input() { } + static Input fromAttrs(Attrs && attrs); - virtual bool operator ==(const Input & other) const { return false; } + ParsedURL toURL() const; + + std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const; + + std::string to_string() const; + + Attrs toAttrs() const; /* Check whether this is a "direct" input, that is, not one that goes through a registry. */ - virtual bool isDirect() const { return true; } + bool isDirect() const { return direct; } /* Check whether this is an "immutable" input, that is, one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } + bool isImmutable() const { return immutable; } - virtual bool contains(const Input & other) const { return false; } + bool hasAllInfo() const; - virtual std::optional<std::string> getRef() const { return {}; } + bool operator ==(const Input & other) const; - virtual std::optional<Hash> getRev() const { return {}; } + bool contains(const Input & other) const; - virtual ParsedURL toURL() const = 0; + std::pair<Tree, Input> fetch(ref<Store> store) const; - std::string to_string() const - { - return toURL().to_string(); - } + Input applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const; - Attrs toAttrs() const; + void clone(const Path & destDir) const; - std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; + std::optional<Path> getSourcePath() const; -private: + void markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const; - virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; + StorePath computeStorePath(Store & store) const; - virtual Attrs toAttrsInternal() const = 0; + // Convenience functions for common attributes. + std::string getType() const; + std::optional<Hash> getNarHash() const; + std::optional<std::string> getRef() const; + std::optional<Hash> getRev() const; + std::optional<uint64_t> getRevCount() const; + std::optional<time_t> getLastModified() const; }; struct InputScheme { - virtual ~InputScheme() { } + virtual ~InputScheme() + { } - virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0; + virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0; -}; + virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0; + + virtual ParsedURL toURL(const Input & input); + + virtual bool hasAllInfo(const Input & input) = 0; -std::unique_ptr<Input> inputFromURL(const ParsedURL & url); + virtual Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev); -std::unique_ptr<Input> inputFromURL(const std::string & url); + virtual void clone(const Input & input, const Path & destDir); -std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs); + virtual std::optional<Path> getSourcePath(const Input & input); + + virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg); + + virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0; +}; -void registerInputScheme(std::unique_ptr<InputScheme> && fetcher); +void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); struct DownloadFileResult { @@ -92,12 +124,14 @@ DownloadFileResult downloadFile( ref<Store> store, const std::string & url, const std::string & name, - bool immutable); + bool immutable, + const Headers & headers = {}); -Tree downloadTarball( +std::pair<Tree, time_t> downloadTarball( ref<Store> store, const std::string & url, const std::string & name, - bool immutable); + bool immutable, + const Headers & headers = {}); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 75ce5ee8b..a6411b02b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include <sys/time.h> @@ -22,80 +23,152 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional<std::string> ref; - std::optional<Hash> rev; - bool shallow = false; - bool submodules = false; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else if (name == "shallow") + attrs.emplace(name, Explicit<bool> { value == "1" }); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } - GitInput(const ParsedURL & url) : url(url) - { } + std::optional<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; - std::string type() const override { return "git"; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (std::regex_search(*ref, badGitRefRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } + + Input input; + input.attrs = attrs; + return input; + } - bool operator ==(const Input & other) const override + ParsedURL toURL(const Input & input) override { - auto other2 = dynamic_cast<const GitInput *>(&other); + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; + } + + bool hasAllInfo(const Input & input) override + { + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); } - bool isImmutable() const override + Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) override { - return (bool) rev || narHash; + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; } - std::optional<std::string> getRef() const override { return ref; } + void clone(const Input & input, const Path & destDir) override + { + auto [isLocal, actualUrl] = getActualUrl(input); + + Strings args = {"clone"}; + + args.push_back(actualUrl); - std::optional<Hash> getRev() const override { return rev; } + if (auto ref = input.getRef()) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented"); + + args.push_back(destDir); - ParsedURL toURL() const override + runProgram("git", true, args); + } + + std::optional<Path> getSourcePath(const Input & input) override { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) + return url.path; + return {}; } - Attrs toAttrsInternal() const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + auto sourcePath = getSourcePath(input); + assert(sourcePath); + + runProgram("git", true, + { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } - std::pair<bool, std::string> getActualUrl() const + std::pair<bool, std::string> getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared<GitInput>(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -106,39 +179,35 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair<Tree, std::shared_ptr<const Input>> + -> std::pair<Tree, Input> { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { - Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -197,44 +266,44 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } - }; + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - return {std::move(tree), input}; + return { + Tree(store->printStorePath(storePath), std::move(storePath)), + input + }; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; } else { if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -248,18 +317,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -282,9 +351,10 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - auto fetchRef = input->ref->compare(0, 5, "refs/") == 0 - ? *input->ref - : "refs/heads/" + *input->ref; + auto ref = input.getRef(); + auto fetchRef = ref->compare(0, 5, "refs/") == 0 + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; @@ -300,8 +370,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -311,7 +381,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -333,7 +403,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -342,7 +412,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -352,18 +422,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -382,60 +452,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (std::regex_search(*ref, badGitRefRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); }); +static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0bee1d6b3..92ff224f7 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -3,86 +3,180 @@ #include "fetchers.hh" #include "globals.hh" #include "store-api.hh" +#include "types.hh" +#include "url-parts.hh" +#include <optional> #include <nlohmann/json.hpp> namespace nix::fetchers { -std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +struct DownloadUrl +{ + std::string url; + Headers headers; +}; + +// A github or gitlab host +const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +std::regex hostRegex(hostRegexS, std::regex::ECMAScript); -struct GitHubInput : Input +struct GitArchiveInputScheme : InputScheme { - std::string owner; - std::string repo; - std::optional<std::string> ref; - std::optional<Hash> rev; + virtual std::string type() = 0; - std::string type() const override { return "github"; } + virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0; - bool operator ==(const Input & other) const override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast<const GitHubInput *>(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != type()) return {}; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + + std::optional<Hash> rev; + std::optional<std::string> ref; + std::optional<std::string> host_url; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + rev = Hash::parseAny(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + ref = path[2]; + else + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (rev) + throw BadURL("URL '%s' contains multiple commit hashes", url.url); + rev = Hash::parseAny(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + if (ref) + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + ref = value; + } + else if (name == "host") { + if (!std::regex_match(value, hostRegex)) + throw BadURL("URL '%s' contains an invalid instance host", url.url); + host_url = value; + } + // FIXME: barf on unsupported attributes + } + + if (ref && rev) + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); + + Input input; + input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("owner", path[0]); + input.attrs.insert_or_assign("repo", path[1]); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + if (host_url) input.attrs.insert_or_assign("host", *host_url); + + return input; } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != type()) return {}; - std::optional<std::string> getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") + throw Error("unsupported input attribute '%s'", name); - std::optional<Hash> getRev() const override { return rev; } + getStrAttr(attrs, "owner"); + getStrAttr(attrs, "repo"); - ParsedURL toURL() const override + Input input; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto ref = input.getRef(); + auto rev = input.getRev(); auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(Base16, false); return ParsedURL { - .scheme = "github", + .scheme = type(), .path = path, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + Input applyOverrides( + const Input & _input, + std::optional<std::string> ref, + std::optional<Hash> rev) override { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); + auto input(_input); + if (rev && ref) + throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'", + rev->gitRev(), *ref, input.to_string()); + if (rev) { + input.attrs.insert_or_assign("rev", rev->gitRev()); + input.attrs.erase("ref"); + } + if (ref) { + input.attrs.insert_or_assign("ref", *ref); + input.attrs.erase("rev"); + } + return input; + } + + std::optional<std::string> getAccessToken(const std::string & host) const + { + auto tokens = settings.accessTokens.get(); + if (auto token = get(tokens, host)) + return *token; + return {}; + } - if (!rev) { - auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false).storePath))); - rev = Hash(std::string { json["sha"] }, htSHA1); - debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + Headers makeHeadersWithAuthTokens(const std::string & host) const + { + Headers headers; + auto accessToken = getAccessToken(host); + if (accessToken) { + auto hdr = accessHeaderFromToken(*accessToken); + if (hdr) + headers.push_back(*hdr); + else + warn("Unrecognized access token for host '%s'", host); } + return headers; + } - auto input = std::make_shared<GitHubInput>(*this); - input->ref = {}; - input->rev = *rev; + virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0; + + virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override + { + Input input(_input); + + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); + + auto rev = input.getRev(); + if (!rev) rev = getRevFromRef(store, input); + + input.attrs.erase("ref"); + input.attrs.insert_or_assign("rev", rev->gitRev()); Attrs immutableAttrs({ {"type", "git-tarball"}, @@ -90,36 +184,25 @@ struct GitHubInput : Input }); if (auto res = getCache()->lookup(store, immutableAttrs)) { + input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { - Tree{ - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, - }, + Tree(store->toRealPath(res->second), std::move(res->second)), input }; } - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. + auto url = getDownloadUrl(input); - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); + auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; - - auto tree = downloadTarball(store, url, "source", true); + input.attrs.insert_or_assign("lastModified", lastModified); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} + {"lastModified", lastModified} }, tree.storePath, true); @@ -128,68 +211,130 @@ struct GitHubInput : Input } }; -struct GitHubInputScheme : InputScheme +struct GitHubInputScheme : GitArchiveInputScheme { - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + std::string type() override { return "github"; } + + std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override { - if (url.scheme != "github") return nullptr; + // Github supports PAT/OAuth2 tokens and HTTP Basic + // Authentication. The former simply specifies the token, the + // latter can use the token as the password. Only the first + // is used here. See + // https://developer.github.com/v3/#authentication and + // https://docs.github.com/en/developers/apps/authorizing-oath-apps + return std::pair<std::string, std::string>("Authorization", fmt("token %s", token)); + } - auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); - auto input = std::make_unique<GitHubInput>(); + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override + { + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + + Headers headers = makeHeadersWithAuthTokens(host); + + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false, headers).storePath))); + auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); + debug("HEAD revision for '%s' is %s", url, rev.gitRev()); + return rev; + } - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("GitHub URL '%s' is invalid", url.url); + DownloadUrl getDownloadUrl(const Input & input) const override + { + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl { url, headers }; + } - if (input->ref && input->rev) - throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + void clone(const Input & input, const Path & destDir) override + { + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) + .clone(destDir); + } +}; - input->owner = path[0]; - input->repo = path[1]; +struct GitLabInputScheme : GitArchiveInputScheme +{ + std::string type() override { return "gitlab"; } - return input; + std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override + { + // Gitlab supports 4 kinds of authorization, two of which are + // relevant here: OAuth2 and PAT (Private Access Token). The + // user can indicate which token is used by specifying the + // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT". + // If the <TYPE> is unrecognized, this will fall back to + // treating this simply has <HDRNAME>:<HDRVAL>. See + // https://docs.gitlab.com/12.10/ee/api/README.html#authentication + auto fldsplit = token.find_first_of(':'); + // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ... + if ("OAuth2" == token.substr(0, fldsplit)) + return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1))); + if ("PAT" == token.substr(0, fldsplit)) + return std::make_pair("Private-token", token.substr(fldsplit+1)); + warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); + return std::nullopt; } - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { - if (maybeGetStrAttr(attrs, "type") != "github") return {}; + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); + // See rate limiting note below + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + + Headers headers = makeHeadersWithAuthTokens(host); + + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false, headers).storePath))); + auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); + debug("HEAD revision for '%s' is %s", url, rev.gitRev()); + return rev; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported GitHub input attribute '%s'", name); - - auto input = std::make_unique<GitHubInput>(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; + DownloadUrl getDownloadUrl(const Input & input) const override + { + // This endpoint has a rate limit threshold that may be + // server-specific and vary based whether the user is + // authenticated via an accessToken or not, but the usual rate + // is 10 reqs/sec/ip-addr. See + // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); + + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl { url, headers }; + } + + void clone(const Input & input, const Path & destDir) override + { + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); + // FIXME: get username somewhere + Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) + .clone(destDir); } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); }); +static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); }); +static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); }); } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc new file mode 100644 index 000000000..10e59919a --- /dev/null +++ b/src/libfetchers/indirect.cc @@ -0,0 +1,105 @@ +#include "fetchers.hh" +#include "url-parts.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInputScheme : InputScheme +{ + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return {}; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + + std::optional<Hash> rev; + std::optional<std::string> ref; + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + rev = Hash::parseAny(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + rev = Hash::parseAny(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + std::string id = path[0]; + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + // FIXME: forbid query params? + + Input input; + input.direct = false; + input.attrs.insert_or_assign("type", "indirect"); + input.attrs.insert_or_assign("id", id); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + + return input; + } + + std::optional<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") + throw Error("unsupported indirect input attribute '%s'", name); + + auto id = getStrAttr(attrs, "id"); + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + Input input; + input.direct = false; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + ParsedURL url; + url.scheme = "flake"; + url.path = getStrAttr(input.attrs, "id"); + if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; }; + if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + bool hasAllInfo(const Input & input) override + { + return false; + } + + Input applyOverrides( + const Input & _input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto input(_input); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + return input; + } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } +}; + +static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); }); + +} diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2e0d4bf4d..7d3d52751 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include <sys/time.h> @@ -10,76 +11,124 @@ using namespace std::string_literals; namespace nix::fetchers { -struct MercurialInput : Input +struct MercurialInputScheme : InputScheme { - ParsedURL url; - std::optional<std::string> ref; - std::optional<Hash> rev; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return {}; - MercurialInput(const ParsedURL & url) : url(url) - { } + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); - std::string type() const override { return "hg"; } + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } - bool operator ==(const Input & other) const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - auto other2 = dynamic_cast<const MercurialInput *>(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + throw Error("unsupported Mercurial input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + } + + Input input; + input.attrs = attrs; + return input; } - bool isImmutable() const override + ParsedURL toURL(const Input & input) override { - return (bool) rev || narHash; + auto url = parseURL(getStrAttr(input.attrs, "url")); + url.scheme = "hg+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + return url; } - std::optional<std::string> getRef() const override { return ref; } + bool hasAllInfo(const Input & input) override + { + // FIXME: ugly, need to distinguish between dirty and clean + // default trees. + return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); + } - std::optional<Hash> getRev() const override { return rev; } + Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + return res; + } - ParsedURL toURL() const override + std::optional<Path> getSourcePath(const Input & input) override { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - return url; + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) + return url.path; + return {}; } - Attrs toAttrsInternal() const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + auto sourcePath = getSourcePath(input); + assert(sourcePath); + + // FIXME: shut up if file is already tracked. + runProgram("hg", true, + { "add", *sourcePath + "/" + std::string(file) }); + + if (commitMsg) + runProgram("hg", true, + { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } - std::pair<bool, std::string> getActualUrl() const + std::pair<bool, std::string> getActualUrl(const Input & input) const { + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; return {isLocal, isLocal ? url.path : url.base}; } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared<MercurialInput>(*this); + Input input(_input); - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; @@ -94,7 +143,7 @@ struct MercurialInput : Input if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); auto files = tokenizeString<std::set<std::string>>( runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); @@ -116,60 +165,54 @@ struct MercurialInput : Input auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); - return {Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - }, input}; + return { + Tree(store->printStorePath(storePath), std::move(storePath)), + input + }; } } - if (!input->ref) input->ref = "default"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); auto getImmutableAttrs = [&]() { return Attrs({ {"type", "hg"}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair<Tree, std::shared_ptr<const Input>> + -> std::pair<Tree, Input> { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { - Tree{ - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; - if (input->rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); Attrs mutableAttrs({ {"type", "hg"}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -178,10 +221,10 @@ struct MercurialInput : Input /* If this is a commit hash that we already have, we don't have to pull again. */ - if (!(input->rev + if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); @@ -210,9 +253,9 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; + input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); @@ -220,18 +263,18 @@ struct MercurialInput : Input Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"revCount", (int64_t) revCount}, }); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -250,54 +293,6 @@ struct MercurialInput : Input } }; -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); }); +static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ba2cc192e..bcb904c0d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -3,65 +3,86 @@ namespace nix::fetchers { -struct PathInput : Input +struct PathInputScheme : InputScheme { - Path path; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return {}; - /* Allow the user to pass in "fake" tree info attributes. This is - useful for making a pinned tree work the same as the repository - from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - std::optional<Hash> rev; - std::optional<uint64_t> revCount; - std::optional<time_t> lastModified; + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - std::string type() const override { return "path"; } + Input input; + input.attrs.insert_or_assign("type", "path"); + input.attrs.insert_or_assign("path", url.path); - std::optional<Hash> getRev() const override { return rev; } + for (auto & [name, value] : url.query) + if (name == "rev" || name == "narHash") + input.attrs.insert_or_assign(name, value); + else if (name == "revCount" || name == "lastModified") { + uint64_t n; + if (!string2Int(value, n)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input.attrs.insert_or_assign(name, n); + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const PathInput *>(&other); - return - other2 - && path == other2->path - && rev == other2->rev - && revCount == other2->revCount - && lastModified == other2->lastModified; + return input; } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) narHash; + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree + work the same as the repository from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") + // checked in Input::fromAttrs + ; + else + throw Error("unsupported path input attribute '%s'", name); + + Input input; + input.attrs = attrs; + return input; } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - auto query = attrsToQuery(toAttrsInternal()); + auto query = attrsToQuery(input.attrs); query.erase("path"); + query.erase("type"); return ParsedURL { .scheme = "path", - .path = path, + .path = getStrAttr(input.attrs, "path"), .query = query, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::optional<Path> getSourcePath(const Input & input) override { - Attrs attrs; - attrs.emplace("path", path); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (revCount) - attrs.emplace("revCount", *revCount); - if (lastModified) - attrs.emplace("lastModified", *lastModified); - return attrs; + return getStrAttr(input.attrs, "path"); } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - auto input = std::make_shared<PathInput>(*this); + // nothing to do + } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + auto path = getStrAttr(input.attrs, "path"); // FIXME: check whether access to 'path' is allowed. @@ -74,75 +95,13 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); - return - { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - .info = TreeInfo { - .revCount = revCount, - .lastModified = lastModified - } - }, - input - }; - } - -}; - -struct PathInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "path") return nullptr; - - auto input = std::make_unique<PathInput>(); - input->path = url.path; - - for (auto & [name, value] : url.query) - if (name == "rev") - input->rev = Hash(value, htSHA1); - else if (name == "revCount") { - uint64_t revCount; - if (!string2Int(value, revCount)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->revCount = revCount; - } - else if (name == "lastModified") { - time_t lastModified; - if (!string2Int(value, lastModified)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->lastModified = lastModified; - } - else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - - return input; - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - auto input = std::make_unique<PathInput>(); - input->path = getStrAttr(attrs, "path"); - - for (auto & [name, value] : attrs) - if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); - else if (name == "revCount") - input->revCount = getIntAttr(attrs, "revCount"); - else if (name == "lastModified") - input->lastModified = getIntAttr(attrs, "lastModified"); - else if (name == "type" || name == "path") - ; - else - throw Error("unsupported path input attribute '%s'", name); - - return input; + return { + Tree(store->toRealPath(*storePath), std::move(*storePath)), + input + }; } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); }); +static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); }); } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc new file mode 100644 index 000000000..4367ee810 --- /dev/null +++ b/src/libfetchers/registry.cc @@ -0,0 +1,212 @@ +#include "registry.hh" +#include "fetchers.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +std::shared_ptr<Registry> Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared<Registry>(type); + + if (!pathExists(path)) + return std::make_shared<Registry>(type); + + try { + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + + if (version == 2) { + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Attrs extraAttrs; + auto j = toAttrs.find("dir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } + auto exact = i.find("exact"); + registry->entries.push_back( + Entry { + .from = Input::fromAttrs(jsonToAttrs(i["from"])), + .to = Input::fromAttrs(std::move(toAttrs)), + .extraAttrs = extraAttrs, + .exact = exact != i.end() && exact.value() + }); + } + } + + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + + } catch (nlohmann::json::exception & e) { + warn("cannot parse flake registry '%s': %s", path, e.what()); + } catch (Error & e) { + warn("cannot read flake registry '%s': %s", path, e.what()); + } + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json arr; + for (auto & entry : entries) { + nlohmann::json obj; + obj["from"] = attrsToJson(entry.from.toAttrs()); + obj["to"] = attrsToJson(entry.to.toAttrs()); + if (!entry.extraAttrs.empty()) + obj["to"].update(attrsToJson(entry.extraAttrs)); + if (entry.exact) + obj["exact"] = true; + arr.emplace_back(std::move(obj)); + } + + nlohmann::json json; + json["version"] = 2; + json["flakes"] = std::move(arr); + + createDirs(dirOf(path)); + writeFile(path, json.dump(2)); +} + +void Registry::add( + const Input & from, + const Input & to, + const Attrs & extraAttrs) +{ + entries.emplace_back( + Entry { + .from = from, + .to = to, + .extraAttrs = extraAttrs + }); +} + +void Registry::remove(const Input & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (i->from == input) + i = entries.erase(i); + else + ++i; +} + +static Path getSystemRegistryPath() +{ + return settings.nixConfDir + "/registry.json"; +} + +static std::shared_ptr<Registry> getSystemRegistry() +{ + static auto systemRegistry = + Registry::read(getSystemRegistryPath(), Registry::System); + return systemRegistry; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr<Registry> getUserRegistry() +{ + static auto userRegistry = + Registry::read(getUserRegistryPath(), Registry::User); + return userRegistry; +} + +static std::shared_ptr<Registry> flagRegistry = + std::make_shared<Registry>(Registry::Flag); + +std::shared_ptr<Registry> getFlagRegistry() +{ + return flagRegistry; +} + +void overrideRegistry( + const Input & from, + const Input & to, + const Attrs & extraAttrs) +{ + flagRegistry->add(from, to, extraAttrs); +} + +static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry.get(); + + if (!hasPrefix(path, "/")) { + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) + store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); + path = store->toRealPath(storePath); + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref<Store> store) +{ + Registries registries; + registries.push_back(getFlagRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getSystemRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::pair<Input, Attrs> lookupInRegistries( + ref<Store> store, + const Input & _input) +{ + Attrs extraAttrs; + int n = 0; + Input input(_input); + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + if (entry.exact) { + if (entry.from == input) { + input = entry.to; + extraAttrs = entry.extraAttrs; + goto restart; + } + } else { + if (entry.from.contains(input)) { + input = entry.to.applyOverrides( + !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(), + !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>()); + extraAttrs = entry.extraAttrs; + goto restart; + } + } + } + } + + if (!input.isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input.to_string()); + + debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string()); + + return {input, extraAttrs}; +} + +} diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh new file mode 100644 index 000000000..1077af020 --- /dev/null +++ b/src/libfetchers/registry.hh @@ -0,0 +1,64 @@ +#pragma once + +#include "types.hh" +#include "fetchers.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + System = 2, + Global = 3, + }; + + RegistryType type; + + struct Entry + { + Input from, to; + Attrs extraAttrs; + bool exact = false; + }; + + std::vector<Entry> entries; + + Registry(RegistryType type) + : type(type) + { } + + static std::shared_ptr<Registry> read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const Input & from, + const Input & to, + const Attrs & extraAttrs); + + void remove(const Input & input); +}; + +typedef std::vector<std::shared_ptr<Registry>> Registries; + +std::shared_ptr<Registry> getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref<Store> store); + +void overrideRegistry( + const Input & from, + const Input & to, + const Attrs & extraAttrs); + +std::pair<Input, Attrs> lookupInRegistries( + ref<Store> store, + const Input & input); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f5356f0af..8c0f20475 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" +#include "types.hh" namespace nix::fetchers { @@ -12,7 +13,8 @@ DownloadFileResult downloadFile( ref<Store> store, const std::string & url, const std::string & name, - bool immutable) + bool immutable, + const Headers & headers) { // FIXME: check store @@ -37,6 +39,7 @@ DownloadFileResult downloadFile( return useCached(); FileTransferRequest request(url); + request.headers = headers; if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); FileTransferResult res; @@ -67,8 +70,10 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + hashString(htSHA256, *sink.s), + }; info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Flat, @@ -105,11 +110,12 @@ DownloadFileResult downloadFile( }; } -Tree downloadTarball( +std::pair<Tree, time_t> downloadTarball( ref<Store> store, const std::string & url, const std::string & name, - bool immutable) + bool immutable, + const Headers & headers) { Attrs inAttrs({ {"type", "tarball"}, @@ -120,15 +126,12 @@ Tree downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), - }, + return { + Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)), + getIntAttr(cached->infoAttrs, "lastModified") }; - auto res = downloadFile(store, url, name, immutable); + auto res = downloadFile(store, url, name, immutable, headers); std::optional<StorePath> unpackedStorePath; time_t lastModified; @@ -160,119 +163,74 @@ Tree downloadTarball( *unpackedStorePath, immutable); - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, - }, + return { + Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)), + lastModified, }; } -struct TarballInput : Input -{ - ParsedURL url; - std::optional<Hash> hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const TarballInput *>(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (hash) - attrs.emplace("hash", hash->to_string(SRI, true)); - return attrs; - } - - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared<TarballInput>(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - struct TarballInputScheme : InputScheme { - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique<TarballInput>(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } - + return {}; + + Input input; + input.attrs.insert_or_assign("type", "tarball"); + input.attrs.insert_or_assign("url", url.to_string()); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") throw Error("unsupported tarball input attribute '%s'", name); - auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - input->hash = newHashAllowEmpty(*hash, {}); - + Input input; + input.attrs = attrs; + //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + /* + else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) + url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true)); + */ + return url; + } + + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + return {std::move(tree), input}; + } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); +static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); } diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc deleted file mode 100644 index b2d8cfc8d..000000000 --- a/src/libfetchers/tree-info.cc +++ /dev/null @@ -1,14 +0,0 @@ -#include "tree-info.hh" -#include "store-api.hh" - -#include <nlohmann/json.hpp> - -namespace nix::fetchers { - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); -} - -} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh deleted file mode 100644 index 2c7347281..000000000 --- a/src/libfetchers/tree-info.hh +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include "path.hh" -#include "hash.hh" - -#include <nlohmann/json_fwd.hpp> - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional<uint64_t> revCount; - std::optional<time_t> lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; -}; - -} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 051668e53..3411e2d7a 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -28,21 +28,31 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "option", - .description = "set a Nix configuration option (overriding nix.conf)", + .description = "set a Nix configuration option (overriding `nix.conf`)", .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - warn(e.what()); + if (!completions) + warn(e.what()); } }}, + .completer = [](size_t index, std::string_view prefix) { + if (index == 0) { + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + if (hasPrefix(s.first, prefix)) + completions->insert(s.first); + } + } }); addFlag({ .longName = "log-format", - .description = "format of log output; \"raw\", \"internal-json\", \"bar\" " - "or \"bar-with-logs\"", + .description = "format of log output; `raw`, `internal-json`, `bar` " + "or `bar-with-logs`", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index fa18f991d..0a7291780 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,12 +1,13 @@ #include "loggers.hh" #include "progress-bar.hh" +#include "util.hh" namespace nix { LogFormat defaultLogFormat = LogFormat::raw; LogFormat parseLogFormat(const std::string & logFormatStr) { - if (logFormatStr == "raw") + if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS")) return LogFormat::raw; else if (logFormatStr == "raw-with-logs") return LogFormat::rawWithLogs; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 3f7d99a1d..07b45b3b5 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -256,7 +256,7 @@ public: } else if (type == resBuildLogLine || type == resPostBuildLogLine) { - auto lastLine = trim(getS(fields, 0)); + auto lastLine = chomp(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); @@ -362,7 +362,7 @@ public: auto width = getWindowSize().second; if (width <= 0) width = std::numeric_limits<decltype(width)>::max(); - writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); + writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); } std::string getStatus(State & state) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 52718c231..2247aeca4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl) { - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); @@ -45,7 +45,7 @@ void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & pa void printMissing(ref<Store> store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) + uint64_t downloadSize, uint64_t narSize, Verbosity lvl) { if (!willBuild.empty()) { if (willBuild.size() == 1) @@ -277,6 +277,8 @@ void printVersion(const string & programName) #if HAVE_SODIUM cfg.push_back("signed-caches"); #endif + std::cout << "System type: " << settings.thisSystem << "\n"; + std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; std::cout << "User configuration files: " << @@ -384,18 +386,12 @@ RunPager::~RunPager() } -string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - - PrintFreed::~PrintFreed() { if (show) - std::cout << format("%1% store paths deleted, %2% freed\n") - % results.paths.size() - % showBytes(results.bytesFreed); + std::cout << fmt("%d store paths deleted, %s freed\n", + results.paths.size(), + showBytes(results.bytesFreed)); } Exit::~Exit() { } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index f558247c0..ffae5d796 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -47,7 +47,7 @@ void printMissing( void printMissing(ref<Store> store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); + uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo); string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end); @@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt; /* GC helpers. */ -string showBytes(unsigned long long bytes); +string showBytes(uint64_t bytes); struct GCResults; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..f6224d6a0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,17 +11,20 @@ #include "nar-accessor.hh" #include "json.hh" #include "thread-pool.hh" +#include "callback.hh" #include <chrono> #include <future> #include <regex> +#include <fstream> #include <nlohmann/json.hpp> namespace nix { BinaryCacheStore::BinaryCacheStore(const Params & params) - : Store(params) + : BinaryCacheStoreConfig(params) + , Store(params) { if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); @@ -57,6 +60,13 @@ void BinaryCacheStore::init() } } +void BinaryCacheStore::upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType) +{ + upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType); +} + void BinaryCacheStore::getFile(const std::string & path, Callback<std::shared_ptr<std::string>> callback) noexcept { @@ -113,13 +123,71 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) +AutoCloseFD openFile(const Path & path) +{ + auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return fd; +} + +struct FileSource : FdSource +{ + AutoCloseFD fd2; + + FileSource(const Path & path) + : fd2(openFile(path)) + { + fd = fd2.get(); + } +}; + +ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, + std::function<ValidPathInfo(HashResult)> mkInfo) { - // FIXME: See if we can use the original source to reduce memory usage. - auto nar = make_ref<std::string>(narSource.drain()); + auto [fdTemp, fnTemp] = createTempFile(); - if (!repair && isValidPath(info.path)) return; + AutoDelete autoDelete(fnTemp); + + auto now1 = std::chrono::steady_clock::now(); + + /* Read the NAR simultaneously into a CompressionSink+FileSink (to + write the compressed NAR to disk), into a HashSink (to get the + NAR hash), and into a NarAccessor (to get the NAR listing). */ + HashSink fileHashSink { htSHA256 }; + std::shared_ptr<FSAccessor> narAccessor; + HashSink narHashSink { htSHA256 }; + { + FdSink fileSink(fdTemp.get()); + TeeSink teeSinkCompressed { fileSink, fileHashSink }; + auto compressionSink = makeCompressionSink(compression, teeSinkCompressed); + TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; + TeeSource teeSource { narSource, teeSinkUncompressed }; + narAccessor = makeNarAccessor(teeSource); + compressionSink->finish(); + fileSink.flush(); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto info = mkInfo(narHashSink.finish()); + auto narInfo = make_ref<NarInfo>(info); + narInfo->compression = compression; + auto [fileHash, fileSize] = fileHashSink.finish(); + narInfo->fileHash = fileHash; + narInfo->fileSize = fileSize; + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), info.narSize, + ((1.0 - (double) fileSize / info.narSize) * 100.0), + duration); /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -132,23 +200,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource printStorePath(info.path), printStorePath(ref)); } - assert(nar->compare(0, narMagic.size(), narMagic) == 0); - - auto narInfo = make_ref<NarInfo>(info); - - narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); - - if (info.narHash && info.narHash != narInfo->narHash) - throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); - - auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); - - auto narAccessor = makeNarAccessor(nar); - - if (accessor_) - accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -160,33 +211,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource { auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); + listNar(res, ref<FSAccessor>(narAccessor), "", true); } } - upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json"); } - /* Compress the NAR. */ - narInfo->compression = compression; - auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); - auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); - narInfo->fileSize = narCompressed->size(); - - auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", - printStorePath(narInfo->path), narInfo->narSize, - ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), - duration); - - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/<build-id>' that specify the NAR file and member containing the debug info. */ @@ -247,12 +278,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + upsertFile(narInfo->url, + std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary), + "application/x-nix-nar"); } else stats.narWriteAverted++; - stats.narWriteBytes += nar->size(); - stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteBytes += info.narSize; + stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -261,6 +294,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource writeNarInfo(narInfo); stats.narInfoWrite++; + + return narInfo; +} + +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + if (!repair && isValidPath(info.path)) { + // FIXME: copyNAR -> null sink + narSource.drain(); + return; + } + + addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { + /* FIXME reinstate these, once we can correctly do hash modulo sink as + needed. We need to throw here in case we uploaded a corrupted store path. */ + // assert(info.narHash == nar.first); + // assert(info.narSize == nar.second); + return info; + }}); +} + +StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + unsupported("addToStoreFromDump"); + return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { + makeFixedOutputPath(method, nar.first, name), + nar.first, + }; + info.narSize = nar.second; + return info; + })->path; } bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) @@ -275,14 +343,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast<const NarInfo>(); - uint64_t narSize = 0; + LengthSink narSize; + TeeSink tee { sink, narSize }; - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - narSize += len; - }); - - auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + auto decompressor = makeDecompressionSink(info->compression, tee); try { getFile(info->url, *decompressor); @@ -294,7 +358,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME - stats.narReadBytes += narSize; + stats.narReadBytes += narSize.length; } void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, @@ -332,44 +396,52 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { - // FIXME: some cut&paste from LocalStore::addToStore(). + /* FIXME: Make BinaryCacheStore::addToStoreCommon support + non-recursive+sha256 so we can just use the default + implementation of this method in terms of addToStoreFromDump. */ - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - Hash h; + HashSink sink { hashAlgo }; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); - h = hashString(hashAlgo, *sink.s); } else { - auto s = readFile(srcPath); - dumpString(s, sink); - h = hashString(hashAlgo, s); + readFile(srcPath, sink); } + auto h = sink.finish().first; - ValidPathInfo info(makeFixedOutputPath(method, h, name)); - - auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); - - return std::move(info.path); + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink, filter); + }); + return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { + makeFixedOutputPath(method, h, name), + nar.first, + }; + info.narSize = nar.second; + info.ca = FixedOutputHash { + .method = method, + .hash = h, + }; + return info; + })->path; } StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info(computeStorePathForText(name, s, references)); - info.references = references; - - if (repair || !isValidPath(info.path)) { - StringSink sink; - dumpString(s, sink); - auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); - } - - return std::move(info.path); + auto textHash = hashString(htSHA256, s); + auto path = makeTextPath(name, textHash, references); + + if (!repair && isValidPath(path)) + return path; + + auto source = StringSource { s }; + return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { path, nar.first }; + info.narSize = nar.second; + info.ca = TextHash { textHash }; + info.references = references; + return info; + })->path; } ref<FSAccessor> BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 52ef8aa7a..5224d7ec8 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -11,17 +11,21 @@ namespace nix { struct NarInfo; -class BinaryCacheStore : public Store +struct BinaryCacheStoreConfig : virtual StoreConfig { -public: - - const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; - const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; - const Setting<bool> parallelCompression{this, false, "parallel-compression", + using StoreConfig::StoreConfig; + + const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; + const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", "enable multi-threading compression, available for xz only currently"}; +}; + +class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +{ private: @@ -36,9 +40,13 @@ public: virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) = 0; + void upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType); + /* Note: subclasses must implement at least one of the two following getFile() methods. */ @@ -54,7 +62,7 @@ public: public: - virtual void init(); + virtual void init() override; private: @@ -64,6 +72,10 @@ private: void writeNarInfo(ref<NarInfo> narInfo); + ref<const ValidPathInfo> addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, + std::function<ValidPathInfo(HashResult)> mkInfo); + public: bool isValidPathUncached(const StorePath & path) override; @@ -75,8 +87,10 @@ public: { unsupported("queryPathFromHashPart"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e927a65f0..4c88504c4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,6 +17,8 @@ #include "daemon.hh" #include "worker-protocol.hh" #include "user-lock.hh" +#include "topo-sort.hh" +#include "callback.hh" #include <algorithm> #include <iostream> @@ -295,10 +297,22 @@ public: ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); + + /* derivation goal */ +private: + std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( + const StorePath & drvPath, const StringSet & wantedOutputs, + std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); +public: + std::shared_ptr<DerivationGoal> makeDerivationGoal( + const StorePath & drvPath, + const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + + /* substitution goal */ + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -606,6 +620,33 @@ typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; +/* Unless we are repairing, we don't both to test validity and just assume it, + so the choices are `Absent` or `Valid`. */ +enum struct PathStatus { + Corrupt, + Absent, + Valid, +}; + +struct InitialOutputStatus { + StorePath path; + PathStatus status; + /* Valid in the store, and additionally non-corrupt if we are repairing */ + bool isValid() const { + return status == PathStatus::Valid; + } + /* Merely present, allowed to be corrupt */ + bool isPresent() const { + return status == PathStatus::Corrupt + || status == PathStatus::Valid; + } +}; + +struct InitialOutput { + bool wanted; + std::optional<InitialOutputStatus> known; +}; + class DerivationGoal : public Goal { private: @@ -633,19 +674,14 @@ private: /* The remainder is state held during the build. */ - /* Locks on the output paths. */ + /* Locks on (fixed) output paths. */ PathLocks outputLocks; /* All input paths (that is, the union of FS closures of the immediate input paths). */ StorePathSet inputPaths; - /* Outputs that are already valid. If we're repairing, these are - the outputs that are valid *and* not corrupt. */ - StorePathSet validPaths; - - /* Outputs that are corrupt or not valid. */ - StorePathSet missingPaths; + std::map<std::string, InitialOutput> initialOutputs; /* User selected for running the builder. */ std::unique_ptr<UserLock> buildUser; @@ -702,8 +738,8 @@ private: /* RAII object to delete the chroot directory. */ std::shared_ptr<AutoDelete> autoDelChroot; - /* Whether this is a fixed-output derivation. */ - bool fixedOutput; + /* The sort of derivation we are building. */ + DerivationType derivationType; /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; @@ -735,6 +771,31 @@ private: typedef map<StorePath, StorePath> RedirectedOutputs; RedirectedOutputs redirectedOutputs; + /* The outputs paths used during the build. + + - Input-addressed derivations or fixed content-addressed outputs are + sometimes built when some of their outputs already exist, and can not + be hidden via sandboxing. We use temporary locations instead and + rewrite after the build. Otherwise the regular predetermined paths are + put here. + + - Floating content-addressed derivations do not know their final build + output paths until the outputs are hashed, so random locations are + used, and then renamed. The randomness helps guard against hidden + self-references. + */ + OutputPathMap scratchOutputs; + + /* The final output paths of the build. + + - For input-addressed derivations, always the precomputed paths + + - For content-addressed derivations, calcuated from whatever the hash + ends up being. (Note that fixed outputs derivations that produce the + "wrong" output still install that data under its true content-address.) + */ + OutputPathMap finalOutputs; + BuildMode buildMode; /* If we're repairing without a chroot, there may be outputs that @@ -796,10 +857,12 @@ private: friend struct RestrictedStore; public: - DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode = bmNormal); + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); ~DerivationGoal(); /* Whether we need to perform hash rewriting if there are valid output paths. */ @@ -833,13 +896,16 @@ private: void getDerivation(); void loadDerivation(); void haveDerivation(); - void outputsSubstituted(); + void outputsSubstitutionTried(); + void gaveUpOnSubstitution(); void closureRepaired(); void inputsRealised(); void tryToBuild(); void tryLocalBuild(); void buildDone(); + void resolvedFinished(); + /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); @@ -894,13 +960,27 @@ private: void handleEOF(int fd) override; void flushLine(); + /* Wrappers around the corresponding Store methods that first consult the + derivation. This is currently needed because when there is no drv file + there also is no DB entry. */ + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); + OutputPathMap queryDerivationOutputMap(); + /* Return the set of (in)valid paths. */ - StorePathSet checkPathValidity(bool returnValid, bool checkHash); + void checkPathValidity(); /* Forcibly kill the child process, if any. */ void killChild(); - void addHashRewrite(const StorePath & path); + /* Create alternative path calculated from but distinct from the + input, so we can avoid overwriting outputs (or other store paths) + that already exist. */ + StorePath makeFallbackPath(const StorePath & path); + /* Make a path to another based on the output name along with the + derivation hash. */ + /* FIXME add option to randomize, so we can audit whether our + rewrites caught everything */ + StorePath makeFallbackPath(std::string_view outputName); void repairClosure(); @@ -917,8 +997,8 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode) +DerivationGoal::DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) , drvPath(drvPath) @@ -926,7 +1006,9 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath)); + name = fmt( + "building of '%s' from .drv file", + StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -935,15 +1017,18 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode) + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) , drvPath(drvPath) + , wantedOutputs(wantedOutputs) , buildMode(buildMode) { this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); + name = fmt( + "building of '%s' from in-memory derivation", + StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1075,43 +1160,60 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); + if (drv->type() == DerivationType::CAFloating) + settings.requireExperimentalFeature("ca-derivations"); + retrySubstitution = false; - for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + worker.store.addTempRoot(*i.second.second); /* Check what outputs paths are not already valid. */ - auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } /* If they are all valid, then we're done. */ - if (invalidOutputs.size() == 0 && buildMode == bmNormal) { + if (allValid && buildMode == bmNormal) { done(BuildResult::AlreadyValid); return; } parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); - if (parsedDrv->contentAddressed()) { - settings.requireExperimentalFeature("ca-derivations"); - throw Error("ca-derivations isn't implemented yet"); - } - /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known) { + warn("do not know how to query for unknown floating content-addressed derivation output yet"); + /* Nothing to wait for; tail call */ + return DerivationGoal::gaveUpOnSubstitution(); + } + addWaitee(worker.makeSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv))); + } if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstituted(); + outputsSubstitutionTried(); else - state = &DerivationGoal::outputsSubstituted; + state = &DerivationGoal::outputsSubstitutionTried; } -void DerivationGoal::outputsSubstituted() +void DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); @@ -1135,7 +1237,14 @@ void DerivationGoal::outputsSubstituted() return; } - auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + checkPathValidity(); + size_t nrInvalid = 0; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) + nrInvalid++; + } + if (buildMode == bmNormal && nrInvalid == 0) { done(BuildResult::Substituted); return; @@ -1148,9 +1257,14 @@ void DerivationGoal::outputsSubstituted() throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); - /* Otherwise, at least one of the output paths could not be - produced using a substitute. So we have to build instead. */ + /* Nothing to wait for; tail call */ + gaveUpOnSubstitution(); +} +/* At least one of the output paths could not be + produced using a substitute. So we have to build instead. */ +void DerivationGoal::gaveUpOnSubstitution() +{ /* Make sure checkPathValidity() from now on checks all outputs. */ wantedOutputs.clear(); @@ -1183,15 +1297,16 @@ void DerivationGoal::repairClosure() that produced those outputs. */ /* Get the output closure. */ + auto outputs = queryDerivationOutputMap(); StorePathSet outputClosure; - for (auto & i : drv->outputs) { + for (auto & i : outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); + worker.store.computeFSClosure(i.second, outputClosure); } /* Filter out our own outputs (which we have already checked). */ - for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); + for (auto & i : outputs) + outputClosure.erase(i.second); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1201,9 +1316,10 @@ void DerivationGoal::repairClosure() std::map<StorePath, StorePath> outputsToDrv; for (auto & i : inputClosure) if (i.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(i); - for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path, i); + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + for (auto & j : depOutputs) + if (j.second) + outputsToDrv.insert_or_assign(*j.second, i); } /* Check each path (slow!). */ @@ -1265,35 +1381,72 @@ void DerivationGoal::inputsRealised() /* Determine the full set of input paths. */ /* First, the input derivations. */ - if (useDerivation) - for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { + if (useDerivation) { + auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); + + if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { + /* We are be able to resolve this derivation based on the + now-known results of dependencies. If so, we become a stub goal + aliasing that resolved derivation goal */ + std::optional attempt = fullDrv.tryResolve(worker.store); + assert(attempt); + Derivation drvResolved { *std::move(attempt) }; + + auto pathResolved = writeDerivation(worker.store, drvResolved); + /* Add to memotable to speed up downstream goal's queries with the + original derivation. */ + drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); + + auto msg = fmt("Resolved derivation: '%s' -> '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved)); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg, + Logger::Fields { + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved), + }); + + auto resolvedGoal = worker.makeDerivationGoal( + pathResolved, wantedOutputs, buildMode); + addWaitee(resolvedGoal); + + state = &DerivationGoal::resolvedFinished; + return; + } + + for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.store.isValidPath(i.first)); - Derivation inDrv = worker.store.derivationFromPath(i.first); - for (auto & j : i.second) { - auto k = inDrv.outputs.find(j); - if (k != inDrv.outputs.end()) - worker.store.computeFSClosure(k->second.path, inputPaths); - else + assert(worker.store.isValidPath(drvPath)); + auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + for (auto & j : wantedDepOutputs) { + if (outputs.count(j) > 0) { + auto optRealizedInput = outputs.at(j); + if (!optRealizedInput) + throw Error( + "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.computeFSClosure(*optRealizedInput, inputPaths); + } else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); } } + } /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); debug("added input paths %s", worker.store.showPaths(inputPaths)); - /* Is this a fixed-output derivation? */ - fixedOutput = drv->isFixedOutput(); + /* What type of derivation are we building? */ + derivationType = drv->type(); /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; + nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1324,14 +1477,18 @@ void DerivationGoal::tryToBuild() { trace("trying to build"); - /* Obtain locks on all output paths. The locks are automatically - released when we exit this function or Nix crashes. If we - can't acquire the lock, then continue; hopefully some other - goal can start a build, and if not, the main loop will sleep a - few seconds and then retry this goal. */ + /* Obtain locks on all output paths, if the paths are known a priori. + + The locks are automatically released when we exit this function or Nix + crashes. If we can't acquire the lock, then continue; hopefully some + other goal can start a build, and if not, the main loop will sleep a few + seconds and then retry this goal. */ PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.Store::toRealPath(outPath)); + /* FIXME: Should lock something like the drv itself so we don't build same + CA drv concurrently */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) @@ -1350,32 +1507,36 @@ void DerivationGoal::tryToBuild() omitted, but that would be less efficient.) Note that since we now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ - validPaths = checkPathValidity(true, buildMode == bmRepair); - if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); done(BuildResult::AlreadyValid); return; } - missingPaths = drv->outputPaths(); - if (buildMode != bmCheck) - for (auto & i : validPaths) missingPaths.erase(i); - /* If any of the outputs already exist but are not valid, delete them. */ - for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path)) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.Store::toRealPath(i.second.path)); + for (auto & [_, status] : initialOutputs) { + if (!status.known || status.known->isValid()) continue; + auto storePath = status.known->path; + debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); + deletePath(worker.store.Store::toRealPath(storePath)); } /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { case rpAccept: @@ -1403,6 +1564,13 @@ void DerivationGoal::tryToBuild() actLock.reset(); + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1413,13 +1581,6 @@ void DerivationGoal::tryToBuild() return; } - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::tryLocalBuild() { - if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(); @@ -1460,7 +1621,34 @@ void DerivationGoal::tryLocalBuild() { } -void replaceValidPath(const Path & storePath, const Path tmpPath) +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + +void replaceValidPath(const Path & storePath, const Path & tmpPath) { /* We can't atomically replace storePath (the original) with tmpPath (the replacement), so we have to move it out of the @@ -1468,11 +1656,20 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) we're repairing (say) Glibc, we end up with a broken system. */ Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); if (pathExists(storePath)) - rename(storePath.c_str(), oldPath.c_str()); - if (rename(tmpPath.c_str(), storePath.c_str()) == -1) { - rename(oldPath.c_str(), storePath.c_str()); // attempt to recover - throw SysError("moving '%s' to '%s'", tmpPath, storePath); + movePath(storePath, oldPath); + + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreException(); + } + throw; } + deletePath(oldPath); } @@ -1538,13 +1735,13 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; #endif @@ -1553,8 +1750,10 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - for (auto & i : missingPaths) { - auto p = worker.store.printStorePath(i); + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) rename((chrootRootDir + p).c_str(), p.c_str()); } @@ -1584,7 +1783,10 @@ void DerivationGoal::buildDone() fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); + StorePathSet outputPaths; + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); + } std::map<std::string, std::string> hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); @@ -1675,7 +1877,7 @@ void DerivationGoal::buildDone() st = dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - fixedOutput || diskFull ? BuildResult::TransientFailure : + derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1686,6 +1888,9 @@ void DerivationGoal::buildDone() done(BuildResult::Built); } +void DerivationGoal::resolvedFinished() { + done(BuildResult::Built); +} HookReply DerivationGoal::tryBuildHook() { @@ -1756,11 +1961,19 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - writeStorePaths(worker.store, hook->sink, inputPaths); + worker_proto::write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - writeStorePaths(worker.store, hook->sink, missingPaths); + { + StorePathSet missingPaths; + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + missingPaths.insert(status.known->path); + } + worker_proto::write(worker.store, hook->sink, missingPaths); + } hook->sink = FdSink(); hook->toHook.writeSide = -1; @@ -1777,13 +1990,6 @@ HookReply DerivationGoal::tryBuildHook() } -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - int childEntry(void * arg) { ((DerivationGoal *) arg)->runChild(); @@ -1811,8 +2017,15 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) for (auto & j : paths2) { if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + for (auto & k : drv.outputsAndOptPaths(worker.store)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + worker.store.computeFSClosure(*k.second.second, paths); + } } } @@ -1843,7 +2056,7 @@ void linkOrCopy(const Path & from, const Path & to) file (e.g. 32000 of ext3), which is quite possible after a 'nix-store --optimise'. FIXME: actually, why don't we just bind-mount in this case? - + It can also fail with EPERM in BeegFS v7 and earlier versions which don't allow hard-links to other directories */ if (errno != EMLINK && errno != EPERM) @@ -1856,13 +2069,13 @@ void linkOrCopy(const Path & from, const Path & to) void DerivationGoal::startBuilder() { /* Right platform? */ - if (!parsedDrv->canBuildLocally()) + if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep<StringSet>(", ", settings.systemFeatures)); + concatStringsSep<StringSet>(", ", worker.store.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); @@ -1888,7 +2101,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && !noChroot; + useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -1905,9 +2118,64 @@ void DerivationGoal::startBuilder() chownToBuilder(tmpDir); - /* Substitute output placeholders with the actual output paths. */ - for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + for (auto & [outputName, status] : initialOutputs) { + /* Set scratch path we'll actually use during the build. + + If we're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + auto scratchPath = + !status.known + ? makeFallbackPath(outputName) + : !needsHashRewrite() + /* Can always use original path in sandbox */ + ? status.known->path + : !status.known->isPresent() + /* If path doesn't yet exist can just use it */ + ? status.known->path + : buildMode != bmRepair && !status.known->isValid() + /* If we aren't repairing we'll delete a corrupted path, so we + can use original path */ + ? status.known->path + : /* If we are repairing or the path is totally valid, we'll need + to use a temporary path */ + makeFallbackPath(status.known->path); + scratchOutputs.insert_or_assign(outputName, scratchPath); + + /* A non-removed corrupted path needs to be stored here, too */ + if (buildMode == bmRepair && !status.known->isValid()) + redirectedBadOutputs.insert(status.known->path); + + /* Substitute output placeholders with the scratch output paths. + We'll use during the build. */ + inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); + + /* Additional tasks if we know the final path a priori. */ + if (!status.known) continue; + auto fixedFinalPath = status.known->path; + + /* Additional tasks if the final and scratch are both known and + differ. */ + if (fixedFinalPath == scratchPath) continue; + + /* Ensure scratch path is ours to use. */ + deletePath(worker.store.printStorePath(scratchPath)); + + /* Rewrite and unrewrite paths */ + { + std::string h1 { fixedFinalPath.hashPart() }; + std::string h2 { scratchPath.hashPart() }; + inputRewrites[h1] = h2; + } + + redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); + } /* Construct the environment passed to the builder. */ initEnv(); @@ -1933,7 +2201,10 @@ void DerivationGoal::startBuilder() if (!std::regex_match(fileName, regex)) throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - auto storePath = worker.store.parseStorePath(*i++); + auto storePathS = *i++; + if (!worker.store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + auto storePath = worker.store.toStorePath(storePathS).first; /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, @@ -1975,10 +2246,11 @@ void DerivationGoal::startBuilder() for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error("while processing 'sandbox-paths': %s", e.what()); + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -2065,7 +2337,7 @@ void DerivationGoal::startBuilder() "nogroup:x:65534:\n") % sandboxGid).str()); /* Create /etc/hosts with localhost entry. */ - if (!fixedOutput) + if (!(derivationIsImpure(derivationType))) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -2085,10 +2357,7 @@ void DerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - struct stat st; - if (lstat(r.c_str(), &st)) - throw SysError("getting attributes of path '%s'", p); - if (S_ISDIR(st.st_mode)) + if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else linkOrCopy(r, chrootRootDir + p); @@ -2099,8 +2368,15 @@ void DerivationGoal::startBuilder() rebuilding a path that is in settings.dirsInChroot (typically the dependencies of /bin/sh). Throw them out. */ - for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path)); + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + /* If the name isn't known a priori (i.e. floating + content-addressed derivation), the temporary location we use + should be fresh. Freshness means it is impossible that the path + is already in the sandbox, so we don't need to worry about + removing it. */ + if (i.second.second) + dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + } if (useSystemdCgroup) { settings.requireExperimentalFeature("systemd-cgroup"); @@ -2130,33 +2406,8 @@ void DerivationGoal::startBuilder() throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); } - if (needsHashRewrite()) { - - if (pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - /* We're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - if (validPaths.size() > 0) - for (auto & i : validPaths) - addHashRewrite(i); - - /* If we're repairing, then we don't want to delete the - corrupt outputs in advance. So rewrite them as well. */ - if (buildMode == bmRepair) - for (auto & i : missingPaths) - if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { - addHashRewrite(i); - redirectedBadOutputs.insert(i); - } - } + if (needsHashRewrite() && pathExists(homeDir)) + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { printMsg(lvlChatty, format("executing pre-build hook '%1%'") @@ -2293,7 +2544,7 @@ void DerivationGoal::startBuilder() us. */ - if (!fixedOutput) + if (!(derivationIsImpure(derivationType))) privateNetwork = true; userNamespaceSync.create(); @@ -2358,6 +2609,12 @@ void DerivationGoal::startBuilder() userNamespaceSync.readSide = -1; + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + pid_t tmp; if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; @@ -2392,7 +2649,6 @@ void DerivationGoal::startBuilder() /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); - userNamespaceSync.writeSide = -1; } else #endif @@ -2412,9 +2668,12 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { - if (msg.size() == 1) break; - throw Error(string(msg, 1)); + FdSource source(builderOut.readSide.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; } debug(msg); } @@ -2502,7 +2761,7 @@ void DerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -2513,7 +2772,7 @@ void DerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (fixedOutput) { + if (derivationIsImpure(derivationType)) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -2540,8 +2799,11 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + for (auto & i : drv->outputs) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); + } json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2634,18 +2896,23 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } +struct RestrictedStoreConfig : LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + const std::string name() { return "Restricted Store"; } +}; /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore +struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig { ref<LocalStore> next; DerivationGoal & goal; RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal) - : Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) { } Path getRealStoreDir() override @@ -2684,8 +2951,12 @@ struct RestrictedStore : public LocalFSStore void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - OutputPathMap queryDerivationOutputMap(const StorePath & path) override - { throw Error("queryDerivationOutputMap"); } + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); + return next->queryPartialDerivationOutputMap(path); + } std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } @@ -2696,21 +2967,12 @@ struct RestrictedStore : public LocalFSStore { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) override + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override { - next->addToStore(info, narSource, repair, checkSigs, accessor); + next->addToStore(info, narSource, repair, checkSigs); goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override - { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); - goal.addDependency(path); - return path; - } - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) override { @@ -2740,19 +3002,20 @@ struct RestrictedStore : public LocalFSStore StorePathSet newPaths; for (auto & path : paths) { - if (path.path.isDerivation()) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - auto drv = derivationFromPath(path.path); - for (auto & output : drv.outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path); - } else if (!goal.isAllowed(path.path)) + if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } next->buildPaths(paths, buildMode); + for (auto & path : paths) { + if (!path.path.isDerivation()) continue; + auto outputs = next->queryDerivationOutputMap(path.path); + for (auto & output : outputs) + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second); + } + StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) @@ -2780,7 +3043,7 @@ struct RestrictedStore : public LocalFSStore void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override + uint64_t & downloadSize, uint64_t & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are @@ -2849,7 +3112,8 @@ void DerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + daemon::NotTrusted, daemon::Recursive, + [&](Store & store) { store.createUser("nobody", 65535); }); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -2903,9 +3167,7 @@ void DerivationGoal::addDependency(const StorePath & path) if (pathExists(target)) throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - struct stat st; - if (lstat(source.c_str(), &st)) - throw SysError("getting attributes of path '%s'", source); + auto st = lstat(source); if (S_ISDIR(st.st_mode)) { @@ -3108,7 +3370,7 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -3124,7 +3386,7 @@ void DerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (fixedOutput) { + if (derivationIsImpure(derivationType)) { ss.push_back("/etc/resolv.conf"); // Only use nss functions to resolve hosts and @@ -3378,17 +3640,13 @@ void DerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (fixedOutput) + if (derivationIsImpure(derivationType)) sandboxProfile += "(import \"sandbox-network.sb\")\n"; - /* Our rwx outputs */ + /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : missingPaths) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i)); - - /* Also add redirected outputs to the chroot */ - for (auto & i : redirectedOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second)); + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); sandboxProfile += ")\n"; @@ -3474,7 +3732,7 @@ void DerivationGoal::runChild() args.push_back(rewriteStrings(i, inputRewrites)); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\1\n")); + writeFull(STDERR_FILENO, string("\2\n")); /* Execute the program. This should not return. */ if (drv->isBuiltin()) { @@ -3495,7 +3753,7 @@ void DerivationGoal::runChild() throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { - writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, e.what() + std::string("\n")); _exit(1); } } @@ -3504,62 +3762,31 @@ void DerivationGoal::runChild() throw SysError("executing '%1%'", drv->builder); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); + } catch (Error & e) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); _exit(1); } } -/* Parse a list of reference specifiers. Each element must either be - a store path, or the symbolic name of the output of the derivation - (such as `out'). */ -StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) -{ - StorePathSet result; - for (auto & i : paths) { - if (store.isStorePath(i)) - result.insert(store.parseStorePath(i)); - else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - return result; -} - - -static void moveCheckToStore(const Path & src, const Path & dst) -{ - /* For the rename of directory to succeed, we must be running as root or - the directory must be made temporarily writable (to update the - directory's parent link ".."). */ - struct stat st; - if (lstat(src.c_str(), &st) == -1) { - throw SysError("getting attributes of path '%1%'", src); - } - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have - to do anything here. */ + to do anything here. + + We can only early return when the outputs are known a priori. For + floating content-addressed derivations this isn't the case. + */ if (hook) { bool allValid = true; - for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + if (!i.second.second || !worker.store.isValidPath(*i.second.second)) + allValid = false; + } if (allValid) return; } @@ -3580,47 +3807,51 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); for (auto & p : addedPaths) referenceablePaths.insert(p); - /* Check whether the output paths were created, and grep each - output path to determine what other paths it references. Also make all - output paths read-only. */ - for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); - if (!missingPaths.count(i.second.path)) continue; - - Path actualPath = path; - if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path); - if (r != redirectedOutputs.end()) { - auto redirected = worker.store.Store::toRealPath(r->second); - if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path) - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck) - actualPath = redirected; - } - } else if (useChroot) { - actualPath = chrootRootDir + path; - if (pathExists(actualPath)) { - /* Move output paths from the chroot to the Nix store. */ - if (buildMode == bmRepair) - replaceValidPath(path, actualPath); - else - if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); - } - if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); + /* FIXME `needsHashRewrite` should probably be removed and we get to the + real reason why we aren't using the chroot dir */ + auto toRealPathChroot = [&](const Path & p) -> Path { + return useChroot && !needsHashRewrite() + ? chrootRootDir + p + : worker.store.toRealPath(p); + }; + + /* Check whether the output paths were created, and make all + output paths read-only. Then get the references of each output (that we + might need to register), so we can topologically sort them. For the ones + that are most definitely already installed, we just store their final + name so we can also use it in rewrites. */ + StringSet outputsToSort; + struct AlreadyRegistered { StorePath path; }; + struct PerhapsNeedToRegister { StorePathSet refs; }; + std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered; + std::map<std::string, struct stat> outputStats; + for (auto & [outputName, _] : drv->outputs) { + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); + + outputsToSort.insert(outputName); + + /* Updated wanted info to remove the outputs we definitely don't need to register */ + auto & initialInfo = initialOutputs.at(outputName); + + /* Don't register if already valid, and not checking */ + initialInfo.wanted = buildMode == bmCheck + || !(initialInfo.known && initialInfo.known->isValid()); + if (!initialInfo.wanted) { + outputReferencesIfUnregistered.insert_or_assign( + outputName, + AlreadyRegistered { .path = initialInfo.known->path }); + continue; } struct stat st; if (lstat(actualPath.c_str(), &st) == -1) { if (errno == ENOENT) throw BuildError( - "builder for '%s' failed to produce output path '%s'", - worker.store.printStorePath(drvPath), path); + "builder for '%s' failed to produce output path for output '%s' at '%s'", + worker.store.printStorePath(drvPath), outputName, actualPath); throw SysError("getting attributes of path '%s'", actualPath); } @@ -3631,140 +3862,296 @@ void DerivationGoal::registerOutputs() user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); + throw BuildError( + "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", + actualPath, outputName); #endif - /* Apply hash rewriting if necessary. */ - bool rewritten = false; - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) - }); + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, + inodesSeen); - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData( - actualPath, - buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, - inodesSeen); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - rewritten = true; - } + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + auto references = worker.store.parseStorePathSet( + scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - /* Check that fixed-output derivations produced the right - outputs (i.e., the content hash should match the specified - hash). */ - std::optional<ContentAddress> ca; + outputReferencesIfUnregistered.insert_or_assign( + outputName, + PerhapsNeedToRegister { .refs = references }); + outputStats.insert_or_assign(outputName, std::move(st)); + } - if (fixedOutput) { + auto sortedOutputNames = topoSort(outputsToSort, + {[&](const std::string & name) { + return std::visit(overloaded { + /* Since we'll use the already installed versions of these, we + can treat them as leaves and ignore any references they + have. */ + [&](AlreadyRegistered _) { return StringSet {}; }, + [&](PerhapsNeedToRegister refs) { + StringSet referencedOutputs; + /* FIXME build inverted map up front so no quadratic waste here */ + for (auto & r : refs.refs) + for (auto & [o, p] : scratchOutputs) + if (r == p) + referencedOutputs.insert(o); + return referencedOutputs; + }, + }, outputReferencesIfUnregistered.at(name)); + }}, + {[&](const std::string & path, const std::string & parent) { + // TODO with more -vvvv also show the temporary paths for manual inspection. + return BuildError( + "cycle detected in build of '%s' in the references of output '%s' from output '%s'", + worker.store.printStorePath(drvPath), path, parent); + }}); + + std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + + for (auto & outputName : sortedOutputNames) { + auto output = drv->outputs.at(outputName); + auto & scratchPath = scratchOutputs.at(outputName); + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); + + auto finish = [&](StorePath finalStorePath) { + /* Store the final path */ + finalOutputs.insert_or_assign(outputName, finalStorePath); + /* The rewrite rule will be used in downstream outputs that refer to + use. This is why the topological sort is essential to do first + before this for loop. */ + if (scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; + }; - if (i.second.hash->method == FileIngestionMethod::Flat) { + std::optional<StorePathSet> referencesOpt = std::visit(overloaded { + [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> { + finish(skippedFinalPath.path); + return std::nullopt; + }, + [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> { + return r.refs; + }, + }, outputReferencesIfUnregistered.at(outputName)); + + if (!referencesOpt) + continue; + auto references = *referencesOpt; + + auto rewriteOutput = [&]() { + /* Apply hash rewriting if necessary. */ + if (!outputRewrites.empty()) { + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), + }); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + } + }; + + auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { + /* In the CA case, we need the rewritten refs to calculate the + final path, therefore we look for a *non-rewritten + self-reference, and use a bool rather try to solve the + computationally intractable fixed point. */ + std::pair<bool, StorePathSet> res { + false, + {}, + }; + for (auto & r : references) { + auto name = r.name(); + auto origHash = std::string { r.hashPart() }; + if (r == scratchPath) + res.first = true; + else if (outputRewrites.count(origHash) == 0) + res.second.insert(r); + else { + std::string newRef = outputRewrites.at(origHash); + newRef += '-'; + newRef += name; + res.second.insert(StorePath { newRef }); + } + } + return res; + }; + + auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { + auto & st = outputStats.at(outputName); + if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " "since recursive hashing is not enabled (outputHashMode=flat)", - path); + actualPath); } + rewriteOutput(); + /* FIXME optimize and deduplicate with addToStore */ + std::string oldHashPart { scratchPath.hashPart() }; + HashModuloSink caSink { outputHash.hashType, oldHashPart }; + switch (outputHash.method) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + auto got = caSink.finish().first; + auto refs = rewriteRefs(); + HashModuloSink narSink { htSHA256, oldHashPart }; + dumpPath(actualPath, narSink); + auto narHashAndSize = narSink.finish(); + ValidPathInfo newInfo0 { + worker.store.makeFixedOutputPath( + outputHash.method, + got, + outputPathName(drv->name, outputName), + refs.second, + refs.first), + narHashAndSize.first, + }; + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); - /* Check the hash. In hash mode, move the path produced by - the derivation to its content-addressed location. */ - Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); - - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); - - if (i.second.hash->hash != h2) { - - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), - i.second.hash->hash.to_string(SRI, true), - h2.to_string(SRI, true))); - - Path actualDest = worker.store.Store::toRealPath(dest); + assert(newInfo0.ca); + return newInfo0; + }; - if (worker.store.isValidPath(dest)) - std::rethrow_exception(delayedException); + ValidPathInfo newInfo = std::visit(overloaded { + [&](DerivationOutputInputAddressed output) { + /* input-addressed case */ + auto requiredFinalPath = output.path; + /* Preemptively add rewrite rule for final hash, as that is + what the NAR hash will use rather than normalized-self references */ + if (scratchPath != requiredFinalPath) + outputRewrites.insert_or_assign( + std::string { scratchPath.hashPart() }, + std::string { requiredFinalPath.hashPart() }); + rewriteOutput(); + auto narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; + newInfo0.narSize = narHashAndSize.second; + auto refs = rewriteRefs(); + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + return newInfo0; + }, + [&](DerivationOutputCAFixed dof) { + auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { + .method = dof.hash.method, + .hashType = dof.hash.hash.type, + }); - if (actualPath != actualDest) { - PathLocks outputLocks({actualDest}); - deletePath(actualDest); - if (rename(actualPath.c_str(), actualDest.c_str()) == -1) - throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest)); + /* Check wanted hash */ + Hash & wanted = dof.hash.hash; + assert(newInfo0.ca); + auto got = getContentAddressHash(*newInfo0.ca); + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + worker.store.printStorePath(drvPath), + wanted.to_string(SRI, true), + got.to_string(SRI, true))); } + return newInfo0; + }, + [&](DerivationOutputCAFloating dof) { + return newInfoFromCA(dof); + }, + }, output.output); + + /* Calculate where we'll move the output files. In the checking case we + will leave leave them where they are, for now, rather than move to + their usual "final destination" */ + auto finalDestPath = worker.store.printStorePath(newInfo.path); + + /* Lock final output path, if not already locked. This happens with + floating CA derivations and hash-mismatching fixed-output + derivations. */ + PathLocks dynamicOutputLock; + auto optFixedPath = output.path(worker.store, drv->name, outputName); + if (!optFixedPath || + worker.store.printStorePath(*optFixedPath) != finalDestPath) + { + assert(newInfo.ca); + dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); + } - path = worker.store.printStorePath(dest); - actualPath = actualDest; + /* Move files, if needed */ + if (worker.store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); + actualPath = worker.store.toRealPath(finalDestPath); + } else if (buildMode == bmCheck) { + /* Path already exists, and we want to compare, so we leave out + new path in place. */ + } else if (worker.store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + } else { + auto destPath = worker.store.toRealPath(finalDestPath); + movePath(actualPath, destPath); + actualPath = destPath; } - else - assert(worker.store.parseStorePath(path) == dest); - - ca = FixedOutputHash { - .method = i.second.hash->method, - .hash = h2, - }; } - /* Get rid of all weird permissions. This also checks that - all files are owned by the build user, if applicable. */ - canonicalisePathMetaData(actualPath, - buildUser && !rewritten - ? std::optional(buildUser->getUIDRange()) - : std::nullopt, - inodesSeen); - - /* For this output path, find the references to other paths - contained in it. Compute the SHA-256 NAR hash at the same - time. The hash is stored in the database so that we can - verify later on whether nobody has messed with the store. */ - debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; - ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); - if (hash.first != info.narHash) { + if (!worker.store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { - Path dst = worker.store.toRealPath(path + checkSuffix); + auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); - moveCheckToStore(actualPath, dst); + movePath(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - path, dst, worker.store.printStorePath(drvPath), tmpDir); + finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); } /* Since we verified the build, it's now ultimately trusted. */ - if (!info.ultimate) { - info.ultimate = true; - worker.store.signPathInfo(info); - ValidPathInfos infos; - infos.push_back(std::move(info)); - worker.store.registerValidPaths(infos); + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + worker.store.signPathInfo(oldInfo); + worker.store.registerValidPaths({ std::move(oldInfo) }); } continue; @@ -3781,24 +4168,22 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(worker.store.parseStorePath(path)); + worker.markContentsGood(newInfo.path); } - ValidPathInfo info(worker.store.parseStorePath(path)); - info.narHash = hash.first; - info.narSize = hash.second; - info.references = std::move(references); - info.deriver = drvPath; - info.ultimate = true; - info.ca = ca; - worker.store.signPathInfo(info); - - if (!info.references.empty()) { - // FIXME don't we have an experimental feature for fixed output with references? - info.ca = {}; - } + newInfo.deriver = drvPath; + newInfo.ultimate = true; + worker.store.signPathInfo(newInfo); + + finish(newInfo.path); + + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration */ + if (newInfo.ca) + worker.store.registerValidPaths({newInfo}); - infos.emplace(i.first, std::move(info)); + infos.emplace(outputName, std::move(newInfo)); } if (buildMode == bmCheck) return; @@ -3841,8 +4226,8 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); + for (auto & [_, outputStorePath] : finalOutputs) { + auto path = worker.store.printStorePath(outputStorePath); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3859,8 +4244,8 @@ void DerivationGoal::registerOutputs() /* Remove the .check directories if we're done. FIXME: keep them if the result was not determistic? */ if (curRound == nrRounds) { - for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; + for (auto & [_, outputStorePath] : finalOutputs) { + Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; deletePath(prev); } } @@ -3870,7 +4255,9 @@ void DerivationGoal::registerOutputs() outputs, this will fail. */ { ValidPathInfos infos2; - for (auto & i : infos) infos2.push_back(i.second); + for (auto & [outputName, newInfo] : infos) { + infos2.push_back(newInfo); + } worker.store.registerValidPaths(infos2); } @@ -3878,6 +4265,27 @@ void DerivationGoal::registerOutputs() exception now that we have registered the output as valid. */ if (delayedException) std::rethrow_exception(delayedException); + + /* If we made it this far, we are sure the output matches the derivation + (since the delayedException would be a fixed output CA mismatch). That + means it's safe to link the derivation to the output hash. We must do + that for floating CA derivations, which otherwise couldn't be cached, + but it's fine to do in all cases. */ + bool isCaFloating = drv->type() == DerivationType::CAFloating; + + auto drvPathResolved = drvPath; + if (!useDerivation && isCaFloating) { + /* Once a floating CA derivations reaches this point, it + must already be resolved, so we don't bother trying to + downcast drv to get would would just be an empty + inputDrvs field. */ + Derivation drv2 { *drv }; + drvPathResolved = writeDerivation(worker.store, drv2); + } + + if (useDerivation || isCaFloating) + for (auto & [outputName, newInfo] : infos) + worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); } @@ -3946,7 +4354,17 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) { if (!value) return; - auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : *value) { + if (worker.store.isStorePath(i)) + spec.insert(worker.store.parseStorePath(i)); + else if (finalOutputs.count(i)) + spec.insert(finalOutputs.at(i)); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); + } auto used = recursive ? getClosure(info.path).first @@ -4155,31 +4573,67 @@ void DerivationGoal::flushLine() } -StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap() { - StorePathSet result; - for (auto & i : drv->outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + std::map<std::string, std::optional<StorePath>> res; + for (auto & [name, output] : drv->outputs) + res.insert_or_assign(name, output.path(worker.store, drv->name, name)); + return res; + } else { + return worker.store.queryPartialDerivationOutputMap(drvPath); } - return result; +} + +OutputPathMap DerivationGoal::queryDerivationOutputMap() +{ + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + OutputPathMap res; + for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) + res.insert_or_assign(name, *output.second); + return res; + } else { + return worker.store.queryDerivationOutputMap(drvPath); + } +} + + +void DerivationGoal::checkPathValidity() +{ + bool checkHash = buildMode == bmRepair; + for (auto & i : queryPartialDerivationOutputMap()) { + InitialOutput info { + .wanted = wantOutput(i.first, wantedOutputs), + }; + if (i.second) { + auto outputPath = *i.second; + info.known = { + .path = outputPath, + .status = !worker.store.isValidPath(outputPath) + ? PathStatus::Absent + : !checkHash || worker.pathContentsGood(outputPath) + ? PathStatus::Valid + : PathStatus::Corrupt, + }; + } + initialOutputs.insert_or_assign(i.first, info); + } +} + + +StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) +{ + return worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), + Hash(htSHA256), outputPathName(drv->name, outputName)); } -void DerivationGoal::addHashRewrite(const StorePath & path) +StorePath DerivationGoal::makeFallbackPath(const StorePath & path) { - auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); - auto p = worker.store.makeStorePath( + return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), Hash(htSHA256), path.name()); - auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); - deletePath(worker.store.printStorePath(p)); - inputRewrites[h1] = h2; - outputRewrites[h2] = h1; - redirectedOutputs.insert_or_assign(path, std::move(p)); } @@ -4220,6 +4674,10 @@ private: /* The store path that should be realised through a substitute. */ StorePath storePath; + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional<StorePath> subPath; + /* The remaining substituters. */ std::list<ref<Store>> subs; @@ -4253,8 +4711,11 @@ private: typedef void (SubstitutionGoal::*GoalState)(); GoalState state; + /* Content address for recomputing store path */ + std::optional<ContentAddress> ca; + public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4284,10 +4745,11 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) : Goal(worker) , storePath(storePath) , repair(repair) + , ca(ca) { state = &SubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); @@ -4362,14 +4824,18 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - if (sub->storeDir != worker.store.storeDir) { + if (ca) { + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { tryNext(); return; } try { // FIXME: make async - info = sub->queryPathInfo(storePath); + info = sub->queryPathInfo(subPath ? *subPath : storePath); } catch (InvalidPath &) { tryNext(); return; @@ -4388,6 +4854,19 @@ void SubstitutionGoal::tryNext() throw; } + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared<ValidPathInfo>(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + tryNext(); + return; + } + } + /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); @@ -4477,7 +4956,7 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), - storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4587,35 +5066,52 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const StorePath & path, - const StringSet & wantedOutputs, BuildMode buildMode) +std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( + const StorePath & drvPath, + const StringSet & wantedOutputs, + std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) { - GoalPtr goal = derivationGoals[path].lock(); // FIXME - if (!goal) { - goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path, goal); + WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; + GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME + std::shared_ptr<DerivationGoal> goal; + if (!abstract_goal) { + goal = mkDrvGoal(); + abstract_goal_weak = goal; wakeUp(goal); - } else - (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); + } else { + goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal); + assert(goal); + goal->addWantedOutputs(wantedOutputs); + } return goal; } +std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, BuildMode buildMode) +{ + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { + return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode); + }); +} + + std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, BuildMode buildMode) + const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); - wakeUp(goal); - return goal; + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { + return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode); + }); } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca) { - GoalPtr goal = substitutionGoals[path].lock(); // FIXME + WeakGoalPtr & goal_weak = substitutionGoals[path]; + GoalPtr goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path, *this, repair); - substitutionGoals.insert_or_assign(path, goal); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca); + goal_weak = goal; wakeUp(goal); } return goal; @@ -4771,8 +5267,17 @@ void Worker::run(const Goals & _topGoals) waitForInput(); else { if (awake.empty() && 0 == settings.maxBuildJobs) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + { + if (getMachines().empty()) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + else + throw Error("unable to start any build; remote machines may not have " + "all required system features." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + + } assert(!awake.empty()); } } @@ -4866,7 +5371,7 @@ void Worker::waitForInput() std::vector<unsigned char> buffer(4096); for (auto & k : fds2) { if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = read(k, buffer.data(), buffer.size()); + ssize_t rd = ::read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { @@ -4956,7 +5461,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } @@ -4982,7 +5487,7 @@ void Worker::markContentsGood(const StorePath & path) static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths) { StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) @@ -5037,7 +5542,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); BuildResult result; diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index e630cf6f1..4fb5d8a06 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,17 +58,14 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; - /* We always have one output, and if it's a fixed-output derivation (as - checked below) it must be the only output */ - auto & output = drv.outputs.begin()->second; - /* Try the hashed mirrors first. */ - if (output.hash && output.hash->method == FileIngestionMethod::Flat) + if (getAttr("outputHashMode") == "flat") for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto & h = output.hash->hash; - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo")); + Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 9e5e937df..0ae45e46d 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -4,7 +4,10 @@ #include "util.hh" #include <chrono> +#include <cmath> +#include <regex> #include <unordered_set> +#include <thread> #include <dirent.h> diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..90a3ad1f5 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,12 +1,16 @@ +#include "args.hh" #include "content-address.hh" +#include "split.hh" namespace nix { -std::string FixedOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); +std::string FixedOutputHash::printMethodAlgo() const +{ + return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) { +std::string makeFileIngestionPrefix(const FileIngestionMethod m) +{ switch (m) { case FileIngestionMethod::Flat: return ""; @@ -24,11 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) + hash.to_string(Base32, true); } -// FIXME Put this somewhere? -template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; -template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; - -std::string renderContentAddress(ContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) +{ return std::visit(overloaded { [](TextHash th) { return "text:" + th.hash.to_string(Base32, true); @@ -39,47 +40,110 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -ContentAddress parseContentAddress(std::string_view rawCa) { - auto prefixSeparator = rawCa.find(':'); - if (prefixSeparator != string::npos) { - auto prefix = string(rawCa, 0, prefixSeparator); - if (prefix == "text") { - auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); - Hash hash = Hash(string(hashTypeAndHash)); - if (*hash.type != htSHA256) { - throw Error("parseContentAddress: the text hash should have type SHA256"); - } - return TextHash { hash }; - } else if (prefix == "fixed") { - // This has to be an inverse of makeFixedOutputCA - auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); - if (methodAndHash.substr(0,2) == "r:") { - std::string_view hashRaw = methodAndHash.substr(2,string::npos); - return FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = Hash(string(hashRaw)), - }; - } else { - std::string_view hashRaw = methodAndHash; - return FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = Hash(string(hashRaw)), - }; - } - } else { - throw Error("parseContentAddress: format not recognized; has to be text or fixed"); +std::string renderContentAddressMethod(ContentAddressMethod cam) +{ + return std::visit(overloaded { + [](TextHashMethod &th) { + return std::string{"text:"} + printHashType(htSHA256); + }, + [](FixedOutputHashMethod &fshm) { + return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - } else { - throw Error("Not a content address because it lacks an appropriate prefix"); + }, cam); +} + +/* + Parses content address strings up to the hash. + */ +static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +{ + std::string_view wholeInput { rest }; + + std::string_view prefix; + { + auto optPrefix = splitPrefixTo(rest, ':'); + if (!optPrefix) + throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput); + prefix = *optPrefix; } -}; -std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt); + auto parseHashType_ = [&](){ + auto hashTypeRaw = splitPrefixTo(rest, ':'); + if (!hashTypeRaw) + throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput); + HashType hashType = parseHashType(*hashTypeRaw); + return std::move(hashType); + }; + + // Switch on prefix + if (prefix == "text") { + // No parsing of the ingestion method, "text" only support flat. + HashType hashType = parseHashType_(); + if (hashType != htSHA256) + throw Error("text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + return TextHashMethod {}; + } else if (prefix == "fixed") { + // Parse method + auto method = FileIngestionMethod::Flat; + if (splitPrefix(rest, "r:")) + method = FileIngestionMethod::Recursive; + HashType hashType = parseHashType_(); + return FixedOutputHashMethod { + .fileIngestionMethod = method, + .hashType = std::move(hashType), + }; + } else + throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); +} + +ContentAddress parseContentAddress(std::string_view rawCa) { + auto rest = rawCa; + + ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + + return std::visit( + overloaded { + [&](TextHashMethod thm) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) + }); + }, + [&](FixedOutputHashMethod fohMethod) { + return ContentAddress(FixedOutputHash { + .method = fohMethod.fileIngestionMethod, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), + }); + }, + }, caMethod); +} + +ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +{ + std::string_view asPrefix {std::string{caMethod} + ":"}; + return parseContentAddressMethodPrefix(asPrefix); +} + +std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) +{ + return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt); }; -std::string renderContentAddress(std::optional<ContentAddress> ca) { +std::string renderContentAddress(std::optional<ContentAddress> ca) +{ return ca ? renderContentAddress(*ca) : ""; } +Hash getContentAddressHash(const ContentAddress & ca) +{ + return std::visit(overloaded { + [](TextHash th) { + return th.hash; + }, + [](FixedOutputHash fsh) { + return fsh.hash; + } + }, ca); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b..f6a6f5140 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -53,4 +53,25 @@ ContentAddress parseContentAddress(std::string_view rawCa); std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); +Hash getContentAddressHash(const ContentAddress & ca); + +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index a8fb62e0a..99d8add92 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,7 +2,6 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" -#include "local-store.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -81,12 +80,12 @@ struct TunnelLogger : public Logger showErrorInfo(oss, ei, false); StringSink buf; - buf << STDERR_NEXT << oss.str() << "\n"; + buf << STDERR_NEXT << oss.str(); enqueueMsg(*buf.s); } /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ + want to send out stderr to the client. */ void startWork() { auto state(state_.lock()); @@ -102,17 +101,20 @@ struct TunnelLogger : public Logger /* stopWork() means that we're done; stop sending stderr to the client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + void stopWork(const Error * ex = nullptr) { auto state(state_.lock()); state->canSendStderr = false; - if (success) + if (!ex) to << STDERR_LAST; else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; + if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { + to << STDERR_ERROR << *ex; + } else { + to << STDERR_ERROR << ex->what() << ex->status; + } } } @@ -173,31 +175,6 @@ struct TunnelSource : BufferedSource } }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - struct ClientSettings { bool keepFailed; @@ -257,7 +234,7 @@ struct ClientSettings else if (setSubstituters(settings.extraSubstituters)) ; else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } @@ -265,6 +242,23 @@ struct ClientSettings } }; +static void writeValidPathInfo( + ref<Store> store, + unsigned int clientVersion, + Sink & to, + std::shared_ptr<const ValidPathInfo> info) +{ + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + worker_proto::write(*store, to, info->references); + to << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << renderContentAddress(info->ca); + } +} + static void performOp(TunnelLogger * logger, ref<Store> store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -281,11 +275,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryValidPaths: { - auto paths = readStorePaths<StorePathSet>(*store, from); + auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto res = store->queryValidPaths(paths); logger->stopWork(); - writeStorePaths(*store, to, res); + worker_proto::write(*store, to, res); break; } @@ -301,11 +295,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQuerySubstitutablePaths: { - auto paths = readStorePaths<StorePathSet>(*store, from); + auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - writeStorePaths(*store, to, res); + worker_proto::write(*store, to, res); break; } @@ -334,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - writeStorePaths(*store, to, paths); + worker_proto::write(*store, to, paths); break; } @@ -350,9 +344,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryDerivationOutputMap: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - OutputPathMap outputs = store->queryDerivationOutputMap(path); + auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - writeOutputPathMap(*store, to, outputs); + worker_proto::write(*store, to, outputs); break; } @@ -375,52 +369,90 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - std::string s, baseName; - FileIngestionMethod method; - { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) - throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - method = FileIngestionMethod::Recursive; + if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { + auto name = readString(from); + auto camStr = readString(from); + auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + bool repairBool; + from >> repairBool; + auto repair = RepairFlag{repairBool}; + + logger->startWork(); + auto pathInfo = [&]() { + // NB: FramedSource must be out of scope before logger->stopWork(); + ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + FramedSource source(from); + // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. + return std::visit(overloaded { + [&](TextHashMethod &_) { + // We could stream this by changing Store + std::string contents = source.drain(); + auto path = store->addTextToStore(name, contents, refs, repair); + return store->queryPathInfo(path); + }, + [&](FixedOutputHashMethod &fohm) { + if (!refs.empty()) + throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + return store->queryPathInfo(path); + }, + }, contentAddressMethod); + }(); + logger->stopWork(); + + to << store->printStorePath(pathInfo->path); + writeValidPathInfo(store, clientVersion, to, pathInfo); + + } else { + HashType hashAlgo; + std::string baseName; + FileIngestionMethod method; + { + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + hashAlgoRaw = "sha256"; + method = FileIngestionMethod::Recursive; + } + hashAlgo = parseHashType(hashAlgoRaw); } - } - HashType hashAlgo = parseHashType(s); - TeeSource savedNAR(from); - RetrieveRegularNARSink savedRegular; + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); + if (method == FileIngestionMethod::Recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else + parseDump(savedRegular, from); - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, - baseName, - method, - hashAlgo); - logger->stopWork(); + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + logger->stopWork(); - to << store->printStorePath(path); + to << store->printStorePath(path); + } break; } case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - auto refs = readStorePaths<StorePathSet>(*store, from); + auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -442,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopImportPaths: { logger->startWork(); TunnelSource source(from, to); - auto paths = store->importPaths(source, nullptr, + auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; @@ -474,11 +506,63 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv); + readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); + + /* Content-addressed derivations are trustless because their output paths + are verified by their content alone, so any derivation is free to + try to produce such a path. + + Input-addressed derivation output paths, however, are calculated + from the derivation closure that produced them---even knowing the + root derivation is not enough. That the output data actually came + from those derivations is fundamentally unverifiable, but the daemon + trusts itself on that matter. The question instead is whether the + submitted plan has rights to the output paths it wants to fill, and + at least the derivation closure proves that. + + It would have been nice if input-address algorithm merely depended + on the build time closure, rather than depending on the derivation + closure. That would mean input-addressed paths used at build time + would just be trusted and not need their own evidence. This is in + fact fine as the same guarantees would hold *inductively*: either + the remote builder has those paths and already trusts them, or it + needs to build them too and thus their evidence must be provided in + turn. The advantage of this variant algorithm is that the evidence + for input-addressed paths which the remote builder already has + doesn't need to be sent again. + + That said, now that we have floating CA derivations, it is better + that people just migrate to those which also solve this problem, and + others. It's the same migration difficulty with strictly more + benefit. + + Lastly, do note that when we parse fixed-output content-addressed + derivations, we throw out the precomputed output paths and just + store the hashes, so there aren't two competing sources of truth an + attacker could exploit. */ + if (drv.type() == DerivationType::InputAddressed && !trusted) + throw Error("you are not privileged to build input-addressed derivations"); + + /* Make sure that the non-input-addressed derivations that got this far + are in fact content-addressed if we don't trust them. */ + assert(derivationIsCA(drv.type()) || trusted); + + /* Recompute the derivation path when we cannot trust the original. */ + if (!trusted) { + /* Recomputing the derivation path for input-address derivations + makes it harder to audit them after the fact, since we need the + original not-necessarily-resolved derivation to verify the drv + derivation as adequate claim to the input-addressed output + paths. */ + assert(derivationIsCA(drv.type())); + + Derivation drv2; + static_cast<BasicDerivation &>(drv2) = drv; + drvPath = writeDerivation(*store, Derivation { drv2 }); + } + auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; @@ -541,7 +625,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths<StorePathSet>(*store, from); + options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {}); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -602,7 +686,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -610,7 +694,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - writeStorePaths(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -618,16 +702,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQuerySubstitutablePathInfos: { - auto paths = readStorePaths<StorePathSet>(*store, from); - logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); + StorePathCAMap pathsMap = {}; + if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + } else + pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {}); + logger->startWork(); + store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - writeStorePaths(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -637,7 +727,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - writeStorePaths(*store, to, paths); + worker_proto::write(*store, to, paths); break; } @@ -654,15 +744,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); - to << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << renderContentAddress(info->ca); - } + writeValidPathInfo(store, clientVersion, to, info); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; @@ -705,18 +787,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(store->printStorePath(path), to); + dumpPath(store->toRealPath(path), to); break; } case wopAddToStoreNar: { bool repair, dontCheckSigs; - ValidPathInfo info(store->parseStorePath(readString(from))); + auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); + auto narHash = Hash::parseAny(readString(from), htSHA256); + ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths<StorePathSet>(*store, from); + info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = parseContentAddressOpt(readString(from)); @@ -726,24 +809,37 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (!trusted) info.ultimate = false; - std::string saved; - std::unique_ptr<Source> source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique<TunnelSource>(from, to); - else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); - source = std::make_unique<StringSource>(saved); + if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + logger->startWork(); + { + FramedSource source(from); + store->addToStore(info, source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } + logger->stopWork(); } - logger->startWork(); + else { + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique<TunnelSource>(from, to); + else { + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique<StringSource>(std::move(*saved.s)); + } - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + + logger->stopWork(); + } - logger->stopWork(); break; } @@ -753,12 +849,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, targets.push_back(store->parsePathWithOutputs(s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - writeStorePaths(*store, to, willBuild); - writeStorePaths(*store, to, willSubstitute); - writeStorePaths(*store, to, unknown); + worker_proto::write(*store, to, willBuild); + worker_proto::write(*store, to, willSubstitute); + worker_proto::write(*store, to, unknown); to << downloadSize << narSize; break; } @@ -774,8 +870,7 @@ void processConnection( FdSink & to, TrustedFlag trusted, RecursiveFlag recursive, - const std::string & userName, - uid_t userId) + std::function<void(Store &)> authHook) { auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr; @@ -816,15 +911,7 @@ void processConnection( /* If we can't accept clientVersion, then throw an error *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - - store->createUser(userName, userId); + authHook(*store); tunnelLogger->stopWork(); to.flush(); @@ -851,10 +938,11 @@ void processConnection( during addTextToStore() / importPath(). If that happens, just send the error message and exit. */ bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); + tunnelLogger->stopWork(&e); if (!errorAllowed) throw; } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + auto ex = Error("Nix daemon out of memory"); + tunnelLogger->stopWork(&ex); throw; } @@ -863,8 +951,13 @@ void processConnection( assert(!tunnelLogger->state_.lock()->canSendStderr); }; + } catch (Error & e) { + tunnelLogger->stopWork(&e); + to.flush(); + return; } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); + auto ex = Error(e.what()); + tunnelLogger->stopWork(&ex); to.flush(); return; } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 266932013..67755d54e 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,3 +1,5 @@ +#pragma once + #include "serialise.hh" #include "store-api.hh" @@ -12,7 +14,10 @@ void processConnection( FdSink & to, TrustedFlag trusted, RecursiveFlag recursive, - const std::string & userName, - uid_t userId); + /* Arbitrary hook to check authorization / initialize user data / whatever + after the protocol has been negotiated. The idea is that this function + and everything it calls doesn't know about this stuff, and the + `nix-daemon` handles that instead. */ + std::function<void(Store &)> authHook); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..07b4e772b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,16 +4,61 @@ #include "util.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" -#include "istringstream_nocopy.hh" namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const +std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const { - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error("derivation has no output '%s'", id); - return i->second.path; + return std::visit(overloaded { + [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> { + return { doi.path }; + }, + [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> { + return { + dof.path(store, drvName, outputName) + }; + }, + [](DerivationOutputCAFloating dof) -> std::optional<StorePath> { + return std::nullopt; + }, + }, output); +} + + +StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { + return store.makeFixedOutputPath( + hash.method, hash.hash, + outputPathName(drvName, outputName)); +} + + +bool derivationIsCA(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return false; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return true; + }; + // Since enums can have non-variant values, but making a `default:` would + // disable exhaustiveness warnings. + assert(false); +} + +bool derivationIsFixed(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return false; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; + }; + assert(false); +} + +bool derivationIsImpure(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return false; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; + }; + assert(false); } @@ -23,8 +68,8 @@ bool BasicDerivation::isBuiltin() const } -StorePath writeDerivation(ref<Store> store, - const Derivation & drv, std::string_view name, RepairFlag repair) +StorePath writeDerivation(Store & store, + const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) @@ -32,11 +77,11 @@ StorePath writeDerivation(ref<Store> store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - auto suffix = std::string(name) + drvExtension; - auto contents = drv.unparse(*store, false); - return settings.readOnlyMode - ? store->computeStorePathForText(suffix, contents, references) - : store->addTextToStore(suffix, contents, references, repair); + auto suffix = std::string(drv.name) + drvExtension; + auto contents = drv.unparse(store, false); + return readOnly || settings.readOnlyMode + ? store.computeStorePathForText(suffix, contents, references) + : store.addTextToStore(suffix, contents, references, repair); } @@ -68,12 +113,15 @@ static string parseString(std::istream & str) return res; } +static void validatePath(std::string_view s) { + if (s.size() == 0 || s[0] != '/') + throw FormatError("bad path '%1%' in derivation", s); +} static Path parsePath(std::istream & str) { - string s = parseString(str); - if (s.size() == 0 || s[0] != '/') - throw FormatError("bad path '%1%' in derivation", s); + auto s = parseString(str); + validatePath(s); return s; } @@ -101,38 +149,63 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +static DerivationOutput parseDerivationOutput(const Store & store, + std::string_view pathS, std::string_view hashAlgo, std::string_view hash) { - expect(str, ","); auto path = store.parseStorePath(parsePath(str)); - expect(str, ","); auto hashAlgo = parseString(str); - expect(str, ","); const auto hash = parseString(str); - expect(str, ")"); - - std::optional<FixedOutputHash> fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { method = FileIngestionMethod::Recursive; - hashAlgo = string(hashAlgo, 2); + hashAlgo = hashAlgo.substr(2); + } + const auto hashType = parseHashType(hashAlgo); + if (hash != "") { + validatePath(pathS); + return DerivationOutput { + .output = DerivationOutputCAFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed(hash, hashType), + }, + }, + }; + } else { + settings.requireExperimentalFeature("ca-derivations"); + assert(pathS == ""); + return DerivationOutput { + .output = DerivationOutputCAFloating { + .method = std::move(method), + .hashType = std::move(hashType), + }, + }; } - const HashType hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), + } else { + validatePath(pathS); + return DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = store.parseStorePath(pathS), + } }; } +} - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; +static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +{ + expect(str, ","); const auto pathS = parseString(str); + expect(str, ","); const auto hashAlgo = parseString(str); + expect(str, ","); const auto hash = parseString(str); + expect(str, ")"); + + return parseDerivationOutput(store, pathS, hashAlgo, hash); } -static Derivation parseDerivation(const Store & store, const string & s) +Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) { Derivation drv; - istringstream_nocopy str(s); + drv.name = name; + + std::istringstream str(std::move(s)); expect(str, "Derive(["); /* Parse the list of outputs. */ @@ -175,34 +248,6 @@ static Derivation parseDerivation(const Store & store, const string & s) } -Derivation readDerivation(const Store & store, const Path & drvPath) -{ - try { - return parseDerivation(store, readFile(drvPath)); - } catch (FormatError & e) { - throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); - } -} - - -Derivation Store::derivationFromPath(const StorePath & drvPath) -{ - ensurePath(drvPath); - return readDerivation(drvPath); -} - - -Derivation Store::readDerivation(const StorePath & drvPath) -{ - auto accessor = getFSAccessor(); - try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); - } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); - } -} - - static void printString(string & res, std::string_view s) { char buf[s.size() * 2 + 2]; @@ -264,10 +309,23 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); - s += ','; printUnquotedString(s, - i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path)); + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + }, + [&](DerivationOutputCAFixed dof) { + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); + s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + }, + [&](DerivationOutputCAFloating dof) { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, ""); + }, + }, i.second.output); s += ')'; } @@ -319,59 +377,144 @@ bool isDerivation(const string & fileName) } -bool BasicDerivation::isFixedOutput() const +std::string outputPathName(std::string_view drvName, std::string_view outputName) { + std::string res { drvName }; + if (outputName != "out") { + res += "-"; + res += outputName; + } + return res; +} + + +DerivationType BasicDerivation::type() const { - return outputs.size() == 1 && - outputs.begin()->first == "out" && - outputs.begin()->second.hash; + std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; + std::optional<HashType> floatingHashType; + for (auto & i : outputs) { + std::visit(overloaded { + [&](DerivationOutputInputAddressed _) { + inputAddressedOutputs.insert(i.first); + }, + [&](DerivationOutputCAFixed _) { + fixedCAOutputs.insert(i.first); + }, + [&](DerivationOutputCAFloating dof) { + floatingCAOutputs.insert(i.first); + if (!floatingHashType) { + floatingHashType = dof.hashType; + } else { + if (*floatingHashType != dof.hashType) + throw Error("All floating outputs must use the same hash type"); + } + }, + }, i.second.output); + } + + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + throw Error("Must have at least one output"); + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + return DerivationType::InputAddressed; + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + if (fixedCAOutputs.size() > 1) + // FIXME: Experimental feature? + throw Error("Only one fixed output is allowed for now"); + if (*fixedCAOutputs.begin() != "out") + throw Error("Single fixed output must be named \"out\""); + return DerivationType::CAFixed; + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) { + return DerivationType::CAFloating; + } else { + throw Error("Can't mix derivation output types"); + } } DrvHashes drvHashes; +/* pathDerivationModulo and hashDerivationModulo are mutually recursive + */ + +/* Look up the derivation by value and memoize the + `hashDerivationModulo` call. + */ +static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath) +{ + auto h = drvHashes.find(drvPath); + if (h == drvHashes.end()) { + assert(store.isValidPath(drvPath)); + // Cache it + h = drvHashes.insert_or_assign( + drvPath, + hashDerivationModulo( + store, + store.readDerivation(drvPath), + false)).first; + } + return h->second; +} -/* Returns the hash of a derivation modulo fixed-output - subderivations. A fixed-output derivation is a derivation with one - output (`out') for which an expected hash and hash algorithm are - specified (using the `outputHash' and `outputHashAlgo' - attributes). We don't want changes to such derivations to - propagate upwards through the dependency graph, changing output - paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it - (after all, (the hash of) the file being downloaded is unchanged). - So the *output paths* should not change. On the other hand, the - *derivation paths* should change to reflect the new dependency - graph. - - That's what this function does: it returns a hash which is just the - hash of the derivation ATerm, except that any input derivation - paths have been replaced by the result of a recursive call to this - function, and that for fixed-output derivations we return a hash of - its output path. */ -Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +/* See the header for interface details. These are the implementation details. + + For fixed-output derivations, each hash in the map is not the + corresponding output's content hash, but a hash of that hash along + with other constant data. The key point is that the value is a pure + function of the output's contents, and there are no preimage attacks + either spoofing an output's contents for a derivation, or + spoofing a derivation for an output's contents. + + For regular derivations, it looks up each subderivation from its hash + and recurs. If the subderivation is also regular, it simply + substitutes the derivation path with its hash. If the subderivation + is fixed-output, however, it takes each output hash and pretends it + is a derivation hash producing a single "out" output. This is so we + don't leak the provenance of fixed outputs, reducing pointless cache + misses as the build itself won't know this. + */ +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(htSHA256, "fixed:out:" - + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base16, false) + ":" - + store.printStorePath(i->second.path)); + switch (drv.type()) { + case DerivationType::CAFloating: + throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); + case DerivationType::CAFixed: { + std::map<std::string, Hash> outputHashes; + for (const auto & i : drv.outputs) { + auto & dof = std::get<DerivationOutputCAFixed>(i.second.output); + auto hash = hashString(htSHA256, "fixed:out:" + + dof.hash.printMethodAlgo() + ":" + + dof.hash.hash.to_string(Base16, false) + ":" + + store.printStorePath(dof.path(store, drv.name, i.first))); + outputHashes.insert_or_assign(i.first, std::move(hash)); + } + return outputHashes; + } + case DerivationType::InputAddressed: + break; } /* For other derivations, replace the inputs paths with recursive - calls to this function.*/ + calls to this function. */ std::map<std::string, StringSet> inputs2; for (auto & i : drv.inputDrvs) { - auto h = drvHashes.find(i.first); - if (h == drvHashes.end()) { - assert(store.isValidPath(i.first)); - h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, - store.readDerivation(i.first), false)).first; - } - inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); + const auto & res = pathDerivationModulo(store, i.first); + std::visit(overloaded { + // Regular non-CA derivation, replace derivation + [&](Hash drvHash) { + inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); + }, + // CA derivation's output hashes + [&](CaOutputHashes outputHashes) { + std::set<std::string> justOut = { "out" }; + for (auto & output : i.second) { + /* Put each one in with a single "out" output.. */ + const auto h = outputHashes.at(output); + inputs2.insert_or_assign( + h.to_string(Base16, false), + justOut); + } + }, + }, res); } return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); @@ -392,38 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -StorePathSet BasicDerivation::outputPaths() const -{ - StorePathSet paths; - for (auto & i : outputs) - paths.insert(i.second.path); - return paths; -} - static DerivationOutput readDerivationOutput(Source & in, const Store & store) { - auto path = store.parseStorePath(readString(in)); - auto hashAlgo = readString(in); - auto hash = readString(in); + const auto pathS = readString(in); + const auto hashAlgo = readString(in); + const auto hash = readString(in); - std::optional<FixedOutputHash> fsh; - if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (string(hashAlgo, 0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = string(hashAlgo, 2); - } - auto hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }; - } - - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; + return parseDerivationOutput(store, pathS, hashAlgo, hash); } StringSet BasicDerivation::outputNames() const @@ -434,9 +552,30 @@ StringSet BasicDerivation::outputNames() const return names; } +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { + DerivationOutputsAndOptPaths outsAndOptPaths; + for (auto output : outputs) + outsAndOptPaths.insert(std::make_pair( + output.first, + std::make_pair(output.second, output.second.path(store, name, output.first)) + ) + ); + return outsAndOptPaths; +} + +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + auto nameWithSuffix = drvPath.name(); + constexpr std::string_view extension = ".drv"; + assert(hasSuffix(nameWithSuffix, extension)); + nameWithSuffix.remove_suffix(extension.size()); + return nameWithSuffix; +} + -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { + drv.name = name; + drv.outputs.clear(); auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { @@ -445,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = readStorePaths<StorePathSet>(store, in); + drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {}); in >> drv.platform >> drv.builder; drv.args = readStrings<Strings>(in); @@ -464,16 +603,26 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr { out << drv.outputs.size(); for (auto & i : drv.outputs) { - out << i.first - << store.printStorePath(i.second.path); - if (i.second.hash) { - out << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); - } else { - out << "" << ""; - } + out << i.first; + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + out << store.printStorePath(doi.path) + << "" + << ""; + }, + [&](DerivationOutputCAFixed dof) { + out << store.printStorePath(dof.path(store, drv.name, i.first)) + << dof.hash.printMethodAlgo() + << dof.hash.hash.to_string(Base16, false); + }, + [&](DerivationOutputCAFloating dof) { + out << "" + << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << ""; + }, + }, i.second.output); } - writeStorePaths(store, out, drv.inputSrcs); + worker_proto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) @@ -487,5 +636,65 @@ std::string hashPlaceholder(const std::string & outputName) return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); } +std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) +{ + auto drvNameWithExtension = drvPath.name(); + auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); + auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); + return "/" + hashString(htSHA256, clearText).to_string(Base32, false); +} + + +// N.B. Outputs are left unchanged +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + + debug("Rewriting the derivation"); + + for (auto &rewrite: rewrites) { + debug("rewriting %s as %s", rewrite.first, rewrite.second); + } + + drv.builder = rewriteStrings(drv.builder, rewrites); + for (auto & arg: drv.args) { + arg = rewriteStrings(arg, rewrites); + } + + StringPairs newEnv; + for (auto & envVar: drv.env) { + auto envName = rewriteStrings(envVar.first, rewrites); + auto envValue = rewriteStrings(envVar.second, rewrites); + newEnv.emplace(envName, envValue); + } + drv.env = newEnv; +} + + +Sync<DrvPathResolutions> drvPathResolutions; + +std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { + BasicDerivation resolved { *this }; + + // Input paths that we'll want to rewrite in the derivation + StringMap inputRewrites; + + for (auto & input : inputDrvs) { + auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); + StringSet newOutputNames; + for (auto & outputName : input.second) { + auto actualPathOpt = inputDrvOutputs.at(outputName); + if (!actualPathOpt) + return std::nullopt; + auto actualPath = *actualPathOpt; + inputRewrites.emplace( + downstreamPlaceholder(store, input.first, outputName), + store.printStorePath(actualPath)); + resolved.inputSrcs.insert(std::move(actualPath)); + } + } + + rewriteDerivation(store, resolved, inputRewrites); + + return resolved; +} } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 68c53c1ff..d48266774 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -4,8 +4,10 @@ #include "types.hh" #include "hash.hh" #include "content-address.hh" +#include "sync.hh" #include <map> +#include <variant> namespace nix { @@ -13,20 +15,82 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutput +/* The traditional non-fixed-output derivation type. */ +struct DerivationOutputInputAddressed { + /* Will need to become `std::optional<StorePath>` once input-addressed + derivations are allowed to depend on cont-addressed derivations */ StorePath path; - std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */ +}; + +/* Fixed-output derivations, whose output paths are content addressed + according to that fixed output. */ +struct DerivationOutputCAFixed +{ + FixedOutputHash hash; /* hash used for expected hash computation */ + StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; +}; + +/* Floating-output derivations, whose output paths are content addressed, but + not fixed, and so are dynamically calculated from whatever the output ends + up being. */ +struct DerivationOutputCAFloating +{ + /* information used for expected hash computation */ + FileIngestionMethod method; + HashType hashType; +}; + +struct DerivationOutput +{ + std::variant< + DerivationOutputInputAddressed, + DerivationOutputCAFixed, + DerivationOutputCAFloating + > output; + std::optional<HashType> hashAlgoOpt(const Store & store) const; + /* Note, when you use this function you should make sure that you're passing + the right derivation name. When in doubt, you should use the safer + interface provided by BasicDerivation::outputsAndOptPaths */ + std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; +/* These are analogues to the previous DerivationOutputs data type, but they + also contains, for each output, the (optional) store path in which it would + be written. To calculate values of these types, see the corresponding + functions in BasicDerivation */ +typedef std::map<string, std::pair<DerivationOutput, StorePath>> + DerivationOutputsAndPaths; +typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>> + DerivationOutputsAndOptPaths; + /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ typedef std::map<StorePath, StringSet> DerivationInputs; typedef std::map<string, string> StringPairs; +enum struct DerivationType : uint8_t { + InputAddressed, + CAFixed, + CAFloating, +}; + +/* Do the outputs of the derivation have paths calculated from their content, + or from the derivation itself? */ +bool derivationIsCA(DerivationType); + +/* Is the content of the outputs fixed a-priori via a hash? Never true for + non-CA derivations. */ +bool derivationIsFixed(DerivationType); + +/* Is the derivation impure and needs to access non-deterministic resources, or + pure and can be sandboxed? Note that whether or not we actually sandbox the + derivation is controlled separately. Never true for non-CA derivations. */ +bool derivationIsImpure(DerivationType); + struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ @@ -35,24 +99,25 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; + std::string name; - BasicDerivation() { } + BasicDerivation() = default; virtual ~BasicDerivation() { }; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - const StorePath & findOutput(const std::string & id) const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ - bool isFixedOutput() const; - - /* Return the output paths of a derivation. */ - StorePathSet outputPaths() const; + DerivationType type() const; /* Return the output names of a derivation. */ StringSet outputNames() const; + + /* Calculates the maps that contains all the DerivationOutputs, but + augmented with knowledge of the Store paths they would be written + into. */ + DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + + static std::string_view nameFromPath(const StorePath & storePath); }; struct Derivation : BasicDerivation @@ -63,7 +128,17 @@ struct Derivation : BasicDerivation std::string unparse(const Store & store, bool maskOutputs, std::map<std::string, StringSet> * actualInputs = nullptr) const; - Derivation() { } + /* Return the underlying basic derivation but with these changes: + + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. + + 2. Input placeholders are replaced with realized input store paths. */ + std::optional<BasicDerivation> tryResolve(Store & store); + + Derivation() = default; + Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } + Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } }; @@ -72,30 +147,96 @@ class Store; enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ -StorePath writeDerivation(ref<Store> store, - const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); +StorePath writeDerivation(Store & store, + const Derivation & drv, + RepairFlag repair = NoRepair, + bool readOnly = false); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath); +Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove bool isDerivation(const string & fileName); -Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +/* Calculate the name that will be used for the store path for this + output. + + This is usually <drv-name>-<output-name>, but is just <drv-name> when + the output name is "out". */ +std::string outputPathName(std::string_view drvName, std::string_view outputName); + +// known CA drv's output hashes, current just for fixed-output derivations +// whose output hashes are always known since they are fixed up-front. +typedef std::map<std::string, Hash> CaOutputHashes; + +typedef std::variant< + Hash, // regular DRV normalized hash + CaOutputHashes +> DrvHashModulo; + +/* Returns hashes with the details of fixed-output subderivations + expunged. + + A fixed-output derivation is a derivation whose outputs have a + specified content hash and hash algorithm. (Currently they must have + exactly one output (`out'), which is specified using the `outputHash' + and `outputHashAlgo' attributes, but the algorithm doesn't assume + this.) We don't want changes to such derivations to propagate upwards + through the dependency graph, changing output paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it---after + all, (the hash of) the file being downloaded is unchanged. So the + *output paths* should not change. On the other hand, the *derivation + paths* should change to reflect the new dependency graph. + + For fixed-output derivations, this returns a map from the name of + each output to its hash, unique up to the output's contents. + + For regular derivations, it returns a single hash of the derivation + ATerm, after subderivations have been likewise expunged from that + derivation. + */ +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Memoisation of hashDerivationModulo(). */ -typedef std::map<StorePath, Hash> DrvHashes; +typedef std::map<StorePath, DrvHashModulo> DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe +/* Memoisation of `readDerivation(..).resove()`. */ +typedef std::map< + StorePath, + std::optional<StorePath> +> DrvPathResolutions; + +// FIXME: global, though at least thread-safe. +// FIXME: arguably overlaps with hashDerivationModulo memo table. +extern Sync<DrvPathResolutions> drvPathResolutions; + bool wantOutput(const string & output, const std::set<string> & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +/* This creates an opaque and almost certainly unique string + deterministically from the output name. + + It is used as a placeholder to allow derivations to refer to their + own outputs without needing to use the hash of a derivation in + itself, making the hash near-impossible to calculate. */ std::string hashPlaceholder(const std::string & outputName); +/* This creates an opaque and almost certainly unique string + deterministically from a derivation path and output name. + + It is used as a placeholder to allow derivations to refer to + content-addressed paths whose content --- and thus the path + themselves --- isn't yet known. This occurs when a derivation has a + dependency which is a CA derivation. */ +std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); + } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc new file mode 100644 index 000000000..98b745c3a --- /dev/null +++ b/src/libstore/dummy-store.cc @@ -0,0 +1,67 @@ +#include "store-api.hh" +#include "callback.hh" + +namespace nix { + +struct DummyStoreConfig : virtual StoreConfig { + using StoreConfig::StoreConfig; + + const std::string name() override { return "Dummy Store"; } +}; + +struct DummyStore : public Store, public virtual DummyStoreConfig +{ + DummyStore(const std::string scheme, const std::string uri, const Params & params) + : DummyStore(params) + { } + + DummyStore(const Params & params) + : StoreConfig(params) + , Store(params) + { } + + string getUri() override + { + return *uriSchemes().begin(); + } + + void queryPathInfoUncached(const StorePath & path, + Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override + { + callback(nullptr); + } + + static std::set<std::string> uriSchemes() { + return {"dummy"}; + } + + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override + { unsupported("addToStore"); } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override + { unsupported("addToStore"); } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override + { unsupported("addTextToStore"); } + + void narFromPath(const StorePath & path, Sink & sink) override + { unsupported("narFromPath"); } + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { unsupported("buildDerivation"); } +}; + +static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; + +} diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..02c839520 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -7,24 +7,6 @@ namespace nix { -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - void Store::exportPaths(const StorePathSet & paths, Sink & sink) { auto sorted = topoSortPaths(paths); @@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashAndWriteSink hashAndWriteSink(sink); + HashSink hashSink(htSHA256); + TeeSink teeSink(sink, hashSink); - narFromPath(path, hashAndWriteSink); + narFromPath(path, teeSink); /* Refuse to export paths that have changed. This prevents filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + Hash hash = hashSink.currentHash().first; + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); - hashAndWriteSink + teeSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); - hashAndWriteSink + worker_proto::write(*this, teeSink, info->references); + teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; } -StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) +StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) { StorePaths res; while (true) { @@ -77,33 +60,36 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info(parseStorePath(readString(source))); + auto path = parseStorePath(readString(source)); //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths<StorePathSet>(*this, source); - + auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {}); auto deriver = readString(source); + auto narHash = hashString(htSHA256, *saved.s); + + ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.references = references; + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; - addToStore(info, source, NoRepair, checkSigs, accessor); + auto source = StringSource { *saved.s }; + addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 531b85af8..c2c65af05 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,6 +5,7 @@ #include "s3.hh" #include "compression.hh" #include "finally.hh" +#include "callback.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -22,6 +23,7 @@ #include <queue> #include <random> #include <thread> +#include <regex> using namespace std::string_literals; @@ -29,7 +31,7 @@ namespace nix { FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&fileTransferSettings); +static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -56,7 +58,7 @@ struct curlFileTransfer : public FileTransfer Callback<FileTransferResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; + std::string statusMsg; unsigned int attempt = 0; @@ -111,6 +113,9 @@ struct curlFileTransfer : public FileTransfer requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + for (auto it = request.headers.begin(); it != request.headers.end(); ++it){ + requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str()); + } } ~TransferItem() @@ -123,7 +128,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -144,6 +149,7 @@ struct curlFileTransfer : public FileTransfer LambdaSink finalSink; std::shared_ptr<CompressionSink> decompressionSink; + std::optional<StringSink> errorSink; std::exception_ptr writeException; @@ -153,9 +159,19 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + errorSink = StringSink { }; + } + } + if (errorSink) + (*errorSink)((unsigned char *) contents, realSize); (*decompressionSink)((unsigned char *) contents, realSize); return realSize; @@ -175,12 +191,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); + std::smatch match; + if (std::regex_match(line, match, statusLine)) { result.etag = ""; - auto ss = tokenizeString<vector<string>>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); result.bodySize = 0; + statusMsg = trim(match[1]); acceptRanges = false; encoding = ""; } else { @@ -194,7 +211,9 @@ struct curlFileTransfer : public FileTransfer the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - if (result.etag == request.expectedETag && status == "200") { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + if (result.etag == request.expectedETag && httpStatus == 200) { debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -408,16 +427,21 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr<std::string> response; + if (errorSink) + response = errorSink->s; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) + response, + fmt("unable to %s '%s': HTTP error %d ('%s')", + request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, + response, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -675,7 +699,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else @@ -820,6 +844,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } } +template<typename... Args> +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how + // to print different messages for different verbosity levels. For now + // we add some heuristics for detecting when we want to show the response. + if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0..c89c51a21 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -17,15 +17,30 @@ struct FileTransferSettings : Config Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", "String appended to the user agent in HTTP requests."}; - Setting<size_t> httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", + Setting<size_t> httpConnections{ + this, 25, "http-connections", + R"( + The maximum number of parallel TCP connections used to fetch + files from binary caches and by other downloads. It defaults + to 25. 0 means no limit. + )", {"binary-caches-parallel-connections"}}; - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - - Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", - "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + Setting<unsigned long> connectTimeout{ + this, 0, "connect-timeout", + R"( + The timeout (in seconds) for establishing connections in the + binary cache substituter. It corresponds to `curl`’s + `--connect-timeout` option. + )"}; + + Setting<unsigned long> stalledDownloadTimeout{ + this, 300, "stalled-download-timeout", + R"( + The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this + timeout's duration. + )"}; Setting<unsigned int> tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; @@ -36,6 +51,7 @@ extern FileTransferSettings fileTransferSettings; struct FileTransferRequest { std::string uri; + Headers headers; std::string expectedETag; bool verifyTLS = true; bool head = false; @@ -103,10 +119,12 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr<string> response; // intentionally optional + template<typename... Args> - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 57fb20845..518a357ef 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -85,8 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const StorePath & storePath, - const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) +Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); @@ -95,47 +94,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", gcRoot); - if (indirect) { - /* Don't clobber the link if it already exists and doesn't - point to the Nix store. */ - if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error("cannot create symlink '%1%'; already exists", gcRoot); - makeSymlink(gcRoot, printStorePath(storePath)); - addIndirectRoot(gcRoot); - } - - else { - if (!allowOutsideRootsDir) { - Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); - - if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error( - "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'", - gcRoot, rootsDir); - } - - if (baseNameOf(gcRoot) == std::string(storePath.to_string())) - writeFile(gcRoot, ""); - else - makeSymlink(gcRoot, printStorePath(storePath)); - } - - /* Check that the root can be found by the garbage collector. - !!! This can be very slow on machines that have many roots. - Instead of reading all the roots, it would be more efficient to - check if the root is in a directory in or linked from the - gcroots directory. */ - if (settings.checkRootReachability) { - auto roots = findRoots(false); - if (roots[storePath].count(gcRoot) == 0) - logWarning({ - .name = "GC root", - .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)) - }); - } + /* Don't clobber the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error("cannot create symlink '%1%'; already exists", gcRoot); + makeSymlink(gcRoot, printStorePath(storePath)); + addIndirectRoot(gcRoot); /* Grab the global GC root, causing us to block while a GC is in progress. This prevents the set of permanent roots from @@ -262,11 +226,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { - auto storePath = maybeParseStorePath(toStorePath(target)); - if (storePath && isValidPath(*storePath)) - roots[std::move(*storePath)].emplace(path); - else - printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + try { + auto storePath = toStorePath(target).first; + if (isValidPath(storePath)) + roots[std::move(storePath)].emplace(path); + else + printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + } catch (BadStorePath &) { } }; try { @@ -472,15 +438,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) for (auto & [target, links] : unchecked) { if (!isInStore(target)) continue; - Path pathS = toStorePath(target); - if (!isStorePath(pathS)) continue; - auto path = parseStorePath(pathS); - if (!isValidPath(path)) continue; - debug("got additional root '%1%'", pathS); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); + try { + auto path = toStorePath(target).first; + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", printStorePath(path)); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); + } catch (BadStorePath &) { } } } @@ -498,7 +464,7 @@ struct LocalStore::GCState StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; - unsigned long long bytesInvalidated; + uint64_t bytesInvalidated; bool moveToTrash = true; bool shouldDelete; GCState(const GCOptions & options, GCResults & results) @@ -516,7 +482,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, void LocalStore::deleteGarbage(GCState & state, const Path & path) { - unsigned long long bytesFreed; + uint64_t bytesFreed; deletePath(path, bytesFreed); state.results.bytesFreed += bytesFreed; } @@ -526,7 +492,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) { checkInterrupt(); - unsigned long long size = 0; + uint64_t size = 0; auto storePath = maybeParseStorePath(path); if (storePath && isValidPath(*storePath)) { @@ -608,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto /* If keep-derivations is set and this is a derivation, then don't delete the derivation if any of the outputs are alive. */ if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == path + ) + incoming.insert(*maybeOutPath); } /* If keep-outputs is set, then don't delete this path if there @@ -685,7 +654,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", linksDir); - long long actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { @@ -694,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (name == "." || name == "..") continue; Path path = linksDir + "/" + name; - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError("statting '%1%'", path); + auto st = lstat(path); if (st.st_nlink != 1) { actualSize += st.st_size; @@ -715,10 +682,10 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - long long overhead = st.st_blocks * 512ULL; + auto overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 683fa5196..1238dc530 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "abstract-setting-to-json.hh" #include <algorithm> #include <map> @@ -9,6 +10,8 @@ #include <dlfcn.h> #include <sys/utsname.h> +#include <nlohmann/json.hpp> + namespace nix { @@ -22,7 +25,7 @@ namespace nix { Settings settings; -static GlobalConfig::Register r1(&settings); +static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) @@ -39,6 +42,7 @@ Settings::Settings() { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; + allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (caFile == "") { @@ -144,6 +148,12 @@ bool Settings::isWSL1() const string nixVersion = PACKAGE_VERSION; +NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { + {SandboxMode::smEnabled, true}, + {SandboxMode::smRelaxed, "relaxed"}, + {SandboxMode::smDisabled, false}, +}); + template<> void BaseSetting<SandboxMode>::set(const std::string & str) { if (str == "true") value = smEnabled; @@ -160,11 +170,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const else abort(); } -template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) -{ - AbstractSetting::toJSON(out); -} - template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) { args.addFlag({ diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5cf73c7b4..aa1fbdeb7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -80,73 +80,178 @@ public: Setting<bool> keepGoing{this, false, "keep-going", "Whether to keep building derivations when another build fails."}; - Setting<bool> tryFallback{this, false, "fallback", - "Whether to fall back to building when substitution fails.", + Setting<bool> tryFallback{ + this, false, "fallback", + R"( + If set to `true`, Nix will fall back to building from source if a + binary substitute fails. This is equivalent to the `--fallback` + flag. The default is `false`. + )", {"build-fallback"}}; /* Whether to show build log output in real time. */ bool verboseBuild = true; Setting<size_t> logLines{this, 10, "log-lines", - "If verbose-build is false, the number of lines of the tail of " + "If `verbose-build` is false, the number of lines of the tail of " "the log to show if a build fails."}; - MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", - "Maximum number of parallel build jobs. \"auto\" means use number of cores.", + MaxBuildJobsSetting maxBuildJobs{ + this, 1, "max-jobs", + R"( + This option defines the maximum number of jobs that Nix will try to + build in parallel. The default is `1`. The special value `auto` + causes Nix to use the number of CPUs in your system. `0` is useful + when using remote builders to prevent any local builds (except for + `preferLocalBuild` derivation attribute which executes locally + regardless). It can be overridden using the `--max-jobs` (`-j`) + command line switch. + )", {"build-max-jobs"}}; - Setting<unsigned int> buildCores{this, getDefaultCores(), "cores", - "Number of CPU cores to utilize in parallel within a build, " - "i.e. by passing this number to Make via '-j'. 0 means that the " - "number of actual CPU cores on the local host ought to be " - "auto-detected.", {"build-cores"}}; + Setting<unsigned int> buildCores{ + this, getDefaultCores(), "cores", + R"( + Sets the value of the `NIX_BUILD_CORES` environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + `enableParallelBuilding` is set to `true`, the builder passes the + `-jN` flag to GNU Make. It can be overridden using the `--cores` + command line switch and defaults to `1`. The value `0` means that + the builder should use all available CPU cores in the system. + )", + {"build-cores"}}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ bool readOnlyMode = false; - Setting<std::string> thisSystem{this, SYSTEM, "system", - "The canonical Nix system name."}; - - Setting<time_t> maxSilentTime{this, 0, "max-silent-time", - "The maximum time in seconds that a builer can go without " - "producing any output on stdout/stderr before it is killed. " - "0 means infinity.", + Setting<std::string> thisSystem{ + this, SYSTEM, "system", + R"( + This option specifies the canonical Nix system name of the current + installation, such as `i686-linux` or `x86_64-darwin`. Nix can only + build derivations whose `system` attribute equals the value + specified here. In general, it never makes sense to modify this + value from its default, since you can use it to ‘lie’ about the + platform you are building on (e.g., perform a Mac OS build on a + Linux machine; the result would obviously be wrong). It only makes + sense if the Nix binaries can run on multiple platforms, e.g., + ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + + It defaults to the canonical Nix system name detected by `configure` + at build time. + )"}; + + Setting<time_t> maxSilentTime{ + this, 0, "max-silent-time", + R"( + This option defines the maximum number of seconds that a builder can + go without producing any data on standard output or standard error. + This is useful (for instance in an automated build system) to catch + builds that are stuck in an infinite loop, or to catch remote builds + that are hanging due to network problems. It can be overridden using + the `--max-silent-time` command line switch. + + The value `0` means that there is no timeout. This is also the + default. + )", {"build-max-silent-time"}}; - Setting<time_t> buildTimeout{this, 0, "timeout", - "The maximum duration in seconds that a builder can run. " - "0 means infinity.", {"build-timeout"}}; + Setting<time_t> buildTimeout{ + this, 0, "timeout", + R"( + This option defines the maximum number of seconds that a builder can + run. This is useful (for instance in an automated build system) to + catch builds that are stuck in an infinite loop but keep writing to + their standard output or standard error. It can be overridden using + the `--timeout` command line switch. + + The value `0` means that there is no timeout. This is also the + default. + )", + {"build-timeout"}}; PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", "The path of the helper program that executes builds to remote machines."}; - Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", - "A semicolon-separated list of build machines, in the format of nix.machines."}; - - Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes", - "Whether build machines should use their own substitutes for obtaining " - "build dependencies if possible, rather than waiting for this host to " - "upload them."}; + Setting<std::string> builders{ + this, "@" + nixConfDir + "/machines", "builders", + "A semicolon-separated list of build machines, in the format of `nix.machines`."}; + + Setting<bool> buildersUseSubstitutes{ + this, false, "builders-use-substitutes", + R"( + If set to `true`, Nix will instruct remote build machines to use + their own binary substitutes if available. In practical terms, this + means that remote hosts will fetch as many build dependencies as + possible from their own substitutes (e.g, from `cache.nixos.org`), + instead of waiting for this host to upload them all. This can + drastically reduce build times if the network connection between + this computer and the remote build host is slow. + )"}; Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; - Setting<bool> fsyncMetadata{this, true, "fsync-metadata", - "Whether SQLite should use fsync()."}; + Setting<bool> fsyncMetadata{ + this, true, "fsync-metadata", + R"( + If set to `true`, changes to the Nix store metadata (in + `/nix/var/nix/db`) are synchronously flushed to disk. This improves + robustness in case of system crashes, but reduces performance. The + default is `true`. + )"}; Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", - "Whether to call sync() before registering a path as valid."}; - - Setting<bool> useSubstitutes{this, true, "substitute", - "Whether to use substitutes.", + "Whether to call `sync()` before registering a path as valid."}; + + Setting<bool> useSubstitutes{ + this, true, "substitute", + R"( + If set to `true` (default), Nix will use binary substitutes if + available. This option can be disabled to force building from + source. + )", {"build-use-substitutes"}}; - Setting<std::string> buildUsersGroup{this, "", "build-users-group", - "The Unix group that contains the build users."}; + Setting<std::string> buildUsersGroup{ + this, "", "build-users-group", + R"( + This options specifies the Unix group containing the Nix build user + accounts. In multi-user Nix installations, builds should not be + performed by the Nix account since that would allow users to + arbitrarily modify the Nix store and database by supplying specially + crafted builders; and they cannot be performed by the calling user + since that would allow him/her to influence the build result. + + Therefore, if this option is non-empty and specifies a valid group, + builds will be performed under the user accounts that are a member + of the group specified here (as listed in `/etc/group`). Those user + accounts should not be used for any other purpose\! + + Nix will never run two builds under the same user account at the + same time. This is to prevent an obvious security hole: a malicious + user writing a Nix expression that modifies the build result of a + legitimate Nix expression being built by another user. Therefore it + is good to have as many Nix build user accounts as you can spare. + (Remember: uids are cheap.) + + The build users should have permission to create files in the Nix + store, but not delete them. Therefore, `/nix/store` should be owned + by the Nix account, its group should be the group specified here, + and its mode should be `1775`. + + If the build users group is empty, builds will be performed under + the uid of the Nix process (that is, the uid of the caller if + `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if + `NIX_REMOTE` is `daemon`). Obviously, this should not be used in + multi-user settings with untrusted users. + )"}; #if __linux__ Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids", @@ -165,17 +270,32 @@ public: "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; - Setting<bool> keepLog{this, true, "keep-build-log", - "Whether to store build logs.", + Setting<bool> keepLog{ + this, true, "keep-build-log", + R"( + If set to `true` (the default), Nix will write the build log of a + derivation (i.e. the standard output and error of its builder) to + the directory `/nix/var/log/nix/drvs`. The build log can be + retrieved using the command `nix-store -l path`. + )", {"build-keep-log"}}; - Setting<bool> compressLog{this, true, "compress-build-log", - "Whether to compress logs.", + Setting<bool> compressLog{ + this, true, "compress-build-log", + R"( + If set to `true` (the default), build logs written to + `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2. + Otherwise, they will not be compressed. + )", {"build-compress-log"}}; - Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size", - "Maximum number of bytes a builder can write to stdout/stderr " - "before being killed (0 means no limit).", + Setting<unsigned long> maxLogSize{ + this, 0, "max-build-log-size", + R"( + This option defines the maximum number of bytes that a builder can + write to its stdout/stderr. If the builder exceeds this limit, it’s + killed. A value of `0` (the default) means that there is no limit. + )", {"build-max-log-size"}}; /* When buildRepeat > 0 and verboseBuild == true, whether to print @@ -186,57 +306,156 @@ public: Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; - Setting<bool> checkRootReachability{this, false, "gc-check-reachability", - "Whether to check if new GC roots can in fact be found by the " - "garbage collector."}; - - Setting<bool> gcKeepOutputs{this, false, "keep-outputs", - "Whether the garbage collector should keep outputs of live derivations.", + Setting<bool> gcKeepOutputs{ + this, false, "keep-outputs", + R"( + If `true`, the garbage collector will keep the outputs of + non-garbage derivations. If `false` (default), outputs will be + deleted unless they are GC roots themselves (or reachable from other + roots). + + In general, outputs must be registered as roots separately. However, + even if the output of a derivation is registered as a root, the + collector will still delete store paths that are used only at build + time (e.g., the C compiler, or source tarballs downloaded from the + network). To prevent it from doing so, set this option to `true`. + )", {"gc-keep-outputs"}}; - Setting<bool> gcKeepDerivations{this, true, "keep-derivations", - "Whether the garbage collector should keep derivers of live paths.", + Setting<bool> gcKeepDerivations{ + this, true, "keep-derivations", + R"( + If `true` (default), the garbage collector will keep the derivations + from which non-garbage store paths were built. If `false`, they will + be deleted unless explicitly registered as a root (or reachable from + other roots). + + Keeping derivation around is useful for querying and traceability + (e.g., it allows you to ask with what dependencies or options a + store path was built), so by default this option is on. Turn it off + to save a bit of disk space (or a lot if `keep-outputs` is also + turned on). + )", {"gc-keep-derivations"}}; - Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", - "Whether to automatically replace files with identical contents with hard links."}; - - Setting<bool> envKeepDerivations{this, false, "keep-env-derivations", - "Whether to add derivations as a dependency of user environments " - "(to prevent them from being GCed).", + Setting<bool> autoOptimiseStore{ + this, false, "auto-optimise-store", + R"( + If set to `true`, Nix automatically detects files in the store + that have identical contents, and replaces them with hard links to + a single copy. This saves disk space. If set to `false` (the + default), you can still run `nix-store --optimise` to get rid of + duplicate files. + )"}; + + Setting<bool> envKeepDerivations{ + this, false, "keep-env-derivations", + R"( + If `false` (default), derivations are not stored in Nix user + environments. That is, the derivations of any build-time-only + dependencies may be garbage-collected. + + If `true`, when you add a Nix derivation to a user environment, the + path of the derivation is stored in the user environment. Thus, the + derivation will not be garbage-collected until the user environment + generation is deleted (`nix-env --delete-generations`). To prevent + build-time-only dependencies from being collected, you should also + turn on `keep-outputs`. + + The difference between this option and `keep-derivations` is that + this one is “sticky”: it applies to any user environment created + while this option was enabled, while `keep-derivations` only applies + at the moment the garbage collector is run. + )", {"env-keep-derivations"}}; /* Whether to lock the Nix client and worker to the same CPU. */ bool lockCPU; - Setting<SandboxMode> sandboxMode{this, + Setting<SandboxMode> sandboxMode{ + this, #if __linux__ smEnabled #else smDisabled #endif , "sandbox", - "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", + R"( + If set to `true`, builds will be performed in a *sandboxed + environment*, i.e., they’re isolated from the normal file system + hierarchy and will only see their dependencies in the Nix store, + the temporary build directory, private versions of `/proc`, + `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths + configured with the `sandbox-paths` option. This is useful to + prevent undeclared dependencies on files in directories such as + `/usr/bin`. In addition, on Linux, builds run in private PID, + mount, network, IPC and UTS namespaces to isolate them from other + processes in the system (except that fixed-output derivations do + not run in private network namespace to ensure they can access the + network). + + Currently, sandboxing only work on Linux and macOS. The use of a + sandbox requires that Nix is run as root (so you should use the + “build users” feature to perform the actual builds under different + users than root). + + If this option is set to `relaxed`, then fixed-output derivations + and derivations that have the `__noChroot` attribute set to `true` + do not run in sandboxes. + + The default is `true` on Linux and `false` on all other platforms. + )", {"build-use-chroot", "build-use-sandbox"}}; - Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths", - "The paths to make available inside the build sandbox.", + Setting<PathSet> sandboxPaths{ + this, {}, "sandbox-paths", + R"( + A list of paths bind-mounted into Nix sandbox environments. You can + use the syntax `target=source` to mount a path in a different + location in the sandbox; for instance, `/bin=/nix-bin` will mount + the path `/nix-bin` as `/bin` inside the sandbox. If *source* is + followed by `?`, then it is not an error if *source* does not exist; + for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will + only be mounted in the sandbox if it exists in the host filesystem. + + Depending on how Nix was built, the default value for this option + may be empty or provide `/bin/sh` as a bind-mount of `bash`. + )", {"build-chroot-dirs", "build-sandbox-paths"}}; Setting<bool> sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", - "Additional paths to make available inside the build sandbox.", + Setting<PathSet> extraSandboxPaths{ + this, {}, "extra-sandbox-paths", + R"( + A list of additional paths appended to `sandbox-paths`. Useful if + you want to extend its default value. + )", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting<size_t> buildRepeat{this, 0, "repeat", - "The number of times to repeat a build in order to verify determinism.", + Setting<size_t> buildRepeat{ + this, 0, "repeat", + R"( + How many times to repeat builds to check whether they are + deterministic. The default value is 0. If the value is non-zero, + every build is repeated the specified number of times. If the + contents of any of the runs differs from the previous ones and + `enforce-determinism` is true, the build is rejected and the + resulting store paths are not registered as “valid” in Nix’s + database. + )", {"build-repeat"}}; #if __linux__ - Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", - "The size of /dev/shm in the build sandbox."}; + Setting<std::string> sandboxShmSize{ + this, "50%", "sandbox-dev-shm-size", + R"( + This option determines the maximum size of the `tmpfs` filesystem + mounted on `/dev/shm` in Linux sandboxes. For the format, see the + description of the `size` option of `tmpfs` in mount8. The default + is `50%`. + )"}; Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", "The build directory inside the sandbox."}; @@ -250,121 +469,457 @@ public: "Whether to log Darwin sandbox access violations to the system log."}; #endif - Setting<bool> runDiffHook{this, false, "run-diff-hook", - "Whether to run the program specified by the diff-hook setting " - "repeated builds produce a different result. Typically used to " - "plug in diffoscope."}; + Setting<bool> runDiffHook{ + this, false, "run-diff-hook", + R"( + If true, enable the execution of the `diff-hook` program. - PathSetting diffHook{this, true, "", "diff-hook", - "A program that prints out the differences between the two paths " - "specified on its command line."}; + When using the Nix daemon, `run-diff-hook` must be set in the + `nix.conf` configuration file, and cannot be passed at the command + line. + )"}; - Setting<bool> enforceDeterminism{this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output."}; + PathSetting diffHook{ + this, true, "", "diff-hook", + R"( + Absolute path to an executable capable of diffing build + results. The hook is executed if `run-diff-hook` is true, and the + output of a build is known to not be the same. This program is not + executed to determine if two results are the same. - Setting<Strings> trustedPublicKeys{this, - {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, - "trusted-public-keys", - "Trusted public keys for secure substitution.", - {"binary-cache-public-keys"}}; + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + + The diff hook program receives three parameters: + + 1. A path to the previous build's results + + 2. A path to the current build's results + + 3. The path to the build's derivation - Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", - "Secret keys with which to sign local builds."}; + 4. The path to the build's scratch directory. This directory will + exist only if the build was run with `--keep-failed`. - Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", - "How long downloaded files are considered up-to-date."}; + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. - Setting<bool> requireSigs{this, true, "require-sigs", - "Whether to check that any non-content-addressed path added to the " - "Nix store has a valid signature (that is, one signed using a key " - "listed in 'trusted-public-keys'."}; + When using the Nix daemon, `diff-hook` must be set in the `nix.conf` + configuration file, and cannot be passed at the command line. + )"}; - Setting<StringSet> extraPlatforms{this, + Setting<bool> enforceDeterminism{ + this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output. See `repeat`."}; + + Setting<Strings> trustedPublicKeys{ + this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "trusted-public-keys", + R"( + A whitespace-separated list of public keys. When paths are copied + from another Nix store (such as a binary cache), they must be + signed with one of these keys. For example: + `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + )", + {"binary-cache-public-keys"}}; + + Setting<Strings> secretKeyFiles{ + this, {}, "secret-key-files", + R"( + A whitespace-separated list of files containing secret (private) + keys. These are used to sign locally-built paths. They can be + generated using `nix-store --generate-binary-cache-key`. The + corresponding public key can be distributed to other users, who + can add it to `trusted-public-keys` in their `nix.conf`. + )"}; + + Setting<unsigned int> tarballTtl{ + this, 60 * 60, "tarball-ttl", + R"( + The number of seconds a downloaded tarball is considered fresh. If + the cached tarball is stale, Nix will check whether it is still up + to date using the ETag header. Nix will download a new version if + the ETag header is unsupported, or the cached ETag doesn't match. + + Setting the TTL to `0` forces Nix to always check if the tarball is + up to date. + + Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`. + + Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`, + `fetchTarball`, and `fetchurl` respect this TTL. + )"}; + + Setting<bool> requireSigs{ + this, true, "require-sigs", + R"( + If set to `true` (the default), any non-content-addressed path added + or copied to the Nix store (e.g. when substituting from a binary + cache) must have a valid signature, that is, be signed using one of + the keys listed in `trusted-public-keys` or `secret-key-files`. Set + to `false` to disable signature checking. + )"}; + + Setting<StringSet> extraPlatforms{ + this, std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", - "Additional platforms that can be built on the local system. " - "These may be supported natively (e.g. armv7 on some aarch64 CPUs " - "or using hacks like qemu-user."}; - - Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), + R"( + Platforms other than the native one which this machine is capable of + building for. This can be useful for supporting additional + architectures on compatible machines: i686-linux can be built on + x86\_64-linux machines (and the default for this setting reflects + this); armv7 is backwards-compatible with armv6 and armv5tel; some + aarch64 machines can also natively run 32-bit ARM code; and + qemu-user may be used to support non-native platforms (though this + may be slow and buggy). Most values for this are not enabled by + default because build systems will often misdetect the target + platform and generate incompatible code, so you may wish to + cross-check the results of using this option against proper + natively-built versions of your derivations. + )"}; + + Setting<StringSet> systemFeatures{ + this, getDefaultSystemFeatures(), "system-features", - "Optional features that this system implements (like \"kvm\")."}; + R"( + A set of system “features” supported by this machine, e.g. `kvm`. + Derivations can express a dependency on such features through the + derivation attribute `requiredSystemFeatures`. For example, the + attribute - Setting<Strings> substituters{this, + requiredSystemFeatures = [ "kvm" ]; + + ensures that the derivation can only be built on a machine with the + `kvm` feature. + + This setting by default includes `kvm` if `/dev/kvm` is accessible, + and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` + that are used in Nixpkgs to route builds to specific machines. + )"}; + + Setting<Strings> substituters{ + this, nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), "substituters", - "The URIs of substituters (such as https://cache.nixos.org/).", + R"( + A list of URLs of substituters, separated by whitespace. The default + is `https://cache.nixos.org`. + )", {"binary-caches"}}; // FIXME: provide a way to add to option values. - Setting<Strings> extraSubstituters{this, {}, "extra-substituters", - "Additional URIs of substituters.", + Setting<Strings> extraSubstituters{ + this, {}, "extra-substituters", + R"( + Additional binary caches appended to those specified in + `substituters`. When used by unprivileged users, untrusted + substituters (i.e. those not listed in `trusted-substituters`) are + silently ignored. + )", {"extra-binary-caches"}}; - Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", - "Disabled substituters that may be enabled via the substituters option by untrusted users.", + Setting<StringSet> trustedSubstituters{ + this, {}, "trusted-substituters", + R"( + A list of URLs of substituters, separated by whitespace. These are + not used by default, but can be enabled by users of the Nix daemon + by specifying `--option substituters urls` on the command + line. Unprivileged users are only allowed to pass a subset of the + URLs listed in `substituters` and `trusted-substituters`. + )", {"trusted-binary-caches"}}; - Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", - "Which users or groups are trusted to ask the daemon to do unsafe things."}; - - Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", - "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " - "return an invalid path result"}; - - Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", - "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " - "return a valid path result."}; + Setting<Strings> trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + Setting<unsigned int> ttlNegativeNarInfoCache{ + this, 3600, "narinfo-cache-negative-ttl", + R"( + The TTL in seconds for negative lookups. If a store path is queried + from a substituter but was not found, there will be a negative + lookup cached in the local disk cache database for the specified + duration. + )"}; + + Setting<unsigned int> ttlPositiveNarInfoCache{ + this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + R"( + The TTL in seconds for positive lookups. If a store path is queried + from a substituter, the result of the query will be cached in the + local disk cache database including some of the NAR metadata. The + default TTL is a month, setting a shorter TTL for positive lookups + can be useful for binary caches that have frequent garbage + collection, in which case having a more frequent cache invalidation + would prevent trying to pull the path again and failing with a hash + mismatch if the build isn't reproducible. + )"}; /* ?Who we trust to use the daemon in safe ways */ - Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", - "Which users or groups are allowed to connect to the daemon."}; + Setting<Strings> allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; Setting<bool> printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; - Setting<std::string> preBuildHook{this, "", - "pre-build-hook", - "A program to run just before a build to set derivation-specific build settings."}; + Setting<std::string> preBuildHook{ + this, "", "pre-build-hook", + R"( + If set, the path to a program that can set extra derivation-specific + settings for this system. This is used for settings that can't be + captured by the derivation model itself and are too variable between + different versions of the same system to be hard-coded into nix. - Setting<std::string> postBuildHook{this, "", "post-build-hook", - "A program to run just after each successful build."}; + The hook is passed the derivation path and, if sandboxes are + enabled, the sandbox directory. It can then modify the sandbox and + send a series of commands to modify various settings to stdout. The + currently recognized commands are: - Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", - "Path to the netrc file used to obtain usernames/passwords for downloads."}; + - `extra-sandbox-paths` + Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an + empty line. Entries have the same format as `sandbox-paths`. + )"}; + + Setting<std::string> postBuildHook{ + this, "", "post-build-hook", + R"( + Optional. The path to a program to execute after each build. + + This option is only settable in the global `nix.conf`, or on the + command line by trusted users. + + When using the nix-daemon, the daemon executes the hook as `root`. + If the nix-daemon is not involved, the hook runs as the user + executing the nix-build. + + - The hook executes after an evaluation-time build. + + - The hook does not execute on substituted paths. + + - The hook's output always goes to the user's terminal. + + - If the hook fails, the build succeeds but no further builds + execute. + + - The hook executes synchronously, and blocks other builds from + progressing while it runs. + + The program executes with no arguments. The program's environment + contains the following environment variables: + + - `DRV_PATH` + The derivation for the built paths. + + Example: + `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` + + - `OUT_PATHS` + Output paths of the built derivation, separated by a space + character. + + Example: + `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. + )"}; + + Setting<std::string> netrcFile{ + this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", + R"( + If set to an absolute path to a `netrc` file, Nix will use the HTTP + authentication credentials in this file when trying to download from + a remote host through HTTP or HTTPS. Defaults to + `$NIX_CONF_DIR/netrc`. + + The `netrc` file consists of a list of accounts in the following + format: + + machine my-machine + login my-username + password my-password + + For the exact syntax, see [the `curl` + documentation](https://ec.haxx.se/usingcurl-netrc.html). + + > **Note** + > + > This must be an absolute path, and `~` is not resolved. For + > example, `~/.netrc` won't resolve to your home directory's + > `.netrc`. + )"}; /* Path to the SSL CA file used */ Path caFile; #if __linux__ - Setting<bool> filterSyscalls{this, true, "filter-syscalls", - "Whether to prevent certain dangerous system calls, such as " - "creation of setuid/setgid files or adding ACLs or extended " - "attributes. Only disable this if you're aware of the " - "security implications."}; - - Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges", - "Whether builders can acquire new privileges by calling programs with " - "setuid/setgid bits or with file capabilities."}; + Setting<bool> filterSyscalls{ + this, true, "filter-syscalls", + R"( + Whether to prevent certain dangerous system calls, such as + creation of setuid/setgid files or adding ACLs or extended + attributes. Only disable this if you're aware of the + security implications. + )"}; + + Setting<bool> allowNewPrivileges{ + this, false, "allow-new-privileges", + R"( + (Linux-specific.) By default, builders on Linux cannot acquire new + privileges by calling setuid/setgid programs or programs that have + file capabilities. For example, programs such as `sudo` or `ping` + will fail. (Note that in sandbox builds, no such programs are + available unless you bind-mount them into the sandbox via the + `sandbox-paths` option.) You can allow the use of such programs by + enabling this option. This is impure and usually undesirable, but + may be useful in certain scenarios (e.g. to spin up containers or + set up userspace network interfaces in tests). + )"}; #endif - Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", - "A list of servers used by builtins.fetchurl to fetch files by hash."}; - - Setting<uint64_t> minFree{this, 0, "min-free", - "Automatically run the garbage collector when free disk space drops below the specified amount."}; - - Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", - "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<Strings> hashedMirrors{ + this, {}, "hashed-mirrors", + R"( + A list of web servers used by `builtins.fetchurl` to obtain files by + hash. The default is `http://tarballs.nixos.org/`. Given a hash type + *ht* and a base-16 hash *h*, Nix will try to download the file from + *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if + they have disappeared from their original URI. For example, given + the default mirror `http://tarballs.nixos.org/`, when building the + derivation + + ```nix + builtins.fetchurl { + url = "https://example.org/foo-1.2.3.tar.xz"; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; + } + ``` + + Nix will attempt to download this file from + `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae` + first. If it is not available there, if will try the original URI. + )"}; + + Setting<uint64_t> minFree{ + this, 0, "min-free", + R"( + When free disk space in `/nix/store` drops below `min-free` during a + build, Nix performs a garbage-collection until `max-free` bytes are + available or there is no more garbage. A value of `0` (the default) + disables this feature. + )"}; + + Setting<uint64_t> maxFree{ + this, std::numeric_limits<uint64_t>::max(), "max-free", + R"( + When a garbage collection is triggered by the `min-free` option, it + stops as soon as `max-free` bytes are available. The default is + infinity (i.e. delete all garbage). + )"}; Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - Setting<Paths> pluginFiles{this, {}, "plugin-files", - "Plugins to dynamically load at nix initialization time."}; - - Setting<std::string> githubAccessToken{this, "", "github-access-token", - "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting<Paths> pluginFiles{ + this, {}, "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix, allowing them to affect execution through static + initialization. In particular, these plugins may construct static + instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; + + Setting<StringMap> accessTokens{this, {}, "access-tokens", + R"( + Access tokens used to access protected GitHub, GitLab, or + other locations requiring token-based authentication. + + Access tokens are specified as a string made up of + space-separated `host=token` values. The specific token + used is selected by matching the `host` portion against the + "host" specification of the input. The actual use of the + `token` value is determined by the type of resource being + accessed: + + * Github: the token value is the OAUTH-TOKEN string obtained + as the Personal Access Token from the Github server (see + https://docs.github.com/en/developers/apps/authorizing-oath-apps). + + * Gitlab: the token value is either the OAuth2 token or the + Personal Access Token (these are different types tokens + for gitlab, see + https://docs.gitlab.com/12.10/ee/api/README.html#authentication). + The `token` value should be `type:tokenstring` where + `type` is either `OAuth2` or `PAT` to indicate which type + of token is being specified. + + Example `~/.config/nix/nix.conf`: + + ``` + access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + ``` + + Example `~/code/flake.nix`: + + ```nix + input.foo = { + type = "gitlab"; + host = "gitlab.mycompany.com"; + owner = "mycompany"; + repo = "pro"; + }; + ``` + + This example specifies three tokens, one each for accessing + github.com, gitlab.mycompany.com, and sourceforge.net. + + The `input.foo` uses the "gitlab" fetcher, which might + requires specifying the token type along with the token + value. + )"}; Setting<Strings> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; @@ -378,6 +933,25 @@ public: Setting<bool> warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + + Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", + "Maximum size of NARs before spilling them to disk."}; + + Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; + + Setting<bool> allowSymlinkedStore{ + this, false, "allow-symlinked-store", + R"( + If set to `true`, Nix will stop complaining if the store directory + (typically /nix/store) contains symlink components. + + This risks making some builds "impure" because builders sometimes + "canonicalise" paths by resolving all symlink components. Problems + occur if those builds are then deployed to machines where /nix/store + resolves to a different location from that of the build machine. You + can enable this setting if you are sure you're not going to do that. + )"}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 451a64785..9d2a89f96 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,12 +2,20 @@ #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "callback.hh" namespace nix { MakeError(UploadToHTTP, Error); -class HttpBinaryCacheStore : public BinaryCacheStore +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Http Binary Cache Store"; } +}; + +class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig { private: @@ -24,9 +32,12 @@ private: public: HttpBinaryCacheStore( - const Params & params, const Path & _cacheUri) - : BinaryCacheStore(params) - , cacheUri(_cacheUri) + const std::string & scheme, + const Path & _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) + , cacheUri(scheme + "://" + _cacheUri) { if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -55,6 +66,14 @@ public: } } + static std::set<std::string> uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set<std::string>({"http", "https"}); + if (forceHttp) ret.insert("file"); + return ret; + } + protected: void maybeDisable() @@ -85,7 +104,7 @@ protected: checkEnabled(); try { - FileTransferRequest request(cacheUri + "/" + path); + FileTransferRequest request(makeRequest(path)); request.head = true; getFileTransfer()->download(request); return true; @@ -100,11 +119,11 @@ protected: } void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) override { - auto req = FileTransferRequest(cacheUri + "/" + path); - req.data = std::make_shared<string>(data); // FIXME: inefficient + auto req = makeRequest(path); + req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain()); req.mimeType = mimeType; try { getFileTransfer()->upload(req); @@ -115,8 +134,11 @@ protected: FileTransferRequest makeRequest(const std::string & path) { - FileTransferRequest request(cacheUri + "/" + path); - return request; + return FileTransferRequest( + hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") + ? path + : cacheUri + "/" + path); + } void getFile(const std::string & path, Sink & sink) override @@ -159,18 +181,6 @@ protected: }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - if (std::string(uri, 0, 7) != "http://" && - std::string(uri, 0, 8) != "https://" && - (!forceHttp || std::string(uri, 0, 7) != "file://")) - return 0; - auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); - store->init(); - return store; -}); +static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..467169ce8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,21 +6,28 @@ #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" +#include "callback.hh" namespace nix { -static std::string uriScheme = "ssh://"; - -struct LegacySSHStore : public Store +struct LegacySSHStoreConfig : virtual StoreConfig { - const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; + using StoreConfig::StoreConfig; + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "Legacy SSH Store"; } +}; +struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +{ // Hack for getting remote build log output. - const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { @@ -37,8 +44,11 @@ struct LegacySSHStore : public Store SSHMaster master; - LegacySSHStore(const string & host, const Params & params) - : Store(params) + static std::set<std::string> uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const string & scheme, const string & host, const Params & params) + : StoreConfig(params) + , Store(params) , host(host) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), @@ -84,7 +94,7 @@ struct LegacySSHStore : public Store string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } void queryPathInfoUncached(const StorePath & path, @@ -93,6 +103,9 @@ struct LegacySSHStore : public Store try { auto conn(connections->get()); + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; @@ -100,23 +113,27 @@ struct LegacySSHStore : public Store auto p = readString(conn->from); if (p.empty()) return callback(nullptr); - auto info = std::make_shared<ValidPathInfo>(parseStorePath(p)); - assert(path == info->path); + auto path2 = parseStorePath(p); + assert(path == path2); + /* Hash will be set below. FIXME construct ValidPathInfo at end. */ + auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy); PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths<StorePathSet>(*this, conn->from); + info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseContentAddressOpt(readString(conn->from)); - info->sigs = readStrings<StringSet>(conn->from); + if (s == "") + throw Error("NAR hash is now mandatory"); + info->narHash = Hash::parseAnyPrefixed(s); } + info->ca = parseContentAddressOpt(readString(conn->from)); + info->sigs = readStrings<StringSet>(conn->from); auto s = readString(conn->from); assert(s == ""); @@ -126,8 +143,7 @@ struct LegacySSHStore : public Store } void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override + RepairFlag repair, CheckSigsFlag checkSigs) override { debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); @@ -140,7 +156,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -169,7 +185,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - writeStorePaths(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -203,6 +219,24 @@ struct LegacySSHStore : public Store const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } +private: + + void putBuildSettings(Connection & conn) + { + conn.to + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) + conn.to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) + conn.to + << settings.buildRepeat + << settings.enforceDeterminism; + } + +public: + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { @@ -212,16 +246,8 @@ struct LegacySSHStore : public Store << cmdBuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); - conn->to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) - conn->to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->to - << settings.buildRepeat - << settings.enforceDeterminism; + + putBuildSettings(*conn); conn->to.flush(); @@ -235,6 +261,29 @@ struct LegacySSHStore : public Store return status; } + void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override + { + auto conn(connections->get()); + + conn->to << cmdBuildPaths; + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + + putBuildSettings(*conn); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } + } + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } @@ -252,10 +301,10 @@ struct LegacySSHStore : public Store conn->to << cmdQueryClosure << includeOutputs; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})) out.insert(i); } @@ -268,10 +317,10 @@ struct LegacySSHStore : public Store << cmdQueryValidPaths << false // lock << maybeSubstitute; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn->to.flush(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } void connect() override @@ -286,12 +335,6 @@ struct LegacySSHStore : public Store } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 48aca478c..7d979c5c2 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -4,7 +4,14 @@ namespace nix { -class LocalBinaryCacheStore : public BinaryCacheStore +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Local Binary Cache Store"; } +}; + +class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig { private: @@ -13,8 +20,11 @@ private: public: LocalBinaryCacheStore( - const Params & params, const Path & binaryCacheDir) - : BinaryCacheStore(params) + const std::string scheme, + const Path & binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { } @@ -26,13 +36,25 @@ public: return "file://" + binaryCacheDir; } + static std::set<std::string> uriSchemes(); + protected: bool fileExists(const std::string & path) override; void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) override; + std::shared_ptr<std::basic_iostream<char>> istream, + const std::string & mimeType) override + { + auto path2 = binaryCacheDir + "/" + path; + Path tmp = path2 + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + StreamToSourceAdapter source(istream); + writeFile(tmp, source); + if (rename(tmp.c_str(), path2.c_str())) + throw SysError("renaming '%1%' to '%2%'", tmp, path2); + del.cancel(); + } void getFile(const std::string & path, Sink & sink) override { @@ -52,7 +74,9 @@ protected: if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); + paths.insert(parseStorePath( + storeDir + "/" + entry.name.substr(0, entry.name.size() - 8) + + "-" + MissingName)); } return paths; @@ -68,38 +92,19 @@ void LocalBinaryCacheStore::init() BinaryCacheStore::init(); } -static void atomicWrite(const Path & path, const std::string & s) -{ - Path tmp = path + ".tmp." + std::to_string(getpid()); - AutoDelete del(tmp, false); - writeFile(tmp, s); - if (rename(tmp.c_str(), path.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path); - del.cancel(); -} - bool LocalBinaryCacheStore::fileExists(const std::string & path) { return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) +std::set<std::string> LocalBinaryCacheStore::uriSchemes() { - atomicWrite(binaryCacheDir + "/" + path, data); + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1") + return {}; + else + return {"file"}; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || - std::string(uri, 0, 7) != "file://") - return 0; - auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); - store->init(); - return store; -}); +static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regLocalBinaryCacheStore; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index dd96d2578..2f1d9663a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { - Path storePath = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + auto storePath = store->toStorePath(path).first; + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80ebe903f..6a5f13996 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" +#include "callback.hh" #include <iostream> #include <algorithm> @@ -42,7 +43,8 @@ namespace nix { LocalStore::LocalStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", "physical path to the Nix store"} @@ -108,12 +110,11 @@ LocalStore::LocalStore(const Params & params) } /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + if (!settings.allowSymlinkedStore) { Path path = realStoreDir; struct stat st; while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); + st = lstat(path); if (S_ISLNK(st.st_mode)) throw Error( "the path '%1%' is a symlink; " @@ -417,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct void canonicaliseTimestampAndPermissions(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); - canonicaliseTimestampAndPermissions(path, st); + canonicaliseTimestampAndPermissions(path, lstat(path)); } @@ -441,9 +439,7 @@ static void canonicalisePathMetaData_( } #endif - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) @@ -479,8 +475,7 @@ static void canonicalisePathMetaData_( ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - assert(!S_ISDIR(st.st_mode)); - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); @@ -526,9 +521,7 @@ void canonicalisePathMetaData( /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); @@ -551,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) { - if (actual != expected) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(actual), printStorePath(expected)); auto j = drv.env.find(varName); if (j == drv.env.end() || parseStorePath(j->second) != actual) throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", @@ -563,24 +553,54 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat }; - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + drv.type(); - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); + std::optional<Hash> h; + for (auto & i : drv.outputs) { + std::visit(overloaded { + [&](DerivationOutputInputAddressed doia) { + if (!h) { + // somewhat expensive so we do lazily + auto temp = hashDerivationModulo(*this, drv, true); + h = std::get<Hash>(temp); + } + StorePath recomputed = makeOutputPath(i.first, *h, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](DerivationOutputCAFixed dof) { + StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + envHasRightPath(path, i.first); + }, + [&](DerivationOutputCAFloating _) { + /* Nothing to check */ + }, + }, i.second.output); } +} + + +void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) +{ + auto state(_state.lock()); + return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); +} + +void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +{ + retrySQLite<void>([&]() { + state.stmtAddDerivationOutput.use() + (deriver) + (outputName) + (printStorePath(output)) + .exec(); + }); - else { - Hash h = hashDerivationModulo(*this, drv, true); - for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); - } } @@ -601,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state, (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); + uint64_t id = state.db.getLastInsertedRowId(); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -617,12 +637,11 @@ uint64_t LocalStore::addValidPath(State & state, registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - for (auto & i : drv.outputs) { - state.stmtAddDerivationOutput.use() - (id) - (i.first) - (printStorePath(i.second.path)) - .exec(); + for (auto & i : drv.outputsAndOptPaths(*this)) { + /* Floating CA derivations have indeterminate output paths until + they are built, so don't register anything in that case */ + if (i.second.second) + linkDeriverToPath(state, id, i.first, *i.second.second); } } @@ -640,25 +659,28 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(path); - callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); if (!useQueryPathInfo.next()) return std::shared_ptr<ValidPathInfo>(); - info->id = useQueryPathInfo.getInt(0); + auto id = useQueryPathInfo.getInt(0); + auto narHash = Hash::dummy; try { - info->narHash = Hash(useQueryPathInfo.getStr(1)); + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); } + auto info = std::make_shared<ValidPathInfo>(path, narHash); + + info->id = id; + info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); @@ -706,7 +728,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); if (!use.next()) - throw Error("path '%s' is not valid", printStorePath(path)); + throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); } @@ -781,17 +803,61 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) } -OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) +std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) { - return retrySQLite<OutputPathMap>([&]() { + auto path = path_; + std::map<std::string, std::optional<StorePath>> outputs; + Derivation drv = readDerivation(path); + for (auto & [outName, _] : drv.outputs) { + outputs.insert_or_assign(outName, std::nullopt); + } + bool haveCached = false; + { + auto resolutions = drvPathResolutions.lock(); + auto resolvedPathOptIter = resolutions->find(path); + if (resolvedPathOptIter != resolutions->end()) { + auto & [_, resolvedPathOpt] = *resolvedPathOptIter; + if (resolvedPathOpt) + path = *resolvedPathOpt; + haveCached = true; + } + } + /* can't just use else-if instead of `!haveCached` because we need to unlock + `drvPathResolutions` before it is locked in `Derivation::resolve`. */ + if (!haveCached && drv.type() == DerivationType::CAFloating) { + /* Try resolve drv and use that path instead. */ + auto attempt = drv.tryResolve(*this); + if (!attempt) + /* If we cannot resolve the derivation, we cannot have any path + assigned so we return the map of all std::nullopts. */ + return outputs; + /* Just compute store path */ + auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true); + /* Store in memo table. */ + /* FIXME: memo logic should not be local-store specific, should have + wrapper-method instead. */ + drvPathResolutions.lock()->insert_or_assign(path, pathResolved); + path = std::move(pathResolved); + } + return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() { auto state(_state.lock()); - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); + uint64_t drvId; + try { + drvId = queryValidPathId(*state, path); + } catch (InvalidPath &) { + /* FIXME? if the derivation doesn't exist, we cannot have a mapping + for it. */ + return outputs; + } + + auto useQueryDerivationOutputs { + state->stmtQueryDerivationOutputs.use() + (drvId) + }; - OutputPathMap outputs; while (useQueryDerivationOutputs.next()) - outputs.emplace( + outputs.insert_or_assign( useQueryDerivationOutputs.getStr(0), parseStorePath(useQueryDerivationOutputs.getStr(1)) ); @@ -853,20 +919,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; for (auto & path : paths) { - if (infos.count(path)) continue; - debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); + auto subPath(path.first); + + // recompute store path so that we can use a different store root + if (path.second) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); try { - auto info = sub->queryPathInfo(path); + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ + infos.insert_or_assign(path.first, SubstitutablePathInfo{ info->deriver, info->references, narInfo ? narInfo->fileSize : 0, @@ -969,11 +1047,8 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { - if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); @@ -983,7 +1058,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); + auto realPath = Store::toRealPath(info.path); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a @@ -1008,11 +1083,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, else hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); - LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { - size_t n = source.read(data, len); - (*hashSink)(data, n); - return n; - }); + TeeSource wrapperSource { source, *hashSink }; restorePath(realPath, wrapperSource); @@ -1040,12 +1111,64 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(hashAlgo, dump); + /* For computing the store path. */ + auto hashSink = std::make_unique<HashSink>(hashAlgo); + TeeSource source { source0, *hashSink }; + + /* Read the source path into memory, but only if it's up to + narBufferSize bytes. If it's larger, write it to a temporary + location in the Nix store. If the subsequently computed + destination store path is already valid, we just delete the + temporary path. Otherwise, we move it to the destination store + path. */ + bool inMemory = false; + + std::string dump; + + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 65536; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } + + std::unique_ptr<AutoDelete> delTempDir; + Path tempPath; + + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; + + auto tempDir = createTempDir(realStoreDir, "add"); + delTempDir = std::make_unique<AutoDelete>(tempDir); + tempPath = tempDir + "/x"; + + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); + + dump.clear(); + } - auto dstPath = makeFixedOutputPath(method, h, name); + auto [hash, size] = hashSink->finish(); + + auto dstPath = makeFixedOutputPath(method, hash, name); addTempRoot(dstPath); @@ -1054,8 +1177,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1065,31 +1187,35 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); + if (inMemory) { + StringSource dumpSource { dump }; + /* Restore from the NAR in memory. */ + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); + } else { + /* Move the temporary path we restored above. */ + if (rename(tempPath.c_str(), realPath.c_str())) + throw Error("renaming '%s' to '%s'", tempPath, realPath); + } - canonicalisePathMetaData(realPath, {}); + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath - optimisePath(realPath); // FIXME: combine with hashPath() + optimisePath(realPath); - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; + ValidPathInfo info { dstPath, narHash.first }; + info.narSize = narHash.second; + info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1100,24 +1226,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } -StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - Path srcPath(absPath(_srcPath)); - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref<std::string>(readFile(srcPath)); - - return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); -} - - StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { @@ -1128,8 +1236,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1149,8 +1256,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info(dstPath); - info.narHash = narHash; + ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s->size(); info.references = references; info.ca = TextHash { .hash = hash }; @@ -1265,9 +1371,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr<AbstractHashSink> hashSink; if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique<HashSink>(*info->narHash.type); + hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1387,7 +1493,7 @@ static void makeMutable(const Path & path) { checkInterrupt(); - struct stat st = lstat(path); + auto st = lstat(path); if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 79b415875..118c7149b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -23,18 +23,26 @@ namespace nix { const int nixSchemaVersion = 10; -struct Derivation; - - struct OptimiseStats { unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; + uint64_t bytesFreed = 0; + uint64_t blocksFreed = 0; +}; + +struct LocalStoreConfig : virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + + Setting<bool> requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + + const std::string name() override { return "Local Store"; } }; -class LocalStore : public LocalFSStore +class LocalStore : public LocalFSStore, public virtual LocalStoreConfig { private: @@ -98,10 +106,6 @@ public: private: - Setting<bool> requireSigs{(Store*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const PublicKeys & getPublicKeys(); public: @@ -133,29 +137,20 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - OutputPathMap queryDerivationOutputMap(const StorePath & path) override; + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, + void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -291,6 +286,11 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); + /* Register the store path 'output' as the output named 'outputName' of + derivation 'deriver'. */ + void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output); + void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index aec4ed493..d266c8efe 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -61,3 +61,6 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libstore/builtins/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index f848582da..7db2556f4 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,7 @@ #include "machines.hh" #include "util.hh" #include "globals.hh" +#include "store-api.hh" #include <algorithm> @@ -48,6 +49,29 @@ bool Machine::mandatoryMet(const std::set<string> & features) const { }); } +ref<Store> Machine::openStore() const { + Store::Params storeParams; + if (hasPrefix(storeUri, "ssh://")) { + storeParams["max-connections"] = "1"; + storeParams["log-fd"] = "4"; + if (sshKey != "") + storeParams["ssh-key"] = sshKey; + } + { + auto & fs = storeParams["system-features"]; + auto append = [&](auto feats) { + for (auto & f : feats) { + if (fs.size() > 0) fs += ' '; + fs += f; + } + }; + append(supportedFeatures); + append(mandatoryFeatures); + } + + return nix::openStore(storeUri, storeParams); +} + void parseMachines(const std::string & s, Machines & machines) { for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index de92eb924..341d9bd97 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -4,6 +4,8 @@ namespace nix { +class Store; + struct Machine { const string storeUri; @@ -28,6 +30,8 @@ struct Machine { decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + + ref<Store> openStore() const; }; typedef std::vector<Machine> Machines; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,7 +4,8 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" - +#include "topo-sort.hh" +#include "callback.hh" namespace nix { @@ -108,9 +109,19 @@ void Store::computeFSClosure(const StorePath & startPath, } +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) +{ + auto out = drv.outputs.find("out"); + if (out != drv.outputs.end()) { + if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output)) + return v->hash; + } + return std::nullopt; +} + void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) + uint64_t & downloadSize_, uint64_t & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -122,8 +133,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, { std::unordered_set<std::string> done; StorePathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; + uint64_t & downloadSize; + uint64_t & narSize; }; struct DrvState @@ -157,7 +168,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); + querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -192,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, return; } + PathSet invalid; + /* true for regular derivations, and CA derivations for which we + have a trust mapping for all wanted outputs. */ + auto knownOutputPaths = true; + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { + if (!pathOpt) { + knownOutputPaths = false; + break; + } + if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) + invalid.insert(printStorePath(*pathOpt)); + } + if (knownOutputPaths && invalid.empty()) return; + auto drv = make_ref<Derivation>(derivationFromPath(path.path)); ParsedDerivation parsedDrv(StorePath(path.path), *drv); - PathSet invalid; - for (auto & j : drv->outputs) - if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path)) - invalid.insert(printStorePath(j.second.path)); - if (invalid.empty()) return; - - if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); @@ -214,7 +232,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path.path}, infos); + querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); @@ -246,41 +264,21 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePaths Store::topoSortPaths(const StorePathSet & paths) { - StorePaths sorted; - StorePathSet visited, parents; - - std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit; - - dfsVisit = [&](const StorePath & path, const StorePath * parent) { - if (parents.count(path)) - throw BuildError("cycle detected in the references of '%s' from '%s'", - printStorePath(path), printStorePath(*parent)); - - if (!visited.insert(path).second) return; - parents.insert(path); - - StorePathSet references; - try { - references = queryPathInfo(path)->references; - } catch (InvalidPath &) { - } - - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) - dfsVisit(i, &path); - - sorted.push_back(path); - parents.erase(path); - }; - - for (auto & i : paths) - dfsVisit(i, nullptr); - - std::reverse(sorted.begin(), sorted.end()); - - return sorted; + return topoSort(paths, + {[&](const StorePath & path) { + StorePathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath &) { + } + return references; + }}, + {[&](const StorePath & path, const StorePath & parent) { + return BuildError( + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); + }}); } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index d1c8a6101..41e28dc99 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,10 +1,18 @@ #include "names.hh" #include "util.hh" +#include <regex> + namespace nix { +struct Regex +{ + std::regex regex; +}; + + DrvName::DrvName() { name = ""; @@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0) } +DrvName::~DrvName() +{ } + + bool DrvName::matches(DrvName & n) { if (name != "*") { - if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); - if (!std::regex_match(n.name, *regex)) return false; + if (!regex) { + regex = std::make_unique<Regex>(); + regex->regex = std::regex(name, std::regex::extended); + } + if (!std::regex_match(n.name, regex->regex)) return false; } if (version != "" && version != n.version) return false; return true; @@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs) { DrvNames result; for (auto & i : opArgs) - result.push_back(DrvName(i)); + result.emplace_back(i); return result; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 00e14b8c7..bc62aac93 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -3,10 +3,11 @@ #include <memory> #include "types.hh" -#include <regex> namespace nix { +struct Regex; + struct DrvName { string fullName; @@ -16,10 +17,12 @@ struct DrvName DrvName(); DrvName(std::string_view s); + ~DrvName(); + bool matches(DrvName & n); private: - std::unique_ptr<std::regex> regex; + std::unique_ptr<Regex> regex; }; typedef list<DrvName> DrvNames; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ca663d837..a9efdd0b6 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -18,7 +18,7 @@ struct NarMember /* If this is a regular file, position of the contents of this file in the NAR. */ - size_t start = 0, size = 0; + uint64_t start = 0, size = 0; std::string target; @@ -34,20 +34,23 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, StringSource + struct NarIndexer : ParseSink, Source { NarAccessor & acc; + Source & source; std::stack<NarMember *> parents; - std::string currentStart; bool isExec = false; - NarIndexer(NarAccessor & acc, const std::string & nar) - : StringSource(nar), acc(acc) + uint64_t pos = 0; + + NarIndexer(NarAccessor & acc, Source & source) + : acc(acc), source(source) { } - void createMember(const Path & path, NarMember member) { + void createMember(const Path & path, NarMember member) + { size_t level = std::count(path.begin(), path.end(), '/'); while (parents.size() > level) parents.pop(); @@ -77,33 +80,40 @@ struct NarAccessor : public FSAccessor parents.top()->isExecutable = true; } - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits<size_t>::max()); - parents.top()->size = (size_t)size; + assert(size <= std::numeric_limits<uint64_t>::max()); + parents.top()->size = (uint64_t) size; parents.top()->start = pos; } - void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); - } - } + void receiveContents(unsigned char * data, size_t len) override + { } void createSymlink(const Path & path, const string & target) override { createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } + + size_t read(unsigned char * data, size_t len) override + { + auto n = source.read(data, len); + pos += n; + return n; + } }; NarAccessor(ref<const std::string> nar) : nar(nar) { - NarIndexer indexer(*this, *nar); + StringSource source(*nar); + NarIndexer indexer(*this, source); + parseDump(indexer, indexer); + } + + NarAccessor(Source & source) + { + NarIndexer indexer(*this, source); parseDump(indexer, indexer); } @@ -219,6 +229,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) return make_ref<NarAccessor>(nar); } +ref<FSAccessor> makeNarAccessor(Source & source) +{ + return make_ref<NarAccessor>(source); +} + ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 2871199de..8af1272f6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -6,10 +6,14 @@ namespace nix { +struct Source; + /* Return an object that provides access to the contents of a NAR file. */ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); +ref<FSAccessor> makeNarAccessor(Source & source); + /* Create a NAR accessor from a NAR listing (in the format produced by listNar()). The callback getNarBytes(offset, length) is used by the readFile() method of the accessor to get the contents of files diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..8541cc51f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -189,13 +189,14 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart)); + auto narInfo = make_ref<NarInfo>( + StorePath(hashPart + "-" + namePart), + Hash::parseAnyPrefixed(queryNAR.getStr(6))); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) - narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4)); narInfo->fileSize = queryNAR.getInt(5); - narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) narInfo->references.insert(StorePath(r)); @@ -230,7 +231,7 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) (info->narHash.to_string(Base32, true)) (info->narSize) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 04550ed97..3454f34bb 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,36 +1,37 @@ #include "globals.hh" #include "nar-info.hh" +#include "store-api.hh" namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { - return Hash(s); + return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; bool havePath = false; + bool haveNarHash = false; size_t pos = 0; while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } - else if (name == "NarHash") + else if (name == "NarHash") { narHash = parseHashField(value); + haveNarHash = true; + } else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString<Strings>(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } @@ -77,7 +80,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -87,8 +90,8 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..39ced76e5 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -2,20 +2,22 @@ #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "path-info.hh" namespace nix { +class Store; + struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional<Hash> fileHash; uint64_t fileSize = 0; std::string system; NarInfo() = delete; - NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } + NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b2b2412a3..a0d482ddf 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -17,9 +17,7 @@ namespace nix { static void makeWritable(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError("changing writability of '%1%'", path); } @@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Yes! We've seen a file with the same contents. Replace the current file with a hard link to that file. */ - struct stat stLink; - if (lstat(linkPath.c_str(), &stLink)) - throw SysError("getting attributes of path '%1%'", linkPath); + auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); @@ -282,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - void LocalStore::optimiseStore() { OptimiseStats stats; optimiseStore(stats); - printInfo( - format("%1% freed by hard-linking %2% files") - % showBytes(stats.bytesFreed) - % stats.filesLinked); + printInfo("%s freed by hard-linking %d files", + showBytes(stats.bytesFreed), + stats.filesLinked); } void LocalStore::optimisePath(const Path & path) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c7797b730..e7b7202d4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -94,7 +94,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const return res; } -bool ParsedDerivation::canBuildLocally() const +bool ParsedDerivation::canBuildLocally(Store & localStore) const { if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform) @@ -102,14 +102,14 @@ bool ParsedDerivation::canBuildLocally() const return false; for (auto & feature : getRequiredSystemFeatures()) - if (!settings.systemFeatures.get().count(feature)) return false; + if (!localStore.systemFeatures.get().count(feature)) return false; return true; } -bool ParsedDerivation::willBuildLocally() const +bool ParsedDerivation::willBuildLocally(Store & localStore) const { - return getBoolAttr("preferLocalBuild") && canBuildLocally(); + return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore); } bool ParsedDerivation::substitutesAllowed() const @@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } -bool ParsedDerivation::contentAddressed() const -{ - return getBoolAttr("__contentAddressed", false); -} - } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0b8e8d031..c9fbe68c4 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,3 +1,5 @@ +#pragma once + #include "store-api.hh" #include <nlohmann/json_fwd.hpp> @@ -29,13 +31,11 @@ public: StringSet getRequiredSystemFeatures() const; - bool canBuildLocally() const; + bool canBuildLocally(Store & localStore) const; - bool willBuildLocally() const; + bool willBuildLocally(Store & localStore) const; bool substitutesAllowed() const; - - bool contentAddressed() const; }; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh new file mode 100644 index 000000000..8ff5c466e --- /dev/null +++ b/src/libstore/path-info.hh @@ -0,0 +1,112 @@ +#pragma once + +#include "crypto.hh" +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" + +#include <string> +#include <optional> + +namespace nix { + + +class Store; + + +struct SubstitutablePathInfo +{ + std::optional<StorePath> deriver; + StorePathSet references; + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ +}; + +typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; + + +struct ValidPathInfo +{ + StorePath path; + std::optional<StorePath> deriver; + // TODO document this + Hash narHash; + StorePathSet references; + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + /* If non-empty, an assertion that the path is content-addressed, + i.e., that the store path is computed from a cryptographic hash + of the contents of the path, plus some other bits of data like + the "name" part of the path. Such a path doesn't need + signatures, since we don't have to trust anybody's claim that + the path is the output of a particular derivation. (In the + extensional store model, we have to trust that the *contents* + of an output path of a derivation were actually produced by + that derivation. In the intensional model, we have to trust + that a particular output path was produced by a derivation; the + path then implies the contents.) + + Ideally, the content-addressability assertion would just be a Boolean, + and the store path would be computed from the name component, ‘narHash’ + and ‘references’. However, we support many types of content addresses. + */ + std::optional<ContentAddress> ca; + + bool operator == (const ValidPathInfo & i) const + { + return + path == i.path + && narHash == i.narHash + && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint(const Store & store) const; + + void sign(const Store & store, const SecretKey & secretKey); + + /* Return true iff the path is verifiably content-addressed. */ + bool isContentAddressed(const Store & store) const; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + + static const size_t maxSigs = std::numeric_limits<size_t>::max(); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys, or maxSigs if the path + is content-addressed. */ + size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + + Strings shortRefs() const; + + ValidPathInfo(const ValidPathInfo & other) = default; + + ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; + ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + + virtual ~ValidPathInfo() { } +}; + +typedef list<ValidPathInfo> ValidPathInfos; + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index b3d8ce95c..dc9dc3897 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,8 +2,6 @@ namespace nix { -MakeError(BadStorePath, Error); - static void checkName(std::string_view path, std::string_view name) { if (name.empty()) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index e43a8b50c..b03a0f69d 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -64,6 +64,8 @@ typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; typedef std::map<string, StorePath> OutputPathMap; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 6cfe393a4..c3809bad7 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -12,30 +12,24 @@ namespace nix { -static bool cmpGensByNumber(const Generation & a, const Generation & b) -{ - return a.number < b.number; -} - - /* Parse a generation name of the format `<profilename>-<number>-link'. */ -static int parseName(const string & profileName, const string & name) +static std::optional<GenerationNumber> parseName(const string & profileName, const string & name) { - if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + if (string(name, 0, profileName.size() + 1) != profileName + "-") return {}; string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); - if (p == string::npos) return -1; - int n; + if (p == string::npos) return {}; + unsigned int n; if (string2Int(string(s, 0, p), n) && n >= 0) return n; else - return -1; + return {}; } -Generations findGenerations(Path profile, int & curGen) +std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile) { Generations gens; @@ -43,30 +37,31 @@ Generations findGenerations(Path profile, int & curGen) auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { - int n; - if ((n = parseName(profileName, i.name)) != -1) { - Generation gen; - gen.path = profileDir + "/" + i.name; - gen.number = n; - struct stat st; - if (lstat(gen.path.c_str(), &st) != 0) - throw SysError("statting '%1%'", gen.path); - gen.creationTime = st.st_mtime; - gens.push_back(gen); + if (auto n = parseName(profileName, i.name)) { + auto path = profileDir + "/" + i.name; + gens.push_back({ + .number = *n, + .path = path, + .creationTime = lstat(path).st_mtime + }); } } - gens.sort(cmpGensByNumber); + gens.sort([](const Generation & a, const Generation & b) + { + return a.number < b.number; + }); - curGen = pathExists(profile) + return { + gens, + pathExists(profile) ? parseName(profileName, readLink(profile)) - : -1; - - return gens; + : std::nullopt + }; } -static void makeName(const Path & profile, unsigned int num, +static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { Path prefix = (format("%1%-%2%") % profile % num).str(); @@ -74,18 +69,17 @@ static void makeName(const Path & profile, unsigned int num, } -Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) +Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath) { /* The new generation number should be higher than old the previous ones. */ - int dummy; - Generations gens = findGenerations(profile, dummy); + auto [gens, dummy] = findGenerations(profile); - unsigned int num; + GenerationNumber num; if (gens.size() > 0) { Generation last = gens.back(); - if (readLink(last.path) == outPath) { + if (readLink(last.path) == store->printStorePath(outPath)) { /* We only create a new generation symlink if it differs from the last one. @@ -108,7 +102,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - store->addPermRoot(store->parseStorePath(outPath), generation, false, true); + store->addPermRoot(outPath, generation); return generation; } @@ -121,7 +115,7 @@ static void removeFile(const Path & path) } -void deleteGeneration(const Path & profile, unsigned int gen) +void deleteGeneration(const Path & profile, GenerationNumber gen) { Path generation; makeName(profile, gen, generation); @@ -129,7 +123,7 @@ void deleteGeneration(const Path & profile, unsigned int gen) } -static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) +static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) printInfo(format("would remove generation %1%") % gen); @@ -140,31 +134,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu } -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun) +void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); - if (gensToDelete.find(curGen) != gensToDelete.end()) + if (gensToDelete.count(*curGen)) throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { - if (gensToDelete.find(i.number) == gensToDelete.end()) continue; + if (!gensToDelete.count(i.number)) continue; deleteGeneration2(profile, i.number, dryRun); } } -void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; bool fromCurGen = false; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto i = gens.rbegin(); i != gens.rend(); ++i) { if (i->number == curGen) { fromCurGen = true; @@ -186,8 +178,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto & i : gens) if (i.number != curGen) @@ -200,8 +191,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); bool canDelete = false; for (auto i = gens.rbegin(); i != gens.rend(); ++i) diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 78645d8b6..be55a65d4 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -8,38 +8,35 @@ namespace nix { +class StorePath; + + +typedef unsigned int GenerationNumber; struct Generation { - int number; + GenerationNumber number; Path path; time_t creationTime; - Generation() - { - number = -1; - } - operator bool() const - { - return number != -1; - } }; -typedef list<Generation> Generations; +typedef std::list<Generation> Generations; /* Returns the list of currently present generations for the specified - profile, sorted by generation number. */ -Generations findGenerations(Path profile, int & curGen); + profile, sorted by generation number. Also returns the number of + the current generation. */ +std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); class LocalFSStore; -Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath); +Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath); -void deleteGeneration(const Path & profile, unsigned int gen); +void deleteGeneration(const Path & profile, GenerationNumber gen); -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); +void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun); -void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); void deleteOldGenerations(const Path & profile, bool dryRun); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..d2096cb49 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len, struct RefScanSink : Sink { - HashSink hashSink; StringSet hashes; StringSet seen; string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() { } void operator () (const unsigned char * data, size_t len); }; @@ -62,8 +61,6 @@ struct RefScanSink : Sink void RefScanSink::operator () (const unsigned char * data, size_t len) { - hashSink(data, len); - /* It's possible that a reference spans the previous and current fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ @@ -79,10 +76,20 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair<PathSet, HashResult> scanForReferences(const string & path, + const PathSet & refs) { - RefScanSink sink; + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair<PathSet, HashResult>(found, hash); +} + +PathSet scanForReferences(Sink & toTee, + const string & path, const PathSet & refs) +{ + RefScanSink refsSink; + TeeSink sink { refsSink, toTee }; std::map<string, Path> backMap; /* For efficiency (and a higher hit rate), just search for the @@ -97,7 +104,7 @@ PathSet scanForReferences(const string & path, assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); - sink.hashes.insert(s); + refsSink.hashes.insert(s); backMap[s] = i; } @@ -106,14 +113,12 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - for (auto & i : sink.seen) { + for (auto & i : refsSink.seen) { std::map<string, Path>::iterator j; if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } - hash = sink.hashSink.finish(); - return found; } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..c2efd095c 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,9 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs); + +PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index bd698d781..2d02a181b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) createDirs(cacheDir); } -Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) +Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); + return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, +void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar, ref<FSAccessor> narAccessor) { - nars.emplace(storePath, narAccessor); + nars.emplace(hashPart, narAccessor); if (cacheDir != "") { try { std::ostringstream str; JSONPlaceholder jsonRoot(str); listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(storePath, "ls"), str.str()); + writeFile(makeCacheFile(hashPart, "ls"), str.str()); /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(storePath, "nar"), nar); + writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { ignoreException(); @@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) { auto path = canonPath(path_); - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); + auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - auto i = nars.find(storePath); + auto i = nars.find(std::string(storePath.hashPart())); if (i != nars.end()) return {i->second, restPath}; StringSink sink; std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { - listing = nix::readFile(makeCacheFile(storePath, "ls")); + listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { @@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) return buf; }); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } @@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) *sink.s = nix::readFile(cacheFile); auto narAccessor = makeNarAccessor(sink.s); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } } - store->narFromPath(store->parseStorePath(storePath), sink); + store->narFromPath(storePath, sink); auto narAccessor = makeNarAccessor(sink.s); - addToCache(storePath, *sink.s, narAccessor); + addToCache(storePath.hashPart(), *sink.s, narAccessor); return {narAccessor, restPath}; } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 4afb3be95..347cf5764 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor { ref<Store> store; - std::map<Path, ref<FSAccessor>> nars; + std::map<std::string, ref<FSAccessor>> nars; Path cacheDir; @@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor friend class BinaryCacheStore; - Path makeCacheFile(const Path & storePath, const std::string & ext); + Path makeCacheFile(std::string_view hashPart, const std::string & ext); - void addToCache(const Path & storePath, const std::string & nar, + void addToCache(std::string_view hashPart, const std::string & nar, ref<FSAccessor> narAccessor); public: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 118aadf7e..23b1942ce 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" @@ -9,6 +10,7 @@ #include "pool.hh" #include "finally.hh" #include "logging.hh" +#include "callback.hh" #include <sys/types.h> #include <sys/stat.h> @@ -22,53 +24,82 @@ namespace nix { +namespace worker_proto { -template<> StorePathSet readStorePaths(const Store & store, Source & from) +std::string read(const Store & store, Source & from, Phantom<std::string> _) { - StorePathSet paths; - for (auto & i : readStrings<Strings>(from)) - paths.insert(store.parseStorePath(i)); - return paths; + return readString(from); } +void write(const Store & store, Sink & out, const std::string & str) +{ + out << str; +} -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) + +StorePath read(const Store & store, Source & from, Phantom<StorePath> _) { - out << paths.size(); - for (auto & i : paths) - out << store.printStorePath(i); + return store.parseStorePath(readString(from)); } -std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from) +void write(const Store & store, Sink & out, const StorePath & storePath) { - std::map<string, StorePath> pathMap; - auto rawInput = readStrings<Strings>(from); - if (rawInput.size() % 2) - throw Error("got an odd number of elements from the daemon when trying to read a output path map"); - auto curInput = rawInput.begin(); - while (curInput != rawInput.end()) { - auto thisKey = *curInput++; - auto thisValue = *curInput++; - pathMap.emplace(thisKey, store.parseStorePath(thisValue)); - } - return pathMap; + out << store.printStorePath(storePath); } -void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap) + +ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) { - out << 2*pathMap.size(); - for (auto & i : pathMap) { - out << i.first; - out << store.printStorePath(i.second); - } + return parseContentAddress(readString(from)); } +void write(const Store & store, Sink & out, const ContentAddress & ca) +{ + out << renderContentAddress(ca); +} + + +std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) +{ + auto s = readString(from); + return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s); +} + +void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt) +{ + out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) +{ + return parseContentAddressOpt(readString(from)); +} + +void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) +{ + out << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} + + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : Store(params) + , RemoteStoreConfig(params) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), - [this]() { return openConnectionWrapper(); }, + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, [this](const ref<Connection> & r) { return r->to.good() @@ -95,19 +126,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , RemoteStore(params) { } -UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) - , path(socket_path) +UDSRemoteStore::UDSRemoteStore( + const std::string scheme, + std::string socket_path, + const Params & params) + : UDSRemoteStore(params) { + path.emplace(socket_path); } @@ -151,8 +184,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() conn->startTime = std::chrono::steady_clock::now(); - initConnection(*conn); - return conn; } @@ -260,14 +291,16 @@ struct ConnectionHandle RemoteStore::Connection * operator -> () { return &*handle; } - void processStderr(Sink * sink = 0, Source * source = 0) + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) { - auto ex = handle->processStderr(sink, source); + auto ex = handle->processStderr(sink, source, flush); if (ex) { daemonException = true; std::rethrow_exception(ex); } } + + void withFramedSink(std::function<void(Sink & sink)> fun); }; @@ -296,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } } @@ -308,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } @@ -325,41 +358,46 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) { - if (paths.empty()) return; + if (pathsMap.empty()) return; auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - for (auto & i : paths) { + for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths<StorePathSet>(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i, std::move(info)); + infos.insert_or_assign(i.first, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; - writeStorePaths(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + worker_proto::write(*this, conn->to, paths); + } else + worker_proto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { @@ -367,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths<StorePathSet>(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -376,11 +414,28 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, } +ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref<ValidPathInfo>(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings<StringSet>(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - std::shared_ptr<ValidPathInfo> info; + std::shared_ptr<const ValidPathInfo> info; { auto conn(getConnection()); conn->to << wopQueryPathInfo << printStorePath(path); @@ -396,17 +451,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(StorePath(path)); - auto deriver = readString(conn->from); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<StorePathSet>(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings<StringSet>(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } + info = readValidPathInfo(conn, path); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -419,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})) referrers.insert(i); } @@ -429,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } @@ -441,17 +486,32 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } -OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path) +std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) { - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return readOutputPathMap(*this, conn->from); - + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {}); + } else { + // Fallback for old daemon versions. + // For floating-CA derivations (and their co-dependencies) this is an + // under-approximation as it only returns the paths that can be inferred + // from the derivation itself (and not the ones that are known because + // the have been built), but as old stores don't handle floating-CA + // derivations this shouldn't matter + auto derivation = readDerivation(path); + auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); + std::map<std::string, std::optional<StorePath>> ret; + for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { + ret.emplace(outputName, outputAndPath.second); + } + return ret; + } } std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) @@ -465,8 +525,95 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & } +ref<const ValidPathInfo> RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) +{ + std::optional<ConnectionHandle> conn_(getConnection()); + auto & conn = *conn_; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(caMethod); + worker_proto::write(*this, conn->to, references); + conn->to << repair; + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + worker_proto::write(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + } + }, caMethod); + auto path = parseStorePath(readString(conn->from)); + // Release our connection to prevent a deadlock in queryPathInfo(). + conn_.reset(); + return queryPathInfo(path); + } +} + + +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; +} + + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { auto conn(getConnection()); @@ -480,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -490,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from); + auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); assert(importedPaths.size() <= 1); } @@ -499,68 +646,30 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); - } -} - -StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - - Path srcPath(absPath(_srcPath)); - - conn->to - << wopAddToStore - << name - << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashAlgo); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + conn.withFramedSink([&](Sink & sink) { + copyNAR(source, sink); + }); + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; } - - return parseStorePath(readString(conn->from)); } StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); - - conn.processStderr(); - return parseStorePath(readString(conn->from)); + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } @@ -659,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - writeStorePaths(*this, conn->to, options.pathsToDelete); + worker_proto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -707,7 +816,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); @@ -721,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = readStorePaths<StorePathSet>(*this, conn->from); - willSubstitute = readStorePaths<StorePathSet>(*this, conn->from); - unknown = readStorePaths<StorePathSet>(*this, conn->from); + willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); conn->from >> downloadSize >> narSize; return; } @@ -762,6 +871,18 @@ RemoteStore::Connection::~Connection() } } +void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << printStorePath(path); + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref<FSAccessor> RemoteStore::getFSAccessor() +{ + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); +} static Logger::Fields readFields(Source & from) { @@ -780,9 +901,10 @@ static Logger::Fields readFields(Source & from) } -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) { - to.flush(); + if (flush) + to.flush(); while (true) { @@ -803,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } } else if (msg == STDERR_NEXT) @@ -843,14 +969,49 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> +void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) { - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params); -}); + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink((*this)->to, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + +static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fb2052752..ec04be985 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -16,18 +16,22 @@ struct FdSource; template<typename T> class Pool; struct ConnectionHandle; - -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ -class RemoteStore : public virtual Store +struct RemoteStoreConfig : virtual StoreConfig { -public: + using StoreConfig::StoreConfig; - const Setting<int> maxConnections{(Store*) this, 1, + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(), + const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(), "max-connection-age", "number of seconds to reuse a connection"}; +}; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +{ +public: virtual bool sameMachine() = 0; @@ -51,21 +55,28 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - OutputPathMap queryDerivationOutputMap(const StorePath & path) override; + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, + void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + /* Add a content-addressable store path. `dump` will be drained. */ + ref<const ValidPathInfo> addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair); - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; + /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + + void addToStore(const ValidPathInfo & info, Source & nar, + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -95,7 +106,7 @@ public: void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + uint64_t & downloadSize, uint64_t & narSize) override; void connect() override; @@ -103,8 +114,6 @@ public: void flushBadConnections(); -protected: - struct Connection { AutoCloseFD fd; @@ -115,11 +124,13 @@ protected: virtual ~Connection(); - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; ref<Connection> openConnectionWrapper(); +protected: + virtual ref<Connection> openConnection() = 0; void initConnection(Connection & conn); @@ -132,24 +143,56 @@ protected: friend struct ConnectionHandle; + virtual ref<FSAccessor> getFSAccessor() override; + + virtual void narFromPath(const StorePath & path, Sink & sink) override; + + ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path); + private: std::atomic_bool failed{false}; }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore +struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +{ + UDSRemoteStoreConfig(const Store::Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + { + } + + UDSRemoteStoreConfig() + : UDSRemoteStoreConfig(Store::Params({})) + { + } + + const std::string name() override { return "Local Daemon Store"; } +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig { public: UDSRemoteStore(const Params & params); - UDSRemoteStore(std::string path, const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, const Params & params); std::string getUri() override; + static std::set<std::string> uriSchemes() + { return {"unix"}; } + bool sameMachine() override { return true; } + ref<FSAccessor> getFSAccessor() override + { return LocalFSStore::getFSAccessor(); } + + void narFromPath(const StorePath & path, Sink & sink) override + { LocalFSStore::narFromPath(path, sink); } + private: ref<RemoteStore::Connection> openConnection() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 427dd48ce..552c4aac7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,7 +7,6 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" -#include "istringstream_nocopy.hh" #include <aws/core/Aws.h> #include <aws/core/VersionConfig.h> @@ -173,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; - const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; + const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; + const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; const Setting<bool> multipartUpload{ - this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting<uint64_t> bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + const std::string name() override { return "S3 Binary Cache Store"; } +}; + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +{ std::string bucketName; Stats stats; @@ -194,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore S3Helper s3Helper; S3BinaryCacheStoreImpl( - const Params & params, const std::string & bucketName) - : S3BinaryCacheStore(params) + const std::string & scheme, + const std::string & bucketName, + const Params & params) + : StoreConfig(params) + , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { @@ -262,11 +270,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr<TransferManager> transferManager; std::once_flag transferManagerCreated; - void uploadFile(const std::string & path, const std::string & data, + void uploadFile(const std::string & path, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType, const std::string & contentEncoding) { - auto stream = std::make_shared<istringstream_nocopy>(data); + istream->seekg(0, istream->end); + auto size = istream->tellg(); + istream->seekg(0, istream->beg); auto maxThreads = std::thread::hardware_concurrency(); @@ -307,7 +318,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr<TransferHandle> transferHandle = transferManager->UploadFile( - stream, bucketName, path, mimeType, + istream, bucketName, path, mimeType, Aws::Map<Aws::String, Aws::String>(), nullptr /*, contentEncoding */); @@ -333,9 +344,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore if (contentEncoding != "") request.SetContentEncoding(contentEncoding); - auto stream = std::make_shared<istringstream_nocopy>(data); - - request.SetBody(stream); + request.SetBody(istream); auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request)); @@ -347,25 +356,32 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) .count(); - printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % - bucketName % path % data.size() % duration); + printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, size, duration); stats.putTimeMs += duration; - stats.putBytes += data.size(); + stats.putBytes += std::max(size, (decltype(size)) 0); stats.put++; } - void upsertFile(const std::string & path, const std::string & data, + void upsertFile(const std::string & path, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) override { + auto compress = [&](std::string compression) + { + auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain()); + return std::make_shared<std::stringstream>(std::move(*compressed)); + }; + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + uploadFile(path, compress(lsCompression), mimeType, lsCompression); else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + uploadFile(path, compress(logCompression), mimeType, logCompression); else - uploadFile(path, data, mimeType, ""); + uploadFile(path, istream, mimeType, ""); } void getFile(const std::string & path, Sink & sink) override @@ -410,7 +426,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); } marker = res.GetNextMarker(); @@ -419,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return paths; } + static std::set<std::string> uriSchemes() { return {"s3"}; } + }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, 5) != "s3://") return 0; - auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); - store->init(); - return store; -}); +static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore; } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 76c822c4e..31a1f0cac 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt) }); } +uint64_t SQLite::getLastInsertedRowId() +{ + return sqlite3_last_insert_rowid(db); +} + void SQLiteStmt::create(sqlite3 * db, const string & sql) { checkInterrupt(); @@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use() sqlite3_reset(stmt); } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throwSQLiteError(stmt.db, "binding argument"); } else bind(); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index dd81ab051..99f0d56ce 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -26,6 +26,8 @@ struct SQLite void isCache(); void exec(const std::string & stmt); + + uint64_t getLastInsertedRowId(); }; /* RAII wrapper to create and destroy SQLite prepared statements. */ @@ -54,7 +56,7 @@ struct SQLiteStmt ~Use(); /* Bind the next parameter. */ - Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596..08d0bd565 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -8,19 +8,25 @@ namespace nix { -static std::string uriScheme = "ssh-ng://"; +struct SSHStoreConfig : virtual RemoteStoreConfig +{ + using RemoteStoreConfig::RemoteStoreConfig; + + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "SSH Store"; } +}; -class SSHStore : public RemoteStore +class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig { public: - const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"}; - - SSHStore(const std::string & host, const Params & params) - : Store(params) + SSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , Store(params) , RemoteStore(params) , host(host) , master( @@ -32,18 +38,16 @@ public: { } + static std::set<std::string> uriSchemes() { return {"ssh-ng"}; } + std::string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } bool sameMachine() override { return false; } - void narFromPath(const StorePath & path, Sink & sink) override; - - ref<FSAccessor> getFSAccessor() override; - private: struct Connection : RemoteStore::Connection @@ -68,19 +72,6 @@ private: }; }; -void SSHStore::narFromPath(const StorePath & path, Sink & sink) -{ - auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); - conn->processStderr(); - copyNAR(conn->from, sink); -} - -ref<FSAccessor> SSHStore::getFSAccessor() -{ - return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); -} - ref<RemoteStore::Connection> SSHStore::openConnection() { auto conn = make_ref<Connection>(); @@ -89,16 +80,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection() + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - initConnection(*conn); return conn; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0a8bc9f6..1bbc74db8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,15 +1,14 @@ #include "crypto.hh" +#include "fs-accessor.hh" #include "globals.hh" #include "store-api.hh" #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "json.hh" -#include "derivations.hh" #include "url.hh" - -#include <future> - +#include "archive.hh" +#include "callback.hh" namespace nix { @@ -20,15 +19,15 @@ bool Store::isInStore(const Path & path) const } -Path Store::toStorePath(const Path & path) const +std::pair<StorePath, Path> Store::toStorePath(const Path & path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) - return path; + return {parseStorePath(path), ""}; else - return Path(path, 0, slash); + return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; } @@ -41,14 +40,14 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw NotInStore("path '%1%' is not in the Nix store", path); + throw BadStorePath("path '%1%' is not in the Nix store", path); return path; } StorePath Store::followLinksToStorePath(std::string_view path) const { - return parseStorePath(toStorePath(followLinksToStore(path))); + return toStorePath(followLinksToStore(path)).first; } @@ -139,21 +138,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p */ -StorePath Store::makeStorePath(const string & type, - const Hash & hash, std::string_view name) const +StorePath Store::makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); + string s = std::string { type } + ":" + std::string { hash } + + ":" + storeDir + ":" + std::string { name }; auto h = compressHash(hashString(htSHA256, s), 20); return StorePath(h, name); } -StorePath Store::makeOutputPath(const string & id, +StorePath Store::makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const +{ + return makeStorePath(type, hash.to_string(Base16, true), name); +} + + +StorePath Store::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { - return makeStorePath("output:" + id, hash, - std::string(name) + (id == "out" ? "" : "-" + id)); + return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); } @@ -192,6 +198,19 @@ StorePath Store::makeFixedOutputPath( } } +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references, bool hasSelfReference) const +{ + // New template + return std::visit(overloaded { + [&](TextHash th) { + return makeTextPath(name, th.hash, references); + }, + [&](FixedOutputHash fsh) { + return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + } + }, ca); +} StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const @@ -221,8 +240,111 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +StorePath Store::addToStore(const string & name, const Path & _srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} + + +/* +The aim of this function is to compute in one pass the correct ValidPathInfo for +the files that we are trying to add to the store. To accomplish that in one +pass, given the different kind of inputs that we can take (normal nar archives, +nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks +and aliases. Also, since the dataflow is obfuscated by this, we include here a +graphviz diagram: + +digraph graphname { + node [shape=box] + fileSource -> narSink + narSink [style=dashed] + narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"] + narSink -> narHashSink [style = dashed, label = "else"] + unsualHashTee -> narHashSink + unsualHashTee -> caHashSink + fileSource -> parseSink + parseSink [style=dashed] + parseSink-> fileSink [style = dashed, label = "Flat"] + parseSink -> blank [style = dashed, label = "Recursive"] + fileSink -> caHashSink +} +*/ +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional<Hash> expectedCAHash) +{ + HashSink narHashSink { htSHA256 }; + HashSink caHashSink { hashAlgo }; + + /* Note that fileSink and unusualHashTee must be mutually exclusive, since + they both write to caHashSink. Note that that requisite is currently true + because the former is only used in the flat case. */ + RetrieveRegularNARSink fileSink { caHashSink }; + TeeSink unusualHashTee { narHashSink, caHashSink }; + + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + ? static_cast<Sink &>(unusualHashTee) + : narHashSink; + + /* Functionally, this means that fileSource will yield the content of + srcPath. The fact that we use scratchpadSink as a temporary buffer here + is an implementation detail. */ + auto fileSource = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + + /* tapped provides the same data as fileSource, but we also write all the + information to narSink. */ + TeeSource tapped { *fileSource, narSink }; + + ParseSink blank; + auto & parseSink = method == FileIngestionMethod::Flat + ? fileSink + : blank; + + /* The information that flows from tapped (besides being replicated in + narSink), is now put in parseSink. */ + parseDump(parseSink, tapped); + + /* We extract the result of the computation from the sink by calling + finish. */ + auto [narHash, narSize] = narHashSink.finish(); + + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? narHash + : caHashSink.finish().first; + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + ValidPathInfo info { + makeFixedOutputPath(method, hash, name), + narHash, + }; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) - : Config(params) + : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { } @@ -242,6 +364,17 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { + auto resp = queryPartialDerivationOutputMap(path); + OutputPathMap result; + for (auto & [outName, optOutPath] : resp) { + if (!optOutPath) + throw Error("output '%s' has no store path mapped to it", outName); + result.insert_or_assign(outName, *optOutPath); + } + return result; +} + StorePathSet Store::queryDerivationOutputs(const StorePath & path) { auto outputMap = this->queryDerivationOutputMap(path); @@ -316,6 +449,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath) } +static bool goodStorePath(const StorePath & expected, const StorePath & actual) +{ + return + expected.hashPart() == actual.hashPart() + && (expected.name() == Store::MissingName || expected.name() == actual.name()); +} + + void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { @@ -343,7 +484,7 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || - res.second->path != storePath) + !goodStorePath(storePath, res.second->path)) throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref<const ValidPathInfo>(res.second)); @@ -355,7 +496,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -368,9 +509,11 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - if (!info || info->path != parseStorePath(storePath)) { + auto storePath = parseStorePath(storePathS); + + if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); + throw InvalidPath("path '%s' is not valid", storePathS); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -515,7 +658,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -591,18 +734,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); + // recompute store path on the chance dstStore does it differently + if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; + info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(info->path == info2->path); info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; } if (info->ultimate) { @@ -612,12 +750,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, } auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); + LambdaSink progressSink([&](const unsigned char * data, size_t len) { total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath(storePath, wrapperSink); + TeeSink tee { sink, progressSink }; + srcStore->narFromPath(storePath, tee); }, [&]() { throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); @@ -626,16 +764,20 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, } -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, +std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); - PathSet missing; + StorePathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); + if (!valid.count(path)) missing.insert(path); - if (missing.empty()) return; + std::map<StorePath, StorePath> pathsMap; + for (auto & path : storePaths) + pathsMap.insert_or_assign(path, path); + + if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -650,30 +792,49 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st ThreadPool pool; - processGraph<Path>(pool, - PathSet(missing.begin(), missing.end()), + processGraph<StorePath>(pool, + StorePathSet(missing.begin(), missing.end()), + + [&](const StorePath & storePath) { + auto info = srcStore->queryPathInfo(storePath); + auto storePathForDst = storePath; + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); + if (storePathForDst != storePath) + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + } + pathsMap.insert_or_assign(storePath, storePathForDst); - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + if (dstStore->isValidPath(storePath)) { nrDone++; showProgress(); - return PathSet(); + return StorePathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); - bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return srcStore->printStorePathSet(info->references); + return info->references; }, - [&](const Path & storePathS) { + [&](const StorePath & storePath) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); + auto info = srcStore->queryPathInfo(storePath); + + auto storePathForDst = storePath; + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); + if (storePathForDst != storePath) + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + } + pathsMap.insert_or_assign(storePath, storePathForDst); - if (!dstStore->isValidPath(storePath)) { + if (!dstStore->isValidPath(storePathForDst)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); try { @@ -682,7 +843,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; } @@ -691,6 +852,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrDone++; showProgress(); }); + + return pathsMap; } @@ -704,19 +867,22 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, } -std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) +std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven) { std::string path; getline(str, path); if (str.eof()) { return {}; } - ValidPathInfo info(store.parseStorePath(path)); - if (hashGiven) { + if (!hashGiven) { string s; getline(str, s); - info.narHash = Hash(s, htSHA256); + auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); + uint64_t narSize; + if (!string2Int(s, narSize)) throw Error("number expected"); + hashGiven = { narHash, narSize }; } + ValidPathInfo info(store.parseStorePath(path), hashGiven->first); + info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); @@ -751,8 +917,8 @@ string showPaths(const PathSet & paths) std::string ValidPathInfo::fingerprint(const Store & store) const { - if (narSize == 0 || !narHash) - throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known", + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" @@ -767,10 +933,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -// FIXME Put this somewhere? -template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; -template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; - bool ValidPathInfo::isContentAddressed(const Store & store) const { if (! ca) return false; @@ -826,6 +988,25 @@ Strings ValidPathInfo::shortRefs() const } +Derivation Store::derivationFromPath(const StorePath & drvPath) +{ + ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ + auto accessor = getFSAccessor(); + try { + return parseDerivation(*this, + accessor->readFile(printStorePath(drvPath)), + Derivation::nameFromPath(drvPath)); + } catch (FormatError & e) { + throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); + } +} + } @@ -835,9 +1016,6 @@ Strings ValidPathInfo::shortRefs() const namespace nix { - -RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) { @@ -851,62 +1029,71 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_ return {uri, params}; } -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) -{ - auto [uri, uriParams] = splitUriAndParams(uri_); - auto params = extraParams; - params.insert(uriParams.begin(), uriParams.end()); - - for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri, params); - if (store) { - store->warnUnknownSettings(); - return ref<Store>(store); - } - } - - throw Error("don't know how to open Nix store '%s'", uri); +static bool isNonUriPath(const std::string & spec) { + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; } - -StoreType getStoreType(const std::string & uri, const std::string & stateDir) +std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params) { - if (uri == "daemon") { - return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { - return tLocal; - } else if (uri == "" || uri == "auto") { + if (uri == "" || uri == "auto") { + auto stateDir = get(params, "state").value_or(settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return tLocal; + return std::make_shared<LocalStore>(params); else if (pathExists(settings.nixDaemonSocketFile)) - return tDaemon; + return std::make_shared<UDSRemoteStore>(params); else - return tLocal; + return std::make_shared<LocalStore>(params); + } else if (uri == "daemon") { + return std::make_shared<UDSRemoteStore>(params); + } else if (uri == "local") { + return std::make_shared<LocalStore>(params); + } else if (isNonUriPath(uri)) { + Store::Params params2 = params; + params2["root"] = absPath(uri); + return std::make_shared<LocalStore>(params2); } else { - return tOther; + return nullptr; } } - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) { - switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { - case tDaemon: - return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); - case tLocal: { - Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; - return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); + auto params = extraParams; + try { + auto parsedUri = parseURL(uri_); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + for (auto implem : *Implementations::registered) { + if (implem.uriSchemes.count(parsedUri.scheme)) { + auto store = implem.create(parsedUri.scheme, baseURI, params); + if (store) { + store->init(); + store->warnUnknownSettings(); + return ref<Store>(store); + } + } } - default: - return nullptr; } -}); + catch (BadURL &) { + auto [uri, uriParams] = splitUriAndParams(uri_); + params.insert(uriParams.begin(), uriParams.end()); + if (auto store = openFromNonUri(uri, params)) { + store->warnUnknownSettings(); + return ref<Store>(store); + } + } + + throw Error("don't know how to open Nix store '%s'", uri_); +} std::list<ref<Store>> getDefaultSubstituters() { @@ -940,5 +1127,6 @@ std::list<ref<Store>> getDefaultSubstituters() return stores; } +std::vector<StoreFactory> * Implementations::registered = 0; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b122e05d6..450c0f554 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -4,12 +4,12 @@ #include "hash.hh" #include "content-address.hh" #include "serialise.hh" -#include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" #include "config.hh" #include "derivations.hh" +#include "path-info.hh" #include <atomic> #include <limits> @@ -24,6 +24,31 @@ namespace nix { +/** + * About the class hierarchy of the store implementations: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting<T>` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation<Foo, FooConfig> regStore; + * ``` + */ MakeError(SubstError, Error); MakeError(BuildError, Error); // denotes a permanent build failure @@ -31,8 +56,9 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(NotInStore, Error); +MakeError(BadStorePath, Error); +MakeError(InvalidStoreURI, Error); class FSAccessor; class NarInfoDiskCache; @@ -85,7 +111,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; + uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; }; @@ -97,98 +123,10 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + uint64_t bytesFreed = 0; }; -struct SubstitutablePathInfo -{ - std::optional<StorePath> deriver; - StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - -struct ValidPathInfo -{ - StorePath path; - std::optional<StorePath> deriver; - Hash narHash; - StorePathSet references; - time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only - - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ - bool ultimate = false; - - StringSet sigs; // note: not necessarily verified - - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ - std::optional<ContentAddress> ca; - - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint(const Store & store) const; - - void sign(const Store & store, const SecretKey & secretKey); - - /* Return true iff the path is verifiably content-addressed. */ - bool isContentAddressed(const Store & store) const; - - static const size_t maxSigs = std::numeric_limits<size_t>::max(); - - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ - size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - - /* Verify a single signature. */ - bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; - - Strings shortRefs() const; - - ValidPathInfo(const ValidPathInfo & other) = default; - - ValidPathInfo(StorePath && path) : path(std::move(path)) { }; - ValidPathInfo(const StorePath & path) : path(path) { }; - - virtual ~ValidPathInfo() { } -}; - -typedef list<ValidPathInfo> ValidPathInfos; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; @@ -232,12 +170,33 @@ struct BuildResult } }; - -class Store : public std::enable_shared_from_this<Store>, public Config +struct StoreConfig : public Config { -public: - - typedef std::map<std::string, std::string> Params; + using Config::Config; + + /** + * When constructing a store implementation, we pass in a map `params` of + * parameters that's supposed to initialize the associated config. + * To do that, we must use the `StoreConfig(StringMap & params)` + * constructor, so we'd like to `delete` its default constructor to enforce + * it. + * + * However, actually deleting it means that all the subclasses of + * `StoreConfig` will have their default constructor deleted (because it's + * supposed to call the deleted default constructor of `StoreConfig`). But + * because we're always using virtual inheritance, the constructors of + * child classes will never implicitely call this one, so deleting it will + * be more painful than anything else. + * + * So we `assert(false)` here to ensure at runtime that the right + * constructor is always called without having to redefine a custom + * constructor for each `*Config` class. + */ + StoreConfig() { assert(false); } + + virtual ~StoreConfig() { } + + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, "store", "path to the Nix store"}; @@ -251,6 +210,18 @@ public: Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting<StringSet> systemFeatures{this, settings.systemFeatures, + "system-features", + "Optional features that the system this store builds on implements (like \"kvm\")."}; + +}; + +class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig +{ +public: + + typedef std::map<std::string, std::string> Params; + protected: struct PathInfoCacheValue { @@ -284,6 +255,11 @@ protected: Store(const Params & params); public: + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init() {}; virtual ~Store() { } @@ -317,9 +293,9 @@ public: the Nix store. */ bool isStorePath(std::string_view path) const; - /* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ - Path toStorePath(const Path & path) const; + /* Split a path like /nix/store/<hash>-<name>/<bla> into + /nix/store/<hash>-<name> and /<bla>. */ + std::pair<StorePath, Path> toStorePath(const Path & path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; @@ -331,10 +307,12 @@ public: StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; /* Constructs a unique store path name. */ - StorePath makeStorePath(const string & type, + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const; - StorePath makeOutputPath(const string & id, + StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; StorePath makeFixedOutputPath(FileIngestionMethod method, @@ -343,7 +321,11 @@ public: bool hasSelfReference = false) const; StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const; + const StorePathSet & references = {}) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -384,13 +366,16 @@ public: SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be omitted - (i.e. you'll get /nix/store/<hash> rather than + backends, the name part of store paths may be replaced by 'x' + (i.e. you'll get /nix/store/<hash>-x rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the - full store path. */ + full store path. FIXME: should return a set of + std::variant<StorePath, HashPart> to get rid of this hack. */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + constexpr static const char * MissingName = "x"; + /* Query information about a valid path. It is permitted to omit the name part of the store path. */ ref<const ValidPathInfo> queryPathInfo(const StorePath & path); @@ -420,9 +405,15 @@ public: /* Query the outputs of the derivation denoted by `path'. */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation */ - virtual OutputPathMap queryDerivationOutputMap(const StorePath & path) - { unsupported("queryDerivationOutputMap"); } + /* Query the mapping outputName => outputPath for the given derivation. All + outputs are mentioned so ones mising the mapping are mapped to + `std::nullopt`. */ + virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) + { unsupported("queryPartialDerivationOutputMap"); } + + /* Query the mapping outputName=>outputPath for the given derivation. + Assume every output has a mapping and throw an exception otherwise. */ + OutputPathMap queryDerivationOutputMap(const StorePath & path); /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ @@ -432,15 +423,15 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathSet & paths, + sizes) of a map of paths to their optional ca values. If a path + does not have substitute info, it's omitted from the resulting + ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) = 0; + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. @@ -448,14 +439,24 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional<Hash> expectedCAHash = {}); + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). + `dump` may be drained */ // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) - { - throw Error("addToStoreFromDump() is not supported by this store"); - } + { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ @@ -478,8 +479,38 @@ public: BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). Note that ‘drvPath’ is only used for - informational purposes. */ + on-disk .drv file). + + ‘drvPath’ is used to deduplicate worker goals so it is imperative that + is correct. That said, it doesn't literally need to be store path that + would be calculated from writing this derivation to the store: it is OK + if it instead is that of a Derivation which would resolve to this (by + taking the outputs of it's input derivations and adding them as input + sources) such that the build time referenceable-paths are the same. + + In the input-addressed case, we usually *do* use an "original" + unresolved derivations's path, as that is what will be used in the + `buildPaths` case. Also, the input-addressed output paths are verified + only by that contents of that specific unresolved derivation, so it is + nice to keep that information around so if the original derivation is + ever obtained later, it can be verified whether the trusted user in fact + used the proper output path. + + In the content-addressed case, we want to always use the + resolved drv path calculated from the provided derivation. This serves + two purposes: + + - It keeps the operation trustless, by ruling out a maliciously + invalid drv path corresponding to a non-resolution-equivalent + derivation. + + - For the floating case in particular, it ensures that the derivation + to output mapping respects the resolution equivalence relation, so + one cannot choose different resolution-equivalent derivations to + subvert dependency coherence (i.e. the property that one doesn't end + up with multiple different versions of dependencies without + explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; @@ -516,7 +547,7 @@ public: - The collector isn't running, or it's just started but hasn't acquired the GC lock yet. In that case we get and release the lock right away, then exit. The collector scans the - permanent root and sees our's. + permanent root and sees ours. In either case the permanent root is seen by the collector. */ virtual void syncWithGC() { }; @@ -600,7 +631,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ @@ -616,8 +647,7 @@ public: the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, - CheckSigsFlag checkSigs = CheckSigs); + StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats { @@ -685,22 +715,25 @@ protected: }; - -class LocalFSStore : public virtual Store +struct LocalFSStoreConfig : virtual StoreConfig { -public: - - // FIXME: the (Store*) cast works around a bug in gcc that causes + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(Store*) this, true, "", + const PathSetting rootDir{(StoreConfig*) this, true, "", "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(Store*) this, false, + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", "directory where Nix will store state"}; - const PathSetting logDir{(Store*) this, false, + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: const static string drvsLogDir; @@ -710,8 +743,7 @@ public: ref<FSAccessor> getFSAccessor() override; /* Register a permanent GC root. */ - Path addPermRoot(const StorePath & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return storeDir; } @@ -731,11 +763,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, /* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order - (i.e. if A is a reference of B, then A is copied before B), but - the set of store paths is not automatically closed; use - copyClosure() for that. */ -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, + in parallel. They are copied in a topologically sorted order (i.e. + if A is a reference of B, then A is copied before B), but the set + of store paths is not automatically closed; use copyClosure() for + that. Returns a map of what each path was copied to the dstStore + as. */ +std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -788,37 +822,48 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -enum StoreType { - tDaemon, - tLocal, - tOther -}; - - -StoreType getStoreType(const std::string & uri = settings.storeUri.get(), - const std::string & stateDir = settings.nixStateDir); - /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); +struct StoreFactory +{ + std::set<std::string> uriSchemes; + std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + std::function<std::shared_ptr<StoreConfig> ()> getConfig; +}; -/* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>( - const std::string & uri, const Store::Params & params)> OpenStore; - -struct RegisterStoreImplementation +struct Implementations { - typedef std::vector<OpenStore> Implementations; - static Implementations * implementations; + static std::vector<StoreFactory> * registered; - RegisterStoreImplementation(OpenStore fun) + template<typename T, typename TConfig> + static void add() { - if (!implementations) implementations = new Implementations; - implementations->push_back(fun); + if (!registered) registered = new std::vector<StoreFactory>(); + StoreFactory factory{ + .uriSchemes = T::uriSchemes(), + .create = + ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> + { return std::make_shared<T>(scheme, uri, params); }), + .getConfig = + ([]() + -> std::shared_ptr<StoreConfig> + { return std::make_shared<TConfig>(StringMap({})); }) + }; + registered->push_back(factory); } }; +template<typename T, typename TConfig> +struct RegisterStoreImplementation +{ + RegisterStoreImplementation() + { + Implementations::add<T, TConfig>(); + } +}; /* Display a set of paths in human-readable form (i.e., between quotes @@ -829,9 +874,11 @@ string showPaths(const PathSet & paths); std::optional<ValidPathInfo> decodeValidPathInfo( const Store & store, std::istream & str, - bool hashGiven = false); + std::optional<HashResult> hashGiven = std::nullopt); /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); + } diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc index fb2a45f48..704bb4ad0 100644 --- a/src/libstore/user-lock.cc +++ b/src/libstore/user-lock.cc @@ -3,6 +3,9 @@ #include "pathlocks.hh" #include "cgroup.hh" +#include <pwd.h> +#include <grp.h> + namespace nix { struct SimpleUserLock : UserLock diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 8b538f6da..b3705578e 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x116 +#define PROTOCOL_VERSION 0x11a #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,7 +18,7 @@ typedef enum { wopQueryReferences = 5, // obsolete wopQueryReferrers = 6, wopAddToStore = 7, - wopAddTextToStore = 8, + wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore wopBuildPaths = 9, wopEnsurePath = 10, wopAddTempRoot = 11, @@ -66,10 +66,89 @@ typedef enum { class Store; struct Source; -template<class T> T readStorePaths(const Store & store, Source & from); +/* To guide overloading */ +template<typename T> +struct Phantom {}; -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); +namespace worker_proto { +/* FIXME maybe move more stuff inside here */ + +#define MAKE_WORKER_PROTO(TEMPLATE, T) \ + TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ + TEMPLATE void write(const Store & store, Sink & out, const T & str) + +MAKE_WORKER_PROTO(, std::string); +MAKE_WORKER_PROTO(, StorePath); +MAKE_WORKER_PROTO(, ContentAddress); + +MAKE_WORKER_PROTO(template<typename T>, std::set<T>); + +#define X_ template<typename K, typename V> +#define Y_ std::map<K, V> +MAKE_WORKER_PROTO(X_, Y_); +#undef X_ +#undef Y_ + +/* These use the empty string for the null case, relying on the fact + that the underlying types never serialize to the empty string. + + We do this instead of a generic std::optional<T> instance because + ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + the same reason, we don't have a std::variant<T..> instances (ordinal + tags 0...n). + + We could the generic instances and then these as specializations for + compatability, but that's proven a bit finnicky, and also makes the + worker protocol harder to implement in other languages where such + specializations may not be allowed. + */ +MAKE_WORKER_PROTO(, std::optional<StorePath>); +MAKE_WORKER_PROTO(, std::optional<ContentAddress>); + +template<typename T> +std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _) +{ + std::set<T> resSet; + auto size = readNum<size_t>(from); + while (size--) { + resSet.insert(read(store, from, Phantom<T> {})); + } + return resSet; +} + +template<typename T> +void write(const Store & store, Sink & out, const std::set<T> & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + +template<typename K, typename V> +std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _) +{ + std::map<K, V> resMap; + auto size = readNum<size_t>(from); + while (size--) { + auto k = read(store, from, Phantom<K> {}); + auto v = read(store, from, Phantom<V> {}); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template<typename K, typename V> +void write(const Store & store, Sink & out, const std::map<K, V> & resMap) +{ + out << resMap.size(); + for (auto & i : resMap) { + write(store, out, i.first); + write(store, out, i.second); + } +} + +} } diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh new file mode 100644 index 000000000..b3fbc84f7 --- /dev/null +++ b/src/libutil/abstract-setting-to-json.hh @@ -0,0 +1,15 @@ +#pragma once + +#include <nlohmann/json.hpp> +#include "config.hh" + +namespace nix { +template<typename T> +std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject() +{ + auto obj = AbstractSetting::toJSONObject(); + obj.emplace("value", value); + obj.emplace("defaultValue", defaultValue); + return obj; +} +} diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index a38c2d798..ae741f867 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -11,7 +11,7 @@ namespace nix { #define ANSI_GREEN "\e[32;1m" #define ANSI_YELLOW "\e[33;1m" #define ANSI_BLUE "\e[34;1m" -#define ANSI_MAGENTA "\e[35m;1m" -#define ANSI_CYAN "\e[36m;1m" +#define ANSI_MAGENTA "\e[35;1m" +#define ANSI_CYAN "\e[36;1m" } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 51c88537e..f1479329f 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,11 +27,13 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; + Setting<bool> preallocateContents{this, true, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; -static GlobalConfig::Register r1(&archiveSettings); +static GlobalConfig::Register rArchiveSettings(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; @@ -66,9 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); sink << "("; @@ -150,17 +150,17 @@ static void skipGeneric(Source & source) static void parseContents(ParseSink & sink, Source & source, const Path & path) { - unsigned long long size = readLongLong(source); + uint64_t size = readLongLong(source); sink.preallocateContents(size); - unsigned long long left = size; + uint64_t left = size; std::vector<unsigned char> buf(65536); while (left) { checkInterrupt(); auto n = buf.size(); - if ((unsigned long long)n > left) n = left; + if ((uint64_t)n > left) n = left; source(buf.data(), n); sink.receiveContents(buf.data(), n); left -= n; @@ -323,8 +323,11 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } - void preallocateContents(unsigned long long len) + void preallocateContents(uint64_t len) { + if (!archiveSettings.preallocateContents) + return; + #if HAVE_POSIX_FALLOCATE if (len) { errno = posix_fallocate(fd.get(), 0, len); @@ -338,7 +341,7 @@ struct RestoreSink : ParseSink #endif } - void receiveContents(unsigned char * data, unsigned int len) + void receiveContents(unsigned char * data, size_t len) { writeFull(fd.get(), data, len); } @@ -366,11 +369,7 @@ void copyNAR(Source & source, Sink & sink) ParseSink parseSink; /* null sink; just parse the NAR */ - LambdaSource wrapper([&](unsigned char * data, size_t len) { - auto n = source.read(data, len); - sink(data, n); - return n; - }); + TeeSource wrapper { source, sink }; parseDump(parseSink, wrapper); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 768fe2536..5665732d2 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -57,17 +57,35 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(unsigned char * data, size_t len) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeSink : ParseSink +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink { - TeeSource source; + bool regular = true; + Sink & sink; - TeeSink(Source & source) : source(source) { } + RetrieveRegularNARSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, size_t len) + { + sink(data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } }; void parseDump(ParseSink & sink, Source & source); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 1f0e4f803..2760b830b 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,6 +1,10 @@ #include "args.hh" #include "hash.hh" +#include <glob.h> + +#include <nlohmann/json.hpp> + namespace nix { void Args::addFlag(Flag && flag_) @@ -13,6 +17,20 @@ void Args::addFlag(Flag && flag_) if (flag->shortName) shortFlags[flag->shortName] = flag; } +bool pathCompletions = false; +std::shared_ptr<std::set<std::string>> completions; + +std::string completionMarker = "___COMPLETE___"; + +std::optional<std::string> needsCompletion(std::string_view s) +{ + if (!completions) return {}; + auto i = s.find(completionMarker); + if (i != std::string::npos) + return std::string(s.begin(), i); + return {}; +} + void Args::parseCmdline(const Strings & _cmdline) { Strings pendingArgs; @@ -20,6 +38,14 @@ void Args::parseCmdline(const Strings & _cmdline) Strings cmdline(_cmdline); + if (auto s = getEnv("NIX_GET_COMPLETIONS")) { + size_t n = std::stoi(*s); + assert(n > 0 && n <= cmdline.size()); + *std::next(cmdline.begin(), n - 1) += completionMarker; + completions = std::make_shared<decltype(completions)::element_type>(); + verbosity = lvlError; + } + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -63,7 +89,7 @@ void Args::printHelp(const string & programName, std::ostream & out) for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1 - if (exp.arity == 0) std::cout << "..."; + if (exp.handler.arity == ArityAny) std::cout << "..."; if (exp.optional) std::cout << "?"; } std::cout << "\n"; @@ -99,18 +125,32 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; std::vector<std::string> args; + bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { if (flag.handler.arity == ArityAny) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } + if (flag.completer) + if (auto prefix = needsCompletion(*pos)) { + anyCompleted = true; + flag.completer(n, *prefix); + } args.push_back(*pos++); } - flag.handler.fun(std::move(args)); + if (!anyCompleted) + flag.handler.fun(std::move(args)); return true; }; if (string(*pos, 0, 2) == "--") { + if (auto prefix = needsCompletion(*pos)) { + for (auto & [name, flag] : longFlags) { + if (!hiddenCategories.count(flag->category) + && hasPrefix(name, std::string(*prefix, 2))) + completions->insert("--" + name); + } + } auto i = longFlags.find(string(*pos, 2)); if (i == longFlags.end()) return false; return process("--" + i->first, *i->second); @@ -123,6 +163,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } + if (auto prefix = needsCompletion(*pos)) { + if (prefix == "-") { + completions->insert("--"); + for (auto & [flag, _] : shortFlags) + completions->insert(std::string("-") + flag); + } + } + return false; } @@ -138,12 +186,17 @@ bool Args::processArgs(const Strings & args, bool finish) bool res = false; - if ((exp.arity == 0 && finish) || - (exp.arity > 0 && args.size() == exp.arity)) + if ((exp.handler.arity == ArityAny && finish) || + (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector<std::string> ss; - for (auto & s : args) ss.push_back(s); - exp.handler(std::move(ss)); + for (const auto &[n, s] : enumerate(args)) { + ss.push_back(s); + if (exp.completer) + if (auto prefix = needsCompletion(s)) + exp.completer(n, *prefix); + } + exp.handler.fun(ss); expectedArgs.pop_front(); res = true; } @@ -154,6 +207,50 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } +nlohmann::json Args::toJSON() +{ + auto flags = nlohmann::json::object(); + + for (auto & [name, flag] : longFlags) { + auto j = nlohmann::json::object(); + if (flag->shortName) + j["shortName"] = std::string(1, flag->shortName); + if (flag->description != "") + j["description"] = flag->description; + if (flag->category != "") + j["category"] = flag->category; + if (flag->handler.arity != ArityAny) + j["arity"] = flag->handler.arity; + if (!flag->labels.empty()) + j["labels"] = flag->labels; + flags[name] = std::move(j); + } + + auto args = nlohmann::json::array(); + + for (auto & arg : expectedArgs) { + auto j = nlohmann::json::object(); + j["label"] = arg.label; + j["optional"] = arg.optional; + if (arg.handler.arity != ArityAny) + j["arity"] = arg.handler.arity; + args.push_back(std::move(j)); + } + + auto res = nlohmann::json::object(); + res["description"] = description(); + res["flags"] = std::move(flags); + res["args"] = std::move(args); + return res; +} + +static void hashTypeCompleter(size_t index, std::string_view prefix) +{ + for (auto & type : hashTypes) + if (hasPrefix(type, prefix)) + completions->insert(type); +} + Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { return Flag { @@ -162,7 +259,8 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .labels = {"hash-algo"}, .handler = {[ht](std::string s) { *ht = parseHashType(s); - }} + }}, + .completer = hashTypeCompleter }; } @@ -174,10 +272,42 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .labels = {"hash-algo"}, .handler = {[oht](std::string s) { *oht = std::optional<HashType> { parseHashType(s) }; - }} + }}, + .completer = hashTypeCompleter }; } +static void _completePath(std::string_view prefix, bool onlyDirs) +{ + pathCompletions = true; + glob_t globbuf; + int flags = GLOB_NOESCAPE | GLOB_TILDE; + #ifdef GLOB_ONLYDIR + if (onlyDirs) + flags |= GLOB_ONLYDIR; + #endif + if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { + for (size_t i = 0; i < globbuf.gl_pathc; ++i) { + if (onlyDirs) { + auto st = lstat(globbuf.gl_pathv[i]); + if (!S_ISDIR(st.st_mode)) continue; + } + completions->insert(globbuf.gl_pathv[i]); + } + globfree(&globbuf); + } +} + +void completePath(size_t, std::string_view prefix) +{ + _completePath(prefix, false); +} + +void completeDir(size_t, std::string_view prefix) +{ + _completePath(prefix, true); +} + Strings argvToStrings(int argc, char * * argv) { Strings args; @@ -222,21 +352,47 @@ void Command::printHelp(const string & programName, std::ostream & out) } } +nlohmann::json Command::toJSON() +{ + auto exs = nlohmann::json::array(); + + for (auto & example : examples()) { + auto ex = nlohmann::json::object(); + ex["description"] = example.description; + ex["command"] = chomp(stripIndentation(example.command)); + exs.push_back(std::move(ex)); + } + + auto res = Args::toJSON(); + res["examples"] = std::move(exs); + auto s = doc(); + if (s != "") res.emplace("doc", stripIndentation(s)); + return res; +} + MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { - assert(!command); - auto cmd = ss[0]; - if (auto alias = get(deprecatedAliases, cmd)) { - warn("'%s' is a deprecated alias for '%s'", cmd, *alias); - cmd = *alias; - } - auto i = commands.find(cmd); - if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", cmd); - command = {cmd, i->second()}; - }}); + expectArgs({ + .label = "subcommand", + .optional = true, + .handler = {[=](std::string s) { + assert(!command); + if (auto alias = get(deprecatedAliases, s)) { + warn("'%s' is a deprecated alias for '%s'", s, *alias); + s = *alias; + } + if (auto prefix = needsCompletion(s)) { + for (auto & [name, command] : commands) + if (hasPrefix(name, *prefix)) + completions->insert(name); + } + auto i = commands.find(s); + if (i == commands.end()) + throw UsageError("'%s' is not a recognised command", s); + command = {s, i->second()}; + }} + }); categories[Command::catDefault] = "Available commands"; } @@ -288,4 +444,20 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } +nlohmann::json MultiCommand::toJSON() +{ + auto cmds = nlohmann::json::object(); + + for (auto & [name, commandFun] : commands) { + auto command = commandFun(); + auto j = command->toJSON(); + j["category"] = categories[command->category()]; + cmds[name] = std::move(j); + } + + auto res = Args::toJSON(); + res["commands"] = std::move(cmds); + return res; +} + } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 433d26bba..f41242e17 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -4,12 +4,12 @@ #include <map> #include <memory> +#include <nlohmann/json_fwd.hpp> + #include "util.hh" namespace nix { -MakeError(UsageError, Error); - enum HashType : char; class Args @@ -22,67 +22,74 @@ public: virtual void printHelp(const string & programName, std::ostream & out); + /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } protected: static const size_t ArityAny = std::numeric_limits<size_t>::max(); + struct Handler + { + std::function<void(std::vector<std::string>)> fun; + size_t arity; + + Handler() {} + + Handler(std::function<void(std::vector<std::string>)> && fun) + : fun(std::move(fun)) + , arity(ArityAny) + { } + + Handler(std::function<void()> && handler) + : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); }) + , arity(0) + { } + + Handler(std::function<void(std::string)> && handler) + : fun([handler{std::move(handler)}](std::vector<std::string> ss) { + handler(std::move(ss[0])); + }) + , arity(1) + { } + + Handler(std::function<void(std::string, std::string)> && handler) + : fun([handler{std::move(handler)}](std::vector<std::string> ss) { + handler(std::move(ss[0]), std::move(ss[1])); + }) + , arity(2) + { } + + Handler(std::vector<std::string> * dest) + : fun([=](std::vector<std::string> ss) { *dest = ss; }) + , arity(ArityAny) + { } + + template<class T> + Handler(T * dest) + : fun([=](std::vector<std::string> ss) { *dest = ss[0]; }) + , arity(1) + { } + + template<class T> + Handler(T * dest, const T & val) + : fun([=](std::vector<std::string> ss) { *dest = val; }) + , arity(0) + { } + }; + /* Flags. */ struct Flag { typedef std::shared_ptr<Flag> ptr; - struct Handler - { - std::function<void(std::vector<std::string>)> fun; - size_t arity; - - Handler() {} - - Handler(std::function<void(std::vector<std::string>)> && fun) - : fun(std::move(fun)) - , arity(ArityAny) - { } - - Handler(std::function<void()> && handler) - : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); }) - , arity(0) - { } - - Handler(std::function<void(std::string)> && handler) - : fun([handler{std::move(handler)}](std::vector<std::string> ss) { - handler(std::move(ss[0])); - }) - , arity(1) - { } - - Handler(std::function<void(std::string, std::string)> && handler) - : fun([handler{std::move(handler)}](std::vector<std::string> ss) { - handler(std::move(ss[0]), std::move(ss[1])); - }) - , arity(2) - { } - - template<class T> - Handler(T * dest) - : fun([=](std::vector<std::string> ss) { *dest = ss[0]; }) - , arity(1) - { } - - template<class T> - Handler(T * dest, const T & val) - : fun([=](std::vector<std::string> ss) { *dest = val; }) - , arity(0) - { } - }; - std::string longName; char shortName = 0; std::string description; std::string category; Strings labels; Handler handler; + std::function<void(size_t, std::string_view)> completer; static Flag mkHashTypeFlag(std::string && longName, HashType * ht); static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht); @@ -99,9 +106,9 @@ protected: struct ExpectedArg { std::string label; - size_t arity; // 0 = any - bool optional; - std::function<void(std::vector<std::string>)> handler; + bool optional = false; + Handler handler; + std::function<void(size_t, std::string_view)> completer; }; std::list<ExpectedArg> expectedArgs; @@ -175,22 +182,32 @@ public: }); } + void expectArgs(ExpectedArg && arg) + { + expectedArgs.emplace_back(std::move(arg)); + } + /* Expect a string argument. */ void expectArg(const std::string & label, string * dest, bool optional = false) { - expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { - *dest = ss[0]; - }}); + expectArgs({ + .label = label, + .optional = optional, + .handler = {dest} + }); } /* Expect 0 or more arguments. */ void expectArgs(const std::string & label, std::vector<std::string> * dest) { - expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) { - *dest = std::move(ss); - }}); + expectArgs({ + .label = label, + .handler = {dest} + }); } + virtual nlohmann::json toJSON(); + friend class MultiCommand; }; @@ -205,6 +222,9 @@ struct Command : virtual Args virtual void prepare() { }; virtual void run() = 0; + /* Return documentation about this command, in Markdown format. */ + virtual std::string doc() { return ""; } + struct Example { std::string description; @@ -222,6 +242,8 @@ struct Command : virtual Args virtual Category category() { return catDefault; } void printHelp(const string & programName, std::ostream & out) override; + + nlohmann::json toJSON() override; }; typedef std::map<std::string, std::function<ref<Command>()>> Commands; @@ -247,6 +269,8 @@ public: bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; + + nlohmann::json toJSON() override; }; Strings argvToStrings(int argc, char * * argv); @@ -259,4 +283,13 @@ typedef std::vector<std::pair<std::string, std::string>> Table2; void printTable(std::ostream & out, const Table2 & table); +extern std::shared_ptr<std::set<std::string>> completions; +extern bool pathCompletions; + +std::optional<std::string> needsCompletion(std::string_view s); + +void completePath(size_t, std::string_view prefix); + +void completeDir(size_t, std::string_view prefix); + } diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh new file mode 100644 index 000000000..ef31794be --- /dev/null +++ b/src/libutil/callback.hh @@ -0,0 +1,46 @@ +#pragma once + +#include <future> +#include <functional> + +namespace nix { + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future<T> to pass the value or + exception.) */ +template<typename T> +class Callback +{ + std::function<void(std::future<T>)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: + + Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_exception(exc); + fun(promise.get_future()); + } +}; + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8fc700a2b..521733025 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,8 @@ #include "config.hh" #include "args.hh" -#include "json.hh" +#include "abstract-setting-to-json.hh" + +#include <nlohmann/json.hpp> namespace nix { @@ -131,15 +133,14 @@ void Config::resetOverriden() s.second.setting->overriden = false; } -void Config::toJSON(JSONObject & out) +nlohmann::json Config::toJSON() { + auto res = nlohmann::json::object(); for (auto & s : _settings) if (!s.second.isAlias) { - JSONObject out2(out.object(s.first)); - out2.attr("description", s.second.setting->description); - JSONPlaceholder out3(out2.placeholder("value")); - s.second.setting->toJSON(out3); + res.emplace(s.first, s.second.setting->toJSON()); } + return res; } void Config::convertToArgs(Args & args, const std::string & category) @@ -153,7 +154,7 @@ AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, const std::set<std::string> & aliases) - : name(name), description(description), aliases(aliases) + : name(name), description(stripIndentation(description)), aliases(aliases) { } @@ -162,19 +163,21 @@ void AbstractSetting::setDefault(const std::string & str) if (!overriden) set(str); } -void AbstractSetting::toJSON(JSONPlaceholder & out) +nlohmann::json AbstractSetting::toJSON() { - out.write(to_string()); + return nlohmann::json(toJSONObject()); } -void AbstractSetting::convertToArg(Args & args, const std::string & category) +std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject() { + std::map<std::string, nlohmann::json> obj; + obj.emplace("description", description); + obj.emplace("aliases", aliases); + return obj; } -template<typename T> -void BaseSetting<T>::toJSON(JSONPlaceholder & out) +void AbstractSetting::convertToArg(Args & args, const std::string & category) { - out.write(value); } template<typename T> @@ -255,13 +258,6 @@ template<> std::string BaseSetting<Strings>::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out) -{ - JSONList list(out.list()); - for (auto & s : value) - list.elem(s); -} - template<> void BaseSetting<StringSet>::set(const std::string & str) { value = tokenizeString<StringSet>(str); @@ -272,11 +268,24 @@ template<> std::string BaseSetting<StringSet>::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out) +template<> void BaseSetting<StringMap>::set(const std::string & str) +{ + auto kvpairs = tokenizeString<Strings>(str); + for (auto & s : kvpairs) + { + auto eq = s.find_first_of('='); + if (std::string::npos != eq) + value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + // else ignored + } +} + +template<> std::string BaseSetting<StringMap>::to_string() const { - JSONList list(out.list()); - for (auto & s : value) - list.elem(s); + Strings kvstrs; + std::transform(value.begin(), value.end(), back_inserter(kvstrs), + [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); + return concatStringsSep(" ", kvstrs); } template class BaseSetting<int>; @@ -289,6 +298,7 @@ template class BaseSetting<bool>; template class BaseSetting<std::string>; template class BaseSetting<Strings>; template class BaseSetting<StringSet>; +template class BaseSetting<StringMap>; void PathSetting::set(const std::string & str) { @@ -323,10 +333,12 @@ void GlobalConfig::resetOverriden() config->resetOverriden(); } -void GlobalConfig::toJSON(JSONObject & out) +nlohmann::json GlobalConfig::toJSON() { + auto res = nlohmann::json::object(); for (auto & config : *configRegistrations) - config->toJSON(out); + res.update(config->toJSON()); + return res; } void GlobalConfig::convertToArgs(Args & args, const std::string & category) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 66073546e..1f5f4e7b9 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -4,6 +4,8 @@ #include "types.hh" +#include <nlohmann/json_fwd.hpp> + #pragma once namespace nix { @@ -42,8 +44,6 @@ namespace nix { class Args; class AbstractSetting; -class JSONPlaceholder; -class JSONObject; class AbstractConfig { @@ -97,7 +97,7 @@ public: * Outputs all settings to JSON * - out: JSONObject to write the configuration to */ - virtual void toJSON(JSONObject & out) = 0; + virtual nlohmann::json toJSON() = 0; /** * Converts settings to `Args` to be used on the command line interface @@ -167,7 +167,7 @@ public: void resetOverriden() override; - void toJSON(JSONObject & out) override; + nlohmann::json toJSON() override; void convertToArgs(Args & args, const std::string & category) override; }; @@ -206,7 +206,9 @@ protected: virtual std::string to_string() const = 0; - virtual void toJSON(JSONPlaceholder & out); + nlohmann::json toJSON(); + + virtual std::map<std::string, nlohmann::json> toJSONObject(); virtual void convertToArg(Args & args, const std::string & category); @@ -220,6 +222,7 @@ class BaseSetting : public AbstractSetting protected: T value; + const T defaultValue; public: @@ -229,6 +232,7 @@ public: const std::set<std::string> & aliases = {}) : AbstractSetting(name, description, aliases) , value(def) + , defaultValue(def) { } operator const T &() const { return value; } @@ -251,7 +255,7 @@ public: void convertToArg(Args & args, const std::string & category) override; - void toJSON(JSONPlaceholder & out) override; + std::map<std::string, nlohmann::json> toJSONObject() override; }; template<typename T> @@ -319,7 +323,7 @@ struct GlobalConfig : public AbstractConfig void resetOverriden() override; - void toJSON(JSONObject & out) override; + nlohmann::json toJSON() override; void convertToArgs(Args & args, const std::string & category) override; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index a4ee7afc2..803a72953 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -11,13 +11,13 @@ const std::string nativeSystem = SYSTEM; BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) { - err.traces.push_front(Trace { .pos = e, .hint = hint}); + err.traces.push_front(Trace { .pos = e, .hint = hint }); return *this; } // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). -const string& BaseError::calcWhat() const +const string & BaseError::calcWhat() const { if (what_.has_value()) return *what_; @@ -34,12 +34,12 @@ const string& BaseError::calcWhat() const std::optional<string> ErrorInfo::programName = std::nullopt; -std::ostream& operator<<(std::ostream &os, const hintformat &hf) +std::ostream & operator<<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -string showErrPos(const ErrPos &errPos) +string showErrPos(const ErrPos & errPos) { if (errPos.line > 0) { if (errPos.column > 0) { @@ -53,7 +53,7 @@ string showErrPos(const ErrPos &errPos) } } -std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos) +std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) { if (errPos.line <= 0) return std::nullopt; @@ -92,13 +92,13 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos) return loc; } } - catch (EndOfFile &eof) { + catch (EndOfFile & eof) { if (loc.errLineOfCode.has_value()) return loc; else return std::nullopt; } - catch (std::exception &e) { + catch (std::exception & e) { printError("error reading nix file: %s\n%s", errPos.file, e.what()); return std::nullopt; } @@ -137,10 +137,10 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos) } // print lines of code to the ostream, indicating the error column. -void printCodeLines(std::ostream &out, - const string &prefix, - const ErrPos &errPos, - const LinesOfCode &loc) +void printCodeLines(std::ostream & out, + const string & prefix, + const ErrPos & errPos, + const LinesOfCode & loc) { // previous line of code. if (loc.prevLineOfCode.has_value()) { @@ -186,7 +186,7 @@ void printCodeLines(std::ostream &out, } } -void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) +void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) { if (pos) { @@ -212,7 +212,7 @@ void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) } } -std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace) +std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { auto errwidth = std::max<size_t>(getWindowSize().second, 20); string prefix = ""; @@ -355,26 +355,21 @@ std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool show for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { - try { + out << std::endl << prefix; + out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str(); + + if (iter->pos.has_value() && (*iter->pos)) { + auto pos = iter->pos.value(); out << std::endl << prefix; - out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str(); + printAtPos(prefix, pos, out); - nl = true; - if (*iter->pos) { - auto pos = iter->pos.value(); + auto loc = getCodeLines(pos); + if (loc.has_value()) + { + out << std::endl << prefix; + printCodeLines(out, prefix, pos, *loc); out << std::endl << prefix; - - printAtPos(prefix, pos, out); - auto loc = getCodeLines(pos); - if (loc.has_value()) - { - out << std::endl << prefix; - printCodeLines(out, prefix, pos, *loc); - out << std::endl << prefix; - } } - } catch(const std::bad_optional_access& e) { - out << iter->hint.str() << std::endl; } } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1b0fb43b8..d1b6d82bb 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,8 +1,8 @@ #pragma once - #include "ref.hh" #include "types.hh" +#include "fmt.hh" #include <cstring> #include <list> @@ -10,7 +10,9 @@ #include <map> #include <optional> -#include "fmt.hh" +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> /* Before 4.7, gcc's std::exception uses empty throw() specifiers for * its (virtual) destructor and what() in c++11 mode, in violation of spec @@ -105,7 +107,7 @@ struct Trace { struct ErrorInfo { Verbosity level; string name; - string description; + string description; // FIXME: remove? it seems to be barely used std::optional<hintformat> hint; std::optional<ErrPos> errPos; std::list<Trace> traces; @@ -167,7 +169,7 @@ public: #endif const string & msg() const { return calcWhat(); } - const ErrorInfo & info() { calcWhat(); return err; } + const ErrorInfo & info() const { calcWhat(); return err; } template<typename... Args> BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args) @@ -189,6 +191,8 @@ public: } MakeError(Error, BaseError); +MakeError(UsageError, Error); +MakeError(UnimplementedError, Error); class SysError : public Error { diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index a39de041f..85c0e9429 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,7 +1,6 @@ #pragma once #include <boost/format.hpp> -#include <boost/algorithm/string/replace.hpp> #include <string> #include "ansicolor.hh" @@ -77,11 +76,11 @@ template <class T> struct yellowtxt { yellowtxt(const T &s) : value(s) {} - const T &value; + const T & value; }; template <class T> -std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y) +std::ostream & operator<<(std::ostream & out, const yellowtxt<T> & y) { return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } @@ -89,12 +88,12 @@ std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y) template <class T> struct normaltxt { - normaltxt(const T &s) : value(s) {} - const T &value; + normaltxt(const T & s) : value(s) {} + const T & value; }; template <class T> -std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y) +std::ostream & operator<<(std::ostream & out, const normaltxt<T> & y) { return out << ANSI_NORMAL << y.value; } @@ -102,26 +101,30 @@ std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y) class hintformat { public: - hintformat(const string &format) :fmt(format) + hintformat(const string & format) : fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } - hintformat(const hintformat &hf) - : fmt(hf.fmt) - {} + hintformat(const hintformat & hf) + : fmt(hf.fmt) + { } + + hintformat(format && fmt) + : fmt(std::move(fmt)) + { } template<class T> - hintformat& operator%(const T &value) + hintformat & operator%(const T & value) { fmt % yellowtxt(value); return *this; } template<class T> - hintformat& operator%(const normaltxt<T> &value) + hintformat & operator%(const normaltxt<T> & value) { fmt % value.value; return *this; @@ -136,7 +139,7 @@ private: format fmt; }; -std::ostream& operator<<(std::ostream &os, const hintformat &hf); +std::ostream & operator<<(std::ostream & os, const hintformat & hf); template<typename... Args> inline hintformat hintfmt(const std::string & fs, const Args & ... args) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1a3e7c5d8..4a94f0dfd 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -7,8 +7,8 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" +#include "split.hh" #include "util.hh" -#include "istringstream_nocopy.hh" #include <sys/types.h> #include <sys/stat.h> @@ -17,15 +17,23 @@ namespace nix { -void Hash::init() -{ - assert(type); - switch (*type) { - case htMD5: hashSize = md5HashSize; break; - case htSHA1: hashSize = sha1HashSize; break; - case htSHA256: hashSize = sha256HashSize; break; - case htSHA512: hashSize = sha512HashSize; break; +static size_t regularHashSize(HashType type) { + switch (type) { + case htMD5: return md5HashSize; + case htSHA1: return sha1HashSize; + case htSHA256: return sha256HashSize; + case htSHA512: return sha512HashSize; } + abort(); +} + + +std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" }; + + +Hash::Hash(HashType type) : type(type) +{ + hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -106,17 +114,11 @@ string printHash16or32(const Hash & hash) } -HashType assertInitHashType(const Hash & h) -{ - assert(h.type); - return *h.type; -} - std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(assertInitHashType(*this)); + s += printHashType(type); s += base == SRI ? '-' : ':'; } switch (base) { @@ -134,63 +136,103 @@ std::string Hash::to_string(Base base, bool includeType) const return s; } -Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } -Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { } +Hash Hash::dummy(htSHA256); -Hash::Hash(std::string_view s, std::optional<HashType> type) - : type(type) -{ - size_t pos = 0; +Hash Hash::parseSRI(std::string_view original) { + auto rest = original; + + // Parse the has type before the separater, if there was one. + auto hashRaw = splitPrefixTo(rest, '-'); + if (!hashRaw) + throw BadHash("hash '%s' is not SRI", original); + HashType parsedType = parseHashType(*hashRaw); + + return Hash(rest, parsedType, true); +} + +// Mutates the string to eliminate the prefixes when found +static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest) { bool isSRI = false; - auto sep = s.find(':'); - if (sep == string::npos) { - sep = s.find('-'); - if (sep != string::npos) { - isSRI = true; - } else if (! type) - throw BadHash("hash '%s' does not include a type", s); - } + // Parse the has type before the separater, if there was one. + std::optional<HashType> optParsedType; + { + auto hashRaw = splitPrefixTo(rest, ':'); - if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (!this->type) - throw BadHash("unknown hash type '%s'", hts); - if (type && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); - pos = sep + 1; + if (!hashRaw) { + hashRaw = splitPrefixTo(rest, '-'); + if (hashRaw) + isSRI = true; + } + if (hashRaw) + optParsedType = parseHashType(*hashRaw); } - init(); + return {optParsedType, isSRI}; +} + +Hash Hash::parseAnyPrefixed(std::string_view original) +{ + auto rest = original; + auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); + + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType) + throw BadHash("hash '%s' does not include a type", rest); + + return Hash(rest, *optParsedType, isSRI); +} - size_t size = s.size() - pos; +Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType) +{ + auto rest = original; + auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); + + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType && !optType) + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); + else if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); + + HashType hashType = optParsedType ? *optParsedType : *optType; + return Hash(rest, hashType, isSRI); +} - if (!isSRI && size == base16Len()) { +Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type) +{ + return Hash(s, type, false); +} + +Hash::Hash(std::string_view rest, HashType type, bool isSRI) + : Hash(type) +{ + if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'A' && c <= 'F') return c - 'A' + 10; if (c >= 'a' && c <= 'f') return c - 'a' + 10; - throw BadHash("invalid base-16 hash '%s'", s); + throw BadHash("invalid base-16 hash '%s'", rest); }; for (unsigned int i = 0; i < hashSize; i++) { hash[i] = - parseHexDigit(s[pos + i * 2]) << 4 - | parseHexDigit(s[pos + i * 2 + 1]); + parseHexDigit(rest[i * 2]) << 4 + | parseHexDigit(rest[i * 2 + 1]); } } - else if (!isSRI && size == base32Len()) { + else if (!isSRI && rest.size() == base32Len()) { - for (unsigned int n = 0; n < size; ++n) { - char c = s[pos + size - n - 1]; + for (unsigned int n = 0; n < rest.size(); ++n) { + char c = rest[rest.size() - n - 1]; unsigned char digit; for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", rest); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -200,21 +242,21 @@ Hash::Hash(std::string_view s, std::optional<HashType> type) hash[i + 1] |= digit >> (8 - j); } else { if (digit >> (8 - j)) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", rest); } } } - else if (isSRI || size == base64Len()) { - auto d = base64Decode(s.substr(pos)); + else if (isSRI || rest.size() == base64Len()) { + auto d = base64Decode(rest); if (d.size() != hashSize) - throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", rest); assert(hashSize); memcpy(hash, d.data(), hashSize); } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht) @@ -226,7 +268,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht) warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); return h; } else - return Hash(hashStr, ht); + return Hash::parseAny(hashStr, ht); } @@ -267,7 +309,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) } -Hash hashString(HashType ht, const string & s) +Hash hashString(HashType ht, std::string_view s) { Ctx ctx; Hash hash(ht); @@ -334,7 +376,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h; + Hash h(hash.type); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -342,7 +384,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional<HashType> parseHashTypeOpt(const string & s) +std::optional<HashType> parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -351,7 +393,7 @@ std::optional<HashType> parseHashTypeOpt(const string & s) else return std::optional<HashType> {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 23259dced..6d6eb70ca 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -18,6 +18,8 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; +extern std::set<std::string> hashTypes; + extern const string base32Chars; enum Base : int { Base64, Base32, Base16, SRI }; @@ -25,34 +27,38 @@ enum Base : int { Base64, Base32, Base16, SRI }; struct Hash { - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; - - std::optional<HashType> type = {}; + constexpr static size_t maxHashSize = 64; + size_t hashSize = 0; + uint8_t hash[maxHashSize] = {}; - /* Create an unset hash object. */ - Hash() { }; + HashType type; /* Create a zero-filled hash object. */ - Hash(HashType type) : type(type) { init(); }; + Hash(HashType type); - /* Initialize the hash from a string representation, in the format + /* Parse the hash from a string representation in the format "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a Subresource Integrity hash expression). If the 'type' argument is not present, then the hash type must be specified in the string. */ - Hash(std::string_view s, std::optional<HashType> type); - // type must be provided - Hash(std::string_view s, HashType type); - // hash type must be part of string - Hash(std::string_view s); + static Hash parseAny(std::string_view s, std::optional<HashType> type); + + /* Parse a hash from a string representation like the above, except the + type prefix is mandatory is there is no separate arguement. */ + static Hash parseAnyPrefixed(std::string_view s); + + /* Parse a plain hash that musst not have any prefix indicating the type. + The type is passed in to disambiguate. */ + static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); - void init(); + static Hash parseSRI(std::string_view original); - /* Check whether a hash is set. */ - operator bool () const { return (bool) type; } +private: + /* The type must be provided, the string view must not include <type> + prefix. `isSRI` helps disambigate the various base-* encodings. */ + Hash(std::string_view s, HashType type, bool isSRI); +public: /* Check whether two hash are equal. */ bool operator == (const Hash & h2) const; @@ -96,6 +102,8 @@ struct Hash assert(type == htSHA1); return std::string(to_string(Base16, false), 0, 7); } + + static Hash dummy; }; /* Helper that defaults empty hashes to the 0 hash. */ @@ -105,14 +113,14 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht); string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, std::string_view s); /* Compute the hash of the given file. */ Hash hashFile(HashType ht, const Path & path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ -typedef std::pair<Hash, unsigned long long> HashResult; +typedef std::pair<Hash, uint64_t> HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); @@ -121,9 +129,10 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); + /* Will return nothing on parse error */ -std::optional<HashType> parseHashTypeOpt(const string & s); +std::optional<HashType> parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); @@ -141,7 +150,7 @@ class HashSink : public BufferedSink, public AbstractHashSink private: HashType ht; Ctx * ctx; - unsigned long long bytes; + uint64_t bytes; public: HashSink(HashType ht); diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh deleted file mode 100644 index f7beac578..000000000 --- a/src/libutil/istringstream_nocopy.hh +++ /dev/null @@ -1,92 +0,0 @@ -/* This file provides a variant of std::istringstream that doesn't - copy its string argument. This is useful for large strings. The - caller must ensure that the string object is not destroyed while - it's referenced by this object. */ - -#pragma once - -#include <string> -#include <iostream> - -template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>> -class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits> -{ -public: - typedef std::basic_string<CharT, Traits, Allocator> string_type; - - typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type; - - typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type; - - typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type; - - typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type; - -private: - const string_type & s; - - off_type off; - -public: - basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0} - { - } - -private: - pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) - { - if (which & std::ios_base::in) { - this->off = dir == std::ios_base::beg - ? off - : (dir == std::ios_base::end - ? s.size() + off - : this->off + off); - } - return pos_type(this->off); - } - - pos_type seekpos(pos_type pos, std::ios_base::openmode which) - { - return seekoff(pos, std::ios_base::beg, which); - } - - std::streamsize showmanyc() - { - return s.size() - off; - } - - int_type underflow() - { - if (typename string_type::size_type(off) == s.size()) - return traits_type::eof(); - return traits_type::to_int_type(s[off]); - } - - int_type uflow() - { - if (typename string_type::size_type(off) == s.size()) - return traits_type::eof(); - return traits_type::to_int_type(s[off++]); - } - - int_type pbackfail(int_type ch) - { - if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) - return traits_type::eof(); - - return traits_type::to_int_type(s[--off]); - } - -}; - -template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>> -class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits> -{ - typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type; - buf_type buf; -public: - basic_istringstream_nocopy(const typename buf_type::string_type & s) : - std::basic_iostream<CharT, Traits>(&buf), buf(s) {}; -}; - -typedef basic_istringstream_nocopy<char> istringstream_nocopy; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 90c6afe81..8a6752e22 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -10,7 +10,7 @@ namespace nix { LoggerSettings loggerSettings; -static GlobalConfig::Register r1(&loggerSettings); +static GlobalConfig::Register rLoggerSettings(&loggerSettings); static thread_local ActivityId curActivity = 0; @@ -81,7 +81,6 @@ public: log(ei.level, oss.str()); } - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) @@ -185,6 +184,33 @@ struct JSONLogger : Logger { json["action"] = "msg"; json["level"] = ei.level; json["msg"] = oss.str(); + json["raw_msg"] = ei.hint->str(); + + if (ei.errPos.has_value() && (*ei.errPos)) { + json["line"] = ei.errPos->line; + json["column"] = ei.errPos->column; + json["file"] = ei.errPos->file; + } else { + json["line"] = nullptr; + json["column"] = nullptr; + json["file"] = nullptr; + } + + if (loggerSettings.showTrace.get() && !ei.traces.empty()) { + nlohmann::json traces = nlohmann::json::array(); + for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) { + nlohmann::json stackFrame; + stackFrame["raw_msg"] = iter->hint.str(); + if (iter->pos.has_value() && (*iter->pos)) { + stackFrame["line"] = iter->pos->line; + stackFrame["column"] = iter->pos->column; + stackFrame["file"] = iter->pos->file; + } + traces.push_back(stackFrame); + } + + json["trace"] = traces; + } write(json); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 09619aac6..63cb2b268 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -37,10 +37,12 @@ typedef uint64_t ActivityId; struct LoggerSettings : Config { - Setting<bool> showTrace{this, - false, - "show-trace", - "Whether to show a stack trace on evaluation errors."}; + Setting<bool> showTrace{ + this, false, "show-trace", + R"( + Where Nix should print out a stack trace in case of Nix + expression evaluation errors. + )"}; }; extern LoggerSettings loggerSettings; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c8b71188f..5c9f6f901 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -93,7 +93,7 @@ void Source::operator () (unsigned char * data, size_t len) } -std::string Source::drain() +void Source::drainInto(Sink & sink) { std::string s; std::vector<unsigned char> buf(8192); @@ -101,12 +101,19 @@ std::string Source::drain() size_t n; try { n = read(buf.data(), buf.size()); - s.append((char *) buf.data(), n); + sink(buf.data(), n); } catch (EndOfFile &) { break; } } - return s; +} + + +std::string Source::drain() +{ + StringSink s; + drainInto(s); + return *s.s; } @@ -259,6 +266,24 @@ Sink & operator << (Sink & sink, const StringSet & s) return sink; } +Sink & operator << (Sink & sink, const Error & ex) +{ + auto info = ex.info(); + sink + << "Error" + << info.level + << info.name + << info.description + << (info.hint ? info.hint->str() : "") + << 0 // FIXME: info.errPos + << info.traces.size(); + for (auto & trace : info.traces) { + sink << 0; // FIXME: trace.pos + sink << trace.hint.str(); + } + return sink; +} + void readPadding(size_t len, Source & source) { @@ -312,6 +337,30 @@ template Paths readStrings(Source & source); template PathSet readStrings(Source & source); +Error readError(Source & source) +{ + auto type = readString(source); + assert(type == "Error"); + ErrorInfo info; + info.level = (Verbosity) readInt(source); + info.name = readString(source); + info.description = readString(source); + auto hint = readString(source); + if (hint != "") info.hint = hintformat(std::move(format("%s") % hint)); + auto havePos = readNum<size_t>(source); + assert(havePos == 0); + auto nrTraces = readNum<size_t>(source); + for (size_t i = 0; i < nrTraces; ++i) { + havePos = readNum<size_t>(source); + assert(havePos == 0); + info.traces.push_back(Trace { + .hint = hintformat(std::move(format("%s") % readString(source))) + }); + } + return Error(std::move(info)); +} + + void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; @@ -322,5 +371,18 @@ void StringSink::operator () (const unsigned char * data, size_t len) s->append((const char *) data, len); } +size_t ChainSource::read(unsigned char * data, size_t len) +{ + if (useSecond) { + return source2.read(data, len); + } else { + try { + return source1.read(data, len); + } catch (EndOfFile &) { + useSecond = true; + return this->read(data, len); + } + } +} } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a04118512..d7fe0b81e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -22,8 +22,15 @@ struct Sink } }; +/* Just throws away data. */ +struct NullSink : Sink +{ + void operator () (const unsigned char * data, size_t len) override + { } +}; -/* A buffered abstract sink. */ +/* A buffered abstract sink. Warning: a BufferedSink should not be + used from multiple threads concurrently. */ struct BufferedSink : virtual Sink { size_t bufSize, bufPos; @@ -62,11 +69,14 @@ struct Source virtual bool good() { return true; } + void drainInto(Sink & sink); + std::string drain(); }; -/* A buffered abstract source. */ +/* A buffered abstract source. Warning: a BufferedSource should not be + used from multiple threads concurrently. */ struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; @@ -166,17 +176,30 @@ struct StringSource : Source }; -/* Adapter class of a Source that saves all data read to `s'. */ +/* A sink that writes all incoming data to two other sinks. */ +struct TeeSink : Sink +{ + Sink & sink1, & sink2; + TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } + virtual void operator () (const unsigned char * data, size_t len) + { + sink1(data, len); + sink2(data, len); + } +}; + + +/* Adapter class of a Source that saves all data read to a sink. */ struct TeeSource : Source { Source & orig; - ref<std::string> data; - TeeSource(Source & orig) - : orig(orig), data(make_ref<std::string>()) { } + Sink & sink; + TeeSource(Source & orig, Sink & sink) + : orig(orig), sink(sink) { } size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - this->data->append((const char *) data, n); + sink(data, n); return n; } }; @@ -212,6 +235,17 @@ struct SizedSource : Source } }; +/* A sink that that just counts the number of bytes given to it */ +struct LengthSink : Sink +{ + uint64_t length = 0; + + virtual void operator () (const unsigned char * _, size_t len) + { + length += len; + } +}; + /* Convert a function into a sink. */ struct LambdaSink : Sink { @@ -243,6 +277,19 @@ struct LambdaSource : Source } }; +/* Chain two sources together so after the first is exhausted, the second is + used */ +struct ChainSource : Source +{ + Source & source1, & source2; + bool useSecond = false; + ChainSource(Source & s1, Source & s2) + : source1(s1), source2(s2) + { } + + size_t read(unsigned char * data, size_t len) override; +}; + /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ @@ -274,6 +321,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) Sink & operator << (Sink & sink, const string & s); Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); +Sink & operator << (Sink & in, const Error & ex); MakeError(SerialisationError, Error); @@ -286,16 +334,16 @@ T readNum(Source & source) source(buf, sizeof(buf)); uint64_t n = - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); - - if (n > std::numeric_limits<T>::max()) + ((uint64_t) buf[0]) | + ((uint64_t) buf[1] << 8) | + ((uint64_t) buf[2] << 16) | + ((uint64_t) buf[3] << 24) | + ((uint64_t) buf[4] << 32) | + ((uint64_t) buf[5] << 40) | + ((uint64_t) buf[6] << 48) | + ((uint64_t) buf[7] << 56); + + if (n > (uint64_t)std::numeric_limits<T>::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; @@ -335,5 +383,119 @@ Source & operator >> (Source & in, bool & b) return in; } +Error readError(Source & source); + + +/* An adapter that converts a std::basic_istream into a source. */ +struct StreamToSourceAdapter : Source +{ + std::shared_ptr<std::basic_istream<char>> istream; + + StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream) + : istream(istream) + { } + + size_t read(unsigned char * data, size_t len) override + { + if (!istream->read((char *) data, len)) { + if (istream->eof()) { + if (istream->gcount() == 0) + throw EndOfFile("end of file"); + } else + throw Error("I/O error in StreamToSourceAdapter"); + } + return istream->gcount(); + } +}; + + +/* A source that reads a distinct format of concatenated chunks back into its + logical form, in order to guarantee a known state to the original stream, + even in the event of errors. + + Use with FramedSink, which also allows the logical stream to be terminated + in the event of an exception. +*/ +struct FramedSource : Source +{ + Source & from; + bool eof = false; + std::vector<unsigned char> pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector<unsigned char> data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector<unsigned char>(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } +}; + +/* Write as chunks in the format expected by FramedSource. + + The exception_ptr reference can be used to terminate the stream when you + detect that an error has occurred on the remote end. +*/ +struct FramedSink : nix::BufferedSink +{ + BufferedSink & to; + std::exception_ptr & ex; + + FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex) + { } + + ~FramedSink() + { + try { + to << 0; + to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + to << len; + to(data, len); + }; +}; + } diff --git a/src/libutil/split.hh b/src/libutil/split.hh new file mode 100644 index 000000000..87a23b13e --- /dev/null +++ b/src/libutil/split.hh @@ -0,0 +1,33 @@ +#pragma once + +#include <optional> +#include <string_view> + +#include "util.hh" + +namespace nix { + +// If `separator` is found, we return the portion of the string before the +// separator, and modify the string argument to contain only the part after the +// separator. Otherwise, we return `std::nullopt`, and we leave the argument +// string alone. +static inline std::optional<std::string_view> splitPrefixTo(std::string_view & string, char separator) { + auto sepInstance = string.find(separator); + + if (sepInstance != std::string_view::npos) { + auto prefix = string.substr(0, sepInstance); + string.remove_prefix(sepInstance+1); + return prefix; + } + + return std::nullopt; +} + +static inline bool splitPrefix(std::string_view & string, std::string_view prefix) { + bool res = hasPrefix(string, prefix); + if (res) + string.remove_prefix(prefix.length()); + return res; +} + +} diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 74c59fd31..c7777a21f 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -1,9 +1,9 @@ -#include "json.hh" #include "config.hh" #include "args.hh" #include <sstream> #include <gtest/gtest.h> +#include <nlohmann/json.hpp> namespace nix { @@ -33,7 +33,7 @@ namespace nix { const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, ""); - ASSERT_EQ(iter->second.description, "description"); + ASSERT_EQ(iter->second.description, "description\n"); } TEST(Config, getDefinedOverridenSettingNotSet) { @@ -59,7 +59,7 @@ namespace nix { const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, "value"); - ASSERT_EQ(iter->second.description, "description"); + ASSERT_EQ(iter->second.description, "description\n"); } TEST(Config, getDefinedSettingSet2) { @@ -73,7 +73,7 @@ namespace nix { const auto e = settings.find("name-of-the-setting"); ASSERT_NE(e, settings.end()); ASSERT_EQ(e->second.value, "value"); - ASSERT_EQ(e->second.description, "description"); + ASSERT_EQ(e->second.description, "description\n"); } TEST(Config, addSetting) { @@ -152,29 +152,16 @@ namespace nix { } TEST(Config, toJSONOnEmptyConfig) { - std::stringstream out; - { // Scoped to force the destructor of JSONObject to write the final `}` - JSONObject obj(out); - Config config; - config.toJSON(obj); - } - - ASSERT_EQ(out.str(), "{}"); + ASSERT_EQ(Config().toJSON().dump(), "{}"); } TEST(Config, toJSONOnNonEmptyConfig) { - std::stringstream out; - { // Scoped to force the destructor of JSONObject to write the final `}` - JSONObject obj(out); - - Config config; - std::map<std::string, Config::SettingInfo> settings; - Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; - setting.assign("value"); + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + setting.assign("value"); - config.toJSON(obj); - } - ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#"); + ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#"); } TEST(Config, setSettingAlias) { diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index ef22e9966..7e53f17c6 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -34,6 +34,24 @@ namespace nix { } } + TEST(logEI, jsonOutput) { + SymbolTable testTable; + auto problem_file = testTable.create("random.nix"); + testing::internal::CaptureStderr(); + + makeJSONLogger(*logger)->logEI({ + .name = "error name", + .description = "error without any code lines.", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .errPos = Pos(foFile, problem_file, 02, 13) + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); + } + TEST(logEI, appendingHintsToPreviousError) { MakeError(TestError, Error); @@ -251,18 +269,19 @@ namespace nix { TEST(addTrace, showTracesWithShowTrace) { SymbolTable testTable; auto problem_file = testTable.create(test_file); - auto oneliner_file = testTable.create(one_liner); + auto invalidfilename = testTable.create("invalid filename"); auto e = AssertionError(ErrorInfo { .name = "wat", - .description = "a well-known problem occurred", + .description = "show-traces", .hint = hintfmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42); e.addTrace(std::nullopt, "while doing something without a %1%", "pos"); + e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file"); testing::internal::CaptureStderr(); @@ -271,24 +290,25 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\na well-known problem occurred\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); } TEST(addTrace, hideTracesWithoutShowTrace) { SymbolTable testTable; auto problem_file = testTable.create(test_file); - auto oneliner_file = testTable.create(one_liner); + auto invalidfilename = testTable.create("invalid filename"); auto e = AssertionError(ErrorInfo { .name = "wat", - .description = "a well-known problem occurred", + .description = "hide traces", .hint = hintfmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42); e.addTrace(std::nullopt, "while doing something without a %1%", "pos"); + e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file"); testing::internal::CaptureStderr(); @@ -297,9 +317,10 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\na well-known problem occurred\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n"); } + /* ---------------------------------------------------------------------------- * hintfmt * --------------------------------------------------------------------------*/ diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh new file mode 100644 index 000000000..7418be5e0 --- /dev/null +++ b/src/libutil/topo-sort.hh @@ -0,0 +1,42 @@ +#pragma once + +#include "error.hh" + +namespace nix { + +template<typename T> +std::vector<T> topoSort(std::set<T> items, + std::function<std::set<T>(const T &)> getChildren, + std::function<Error(const T &, const T &)> makeCycleError) +{ + std::vector<T> sorted; + std::set<T> visited, parents; + + std::function<void(const T & path, const T * parent)> dfsVisit; + + dfsVisit = [&](const T & path, const T * parent) { + if (parents.count(path)) throw makeCycleError(path, *parent); + + if (!visited.insert(path).second) return; + parents.insert(path); + + std::set<T> references = getChildren(path); + + for (auto & i : references) + /* Don't traverse into items that don't exist in our starting set. */ + if (i != path && items.count(i)) + dfsVisit(i, &path); + + sorted.push_back(path); + parents.erase(path); + }; + + for (auto & i : items) + dfsVisit(i, nullptr); + + std::reverse(sorted.begin(), sorted.end()); + + return sorted; +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 3af485fa0..55d02bcf9 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -24,6 +24,8 @@ typedef string Path; typedef list<Path> Paths; typedef set<Path> PathSet; +typedef vector<std::pair<string, string>> Headers; + /* Helper class to run code at startup. */ template<typename T> struct OnStartup diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh new file mode 100644 index 000000000..68be15cb0 --- /dev/null +++ b/src/libutil/url-parts.hh @@ -0,0 +1,44 @@ +#pragma once + +#include <string> +#include <regex> + +namespace nix { + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 88c09eef9..c1bab866c 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,4 +1,5 @@ #include "url.hh" +#include "url-parts.hh" #include "util.hh" namespace nix { diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 2ef88ef2a..6e77142e3 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -2,8 +2,6 @@ #include "error.hh" -#include <regex> - namespace nix { struct ParsedURL @@ -29,40 +27,4 @@ std::map<std::string, std::string> decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -// URI stuff. -const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+]+)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; -const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; -const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; -const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; -const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; -const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; -const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; -const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; -const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; -const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; -const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; - -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check -extern std::regex refRegex; - -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition -const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; -extern std::regex badGitRefRegex; - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -extern std::regex revRegex; - -// A ref or revision, or a ref followed by a revision. -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1268b146a..f07e99885 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -23,6 +23,7 @@ #include <sys/types.h> #include <sys/socket.h> #include <sys/wait.h> +#include <sys/time.h> #include <sys/un.h> #include <unistd.h> @@ -79,7 +80,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv) } -Path absPath(Path path, std::optional<Path> dir) +Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks) { if (path[0] != '/') { if (!dir) { @@ -100,7 +101,7 @@ Path absPath(Path path, std::optional<Path> dir) } path = *dir + "/" + path; } - return canonPath(path); + return canonPath(path, resolveSymlinks); } @@ -345,7 +346,6 @@ void writeFile(const Path & path, Source & source, mode_t mode) } } - string readLine(int fd) { string s; @@ -374,7 +374,7 @@ void writeLine(int fd, string s) } -static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) { checkInterrupt(); @@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by } } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(const Path & path, uint64_t & bytesFreed) { Path dir = dirOf(path); if (dir == "") @@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) void deletePath(const Path & path) { - unsigned long long dummy; + uint64_t dummy; deletePath(path, dummy); } -void deletePath(const Path & path, unsigned long long & bytesFreed) +void deletePath(const Path & path, uint64_t & bytesFreed) { //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); bytesFreed = 0; @@ -494,6 +494,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) { Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); @@ -581,20 +582,31 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp); + createSymlink(target, tmp, mtime); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; @@ -1006,12 +1018,14 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) return res; } - +// Output = "standard out" output stream string runProgram(Path program, bool searchPath, const Strings & args, const std::optional<std::string> & input) { RunOptions opts(program, args); opts.searchPath = searchPath; + // This allows you to refer to a program with a pathname relative to the + // PATH variable. opts.input = input; auto res = runProgram(opts); @@ -1022,6 +1036,7 @@ string runProgram(Path program, bool searchPath, const Strings & args, return res.second; } +// Output = error code + "standard out" output stream std::pair<int, std::string> runProgram(const RunOptions & options_) { RunOptions options(options_); @@ -1094,6 +1109,8 @@ void runProgram2(const RunOptions & options) if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. else execv(options.program.c_str(), stringsToCharPtrs(args_).data()); @@ -1433,7 +1450,7 @@ string base64Decode(std::string_view s) char digit = decode[(unsigned char) c]; if (digit == -1) - throw Error("invalid character in Base64 string"); + throw Error("invalid character in Base64 string: '%c'", c); bits += 6; d = d << 6 | digit; @@ -1447,6 +1464,47 @@ string base64Decode(std::string_view s) } +std::string stripIndentation(std::string_view s) +{ + size_t minIndent = 10000; + size_t curIndent = 0; + bool atStartOfLine = true; + + for (auto & c : s) { + if (atStartOfLine && c == ' ') + curIndent++; + else if (c == '\n') { + if (atStartOfLine) + minIndent = std::max(minIndent, curIndent); + curIndent = 0; + atStartOfLine = true; + } else { + if (atStartOfLine) { + minIndent = std::min(minIndent, curIndent); + atStartOfLine = false; + } + } + } + + std::string res; + + size_t pos = 0; + while (pos < s.size()) { + auto eol = s.find('\n', pos); + if (eol == s.npos) eol = s.size(); + if (eol - pos > minIndent) + res.append(s.substr(pos + minIndent, eol - pos - minIndent)); + res.push_back('\n'); + pos = eol + 1; + } + + return res; +} + + +////////////////////////////////////////////////////////////////////// + + static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; @@ -1565,7 +1623,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) struct sockaddr_un addr; addr.sun_family = AF_UNIX; - if (path.size() >= sizeof(addr.sun_path)) + if (path.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%1%' is too long", path); strcpy(addr.sun_path, path.c_str()); @@ -1583,4 +1641,11 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) return fdSocket; } + +string showBytes(uint64_t bytes) +{ + return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); +} + + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3641daaec..129d59a97 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -12,13 +12,9 @@ #include <signal.h> #include <functional> -#include <limits> -#include <cstdio> #include <map> #include <sstream> #include <optional> -#include <future> -#include <iterator> #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -49,7 +45,9 @@ void clearEnv(); /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, std::optional<Path> dir = {}); +Path absPath(Path path, + std::optional<Path> dir = {}, + bool resolveSymlinks = false); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink @@ -123,7 +121,7 @@ void writeLine(int fd, string s); second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); -void deletePath(const Path & path, unsigned long long & bytesFreed); +void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); @@ -147,10 +145,12 @@ Path getDataDir(); Paths createDirs(const Path & path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Wrappers arount read()/write() that read/write exactly the @@ -460,6 +460,12 @@ string base64Encode(std::string_view s); string base64Decode(std::string_view s); +/* Remove common leading whitespace from the lines in the string + 's'. For example, if every line is indented by at least 3 spaces, + then we remove 3 spaces from the start of every line. */ +std::string stripIndentation(std::string_view s); + + /* Get a value for the specified key from an associate container. */ template <class T> std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key) @@ -470,43 +476,8 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_ } -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future<T> to pass the value or - exception.) */ template<typename T> -class Callback -{ - std::function<void(std::future<T>)> fun; - std::atomic_flag done = ATOMIC_FLAG_INIT; - -public: - - Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } - - Callback(Callback && callback) : fun(std::move(callback.fun)) - { - auto prev = callback.done.test_and_set(); - if (prev) done.test_and_set(); - } - - void operator()(T && t) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise<T> promise; - promise.set_value(std::move(t)); - fun(promise.get_future()); - } - - void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise<T> promise; - promise.set_exception(exc); - fun(promise.get_future()); - } -}; +class Callback; /* Start a thread that handles various signals. Also block those signals @@ -597,4 +568,12 @@ constexpr auto enumerate(T && iterable) } +// C++17 std::visit boilerplate +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + + +std::string showBytes(uint64_t bytes); + + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f77de56ea..a79b1086b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -67,7 +67,7 @@ std::vector<string> shellwords(const string & s) return res; } -static void _main(int argc, char * * argv) +static void main_nix_build(int argc, char * * argv) { auto dryRun = false; auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); @@ -98,8 +98,8 @@ static void _main(int argc, char * * argv) // List of environment variables kept for --pure std::set<string> keepVars{ - "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", - "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", + "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", + "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" }; @@ -174,7 +174,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--run-env") // obsolete runEnv = true; - else if (*arg == "--command" || *arg == "--run") { + else if (runEnv && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; @@ -192,7 +192,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (*arg == "--packages" || *arg == "-p") + else if (runEnv && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { @@ -325,7 +325,7 @@ static void _main(int argc, char * * argv) auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -446,7 +446,7 @@ static void _main(int argc, char * * argv) "PATH=%4%:\"$PATH\"; " "SHELL=%5%; " "set +e; " - R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " @@ -487,52 +487,58 @@ static void _main(int argc, char * * argv) std::vector<StorePathWithOutputs> pathsToBuild; - std::map<Path, Path> drvPrefixes; - std::map<Path, Path> resultSymlinks; - std::vector<Path> outPaths; + std::map<StorePath, std::pair<size_t, StringSet>> drvMap; for (auto & drvInfo : drvs) { - auto drvPath = drvInfo.queryDrvPath(); - auto outPath = drvInfo.queryOutPath(); + auto drvPath = store->parseStorePath(drvInfo.queryDrvPath()); auto outputName = drvInfo.queryOutputName(); if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); + throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}}); + pathsToBuild.push_back({drvPath, {outputName}}); - std::string drvPrefix; - auto i = drvPrefixes.find(drvPath); - if (i != drvPrefixes.end()) - drvPrefix = i->second; + auto i = drvMap.find(drvPath); + if (i != drvMap.end()) + i->second.second.insert(outputName); else { - drvPrefix = outLink; - if (drvPrefixes.size()) - drvPrefix += fmt("-%d", drvPrefixes.size() + 1); - drvPrefixes[drvPath] = drvPrefix; + drvMap[drvPath] = {drvMap.size(), {outputName}}; } - - std::string symlink = drvPrefix; - if (outputName != "out") symlink += "-" + outputName; - - resultSymlinks[symlink] = outPath; - outPaths.push_back(outPath); } buildPaths(pathsToBuild); if (dryRun) return; - for (auto & symlink : resultSymlinks) - if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) - store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); + std::vector<StorePath> outPaths; + + for (auto & [drvPath, info] : drvMap) { + auto & [counter, wantedOutputs] = info; + std::string drvPrefix = outLink; + if (counter) + drvPrefix += fmt("-%d", counter + 1); + + auto builtOutputs = store->queryDerivationOutputMap(drvPath); + + for (auto & outputName : wantedOutputs) { + auto outputPath = builtOutputs.at(outputName); + + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { + std::string symlink = drvPrefix; + if (outputName != "out") symlink += "-" + outputName; + store2->addPermRoot(outputPath, absPath(symlink)); + } + + outPaths.push_back(outputPath); + } + } logger->stop(); for (auto & path : outPaths) - std::cout << path << '\n'; + std::cout << store->printStorePath(path) << '\n'; } } -static RegisterLegacyCommand s1("nix-build", _main); -static RegisterLegacyCommand s2("nix-shell", _main); +static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); +static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 3ccf620c9..309970df6 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -76,6 +76,13 @@ static void update(const StringSet & channelNames) auto store = openStore(); + auto [fd, unpackChannelPath] = createTempFile(); + writeFull(fd.get(), + #include "unpack-channel.nix.gen.hh" + ); + fd = -1; + AutoDelete del(unpackChannelPath, false); + // Download each channel. Strings exprs; for (const auto & channel : channels) { @@ -104,7 +111,7 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -125,7 +132,7 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. std::cerr << "unpacking channels...\n"; - Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" }; + Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); @@ -146,7 +153,7 @@ static void update(const StringSet & channelNames) replaceSymlink(profile, channelLink); } -static int _main(int argc, char ** argv) +static int main_nix_channel(int argc, char ** argv) { { // Figure out the name of the `.nix-channels' file to use @@ -243,4 +250,4 @@ static int _main(int argc, char ** argv) } } -static RegisterLegacyCommand s1("nix-channel", _main); +static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); diff --git a/src/nix-channel/unpack-channel.nix b/src/nix-channel/unpack-channel.nix new file mode 100644 index 000000000..10515bc8b --- /dev/null +++ b/src/nix-channel/unpack-channel.nix @@ -0,0 +1,12 @@ +{ name, channelName, src }: + +derivation { + builder = "builtin:unpack-channel"; + + system = "builtin"; + + inherit name channelName src; + + # No point in doing this remotely. + preferLocalBuild = true; +} diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index aa5ada3a6..57092b887 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -49,7 +49,7 @@ void removeOldGenerations(std::string dir) } } -static int _main(int argc, char * * argv) +static int main_nix_collect_garbage(int argc, char * * argv) { { bool removeOld = false; @@ -67,10 +67,8 @@ static int _main(int argc, char * * argv) deleteOlderThan = getArg(*arg, arg, end); } else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--max-freed") { - long long maxFreed = getIntArg<long long>(*arg, arg, end, true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*arg == "--max-freed") + options.maxFreed = std::max(getIntArg<int64_t>(*arg, arg, end, true), (int64_t) 0); else return false; return true; @@ -94,4 +92,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-collect-garbage", _main); +static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index b10184718..10990f7b5 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -4,7 +4,7 @@ using namespace nix; -static int _main(int argc, char ** argv) +static int main_nix_copy_closure(int argc, char ** argv) { { auto gzip = false; @@ -65,4 +65,4 @@ static int _main(int argc, char ** argv) } } -static RegisterLegacyCommand s1("nix-copy-closure", _main); +static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index bcb86cbce..fc6195cf0 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -1,5 +1,6 @@ #include "shared.hh" #include "local-store.hh" +#include "remote-store.hh" #include "util.hh" #include "serialise.hh" #include "archive.hh" @@ -239,7 +240,15 @@ static void daemonLoop(char * * argv) // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + store.createUser(user, peer.uid); + }); exit(0); }, options); @@ -256,7 +265,7 @@ static void daemonLoop(char * * argv) } -static int _main(int argc, char * * argv) +static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; @@ -277,44 +286,28 @@ static int _main(int argc, char * * argv) initPlugins(); if (stdio) { - if (getStoreType() == tDaemon) { - // Forward on this connection to the real daemon - auto socketPath = settings.nixDaemonSocketFile; - auto s = socket(PF_UNIX, SOCK_STREAM, 0); - if (s == -1) - throw SysError("creating Unix domain socket"); - - auto socketDir = dirOf(socketPath); - if (chdir(socketDir.c_str()) == -1) - throw SysError("changing to socket directory '%1%'", socketDir); - - auto socketName = std::string(baseNameOf(socketPath)); - auto addr = sockaddr_un{}; - addr.sun_family = AF_UNIX; - if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket name %1% is too long", socketName); - strcpy(addr.sun_path, socketName.c_str()); - - if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at %1%", socketPath); - - auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; + if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; while (true) { fd_set fds; FD_ZERO(&fds); - FD_SET(s, &fds); + FD_SET(from, &fds); FD_SET(STDIN_FILENO, &fds); if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) throw SysError("waiting for data from client or server"); - if (FD_ISSET(s, &fds)) { - auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from daemon socket to stdout"); else if (res == 0) throw EndOfFile("unexpected EOF from daemon socket"); } if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) @@ -324,7 +317,10 @@ static int _main(int argc, char * * argv) } else { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limitting access to thoses is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); } } else { daemonLoop(argv); @@ -334,4 +330,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-daemon", _main); +static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c992b7d74..e6667e7f5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, { DrvNames selectors = drvNamesFromArgs(args); if (selectors.empty()) - selectors.push_back(DrvName("*")); + selectors.emplace_back("*"); DrvInfos elems; set<unsigned int> done; @@ -381,7 +381,8 @@ static void queryInstSources(EvalState & state, if (path.isDerivation()) { elem.setDrvPath(state.store->printStorePath(path)); - elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); + auto outputs = state.store->queryDerivationOutputMap(path); + elem.setOutPath(state.store->printStorePath(outputs.at("out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); @@ -707,7 +708,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) } debug(format("switching to new user environment")); - Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, drv.queryOutPath()); + Path generation = createGeneration( + ref<LocalFSStore>(store2), globals.profile, + store2->parseStorePath(drv.queryOutPath())); switchLink(globals.profile, generation); } @@ -1208,18 +1211,17 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) } -static const int prevGen = -2; +static constexpr GenerationNumber prevGen = std::numeric_limits<GenerationNumber>::max(); -static void switchGeneration(Globals & globals, int dstGen) +static void switchGeneration(Globals & globals, GenerationNumber dstGen) { PathLocks lock; lockProfile(lock, globals.profile); - int curGen; - Generations gens = findGenerations(globals.profile, curGen); + auto [gens, curGen] = findGenerations(globals.profile); - Generation dst; + std::optional<Generation> dst; for (auto & i : gens) if ((dstGen == prevGen && i.number < curGen) || (dstGen >= 0 && i.number == dstGen)) @@ -1227,18 +1229,16 @@ static void switchGeneration(Globals & globals, int dstGen) if (!dst) { if (dstGen == prevGen) - throw Error("no generation older than the current (%1%) exists", - curGen); + throw Error("no generation older than the current (%1%) exists", curGen.value_or(0)); else throw Error("generation %1% does not exist", dstGen); } - printInfo(format("switching from generation %1% to %2%") - % curGen % dst.number); + printInfo("switching from generation %1% to %2%", curGen.value_or(0), dst->number); if (globals.dryRun) return; - switchLink(globals.profile, dst.path); + switchLink(globals.profile, dst->path); } @@ -1249,7 +1249,7 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - int dstGen; + GenerationNumber dstGen; if (!string2Int(opArgs.front(), dstGen)) throw UsageError("expected a generation number"); @@ -1278,8 +1278,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs PathLocks lock; lockProfile(lock, globals.profile); - int curGen; - Generations gens = findGenerations(globals.profile, curGen); + auto [gens, curGen] = findGenerations(globals.profile); RunPager pager; @@ -1308,14 +1307,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - int max; + GenerationNumber max; if (!string2Int(str_max, max) || max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); } else { - std::set<unsigned int> gens; + std::set<GenerationNumber> gens; for (auto & i : opArgs) { - unsigned int n; + GenerationNumber n; if (!string2Int(i, n)) throw UsageError("invalid generation number '%1%'", i); gens.insert(n); @@ -1331,7 +1330,7 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -static int _main(int argc, char * * argv) +static int main_nix_env(int argc, char * * argv) { { Strings opFlags, opArgs; @@ -1461,4 +1460,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-env", _main); +static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 8e7f09e12..8c6c8af05 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -151,7 +151,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, } debug(format("switching to new user environment")); - Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); + Path generation = createGeneration(ref<LocalFSStore>(store2), profile, + store2->parseStorePath(topLevelOut)); switchLink(profile, generation); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index bf353677a..18a0049a6 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -20,7 +20,6 @@ using namespace nix; static Path gcRoot; static int rootNr = 0; -static bool indirectRoot = false; enum OutputKind { okPlain, okXML, okJSON }; @@ -71,11 +70,11 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (gcRoot == "") printGCWarning(); else { - Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; + Path rootName = absPath(gcRoot); if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); if (store2) - drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot); + drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName); } std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : "")); } @@ -84,7 +83,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, } -static int _main(int argc, char * * argv) +static int main_nix_instantiate(int argc, char * * argv) { { Strings files; @@ -127,7 +126,7 @@ static int _main(int argc, char * * argv) else if (*arg == "--add-root") gcRoot = getArg(*arg, arg, end); else if (*arg == "--indirect") - indirectRoot = true; + ; else if (*arg == "--xml") outputKind = okXML; else if (*arg == "--json") @@ -193,4 +192,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-instantiate", _main); +static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..3bdee55a7 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -48,7 +48,7 @@ string resolveMirrorUri(EvalState & state, string uri) } -static int _main(int argc, char * * argv) +static int main_nix_prefetch_url(int argc, char * * argv) { { HashType ht = htSHA256; @@ -57,6 +57,7 @@ static int _main(int argc, char * * argv) bool fromExpr = false; string attrPath; bool unpack = false; + bool executable = false; string name; struct MyArgs : LegacyArgs, MixEvalArgs @@ -81,6 +82,8 @@ static int _main(int argc, char * * argv) } else if (*arg == "--unpack") unpack = true; + else if (*arg == "--executable") + executable = true; else if (*arg == "--name") name = getArg(*arg, arg, end); else if (*arg != "" && arg->at(0) == '-') @@ -153,14 +156,15 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + std::optional<Hash> expectedHash; + Hash hash(ht); std::optional<StorePath> storePath; if (args.size() == 2) { - expectedHash = Hash(args[1], ht); + expectedHash = Hash::parseAny(args[1], ht); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); if (store->isValidPath(*storePath)) - hash = expectedHash; + hash = *expectedHash; else storePath.reset(); } @@ -174,7 +178,11 @@ static int _main(int argc, char * * argv) /* Download the file. */ { - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); if (!fd) throw SysError("creating temporary file '%s'", tmpFile); FdSink sink(fd.get()); @@ -200,22 +208,12 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - /* FIXME: inefficient; addToStore() will also hash - this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); - - if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error("hash mismatch for '%1%'", uri); - - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - /* Copy the file to the Nix store. FIXME: if RemoteStore - implemented addToStoreFromDump() and downloadFile() - supported a sink, we could stream the download directly - into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); + const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); } stopProgressBar(); @@ -231,4 +229,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-prefetch-url", _main); +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b948380bb..70927925d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -24,6 +24,9 @@ #endif +namespace nix_store { + + using namespace nix; using std::cin; using std::cout; @@ -34,7 +37,6 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs); static Path gcRoot; static int rootNr = 0; -static bool indirectRoot = false; static bool noOutput = false; static std::shared_ptr<Store> store; @@ -65,6 +67,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) if (path.path.isDerivation()) { if (build) store->buildPaths({path}); + auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -77,7 +80,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", store2->printStorePath(path.path), j); - auto outPath = store2->printStorePath(i->second.path); + auto outPath = outputPaths.at(i->first); + auto retPath = store->printStorePath(outPath); if (store2) { if (gcRoot == "") printGCWarning(); @@ -85,10 +89,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) Path rootName = gcRoot; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (i->first != "out") rootName += "-" + i->first; - outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot); + retPath = store2->addPermRoot(outPath, rootName); } } - outputs.insert(outPath); + outputs.insert(retPath); } return outputs; } @@ -104,7 +108,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) Path rootName = gcRoot; rootNr++; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); - return {store2->addPermRoot(path.path, rootName, indirectRoot)}; + return {store2->addPermRoot(path.path, rootName)}; } } return {store->printStorePath(path.path)}; @@ -130,7 +134,7 @@ static void opRealise(Strings opFlags, Strings opArgs) for (auto & i : opArgs) paths.push_back(store->followLinksToStorePathWithOutputs(i)); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -174,10 +178,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -187,7 +191,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); } @@ -208,7 +212,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); } @@ -218,8 +222,13 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, if (useOutput && storePath.isDerivation()) { auto drv = store->derivationFromPath(storePath); StorePathSet outputs; - for (auto & i : drv.outputs) - outputs.insert(i.second.path); + if (forceRealise) + return store->queryDerivationOutputs(storePath); + for (auto & i : drv.outputsAndOptPaths(*store)) { + if (!i.second.second) + throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)"); + outputs.insert(*i.second.second); + } return outputs; } else return {storePath}; @@ -309,11 +318,9 @@ static void opQuery(Strings opFlags, Strings opArgs) case qOutputs: { for (auto & i : opArgs) { - auto i2 = store->followLinksToStorePath(i); - if (forceRealise) realisePath({i2}); - Derivation drv = store->derivationFromPath(i2); - for (auto & j : drv.outputs) - cout << fmt("%1%\n", store->printStorePath(j.second.path)); + auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise); + for (auto & outputPath : outputs) + cout << fmt("%1%\n", store->printStorePath(outputPath)); } break; } @@ -495,7 +502,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) ValidPathInfos infos; while (1) { - auto info = decodeValidPathInfo(*store, cin, hashGiven); + // We use a dummy value because we'll set it below. FIXME be correct by + // construction and avoid dummy value. + auto hashResultOpt = !hashGiven ? std::optional<HashResult> { {Hash::dummy, -1} } : std::nullopt; + auto info = decodeValidPathInfo(*store, cin, hashResultOpt); if (!info) break; if (!store->isValidPath(info->path) || reregister) { /* !!! races */ @@ -572,10 +582,8 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--max-freed") { - long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*i == "--max-freed") + options.maxFreed = std::max(getIntArg<int64_t>(*i, i, opFlags.end(), true), (int64_t) 0); else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -671,7 +679,7 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - auto paths = store->importPaths(source, nullptr, NoCheckSigs); + auto paths = store->importPaths(source, NoCheckSigs); for (auto & i : paths) cout << fmt("%s\n", store->printStorePath(i)) << std::flush; @@ -725,7 +733,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(*info->narHash.type); + HashSink sink(info->narHash.type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -817,7 +825,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = readStorePaths<StorePathSet>(*store, in); + auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {}); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -831,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) if (!path.isDerivation()) paths2.push_back({path}); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths2, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -847,24 +855,26 @@ static void opServe(Strings opFlags, Strings opArgs) } } - writeStorePaths(*store, out, store->queryValidPaths(paths)); + worker_proto::write(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = readStorePaths<StorePathSet>(*store, in); + auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {}); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - writeStorePaths(*store, out, info->references); + worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; + out << info->narHash.to_string(Base32, true) + << renderContentAddress(info->ca) + << info->sigs; } catch (InvalidPath &) { } } @@ -878,14 +888,14 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? + store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(readStorePaths<StorePathSet>(*store, in), out); + store->exportPaths(worker_proto::read(*store, in, Phantom<StorePathSet> {}), out); break; } @@ -914,9 +924,9 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - auto drvPath = store->parseStorePath(readString(in)); // informational only + auto drvPath = store->parseStorePath(readString(in)); BasicDerivation drv; - readDerivation(in, *store, drv); + readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath)); getBuildSettings(); @@ -934,9 +944,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(readStorePaths<StorePathSet>(*store, in), + store->computeFSClosure(worker_proto::read(*store, in, Phantom<StorePathSet> {}), closure, false, includeOutputs); - writeStorePaths(*store, out, closure); + worker_proto::write(*store, out, closure); break; } @@ -944,12 +954,14 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("importing paths is not allowed"); auto path = readString(in); - ValidPathInfo info(store->parseStorePath(path)); auto deriver = readString(in); + ValidPathInfo info { + store->parseStorePath(path), + Hash::parseAny(readString(in), htSHA256), + }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths<StorePathSet>(*store, in); + info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); info.ca = parseContentAddressOpt(readString(in)); @@ -1016,7 +1028,7 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -static int _main(int argc, char * * argv) +static int main_nix_store(int argc, char * * argv) { { Strings opFlags, opArgs; @@ -1080,7 +1092,7 @@ static int _main(int argc, char * * argv) else if (*arg == "--add-root") gcRoot = absPath(getArg(*arg, arg, end)); else if (*arg == "--indirect") - indirectRoot = true; + ; else if (*arg == "--no-output") noOutput = true; else if (*arg != "" && arg->at(0) == '-') { @@ -1112,4 +1124,6 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-store", _main); +static RegisterLegacyCommand r_nix_store("nix-store", main_nix_store); + +} diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d6de16e..7fe87d757 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional<std::string> namePart; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "flat", + .shortName = 0, + .description = "add flat file to the Nix store", + .handler = {&ingestionMethod, FileIngestionMethod::Flat}, + }); } std::string description() override @@ -28,6 +36,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand return "add a path to the Nix store"; } + std::string doc() override + { + return R"( + Copy the file or directory *path* to the Nix store, and + print the resulting store path on standard output. + )"; + } + Examples examples() override { return { @@ -45,12 +61,21 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - info.narHash = narHash; + Hash hash = narHash; + if (ingestionMethod == FileIngestionMethod::Flat) { + HashSink hsink(htSHA256); + readFile(path, hsink); + hash = hsink.finish().first; + } + + ValidPathInfo info { + store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + narHash, + }; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, + .method = ingestionMethod, + .hash = hash, } }; if (!dryRun) { @@ -62,4 +87,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand<CmdAddToStore>("add-to-store"); +static auto rCmdAddToStore = registerCommand<CmdAddToStore>("add-to-store"); diff --git a/src/nix/app.cc b/src/nix/app.cc new file mode 100644 index 000000000..80acbf658 --- /dev/null +++ b/src/nix/app.cc @@ -0,0 +1,46 @@ +#include "installables.hh" +#include "store-api.hh" +#include "eval-inline.hh" +#include "eval-cache.hh" +#include "names.hh" + +namespace nix { + +App Installable::toApp(EvalState & state) +{ + auto [cursor, attrPath] = getCursor(state); + + auto type = cursor->getAttr("type")->getString(); + + if (type == "app") { + auto [program, context] = cursor->getAttr("program")->getStringWithContext(); + + if (!state.store->isInStore(program)) + throw Error("app program '%s' is not in the Nix store", program); + + std::vector<StorePathWithOutputs> context2; + for (auto & [path, name] : context) + context2.push_back({state.store->parseStorePath(path), {name}}); + + return App { + .context = std::move(context2), + .program = program, + }; + } + + else if (type == "derivation") { + auto drvPath = cursor->forceDerivation(); + auto outPath = cursor->getAttr(state.sOutPath)->getString(); + auto outputName = cursor->getAttr(state.sOutputName)->getString(); + auto name = cursor->getAttr(state.sName)->getString(); + return App { + .context = { { drvPath, {outputName} } }, + .program = outPath + "/bin/" + DrvName(name).name, + }; + } + + else + throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); +} + +} diff --git a/src/nix/build.cc b/src/nix/build.cc index 850e09ce8..d85a482db 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,3 +1,4 @@ +#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -8,6 +9,7 @@ using namespace nix; struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; + BuildMode buildMode = bmNormal; CmdBuild() { @@ -17,6 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile .description = "path of the symlink to the build result", .labels = {"path"}, .handler = {&outLink}, + .completer = completePath }); addFlag({ @@ -24,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile .description = "do not create a symlink to the build result", .handler = {&outLink, Path("")}, }); + + addFlag({ + .longName = "rebuild", + .description = "rebuild an already built package and compare the result to the existing store paths", + .handler = {&buildMode, bmCheck}, + }); } std::string description() override @@ -44,31 +53,39 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile }, Example{ "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs.hello" + "nix build --profile /tmp/profile nixpkgs#hello" }, }; } void run(ref<Store> store) override { - auto buildables = build(store, dryRun ? DryRun : Build, installables); + auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); if (dryRun) return; - if (outLink != "") { - for (size_t i = 0; i < buildables.size(); ++i) { - for (auto & output : buildables[i].outputs) - if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink), true); - } - } - } + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) + for (size_t i = 0; i < buildables.size(); ++i) + std::visit(overloaded { + [&](BuildableOpaque bo) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2->addPermRoot(bo.path, absPath(symlink)); + }, + [&](BuildableFromDrv bfd) { + auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); + for (auto & output : builtOutputs) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink)); + } + }, + }, buildables[i]); updateProfile(buildables); } }; -static auto r1 = registerCommand<CmdBuild>("build"); +static auto rCmdBuild = registerCommand<CmdBuild>("build"); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc new file mode 100644 index 000000000..fc41da9e4 --- /dev/null +++ b/src/nix/bundle.cc @@ -0,0 +1,129 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "fs-accessor.hh" + +using namespace nix; + +struct CmdBundle : InstallableCommand +{ + std::string bundler = "github:matthewbauer/nix-bundle"; + std::optional<Path> outLink; + + CmdBundle() + { + addFlag({ + .longName = "bundler", + .description = "use custom bundler", + .labels = {"flake-url"}, + .handler = {&bundler}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + } + + std::string description() override + { + return "bundle an application so that it works outside of the Nix store"; + } + + Examples examples() override + { + return { + Example{ + "To bundle Hello:", + "nix bundle hello" + }, + }; + } + + Category category() override { return catSecondary; } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref<Store> store) override + { + auto evalState = getEvalState(); + + auto app = installable->toApp(*evalState); + store->buildPaths(app.context); + + auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); + const flake::LockFlags lockFlags{ .writeLockFile = false }; + auto bundler = InstallableFlake( + evalState, std::move(bundlerFlakeRef), + Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, + Strings({"bundlers."}), lockFlags); + + Value * arg = evalState->allocValue(); + evalState->mkAttrs(*arg, 2); + + PathSet context; + for (auto & i : app.context) + context.insert("=" + store->printStorePath(i.path)); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); + + arg->attrs->sort(); + + auto vRes = evalState->allocValue(); + evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); + + if (!evalState->isDerivation(*vRes)) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + auto attr1 = vRes->attrs->get(evalState->sDrvPath); + if (!attr1) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); + + auto attr2 = vRes->attrs->get(evalState->sOutPath); + if (!attr2) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); + + store->buildPaths({{drvPath}}); + + auto outPathS = store->printStorePath(outPath); + + auto info = store->queryPathInfo(outPath); + if (!info->references.empty()) + throw Error("'%s' has references; a bundler must not leave any references", outPathS); + + if (!outLink) + outLink = baseNameOf(app.program); + + store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink)); + } +}; + +static auto r2 = registerCommand<CmdBundle>("bundle"); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index c82819af8..eef172cfc 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -25,7 +25,11 @@ struct CmdCatStore : StoreCommand, MixCat { CmdCatStore() { - expectArg("path", &path); + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); } std::string description() override @@ -47,7 +51,11 @@ struct CmdCatNar : StoreCommand, MixCat CmdCatNar() { - expectArg("nar", &narPath); + expectArgs({ + .label = "nar", + .handler = {&narPath}, + .completer = completePath + }); expectArg("path", &path); } @@ -64,5 +72,5 @@ struct CmdCatNar : StoreCommand, MixCat } }; -static auto r1 = registerCommand<CmdCatStore>("cat-store"); -static auto r2 = registerCommand<CmdCatNar>("cat-nar"); +static auto rCmdCatStore = registerCommand<CmdCatStore>("cat-store"); +static auto rCmdCatNar = registerCommand<CmdCatNar>("cat-nar"); diff --git a/src/nix/command.cc b/src/nix/command.cc index 3651a9e9c..ba7de9fdd 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -4,12 +4,25 @@ #include "nixexpr.hh" #include "profiles.hh" +#include <nlohmann/json.hpp> + extern char * * environ __attribute__((weak)); namespace nix { Commands * RegisterCommand::commands = nullptr; +void NixMultiCommand::printHelp(const string & programName, std::ostream & out) +{ + MultiCommand::printHelp(programName, out); +} + +nlohmann::json NixMultiCommand::toJSON() +{ + // FIXME: use Command::toJSON() as well. + return MultiCommand::toJSON(); +} + StoreCommand::StoreCommand() { } @@ -63,7 +76,7 @@ void StorePathsCommand::run(ref<Store> store) } else { - for (auto & p : toStorePaths(store, realiseMode, installables)) + for (auto & p : toStorePaths(store, realiseMode, operateOn, installables)) storePaths.push_back(p); if (recursive) { @@ -80,7 +93,7 @@ void StorePathsCommand::run(ref<Store> store) void StorePathCommand::run(ref<Store> store) { - auto storePaths = toStorePaths(store, NoBuild, installables); + auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables); if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); @@ -108,6 +121,7 @@ MixProfile::MixProfile() .description = "profile to update", .labels = {"path"}, .handler = {&profile}, + .completer = completePath }); } @@ -120,27 +134,35 @@ void MixProfile::updateProfile(const StorePath & storePath) switchLink(profile2, createGeneration( ref<LocalFSStore>(store), - profile2, store->printStorePath(storePath))); + profile2, storePath)); } void MixProfile::updateProfile(const Buildables & buildables) { if (!profile) return; - std::optional<StorePath> result; + std::vector<StorePath> result; for (auto & buildable : buildables) { - for (auto & output : buildable.outputs) { - if (result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second; - } + std::visit(overloaded { + [&](BuildableOpaque bo) { + result.push_back(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) { + /* Output path should be known because we just tried to + build it. */ + assert(output.second); + result.push_back(*output.second); + } + }, + }, buildable); } - if (!result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + if (result.size() != 1) + throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(*result); + updateProfile(result[0]); } MixDefaultProfile::MixDefaultProfile() diff --git a/src/nix/command.hh b/src/nix/command.hh index 959d5f19d..d60c8aeb6 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -4,16 +4,30 @@ #include "args.hh" #include "common-eval-args.hh" #include "path.hh" -#include "eval.hh" +#include "flake/lockfile.hh" +#include "store-api.hh" + +#include <optional> namespace nix { extern std::string programPath; +class EvalState; +struct Pos; +class Store; + static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; +struct NixMultiCommand : virtual MultiCommand, virtual Command +{ + void printHelp(const string & programName, std::ostream & out) override; + + nlohmann::json toJSON() override; +}; + /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { @@ -27,28 +41,64 @@ private: std::shared_ptr<Store> _store; }; -struct SourceExprCommand : virtual StoreCommand, MixEvalArgs +struct EvalCommand : virtual StoreCommand, MixEvalArgs +{ + ref<EvalState> getEvalState(); + + std::shared_ptr<EvalState> evalState; +}; + +struct MixFlakeOptions : virtual Args, EvalCommand +{ + flake::LockFlags lockFlags; + + MixFlakeOptions(); + + virtual std::optional<FlakeRef> getFlakeRefForCompletion() + { return {}; } +}; + +/* How to handle derivations in commands that operate on store paths. */ +enum class OperateOn { + /* Operate on the output path. */ + Output, + /* Operate on the .drv path. */ + Derivation +}; + +struct SourceExprCommand : virtual Args, MixFlakeOptions { - Path file; + std::optional<Path> file; + std::optional<std::string> expr; + + // FIXME: move this; not all commands (e.g. 'nix run') use it. + OperateOn operateOn = OperateOn::Output; SourceExprCommand(); - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); + std::vector<std::shared_ptr<Installable>> parseInstallables( + ref<Store> store, std::vector<std::string> ss); - ref<EvalState> getEvalState(); + std::shared_ptr<Installable> parseInstallable( + ref<Store> store, const std::string & installable); -private: + virtual Strings getDefaultFlakeAttrPaths(); - std::shared_ptr<EvalState> evalState; + virtual Strings getDefaultFlakeAttrPathPrefixes(); - RootValue vSourceExpr; + void completeInstallable(std::string_view prefix); }; -enum RealiseMode { Build, NoBuild, DryRun }; +enum class Realise { + /* Build the derivation. Postcondition: the + derivation outputs exist. */ + Outputs, + /* Don't build the derivation. Postcondition: the store derivation + exists. */ + Derivation, + /* Evaluate in dry-run mode. Postcondition: nothing. */ + Nothing +}; /* A command that operates on a list of "installables", which can be store paths, attribute paths, Nix expressions, etc. */ @@ -56,15 +106,14 @@ struct InstallablesCommand : virtual Args, SourceExprCommand { std::vector<std::shared_ptr<Installable>> installables; - InstallablesCommand() - { - expectArgs("installables", &_installables); - } + InstallablesCommand(); void prepare() override; virtual bool useDefaultInstallables() { return true; } + std::optional<FlakeRef> getFlakeRefForCompletion() override; + private: std::vector<std::string> _installables; @@ -75,16 +124,18 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr<Installable> installable; - InstallableCommand() - { - expectArg("installable", &_installable); - } + InstallableCommand(); void prepare() override; + std::optional<FlakeRef> getFlakeRefForCompletion() override + { + return parseFlakeRef(_installable, absPath(".")); + } + private: - std::string _installable; + std::string _installable{"."}; }; /* A command that operates on zero or more store paths. */ @@ -97,7 +148,7 @@ private: protected: - RealiseMode realiseMode = NoBuild; + Realise realiseMode = Realise::Derivation; public: @@ -141,17 +192,15 @@ static RegisterCommand registerCommand(const std::string & name) return RegisterCommand(name, [](){ return make_ref<T>(); }); } -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables); +Buildables build(ref<Store> store, Realise mode, + std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal); -Buildables build(ref<Store> store, RealiseMode mode, +std::set<StorePath> toStorePaths(ref<Store> store, + Realise mode, OperateOn operateOn, std::vector<std::shared_ptr<Installable>> installables); -std::set<StorePath> toStorePaths(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables); - -StorePath toStorePath(ref<Store> store, RealiseMode mode, +StorePath toStorePath(ref<Store> store, + Realise mode, OperateOn operateOn, std::shared_ptr<Installable> installable); std::set<StorePath> toDerivations(ref<Store> store, @@ -194,4 +243,19 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; +void completeFlakeRef(ref<Store> store, std::string_view prefix); + +void completeFlakeRefWithFragment( + ref<EvalState> evalState, + flake::LockFlags lockFlags, + Strings attrPathPrefixes, + const Strings & defaultFlakeAttrPaths, + std::string_view prefix); + +void printClosureDiff( + ref<Store> store, + const StorePath & beforePath, + const StorePath & afterPath, + std::string_view indent); + } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 64099f476..cb31aac8f 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -45,6 +45,8 @@ struct CmdCopy : StorePathsCommand .description = "whether to try substitutes on the destination store (only supported by SSH)", .handler = {&substitute, Substitute}, }); + + realiseMode = Realise::Outputs; } std::string description() override @@ -70,11 +72,11 @@ struct CmdCopy : StorePathsCommand #ifdef ENABLE_S3 Example{ "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello" }, Example{ "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello" }, #endif }; @@ -87,11 +89,16 @@ struct CmdCopy : StorePathsCommand return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); } - void run(ref<Store> srcStore, StorePaths storePaths) override + void run(ref<Store> store) override { if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); + StorePathsCommand::run(store); + } + + void run(ref<Store> srcStore, StorePaths storePaths) override + { ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), @@ -99,4 +106,4 @@ struct CmdCopy : StorePathsCommand } }; -static auto r1 = registerCommand<CmdCopy>("copy"); +static auto rCmdCopy = registerCommand<CmdCopy>("copy"); diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc new file mode 100644 index 000000000..1dd384c0e --- /dev/null +++ b/src/nix/describe-stores.cc @@ -0,0 +1,44 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +using namespace nix; + +struct CmdDescribeStores : Command, MixJSON +{ + std::string description() override + { + return "show registered store types and their available options"; + } + + Category category() override { return catUtility; } + + void run() override + { + auto res = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + res[storeName] = storeConfig->toJSON(); + } + if (json) { + std::cout << res; + } else { + for (auto & [storeName, storeConfig] : res.items()) { + std::cout << "## " << storeName << std::endl << std::endl; + for (auto & [optionName, optionDesc] : storeConfig.items()) { + std::cout << "### " << optionName << std::endl << std::endl; + std::cout << optionDesc["description"].get<std::string>() << std::endl; + std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl; + if (!optionDesc["aliases"].empty()) + std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl; + } + } + } + } +}; + +static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index eb93f56fc..a46ea39b6 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -15,7 +15,7 @@ struct Var { bool exported = true; bool associative = false; - std::string value; // quoted string or array + std::string quoted; // quoted string or array }; struct BuildEnvironment @@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path) std::smatch match; - if (std::regex_search(pos, file.cend(), match, declareRegex)) { + if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { pos = match[0].second; exported.insert(match[1]); } - else if (std::regex_search(pos, file.cend(), match, varRegex)) { + else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { pos = match[0].second; - res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { pos = match[0].second; - res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); + res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { res.bashFunctions = std::string(pos, file.cend()); break; } @@ -92,6 +92,8 @@ BuildEnvironment readEnvironment(const Path & path) path, file.substr(pos - file.cbegin(), 60)); } + res.env.erase("__output"); + return res; } @@ -124,27 +126,36 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ - auto drvName = std::string(drvPath.name()); - assert(hasSuffix(drvName, ".drv")); - drvName.resize(drvName.size() - 4); - drvName += "-env"; - for (auto & output : drv.outputs) - drv.env.erase(output.first); - drv.env["out"] = ""; - drv.env["outputs"] = "out"; + drv.name += "-env"; + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = hashDerivationModulo(*store, drv, true); - auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); - drv.env["out"] = store->printStorePath(shellOutPath); - auto shellDrvPath2 = writeDerivation(store, drv, drvName); + Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); - /* Build the derivation. */ - store->buildPaths({{shellDrvPath2}}); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } - assert(store->isValidPath(shellOutPath)); + auto shellDrvPath = writeDerivation(*store, drv); - return shellOutPath; + /* Build the derivation. */ + store->buildPaths({{shellDrvPath}}); + + for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { + auto & [_1, optPath] = outputAndOptPath; + assert(optPath); + auto & outPath = *optPath; + assert(store->isValidPath(outPath)); + auto outPathS = store->toRealPath(outPath); + if (lstat(outPathS).st_size) + return outPath; + } + + throw Error("get-env.sh failed to produce an environment"); } struct Common : InstallableCommand, MixProfile @@ -170,8 +181,12 @@ struct Common : InstallableCommand, MixProfile "UID", }; - void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + std::string makeRcScript( + const BuildEnvironment & buildEnvironment, + const Path & outputsDir = absPath(".") + "/outputs") { + std::ostringstream out; + out << "unset shellHook\n"; out << "nix_saved_PATH=\"$PATH\"\n"; @@ -179,9 +194,9 @@ struct Common : InstallableCommand, MixProfile for (auto & i : buildEnvironment.env) { if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { if (i.second.associative) - out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); + out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted); else { - out << fmt("%s=%s\n", i.first, i.second.value); + out << fmt("%s=%s\n", i.first, i.second.quoted); if (i.second.exported) out << fmt("export %s\n", i.first); } @@ -192,13 +207,31 @@ struct Common : InstallableCommand, MixProfile out << buildEnvironment.bashFunctions << "\n"; - // FIXME: set outputs - out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); out << "eval \"$shellHook\"\n"; + + /* Substitute occurrences of output paths. */ + auto outputs = buildEnvironment.env.find("outputs"); + assert(outputs != buildEnvironment.env.end()); + + // FIXME: properly unquote 'outputs'. + StringMap rewrites; + for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) { + auto from = buildEnvironment.env.find(outputName); + assert(from != buildEnvironment.env.end()); + // FIXME: unquote + rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); + } + + return rewriteStrings(out.str(), rewrites); + } + + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; } StorePath getShellOutPath(ref<Store> store) @@ -234,19 +267,57 @@ struct Common : InstallableCommand, MixProfile struct CmdDevelop : Common, MixEnvironment { std::vector<std::string> command; + std::optional<std::string> phase; CmdDevelop() { addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed insted of an interactive shell", + .description = "command and arguments to be executed instead of an interactive shell", .labels = {"command", "args"}, .handler = {[&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; }} }); + + addFlag({ + .longName = "phase", + .description = "phase to run (e.g. `build` or `configure`)", + .labels = {"phase-name"}, + .handler = {&phase}, + }); + + addFlag({ + .longName = "configure", + .description = "run the configure phase", + .handler = {&phase, {"configure"}}, + }); + + addFlag({ + .longName = "build", + .description = "run the build phase", + .handler = {&phase, {"build"}}, + }); + + addFlag({ + .longName = "check", + .description = "run the check phase", + .handler = {&phase, {"check"}}, + }); + + addFlag({ + .longName = "install", + .description = "run the install phase", + .handler = {&phase, {"install"}}, + }); + + addFlag({ + .longName = "installcheck", + .description = "run the installcheck phase", + .handler = {&phase, {"installCheck"}}, + }); } std::string description() override @@ -259,11 +330,15 @@ struct CmdDevelop : Common, MixEnvironment return { Example{ "To get the build environment of GNU hello:", - "nix develop nixpkgs.hello" + "nix develop nixpkgs#hello" + }, + Example{ + "To get the build environment of the default package of flake in the current directory:", + "nix develop" }, Example{ "To store the build environment in a profile:", - "nix develop --profile /tmp/my-shell nixpkgs.hello" + "nix develop --profile /tmp/my-shell nixpkgs#hello" }, Example{ "To use a build environment previously recorded in a profile:", @@ -278,28 +353,56 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - std::ostringstream ss; - makeRcScript(buildEnvironment, ss); + auto script = makeRcScript(buildEnvironment); + + if (verbosity >= lvlDebug) + script += "set -x\n"; - ss << fmt("rm -f '%s'\n", rcFilePath); + script += fmt("rm -f '%s'\n", rcFilePath); - if (!command.empty()) { + if (phase) { + if (!command.empty()) + throw UsageError("you cannot use both '--command' and '--phase'"); + // FIXME: foundMakefile is set by buildPhase, need to get + // rid of that. + script += fmt("foundMakefile=1\n"); + script += fmt("runHook %1%Phase\n", *phase); + script += fmt("exit 0\n", *phase); + } + + else if (!command.empty()) { std::vector<std::string> args; for (auto s : command) args.push_back(shellEscape(s)); - ss << fmt("exec %s\n", concatStringsSep(" ", args)); + script += fmt("exec %s\n", concatStringsSep(" ", args)); } - writeFull(rcFileFd.get(), ss.str()); + writeFull(rcFileFd.get(), script); stopProgressBar(); - auto shell = getEnv("SHELL").value_or("bash"); - setEnviron(); // prevent garbage collection until shell exits setenv("NIX_GCROOT", gcroot.data(), 1); + Path shell = "bash"; + + try { + auto state = getEvalState(); + + auto bashInstallable = std::make_shared<InstallableFlake>( + state, + installable->nixpkgsFlakeRef(), + Strings{"bashInteractive"}, + Strings{"legacyPackages." + settings.thisSystem.get() + "."}, + lockFlags); + + shell = state->store->printStorePath( + toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; + } catch (Error &) { + ignoreException(); + } + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); @@ -323,7 +426,7 @@ struct CmdPrintDevEnv : Common return { Example{ "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs.hello)" + ". <(nix print-dev-env nixpkgs#hello)" }, }; } @@ -336,9 +439,9 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - makeRcScript(buildEnvironment, std::cout); + std::cout << makeRcScript(buildEnvironment); } }; -static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env"); -static auto r2 = registerCommand<CmdDevelop>("develop"); +static auto rCmdPrintDevEnv = registerCommand<CmdPrintDevEnv>("print-dev-env"); +static auto rCmdDevelop = registerCommand<CmdDevelop>("develop"); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc new file mode 100644 index 000000000..30e7b20e1 --- /dev/null +++ b/src/nix/diff-closures.cc @@ -0,0 +1,146 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "common-args.hh" +#include "names.hh" + +#include <regex> + +namespace nix { + +struct Info +{ + std::string outputName; +}; + +// name -> version -> store paths +typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths; + +GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel) +{ + StorePathSet closure; + store->computeFSClosure({toplevel}, closure); + + GroupedPaths groupedPaths; + + for (auto & path : closure) { + /* Strip the output name. Unfortunately this is ambiguous (we + can't distinguish between output names like "bin" and + version suffixes like "unstable"). */ + static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); + std::smatch match; + std::string name(path.name()); + std::string outputName; + if (std::regex_match(name, match, regex)) { + name = match[1]; + outputName = match[2]; + } + + DrvName drvName(name); + groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName }); + } + + return groupedPaths; +} + +std::string showVersions(const std::set<std::string> & versions) +{ + if (versions.empty()) return "∅"; + std::set<std::string> versions2; + for (auto & version : versions) + versions2.insert(version.empty() ? "ε" : version); + return concatStringsSep(", ", versions2); +} + +void printClosureDiff( + ref<Store> store, + const StorePath & beforePath, + const StorePath & afterPath, + std::string_view indent) +{ + auto beforeClosure = getClosureInfo(store, beforePath); + auto afterClosure = getClosureInfo(store, afterPath); + + std::set<std::string> allNames; + for (auto & [name, _] : beforeClosure) allNames.insert(name); + for (auto & [name, _] : afterClosure) allNames.insert(name); + + for (auto & name : allNames) { + auto & beforeVersions = beforeClosure[name]; + auto & afterVersions = afterClosure[name]; + + auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions) + { + uint64_t sum = 0; + for (auto & [_, paths] : versions) + for (auto & [path, _] : paths) + sum += store->queryPathInfo(path)->narSize; + return sum; + }; + + auto beforeSize = totalSize(beforeVersions); + auto afterSize = totalSize(afterVersions); + auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; + auto showDelta = std::abs(sizeDelta) >= 8 * 1024; + + std::set<std::string> removed, unchanged; + for (auto & [version, _] : beforeVersions) + if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version); + + std::set<std::string> added; + for (auto & [version, _] : afterVersions) + if (!beforeVersions.count(version)) added.insert(version); + + if (showDelta || !removed.empty() || !added.empty()) { + std::vector<std::string> items; + if (!removed.empty() || !added.empty()) + items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); + if (showDelta) + items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); + std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items)); + } + } +} + +} + +using namespace nix; + +struct CmdDiffClosures : SourceExprCommand +{ + std::string _before, _after; + + CmdDiffClosures() + { + expectArg("before", &_before); + expectArg("after", &_after); + } + + std::string description() override + { + return "show what packages and versions were added and removed between two closures"; + } + + Category category() override { return catSecondary; } + + Examples examples() override + { + return { + { + "To show what got added and removed between two versions of the NixOS system profile:", + "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", + }, + }; + } + + void run(ref<Store> store) override + { + auto before = parseInstallable(store, _before); + auto beforePath = toStorePath(store, Realise::Outputs, operateOn, before); + auto after = parseInstallable(store, _after); + auto afterPath = toStorePath(store, Realise::Outputs, operateOn, after); + printClosureDiff(store, beforePath, afterPath, ""); + } +}; + +static auto rCmdDiffClosures = registerCommand<CmdDiffClosures>("diff-closures"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 82e92cdd0..4588ac05e 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -49,9 +49,7 @@ struct CmdDoctor : StoreCommand { logger->log("Running checks against store uri: " + store->getUri()); - auto type = getStoreType(); - - if (type < tOther) { + if (store.dynamic_pointer_cast<LocalFSStore>()) { success &= checkNixInPath(); success &= checkProfileRoots(store); } @@ -133,4 +131,4 @@ struct CmdDoctor : StoreCommand } }; -static auto r1 = registerCommand<CmdDoctor>("doctor"); +static auto rCmdDoctor = registerCommand<CmdDoctor>("doctor"); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index e1de71bf8..6fd197531 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -30,4 +30,4 @@ struct CmdDumpPath : StorePathCommand } }; -static auto r1 = registerCommand<CmdDumpPath>("dump-path"); +static auto rDumpPath = registerCommand<CmdDumpPath>("dump-path"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 067d3a973..51c16f5a9 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -20,7 +20,7 @@ struct CmdEdit : InstallableCommand return { Example{ "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs.hello" + "nix edit nixpkgs#hello" }, }; } @@ -45,6 +45,7 @@ struct CmdEdit : InstallableCommand auto args = editorFor(pos); + restoreSignals(); execvp(args.front().c_str(), stringsToCharPtrs(args).data()); std::string command; @@ -53,4 +54,4 @@ struct CmdEdit : InstallableCommand } }; -static auto r1 = registerCommand<CmdEdit>("edit"); +static auto rCmdEdit = registerCommand<CmdEdit>("edit"); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 26e98ac2a..43ce46546 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -12,10 +12,18 @@ using namespace nix; struct CmdEval : MixJSON, InstallableCommand { bool raw = false; + std::optional<std::string> apply; CmdEval() { mkFlag(0, "raw", "print strings unquoted", &raw); + + addFlag({ + .longName = "apply", + .description = "apply a function to each argument", + .labels = {"expr"}, + .handler = {&apply}, + }); } std::string description() override @@ -26,21 +34,25 @@ struct CmdEval : MixJSON, InstallableCommand Examples examples() override { return { - Example{ + { "To evaluate a Nix expression given on the command line:", - "nix eval '(1 + 2)'" + "nix eval --expr '1 + 2'" }, - Example{ + { "To evaluate a Nix expression from a file or URI:", - "nix eval -f channel:nixos-17.09 hello.name" + "nix eval -f ./my-nixpkgs hello.name" }, - Example{ + { "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs.lib.version" + "nix eval --raw nixpkgs#lib.version" }, - Example{ + { "To print the store path of the Hello package:", - "nix eval --raw nixpkgs.hello" + "nix eval --raw nixpkgs#hello" + }, + { + "To get a list of checks in the 'nix' flake:", + "nix eval nix#checks.x86_64-linux --apply builtins.attrNames" }, }; } @@ -57,6 +69,14 @@ struct CmdEval : MixJSON, InstallableCommand auto v = installable->toValue(*state).first; PathSet context; + if (apply) { + auto vApply = state->allocValue(); + state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply); + auto vRes = state->allocValue(); + state->callFunction(*vApply, *v, *vRes, noPos); + v = vRes; + } + if (raw) { stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); @@ -70,4 +90,4 @@ struct CmdEval : MixJSON, InstallableCommand } }; -static auto r1 = registerCommand<CmdEval>("eval"); +static auto rCmdEval = registerCommand<CmdEval>("eval"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc new file mode 100644 index 000000000..d45f13029 --- /dev/null +++ b/src/nix/flake.cc @@ -0,0 +1,968 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "flake/flake.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "attr-path.hh" +#include "fetchers.hh" +#include "registry.hh" +#include "json.hh" +#include "eval-cache.hh" + +#include <nlohmann/json.hpp> +#include <queue> +#include <iomanip> + +using namespace nix; +using namespace nix::flake; + +class FlakeCommand : virtual Args, public MixFlakeOptions +{ + std::string flakeUrl = "."; + +public: + + FlakeCommand() + { + expectArgs({ + .label = "flake-url", + .optional = true, + .handler = {&flakeUrl}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + } + + FlakeRef getFlakeRef() + { + return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + } + + Flake getFlake() + { + auto evalState = getEvalState(); + return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries); + } + + LockedFlake lockFlake() + { + return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + } + + std::optional<FlakeRef> getFlakeRefForCompletion() override + { + return getFlakeRef(); + } +}; + +static void printFlakeInfo(const Store & store, const Flake & flake) +{ + logger->stdout("Resolved URL: %s", flake.resolvedRef.to_string()); + logger->stdout("Locked URL: %s", flake.lockedRef.to_string()); + if (flake.description) + logger->stdout("Description: %s", *flake.description); + logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input.getRev()) + logger->stdout("Revision: %s", rev->to_string(Base16, false)); + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->stdout("Revisions: %s", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + logger->stdout("Last modified: %s", + std::put_time(std::localtime(&*lastModified), "%F %T")); +} + +static nlohmann::json flakeToJson(const Store & store, const Flake & flake) +{ + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = attrsToJson(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); + if (auto rev = flake.lockedRef.input.getRev()) + j["revision"] = rev->to_string(Base16, false); + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; + j["path"] = store.printStorePath(flake.sourceInfo->storePath); + return j; +} + +struct CmdFlakeUpdate : FlakeCommand +{ + std::string description() override + { + return "update flake lock file"; + } + + void run(nix::ref<nix::Store> store) override + { + /* Use --refresh by default for 'nix flake update'. */ + settings.tarballTtl = 0; + + lockFlake(); + } +}; + +static void enumerateOutputs(EvalState & state, Value & vFlake, + std::function<void(const std::string & name, Value & vProvide, const Pos & pos)> callback) +{ + state.forceAttrs(vFlake); + + auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceAttrs(*aOutputs->value); + + for (auto & attr : *aOutputs->value->attrs) + callback(attr.name, *attr.value, *attr.pos); +} + +struct CmdFlakeInfo : FlakeCommand, MixJSON +{ + std::string description() override + { + return "list info about a given flake"; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = getFlake(); + + if (json) { + auto json = flakeToJson(*store, flake); + logger->stdout("%s", json.dump()); + } else + printFlakeInfo(*store, flake); + } +}; + +struct CmdFlakeListInputs : FlakeCommand, MixJSON +{ + std::string description() override + { + return "list flake inputs"; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = lockFlake(); + + if (json) + logger->stdout("%s", flake.lockFile.toJson()); + else { + logger->stdout("%s", flake.flake.lockedRef); + + std::unordered_set<std::shared_ptr<Node>> visited; + + std::function<void(const Node & node, const std::string & prefix)> recurse; + + recurse = [&](const Node & node, const std::string & prefix) + { + for (const auto & [i, input] : enumerate(node.inputs)) { + bool last = i + 1 == node.inputs.size(); + + if (auto lockedNode = std::get_if<0>(&input.second)) { + logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + prefix + (last ? treeLast : treeConn), input.first, + *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); + + bool firstVisit = visited.insert(*lockedNode).second; + + if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); + } else if (auto follows = std::get_if<1>(&input.second)) { + logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", + prefix + (last ? treeLast : treeConn), input.first, + printInputPath(*follows)); + } + } + }; + + visited.insert(flake.lockFile.root); + recurse(*flake.lockFile.root, ""); + } + } +}; + +struct CmdFlakeCheck : FlakeCommand +{ + bool build = true; + + CmdFlakeCheck() + { + addFlag({ + .longName = "no-build", + .description = "do not build checks", + .handler = {&build, false} + }); + } + + std::string description() override + { + return "check whether the flake evaluates and run its tests"; + } + + void run(nix::ref<nix::Store> store) override + { + settings.readOnlyMode = !build; + + auto state = getEvalState(); + auto flake = lockFlake(); + + // FIXME: rewrite to use EvalCache. + + auto checkSystemName = [&](const std::string & system, const Pos & pos) { + // FIXME: what's the format of "system"? + if (system.find('-') == std::string::npos) + throw Error("'%s' is not a valid system type, at %s", system, pos); + }; + + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("flake attribute '%s' is not a derivation", attrPath); + // FIXME: check meta attributes + return store->parseStorePath(drvInfo->queryDrvPath()); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); + throw; + } + }; + + std::vector<StorePathWithOutputs> drvPaths; + + auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + #if 0 + // FIXME + auto app = App(*state, v); + for (auto & i : app.context) { + auto [drvPathS, outputName] = decodeContext(i); + store->parseStorePath(drvPathS); + } + #endif + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); + throw; + } + }; + + auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + throw Error("overlay does not take an argument named 'final'"); + auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); + if (!body || body->matchAttrs || std::string(body->arg) != "prev") + throw Error("overlay does not take an argument named 'prev'"); + // FIXME: if we have a 'nixpkgs' input, use it to + // evaluate the overlay. + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); + throw; + } + }; + + auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type == tLambda) { + if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + throw Error("module must match an open attribute set ('{ config, ... }')"); + } else if (v.type == tAttrs) { + for (auto & attr : *v.attrs) + try { + state->forceValue(*attr.value, *attr.pos); + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("while evaluating the option '%s'", attr.name)); + throw; + } + } else + throw Error("module must be a function or an attribute set"); + // FIXME: if we have a 'nixpkgs' input, use it to + // check the module. + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); + throw; + } + }; + + std::function<void(const std::string & attrPath, Value & v, const Pos & pos)> checkHydraJobs; + + checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceAttrs(v, pos); + + if (state->isDerivation(v)) + throw Error("jobset should not be a derivation at top-level"); + + for (auto & attr : *v.attrs) { + state->forceAttrs(*attr.value, *attr.pos); + if (!state->isDerivation(*attr.value)) + checkHydraJobs(attrPath + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); + throw; + } + }; + + auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking NixOS configuration '%s'", attrPath)); + Bindings & bindings(*state->allocBindings(0)); + auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; + state->forceAttrs(*vToplevel, pos); + if (!state->isDerivation(*vToplevel)) + throw Error("attribute 'config.system.build.toplevel' is not a derivation"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); + throw; + } + }; + + auto checkTemplate = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking template '%s'", attrPath)); + + state->forceAttrs(v, pos); + + if (auto attr = v.attrs->get(state->symbols.create("path"))) { + if (attr->name == state->symbols.create("path")) { + PathSet context; + auto path = state->coerceToPath(*attr->pos, *attr->value, context); + if (!store->isInStore(path)) + throw Error("template '%s' has a bad 'path' attribute"); + // TODO: recursively check the flake in 'path'. + } + } else + throw Error("template '%s' lacks attribute 'path'", attrPath); + + if (auto attr = v.attrs->get(state->symbols.create("description"))) + state->forceStringNoCtx(*attr->value, *attr->pos); + else + throw Error("template '%s' lacks attribute 'description'", attrPath); + + for (auto & attr : *v.attrs) { + std::string name(attr.name); + if (name != "path" && name != "description") + throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); + } + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + throw; + } + }; + + auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda) + throw Error("bundler must be a function"); + if (!v.lambda.fun->formals || + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end()) + throw Error("bundler must take formal arguments 'program' and 'system'"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + throw; + } + }; + + { + Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + enumerateOutputs(*state, + *vFlake, + [&](const std::string & name, Value & vOutput, const Pos & pos) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking flake output '%s'", name)); + + try { + state->forceValue(vOutput, pos); + + if (name == "checks") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + if ((std::string) attr.name == settings.thisSystem.get()) + drvPaths.push_back({drvPath}); + } + } + } + + else if (name == "packages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "apps") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "defaultPackage" || name == "devShell") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkDerivation( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "defaultApp") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "legacyPackages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + // FIXME: do getDerivations? + } + } + + else if (name == "overlay") + checkOverlay(name, vOutput, pos); + + else if (name == "overlays") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkOverlay(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosModule") + checkModule(name, vOutput, pos); + + else if (name == "nixosModules") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkModule(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosConfigurations") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkNixOSConfiguration(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "hydraJobs") + checkHydraJobs(name, vOutput, pos); + + else if (name == "defaultTemplate") + checkTemplate(name, vOutput, pos); + + else if (name == "templates") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkTemplate(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "defaultBundler") + checkBundler(name, vOutput, pos); + + else if (name == "bundlers") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkBundler(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else + warn("unknown flake output '%s'", name); + + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); + throw; + } + }); + } + + if (build && !drvPaths.empty()) { + Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + store->buildPaths(drvPaths); + } + } +}; + +struct CmdFlakeInitCommon : virtual Args, EvalCommand +{ + std::string templateUrl = "templates"; + Path destDir; + + const Strings attrsPathPrefixes{"templates."}; + const LockFlags lockFlags{ .writeLockFile = false }; + + CmdFlakeInitCommon() + { + addFlag({ + .longName = "template", + .shortName = 't', + .description = "the template to use", + .labels = {"template"}, + .handler = {&templateUrl}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + attrsPathPrefixes, + {"defaultTemplate"}, + prefix); + }} + }); + } + + void run(nix::ref<nix::Store> store) override + { + auto flakeDir = absPath(destDir); + + auto evalState = getEvalState(); + + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + + auto installable = InstallableFlake( + evalState, std::move(templateFlakeRef), + Strings{templateName == "" ? "defaultTemplate" : templateName}, + Strings(attrsPathPrefixes), lockFlags); + + auto [cursor, attrPath] = installable.getCursor(*evalState); + + auto templateDir = cursor->getAttr("path")->getString(); + + assert(store->isInStore(templateDir)); + + std::vector<Path> files; + + std::function<void(const Path & from, const Path & to)> copyDir; + copyDir = [&](const Path & from, const Path & to) + { + createDirs(to); + + for (auto & entry : readDirectory(from)) { + auto from2 = from + "/" + entry.name; + auto to2 = to + "/" + entry.name; + auto st = lstat(from2); + if (S_ISDIR(st.st_mode)) + copyDir(from2, to2); + else if (S_ISREG(st.st_mode)) { + auto contents = readFile(from2); + if (pathExists(to2)) { + auto contents2 = readFile(to2); + if (contents != contents2) + throw Error("refusing to overwrite existing file '%s'", to2); + } else + writeFile(to2, contents); + } + else if (S_ISLNK(st.st_mode)) { + auto target = readLink(from2); + if (pathExists(to2)) { + if (readLink(to2) != target) + throw Error("refusing to overwrite existing symlink '%s'", to2); + } else + createSymlink(target, to2); + } + else + throw Error("file '%s' has unsupported type", from2); + files.push_back(to2); + } + }; + + copyDir(templateDir, flakeDir); + + if (pathExists(flakeDir + "/.git")) { + Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; + for (auto & s : files) args.push_back(s); + runProgram("git", true, args); + } + } +}; + +struct CmdFlakeInit : CmdFlakeInitCommon +{ + std::string description() override + { + return "create a flake in the current directory from a template"; + } + + Examples examples() override + { + return { + Example{ + "To create a flake using the default template:", + "nix flake init" + }, + Example{ + "To see available templates:", + "nix flake show templates" + }, + Example{ + "To create a flake from a specific template:", + "nix flake init -t templates#nixos-container" + }, + }; + } + + CmdFlakeInit() + { + destDir = "."; + } +}; + +struct CmdFlakeNew : CmdFlakeInitCommon +{ + std::string description() override + { + return "create a flake in the specified directory from a template"; + } + + CmdFlakeNew() + { + expectArgs({ + .label = "dest-dir", + .handler = {&destDir}, + .completer = completePath + }); + } +}; + +struct CmdFlakeClone : FlakeCommand +{ + Path destDir; + + std::string description() override + { + return "clone flake repository"; + } + + CmdFlakeClone() + { + addFlag({ + .longName = "dest", + .shortName = 'f', + .description = "destination path", + .labels = {"path"}, + .handler = {&destDir} + }); + } + + void run(nix::ref<nix::Store> store) override + { + if (destDir.empty()) + throw Error("missing flag '--dest'"); + + getFlakeRef().resolve(store).input.clone(destDir); + } +}; + +struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun +{ + std::string dstUri; + + CmdFlakeArchive() + { + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri} + }); + } + + std::string description() override + { + return "copy a flake and all its inputs to a store"; + } + + Examples examples() override + { + return { + Example{ + "To copy the dwarffs flake and its dependencies to a binary cache:", + "nix flake archive --to file:///tmp/my-cache dwarffs" + }, + Example{ + "To fetch the dwarffs flake and its dependencies to the local Nix store:", + "nix flake archive dwarffs" + }, + Example{ + "To print the store paths of the flake sources of NixOps without fetching them:", + "nix flake archive --json --dry-run nixops" + }, + }; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = lockFlake(); + + auto jsonRoot = json ? std::optional<JSONObject>(std::cout) : std::nullopt; + + StorePathSet sources; + + sources.insert(flake.flake.sourceInfo->storePath); + if (jsonRoot) + jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); + + // FIXME: use graph output, handle cycles. + std::function<void(const Node & node, std::optional<JSONObject> & jsonObj)> traverse; + traverse = [&](const Node & node, std::optional<JSONObject> & jsonObj) + { + auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional<JSONObject>(); + for (auto & [inputName, input] : node.inputs) { + if (auto inputNode = std::get_if<0>(&input)) { + auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional<JSONObject>(); + auto storePath = + dryRun + ? (*inputNode)->lockedRef.input.computeStorePath(*store) + : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + if (jsonObj3) + jsonObj3->attr("path", store->printStorePath(storePath)); + sources.insert(std::move(storePath)); + traverse(**inputNode, jsonObj3); + } + } + }; + + traverse(*flake.lockFile.root, jsonRoot); + + if (!dryRun && !dstUri.empty()) { + ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + copyPaths(store, dstStore, sources); + } + } +}; + +struct CmdFlakeShow : FlakeCommand +{ + bool showLegacy = false; + + CmdFlakeShow() + { + addFlag({ + .longName = "legacy", + .description = "show the contents of the 'legacyPackages' output", + .handler = {&showLegacy, true} + }); + } + + std::string description() override + { + return "show the outputs provided by a flake"; + } + + void run(nix::ref<nix::Store> store) override + { + auto state = getEvalState(); + auto flake = std::make_shared<LockedFlake>(lockFlake()); + + std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; + + visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) + { + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPath))); + try { + auto recurse = [&]() + { + logger->stdout("%s", headerPrefix); + auto attrs = visitor.getAttrs(); + for (const auto & [i, attr] : enumerate(attrs)) { + bool last = i + 1 == attrs.size(); + auto visitor2 = visitor.getAttr(attr); + auto attrPath2(attrPath); + attrPath2.push_back(attr); + visit(*visitor2, attrPath2, + fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr), + nextPrefix + (last ? treeNull : treeLine)); + } + }; + + auto showDerivation = [&]() + { + auto name = visitor.getAttr(state->sName)->getString(); + + /* + std::string description; + + if (auto aMeta = visitor.maybeGetAttr("meta")) { + if (auto aDescription = aMeta->maybeGetAttr("description")) + description = aDescription->getString(); + } + */ + + logger->stdout("%s: %s '%s'", + headerPrefix, + attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : + attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : + attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : + "package", + name); + }; + + if (attrPath.size() == 0 + || (attrPath.size() == 1 && ( + attrPath[0] == "defaultPackage" + || attrPath[0] == "devShell" + || attrPath[0] == "nixosConfigurations" + || attrPath[0] == "nixosModules" + || attrPath[0] == "defaultApp" + || attrPath[0] == "templates")) + || ((attrPath.size() == 1 || attrPath.size() == 2) + && (attrPath[0] == "checks" + || attrPath[0] == "packages" + || attrPath[0] == "apps")) + ) + { + recurse(); + } + + else if ( + (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) + || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages")) + ) + { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } + + else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") { + if (visitor.isDerivation()) + showDerivation(); + else + recurse(); + } + + else if (attrPath.size() > 0 && attrPath[0] == "legacyPackages") { + if (attrPath.size() == 1) + recurse(); + else if (!showLegacy) + logger->stdout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); + else { + if (visitor.isDerivation()) + showDerivation(); + else if (attrPath.size() <= 2) + // FIXME: handle recurseIntoAttrs + recurse(); + } + } + + else if ( + (attrPath.size() == 2 && attrPath[0] == "defaultApp") || + (attrPath.size() == 3 && attrPath[0] == "apps")) + { + auto aType = visitor.maybeGetAttr("type"); + if (!aType || aType->getString() != "app") + throw EvalError("not an app definition"); + logger->stdout("%s: app", headerPrefix); + } + + else if ( + (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") || + (attrPath.size() == 2 && attrPath[0] == "templates")) + { + auto description = visitor.getAttr("description")->getString(); + logger->stdout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + } + + else { + logger->stdout("%s: %s", + headerPrefix, + attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" : + attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : + attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" : + ANSI_YELLOW "unknown" ANSI_NORMAL); + } + } catch (EvalError & e) { + if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) + throw; + } + }; + + auto cache = openEvalCache(*state, flake); + + visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + } +}; + +struct CmdFlake : NixMultiCommand +{ + CmdFlake() + : MultiCommand({ + {"update", []() { return make_ref<CmdFlakeUpdate>(); }}, + {"info", []() { return make_ref<CmdFlakeInfo>(); }}, + {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }}, + {"check", []() { return make_ref<CmdFlakeCheck>(); }}, + {"init", []() { return make_ref<CmdFlakeInit>(); }}, + {"new", []() { return make_ref<CmdFlakeNew>(); }}, + {"clone", []() { return make_ref<CmdFlakeClone>(); }}, + {"archive", []() { return make_ref<CmdFlakeArchive>(); }}, + {"show", []() { return make_ref<CmdFlakeShow>(); }}, + }) + { + } + + std::string description() override + { + return "manage Nix flakes"; + } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdFlake = registerCommand<CmdFlake>("flake"); diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a25ec43a9..091c0f573 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,9 +1,19 @@ set -e if [ -e .attrs.sh ]; then source .attrs.sh; fi + export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 + if [[ -n $stdenv ]]; then source $stdenv/setup fi -export > $out -set >> $out + +for __output in $outputs; do + if [[ -z $__done ]]; then + export > ${!__output} + set >> ${!__output} + __done=1 + else + echo -n >> ${!__output} + fi +done diff --git a/src/nix/hash.cc b/src/nix/hash.cc index b97c6d21f..1d23bb0e2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -31,7 +31,11 @@ struct CmdHash : Command .labels({"modulus"}) .dest(&modulus); #endif - expectArgs("paths", &paths); + expectArgs({ + .label = "paths", + .handler = {&paths}, + .completer = completePath + }); } std::string description() override @@ -40,6 +44,7 @@ struct CmdHash : Command switch (mode) { case FileIngestionMethod::Flat: d = "print cryptographic hash of a regular file"; + break; case FileIngestionMethod::Recursive: d = "print cryptographic hash of the NAR serialisation of a path"; }; @@ -74,8 +79,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); }); -static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); }); +static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); }); +static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { @@ -103,14 +108,14 @@ struct CmdToBase : Command void run() override { for (auto s : args) - logger->stdout(Hash(s, ht).to_string(base, base == SRI)); + logger->stdout(Hash::parseAny(s, ht).to_string(base, base == SRI)); } }; -static RegisterCommand r3("to-base16", [](){ return make_ref<CmdToBase>(Base16); }); -static RegisterCommand r4("to-base32", [](){ return make_ref<CmdToBase>(Base32); }); -static RegisterCommand r5("to-base64", [](){ return make_ref<CmdToBase>(Base64); }); -static RegisterCommand r6("to-sri", [](){ return make_ref<CmdToBase>(SRI); }); +static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref<CmdToBase>(Base16); }); +static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref<CmdToBase>(Base32); }); +static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref<CmdToBase>(Base64); }); +static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref<CmdToBase>(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -162,4 +167,4 @@ static int compatNixHash(int argc, char * * argv) return 0; } -static RegisterLegacyCommand s1("nix-hash", compatNixHash); +static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 708a0dc88..9bf6b7caa 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,3 +1,4 @@ +#include "installables.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -7,80 +8,261 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" #include <regex> +#include <queue> namespace nix { +void completeFlakeInputPath( + ref<EvalState> evalState, + const FlakeRef & flakeRef, + std::string_view prefix) +{ + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions->insert(input.first); +} -SourceExprCommand::SourceExprCommand() +MixFlakeOptions::MixFlakeOptions() { addFlag({ - .longName = "file", - .shortName = 'f', - .description = "evaluate FILE rather than the default", - .labels = {"file"}, - .handler = {&file} + .longName = "recreate-lock-file", + .description = "recreate lock file from scratch", + .handler = {&lockFlags.recreateLockFile, true} }); -} -Value * SourceExprCommand::getSourceExpr(EvalState & state) -{ - if (vSourceExpr) return *vSourceExpr; + addFlag({ + .longName = "no-update-lock-file", + .description = "do not allow any updates to the lock file", + .handler = {&lockFlags.updateLockFile, false} + }); - auto sToplevel = state.symbols.create("_toplevel"); + addFlag({ + .longName = "no-write-lock-file", + .description = "do not write the newly generated lock file", + .handler = {&lockFlags.writeLockFile, false} + }); + + addFlag({ + .longName = "no-registries", + .description = "don't use flake registries", + .handler = {&lockFlags.useRegistries, false} + }); - vSourceExpr = allocRootValue(state.allocValue()); + addFlag({ + .longName = "commit-lock-file", + .description = "commit changes to the lock file", + .handler = {&lockFlags.commitLockFile, true} + }); - if (file != "") - state.evalFile(lookupFileArg(state, file), **vSourceExpr); + addFlag({ + .longName = "update-input", + .description = "update a specific flake input", + .labels = {"input-path"}, + .handler = {[&](std::string s) { + lockFlags.inputUpdates.insert(flake::parseInputPath(s)); + }}, + .completer = {[&](size_t, std::string_view prefix) { + if (auto flakeRef = getFlakeRefForCompletion()) + completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + }} + }); - else { + addFlag({ + .longName = "override-input", + .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)", + .labels = {"input-path", "flake-url"}, + .handler = {[&](std::string inputPath, std::string flakeRef) { + lockFlags.inputOverrides.insert_or_assign( + flake::parseInputPath(inputPath), + parseFlakeRef(flakeRef, absPath("."))); + }} + }); - /* Construct the installation source from $NIX_PATH. */ + addFlag({ + .longName = "inputs-from", + .description = "use the inputs of the specified flake as registry entries", + .labels = {"flake-url"}, + .handler = {[&](std::string flakeRef) { + auto evalState = getEvalState(); + auto flake = flake::lockFlake( + *evalState, + parseFlakeRef(flakeRef, absPath(".")), + { .writeLockFile = false }); + for (auto & [inputName, input] : flake.lockFile.root->inputs) { + auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes + if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) { + overrideRegistry( + fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + input3->lockedRef.input, + {}); + } + } + }}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getEvalState()->store, prefix); + }} + }); +} - auto searchPath = state.getSearchPath(); +SourceExprCommand::SourceExprCommand() +{ + addFlag({ + .longName = "file", + .shortName = 'f', + .description = "evaluate *file* rather than the default", + .labels = {"file"}, + .handler = {&file}, + .completer = completePath + }); - state.mkAttrs(**vSourceExpr, 1024); + addFlag({ + .longName ="expr", + .description = "evaluate attributes from *expr*", + .labels = {"expr"}, + .handler = {&expr} + }); - mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); + addFlag({ + .longName ="derivation", + .description = "operate on the store derivation rather than its outputs", + .handler = {&operateOn, OperateOn::Derivation}, + }); +} - std::unordered_set<std::string> seen; +Strings SourceExprCommand::getDefaultFlakeAttrPaths() +{ + return {"defaultPackage." + settings.thisSystem.get()}; +} - auto addEntry = [&](const std::string & name) { - if (name == "") return; - if (!seen.insert(name).second) return; - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), - state.getBuiltin("import"), *v2); - }; +Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() +{ + return { + // As a convenience, look for the attribute in + // 'outputs.packages'. + "packages." + settings.thisSystem.get() + ".", + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + "legacyPackages." + settings.thisSystem.get() + "." + }; +} + +void SourceExprCommand::completeInstallable(std::string_view prefix) +{ + if (file) return; // FIXME + + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + getDefaultFlakeAttrPathPrefixes(), + getDefaultFlakeAttrPaths(), + prefix); +} - for (auto & i : searchPath) - /* Hack to handle channels. */ - if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { - for (auto & j : readDirectory(i.second)) - if (j.name != "manifest.nix" - && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) - addEntry(j.name); - } else - addEntry(i.first); - - (*vSourceExpr)->attrs->sort(); +void completeFlakeRefWithFragment( + ref<EvalState> evalState, + flake::LockFlags lockFlags, + Strings attrPathPrefixes, + const Strings & defaultFlakeAttrPaths, + std::string_view prefix) +{ + /* Look for flake output attributes that match the + prefix. */ + try { + auto hash = prefix.find('#'); + if (hash != std::string::npos) { + auto fragment = prefix.substr(hash + 1); + auto flakeRefS = std::string(prefix.substr(0, hash)); + // FIXME: do tilde expansion. + auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); + + auto evalCache = openEvalCache(*evalState, + std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags))); + + auto root = evalCache->getRoot(); + + /* Complete 'fragment' relative to all the + attrpath prefixes as well as the root of the + flake. */ + attrPathPrefixes.push_back(""); + + for (auto & attrPathPrefixS : attrPathPrefixes) { + auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS); + auto attrPathS = attrPathPrefixS + std::string(fragment); + auto attrPath = parseAttrPath(*evalState, attrPathS); + + std::string lastAttr; + if (!attrPath.empty() && !hasSuffix(attrPathS, ".")) { + lastAttr = attrPath.back(); + attrPath.pop_back(); + } + + auto attr = root->findAlongAttrPath(attrPath); + if (!attr) continue; + + for (auto & attr2 : attr->getAttrs()) { + if (hasPrefix(attr2, lastAttr)) { + auto attrPath2 = attr->getAttrPath(attr2); + /* Strip the attrpath prefix. */ + attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2)); + } + } + } + + /* And add an empty completion for the default + attrpaths. */ + if (fragment.empty()) { + for (auto & attrPath : defaultFlakeAttrPaths) { + auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); + if (!attr) continue; + completions->insert(flakeRefS + "#"); + } + } + } + } catch (Error & e) { + warn(e.msg()); } - return *vSourceExpr; + completeFlakeRef(evalState->store, prefix); } -ref<EvalState> SourceExprCommand::getEvalState() +ref<EvalState> EvalCommand::getEvalState() { if (!evalState) evalState = std::make_shared<EvalState>(searchPath, getStore()); return ref<EvalState>(evalState); } +void completeFlakeRef(ref<Store> store, std::string_view prefix) +{ + if (prefix == "") + completions->insert("."); + + completeDir(0, prefix); + + /* Look for registry entries that match the prefix. */ + for (auto & registry : fetchers::getRegistries(store)) { + for (auto & entry : registry->entries) { + auto from = entry.from.to_string(); + if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { + std::string from2(from, 6); + if (hasPrefix(from2, prefix)) + completions->insert(from2); + } else { + if (hasPrefix(from, prefix)) + completions->insert(from); + } + } + } +} + Buildable Installable::toBuildable() { auto buildables = toBuildables(); @@ -89,27 +271,54 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } +std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +Installable::getCursors(EvalState & state) +{ + auto evalCache = + std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state, + [&]() { return toValue(state).first; }); + return {{evalCache->getRoot(), ""}}; +} + +std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> +Installable::getCursor(EvalState & state) +{ + auto cursors = getCursors(state); + if (cursors.empty()) + throw Error("cannot find flake attribute '%s'", what()); + return cursors[0]; +} + struct InstallableStorePath : Installable { ref<Store> store; StorePath storePath; - InstallableStorePath(ref<Store> store, const Path & storePath) - : store(store), storePath(store->parseStorePath(storePath)) { } + InstallableStorePath(ref<Store> store, StorePath && storePath) + : store(store), storePath(std::move(storePath)) { } std::string what() override { return store->printStorePath(storePath); } Buildables toBuildables() override { - std::map<std::string, StorePath> outputs; - outputs.insert_or_assign("out", storePath); - Buildable b{ - .drvPath = storePath.isDerivation() ? storePath : std::optional<StorePath>(), - .outputs = std::move(outputs) - }; - Buildables bs; - bs.push_back(std::move(b)); - return bs; + if (storePath.isDerivation()) { + std::map<std::string, std::optional<StorePath>> outputs; + auto drv = store->readDerivation(storePath); + for (auto & [name, output] : drv.outputsAndOptPaths(*store)) + outputs.emplace(name, output.second); + return { + BuildableFromDrv { + .drvPath = storePath, + .outputs = std::move(outputs) + } + }; + } else { + return { + BuildableOpaque { + .path = storePath, + } + }; + } } std::optional<StorePath> getStorePath() override @@ -118,146 +327,309 @@ struct InstallableStorePath : Installable } }; -struct InstallableValue : Installable +Buildables InstallableValue::toBuildables() +{ + Buildables res; + + std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs; + + // Group by derivation, helps with .all in particular + for (auto & drv : toDerivations()) { + auto outputName = drv.outputName; + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); + drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath); + } + + for (auto & i : drvsToOutputs) + res.push_back(BuildableFromDrv { i.first, i.second }); + + return res; +} + +struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; + RootValue v; + std::string attrPath; - InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath) + : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) + { } - Buildables toBuildables() override + std::string what() override { return attrPath; } + + std::pair<Value *, Pos> toValue(EvalState & state) override { - auto state = cmd.getEvalState(); + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes); + return {vRes, pos}; + } + + virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; +}; - auto v = toValue(*state).first; +std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations() +{ + auto v = toValue(*state).first; - Bindings & autoArgs = *cmd.getAutoArgs(*state); + Bindings & autoArgs = *cmd.getAutoArgs(*state); - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); - Buildables res; + std::vector<DerivationInfo> res; + for (auto & drvInfo : drvInfos) { + res.push_back({ + state->store->parseStorePath(drvInfo.queryDrvPath()), + state->store->parseStorePath(drvInfo.queryOutPath()), + drvInfo.queryOutputName() + }); + } - StorePathSet drvPaths; + return res; +} - for (auto & drv : drvs) { - Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; - drvPaths.insert(*b.drvPath); +std::vector<std::string> InstallableFlake::getActualAttrPaths() +{ + std::vector<std::string> res; - auto outputName = drv.queryOutputName(); - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); - b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath())); + for (auto & s : attrPaths) + res.push_back(s); - res.push_back(std::move(b)); - } + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value); + + return aOutputs->value; +} + +ref<eval_cache::EvalCache> openEvalCache( + EvalState & state, + std::shared_ptr<flake::LockedFlake> lockedFlake) +{ + auto fingerprint = lockedFlake->getFingerprint(); + return make_ref<nix::eval_cache::EvalCache>( + evalSettings.useEvalCache && evalSettings.pureEval + ? std::optional { std::cref(fingerprint) } + : std::nullopt, + state, + [&state, lockedFlake]() + { + /* For testing whether the evaluation cache is + complete. */ + if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") + throw Error("not everything is cached, but evaluation is not allowed"); + + auto vFlake = state.allocValue(); + flake::callFlake(state, *lockedFlake, *vFlake); + + state.forceAttrs(*vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + return aOutputs->value; + }); +} - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{.drvPath = *drvPaths.begin()}; - for (auto & b2 : res) - for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second); - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } else - return res; +static std::string showAttrPaths(const std::vector<std::string> & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; } -}; + return s; +} -struct InstallableExpr : InstallableValue +std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() { - std::string text; + auto lockedFlake = getLockedFlake(); - InstallableExpr(SourceExprCommand & cmd, const std::string & text) - : InstallableValue(cmd), text(text) { } + auto cache = openEvalCache(*state, lockedFlake); + auto root = cache->getRoot(); - std::string what() override { return text; } + for (auto & attrPath : getActualAttrPaths()) { + auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath)); + if (!attr) continue; - std::pair<Value *, Pos> toValue(EvalState & state) override - { - auto v = state.allocValue(); - state.eval(state.parseExprFromString(text, absPath(".")), *v); - return {v, noPos}; + if (!attr->isDerivation()) + throw Error("flake output attribute '%s' is not a derivation", attrPath); + + auto drvPath = attr->forceDerivation(); + + auto drvInfo = DerivationInfo{ + std::move(drvPath), + state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + attr->getAttr(state->sOutputName)->getString() + }; + + return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; } -}; -struct InstallableAttrPath : InstallableValue + throw Error("flake '%s' does not provide attribute %s", + flakeRef, showAttrPaths(getActualAttrPaths())); +} + +std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() { - std::string attrPath; + std::vector<DerivationInfo> res; + res.push_back(std::get<2>(toDerivation())); + return res; +} - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) - { } +std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) +{ + auto lockedFlake = getLockedFlake(); - std::string what() override { return attrPath; } + auto vOutputs = getFlakeOutputs(state, *lockedFlake); - std::pair<Value *, Pos> toValue(EvalState & state) override - { - auto source = cmd.getSourceExpr(state); + auto emptyArgs = state.allocBindings(0); + + for (auto & attrPath : getActualAttrPaths()) { + try { + auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); + state.forceValue(*v); + return {v, pos}; + } catch (AttrPathNotFound & e) { + } + } + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, showAttrPaths(getActualAttrPaths())); +} + +std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +InstallableFlake::getCursors(EvalState & state) +{ + auto evalCache = openEvalCache(state, + std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags))); - Bindings & autoArgs = *cmd.getAutoArgs(state); + auto root = evalCache->getRoot(); - auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; - state.forceValue(*v); + std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res; - return {v, noPos}; + for (auto & attrPath : getActualAttrPaths()) { + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) res.push_back({attr, attrPath}); } -}; -// FIXME: extend -std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; -static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); + return res; +} + +std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) + _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags)); + return _lockedFlake; +} -static std::vector<std::shared_ptr<Installable>> parseInstallables( - SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables) +FlakeRef InstallableFlake::nixpkgsFlakeRef() const { - std::vector<std::shared_ptr<Installable>> result; + auto lockedFlake = getLockedFlake(); - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } } - for (auto & s : ss) { + return Installable::nixpkgsFlakeRef(); +} - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(cmd, s)); +std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( + ref<Store> store, std::vector<std::string> ss) +{ + std::vector<std::shared_ptr<Installable>> result; - else if (s.find("/") != std::string::npos) { + if (file || expr) { + if (file && expr) + throw UsageError("'--file' and '--expr' are exclusive"); - auto path = store->toStorePath(store->followLinksToStore(s)); + // FIXME: backward compatibility hack + if (file) evalSettings.pureEval = false; - if (store->isStorePath(path)) - result.push_back(std::make_shared<InstallableStorePath>(store, path)); + auto state = getEvalState(); + auto vFile = state->allocValue(); + + if (file) + state->evalFile(lookupFileArg(*state, *file), *vFile); + else { + auto e = state->parseExprFromString(*expr, absPath(".")); + state->eval(e, *vFile); } - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); + for (auto & s : ss) + result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s)); + + } else { + + for (auto & s : ss) { + std::exception_ptr ex; + + try { + auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); + result.push_back(std::make_shared<InstallableFlake>( + getEvalState(), std::move(flakeRef), + fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment}, + getDefaultFlakeAttrPathPrefixes(), lockFlags)); + continue; + } catch (...) { + ex = std::current_exception(); + } + + if (s.find('/') != std::string::npos) { + try { + result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s))); + continue; + } catch (BadStorePath &) { + } catch (...) { + if (!ex) + ex = std::current_exception(); + } + } - else - throw UsageError("don't know what to do with argument '%s'", s); + std::rethrow_exception(ex); + + /* + throw Error( + pathExists(s) + ? "path '%s' is not a flake or a store path" + : "don't know how to handle argument '%s'", s); + */ + } } return result; } -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables) +std::shared_ptr<Installable> SourceExprCommand::parseInstallable( + ref<Store> store, const std::string & installable) { - auto installables = parseInstallables(cmd, store, {installable}, false); + auto installables = parseInstallables(store, {installable}); assert(installables.size() == 1); return installables.front(); } -Buildables build(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables) +Buildables build(ref<Store> store, Realise mode, + std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode) { - if (mode != Build) + if (mode == Realise::Nothing) settings.readOnlyMode = true; Buildables buildables; @@ -266,45 +638,70 @@ Buildables build(ref<Store> store, RealiseMode mode, for (auto & i : installables) { for (auto & b : i->toBuildables()) { - if (b.drvPath) { - StringSet outputNames; - for (auto & output : b.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back({*b.drvPath, outputNames}); - } else - for (auto & output : b.outputs) - pathsToBuild.push_back({output.second}); + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path}); + }, + [&](BuildableFromDrv bfd) { + StringSet outputNames; + for (auto & output : bfd.outputs) + outputNames.insert(output.first); + pathsToBuild.push_back({bfd.drvPath, outputNames}); + }, + }, b); buildables.push_back(std::move(b)); } } - if (mode == DryRun) + if (mode == Realise::Nothing) printMissing(store, pathsToBuild, lvlError); - else if (mode == Build) - store->buildPaths(pathsToBuild); + else if (mode == Realise::Outputs) + store->buildPaths(pathsToBuild, bMode); return buildables; } -StorePathSet toStorePaths(ref<Store> store, RealiseMode mode, +StorePathSet toStorePaths(ref<Store> store, + Realise mode, OperateOn operateOn, std::vector<std::shared_ptr<Installable>> installables) { StorePathSet outPaths; - for (auto & b : build(store, mode, installables)) - for (auto & output : b.outputs) - outPaths.insert(output.second); + if (operateOn == OperateOn::Output) { + for (auto & b : build(store, mode, installables)) + std::visit(overloaded { + [&](BuildableOpaque bo) { + outPaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : bfd.outputs) { + if (!output.second) + throw Error("Cannot operate on output of unbuilt CA drv"); + outPaths.insert(*output.second); + } + }, + }, b); + } else { + if (mode == Realise::Nothing) + settings.readOnlyMode = true; + + for (auto & i : installables) + for (auto & b : i->toBuildables()) + if (auto bfd = std::get_if<BuildableFromDrv>(&b)) + outPaths.insert(bfd->drvPath); + } return outPaths; } -StorePath toStorePath(ref<Store> store, RealiseMode mode, +StorePath toStorePath(ref<Store> store, + Realise mode, OperateOn operateOn, std::shared_ptr<Installable> installable) { - auto paths = toStorePaths(store, mode, {installable}); + auto paths = toStorePaths(store, mode, operateOn, {installable}); if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); + throw Error("argument '%s' should evaluate to one store path", installable->what()); return *paths.begin(); } @@ -315,32 +712,70 @@ StorePathSet toDerivations(ref<Store> store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) { - if (!b.drvPath) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - for (auto & output : b.outputs) { - auto derivers = store->queryValidDerivers(output.second); + for (auto & b : i->toBuildables()) + std::visit(overloaded { + [&](BuildableOpaque bo) { + if (!useDeriver) + throw Error("argument '%s' did not evaluate to a derivation", i->what()); + auto derivers = store->queryValidDerivers(bo.path); if (derivers.empty()) throw Error("'%s' does not have a known deriver", i->what()); // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); - } - } else - drvPaths.insert(*b.drvPath); - } + }, + [&](BuildableFromDrv bfd) { + drvPaths.insert(bfd.drvPath); + }, + }, b); return drvPaths; } +InstallablesCommand::InstallablesCommand() +{ + expectArgs({ + .label = "installables", + .handler = {&_installables}, + .completer = {[&](size_t, std::string_view prefix) { + completeInstallable(prefix); + }} + }); +} + void InstallablesCommand::prepare() { - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); + if (_installables.empty() && useDefaultInstallables()) + // FIXME: commands like "nix install" should not have a + // default, probably. + _installables.push_back("."); + installables = parseInstallables(getStore(), _installables); +} + +std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion() +{ + if (_installables.empty()) { + if (useDefaultInstallables()) + return parseFlakeRef(".", absPath(".")); + return {}; + } + return parseFlakeRef(_installables.front(), absPath(".")); +} + +InstallableCommand::InstallableCommand() +{ + expectArgs({ + .label = "installable", + .optional = true, + .handler = {&_installable}, + .completer = {[&](size_t, std::string_view prefix) { + completeInstallable(prefix); + }} + }); } void InstallableCommand::prepare() { - installable = parseInstallable(*this, getStore(), _installable, false); + installable = parseInstallable(getStore(), _installable); } } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 503984220..c7c2f8981 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -3,32 +3,52 @@ #include "util.hh" #include "path.hh" #include "eval.hh" +#include "flake/flake.hh" #include <optional> namespace nix { -struct Buildable -{ - std::optional<StorePath> drvPath; - std::map<std::string, StorePath> outputs; +struct DrvInfo; +struct SourceExprCommand; + +namespace eval_cache { class EvalCache; class AttrCursor; } + +struct BuildableOpaque { + StorePath path; +}; + +struct BuildableFromDrv { + StorePath drvPath; + std::map<std::string, std::optional<StorePath>> outputs; }; +typedef std::variant< + BuildableOpaque, + BuildableFromDrv +> Buildable; + typedef std::vector<Buildable> Buildables; +struct App +{ + std::vector<StorePathWithOutputs> context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + struct Installable { virtual ~Installable() { } virtual std::string what() = 0; - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } + virtual Buildables toBuildables() = 0; Buildable toBuildable(); + App toApp(EvalState & state); + virtual std::pair<Value *, Pos> toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); @@ -40,6 +60,73 @@ struct Installable { return {}; } + + virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + getCursors(EvalState & state); + + std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> + getCursor(EvalState & state); + + virtual FlakeRef nixpkgsFlakeRef() const + { + return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + } }; +struct InstallableValue : Installable +{ + ref<EvalState> state; + + InstallableValue(ref<EvalState> state) : state(state) {} + + struct DerivationInfo + { + StorePath drvPath; + std::optional<StorePath> outPath; + std::string outputName; + }; + + virtual std::vector<DerivationInfo> toDerivations() = 0; + + Buildables toBuildables() override; +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; + + InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef, + Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags) + : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths), + prefixes(prefixes), lockFlags(lockFlags) + { } + + std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector<std::string> getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation(); + + std::vector<DerivationInfo> toDerivations() override; + + std::pair<Value *, Pos> toValue(EvalState & state) override; + + std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + getCursors(EvalState & state) override; + + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const override; +}; + +ref<eval_cache::EvalCache> openEvalCache( + EvalState & state, + std::shared_ptr<flake::LockedFlake> lockedFlake); + } diff --git a/src/nix/local.mk b/src/nix/local.mk index b057b7cc6..f37b73384 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -19,7 +19,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr nix_LIBS = libexpr libmain libfetchers libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ @@ -29,3 +29,5 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh src/nix/develop.cc: src/nix/get-env.sh.gen.hh + +src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh diff --git a/src/nix/log.cc b/src/nix/log.cc index 3fe22f6c2..33a3053f5 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -18,7 +18,7 @@ struct CmdLog : InstallableCommand return { Example{ "To get the build log of GNU Hello:", - "nix log nixpkgs.hello" + "nix log nixpkgs#hello" }, Example{ "To get the build log of a specific path:", @@ -26,7 +26,7 @@ struct CmdLog : InstallableCommand }, Example{ "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs.hello" + "nix log --store https://cache.nixos.org nixpkgs#hello" }, }; } @@ -45,11 +45,14 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { - auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; - for (auto & output : b.outputs) { - if (log) break; - log = sub->getBuildLog(output.second); - } + auto log = std::visit(overloaded { + [&](BuildableOpaque bo) { + return sub->getBuildLog(bo.path); + }, + [&](BuildableFromDrv bfd) { + return sub->getBuildLog(bfd.drvPath); + }, + }, b); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); @@ -61,4 +64,4 @@ struct CmdLog : InstallableCommand } }; -static auto r1 = registerCommand<CmdLog>("log"); +static auto rCmdLog = registerCommand<CmdLog>("log"); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d2157f2d4..baca54431 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -85,7 +85,11 @@ struct CmdLsStore : StoreCommand, MixLs { CmdLsStore() { - expectArg("path", &path); + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); } Examples examples() override @@ -117,7 +121,11 @@ struct CmdLsNar : Command, MixLs CmdLsNar() { - expectArg("nar", &narPath); + expectArgs({ + .label = "nar", + .handler = {&narPath}, + .completer = completePath + }); expectArg("path", &path); } @@ -144,5 +152,5 @@ struct CmdLsNar : Command, MixLs } }; -static auto r1 = registerCommand<CmdLsStore>("ls-store"); -static auto r2 = registerCommand<CmdLsNar>("ls-nar"); +static auto rCmdLsStore = registerCommand<CmdLsStore>("ls-store"); +static auto rCmdLsNar = registerCommand<CmdLsNar>("ls-nar"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 203901168..1e9e07bc0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -7,7 +7,6 @@ #include "legacy.hh" #include "shared.hh" #include "store-api.hh" -#include "progress-bar.hh" #include "filetransfer.hh" #include "finally.hh" #include "loggers.hh" @@ -18,6 +17,8 @@ #include <netdb.h> #include <netinet/in.h> +#include <nlohmann/json.hpp> + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); @@ -69,7 +70,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", .description = "show usage information", - .handler = {[&]() { showHelpAndExit(); }}, + .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ @@ -97,7 +98,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "version", .description = "show version information", - .handler = {[&]() { printVersion(programName); }}, + .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ @@ -141,6 +142,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs printHelp(programName, std::cout); throw Exit(); } + + std::string description() override + { + return "a tool for reproducible and declarative configuration management"; + } }; void mainWrapped(int argc, char * * argv) @@ -165,6 +171,7 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlWarn; settings.verboseBuild = false; + evalSettings.pureEval = true; setLogFormat("bar"); @@ -172,7 +179,46 @@ void mainWrapped(int argc, char * * argv) NixArgs args; - args.parseCmdline(argvToStrings(argc, argv)); + if (argc == 2 && std::string(argv[1]) == "__dump-args") { + std::cout << args.toJSON().dump() << "\n"; + return; + } + + if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + evalSettings.pureEval = false; + EvalState state({}, openStore("dummy://")); + auto res = nlohmann::json::object(); + auto builtins = state.baseEnv.values[0]->attrs; + for (auto & builtin : *builtins) { + auto b = nlohmann::json::object(); + if (builtin.value->type != tPrimOp) continue; + auto primOp = builtin.value->primOp; + if (!primOp->doc) continue; + b["arity"] = primOp->arity; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + res[(std::string) builtin.name] = std::move(b); + } + std::cout << res.dump() << "\n"; + return; + } + + Finally printCompletions([&]() + { + if (completions) { + std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n"); + for (auto & s : *completions) + std::cout << s << "\n"; + } + }); + + try { + args.parseCmdline(argvToStrings(argc, argv)); + } catch (UsageError &) { + if (!completions) throw; + } + + if (completions) return; initPlugins(); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index fb36fc410..df3ec5194 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -10,7 +10,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON { CmdMakeContentAddressable() { - realiseMode = Build; + realiseMode = Realise::Outputs; } std::string description() override @@ -23,7 +23,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON return { Example{ "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix make-content-addressable nixpkgs.hello" + "nix make-content-addressable nixpkgs#hello" }, Example{ "To compute a content-addressable representation of the current NixOS system closure:", @@ -77,10 +77,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + narHash, + }; info.references = std::move(references); if (hasSelfReference) info.references.insert(info.path); - info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, @@ -106,4 +108,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand<CmdMakeContentAddressable>("make-content-addressable"); +static auto rCmdMakeContentAddressable = registerCommand<CmdMakeContentAddressable>("make-content-addressable"); diff --git a/src/nix/markdown.cc b/src/nix/markdown.cc new file mode 100644 index 000000000..40788a42f --- /dev/null +++ b/src/nix/markdown.cc @@ -0,0 +1,50 @@ +#include "markdown.hh" +#include "util.hh" +#include "finally.hh" + +#include <sys/queue.h> +extern "C" { +#include <lowdown.h> +} + +namespace nix { + +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + struct lowdown_opts opts { + .type = LOWDOWN_TERM, + .maxdepth = 20, + .cols = std::min(getWindowSize().second, (unsigned short) 80), + .hmargin = 0, + .vmargin = 0, + .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, + .oflags = 0, + }; + + auto doc = lowdown_doc_new(&opts); + if (!doc) + throw Error("cannot allocate Markdown document"); + Finally freeDoc([&]() { lowdown_doc_free(doc); }); + + size_t maxn = 0; + auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size()); + if (!node) + throw Error("cannot parse Markdown document"); + Finally freeNode([&]() { lowdown_node_free(node); }); + + auto renderer = lowdown_term_new(&opts); + if (!renderer) + throw Error("cannot allocate Markdown renderer"); + Finally freeRenderer([&]() { lowdown_term_free(renderer); }); + + auto buf = lowdown_buf_new(16384); + if (!buf) + throw Error("cannot allocate Markdown output buffer"); + Finally freeBuffer([&]() { lowdown_buf_free(buf); }); + + lowdown_term_rndr(buf, nullptr, renderer, node); + + return std::string(buf->data, buf->size); +} + +} diff --git a/src/nix/markdown.hh b/src/nix/markdown.hh new file mode 100644 index 000000000..78320fcf5 --- /dev/null +++ b/src/nix/markdown.hh @@ -0,0 +1,7 @@ +#include "types.hh" + +namespace nix { + +std::string renderMarkdownToTerminal(std::string_view markdown); + +} diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index b45951879..51a7a9756 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -31,4 +31,4 @@ struct CmdOptimiseStore : StoreCommand } }; -static auto r1 = registerCommand<CmdOptimiseStore>("optimise-store"); +static auto rCmdOptimiseStore = registerCommand<CmdOptimiseStore>("optimise-store"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b89a44f83..63cf885f9 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -40,7 +40,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }, Example{ "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs.rust" + "nix path-info -rsSh nixpkgs#rust" }, Example{ "To check the existence of a path in a binary cache:", @@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; } - void printSize(unsigned long long value) + void printSize(uint64_t value) { if (!humanReadable) { std::cout << fmt("\t%11d", value); @@ -127,4 +127,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand<CmdPathInfo>("path-info"); +static auto rCmdPathInfo = registerCommand<CmdPathInfo>("path-info"); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 127397a29..8db78d591 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -29,4 +29,4 @@ struct CmdPingStore : StoreCommand } }; -static auto r1 = registerCommand<CmdPingStore>("ping-store"); +static auto rCmdPingStore = registerCommand<CmdPingStore>("ping-store"); diff --git a/src/nix/profile.cc b/src/nix/profile.cc new file mode 100644 index 000000000..01aef2f9b --- /dev/null +++ b/src/nix/profile.cc @@ -0,0 +1,470 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "archive.hh" +#include "builtins/buildenv.hh" +#include "flake/flakeref.hh" +#include "../nix-env/user-env.hh" +#include "profiles.hh" + +#include <nlohmann/json.hpp> +#include <regex> + +using namespace nix; + +struct ProfileElementSource +{ + FlakeRef originalRef; + // FIXME: record original attrpath. + FlakeRef resolvedRef; + std::string attrPath; + // FIXME: output names +}; + +struct ProfileElement +{ + StorePathSet storePaths; + std::optional<ProfileElementSource> source; + bool active = true; + // FIXME: priority +}; + +struct ProfileManifest +{ + std::vector<ProfileElement> elements; + + ProfileManifest() { } + + ProfileManifest(EvalState & state, const Path & profile) + { + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + + for (auto & e : json["elements"]) { + ProfileElement element; + for (auto & p : e["storePaths"]) + element.storePaths.insert(state.store->parseStorePath((std::string) p)); + element.active = e["active"]; + if (e.value("uri", "") != "") { + element.source = ProfileElementSource{ + parseFlakeRef(e["originalUri"]), + parseFlakeRef(e["uri"]), + e["attrPath"] + }; + } + elements.emplace_back(std::move(element)); + } + } + + else if (pathExists(profile + "/manifest.nix")) { + // FIXME: needed because of pure mode; ugly. + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->followLinksToStore(profile)); + state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); + } + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = {state.store->parseStorePath(drvInfo.queryOutPath())}; + elements.emplace_back(std::move(element)); + } + } + } + + std::string toJSON(Store & store) const + { + auto array = nlohmann::json::array(); + for (auto & element : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) + paths.push_back(store.printStorePath(path)); + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + if (element.source) { + obj["originalUri"] = element.source->originalRef.to_string(); + obj["uri"] = element.source->resolvedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + } + array.push_back(obj); + } + nlohmann::json json; + json["version"] = 1; + json["elements"] = array; + return json.dump(); + } + + StorePath build(ref<Store> store) + { + auto tempDir = createTempDir(); + + StorePathSet references; + + Packages pkgs; + for (auto & element : elements) { + for (auto & path : element.storePaths) { + if (element.active) + pkgs.emplace_back(store->printStorePath(path), true, 5); + references.insert(path); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON(*store)); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + auto narHash = hashString(htSHA256, *sink.s); + + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + narHash, + }; + info.references = std::move(references); + info.narSize = sink.s->size(); + info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; + + auto source = StringSource { *sink.s }; + store->addToStore(info, source); + + return std::move(info.path); + } +}; + +struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile +{ + std::string description() override + { + return "install a package into a profile"; + } + + Examples examples() override + { + return { + Example{ + "To install a package from Nixpkgs:", + "nix profile install nixpkgs#hello" + }, + Example{ + "To install a package from a specific branch of Nixpkgs:", + "nix profile install nixpkgs/release-19.09#hello" + }, + Example{ + "To install a package from a specific revision of Nixpkgs:", + "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + std::vector<StorePathWithOutputs> pathsToBuild; + + for (auto & installable : installables) { + if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + + ProfileElement element; + if (!drv.outPath) + throw UnimplementedError("CA derivations are not yet supported by 'nix profile'"); + element.storePaths = {*drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable2->flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + + manifest.elements.emplace_back(std::move(element)); + } else + throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what()); + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +class MixProfileElementMatchers : virtual Args +{ + std::vector<std::string> _matchers; + +public: + + MixProfileElementMatchers() + { + expectArgs("elements", &_matchers); + } + + typedef std::variant<size_t, Path, std::regex> Matcher; + + std::vector<Matcher> getMatchers(ref<Store> store) + { + std::vector<Matcher> res; + + for (auto & s : _matchers) { + size_t n; + if (string2Int(s, n)) + res.push_back(n); + else if (store->isStorePath(s)) + res.push_back(s); + else + res.push_back(std::regex(s, std::regex::extended | std::regex::icase)); + } + + return res; + } + + bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers) + { + for (auto & matcher : matchers) { + if (auto n = std::get_if<size_t>(&matcher)) { + if (*n == pos) return true; + } else if (auto path = std::get_if<Path>(&matcher)) { + if (element.storePaths.count(store.parseStorePath(*path))) return true; + } else if (auto regex = std::get_if<std::regex>(&matcher)) { + if (element.source + && std::regex_match(element.source->attrPath, *regex)) + return true; + } + } + + return false; + } +}; + +struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "remove packages from a profile"; + } + + Examples examples() override + { + return { + Example{ + "To remove a package by attribute path:", + "nix profile remove packages.x86_64-linux.hello" + }, + Example{ + "To remove all packages:", + "nix profile remove '.*'" + }, + Example{ + "To remove a package by store path:", + "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" + }, + Example{ + "To remove a package by position:", + "nix profile remove 3" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest oldManifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + ProfileManifest newManifest; + + for (size_t i = 0; i < oldManifest.elements.size(); ++i) { + auto & element(oldManifest.elements[i]); + if (!matches(*store, element, i, matchers)) + newManifest.elements.push_back(std::move(element)); + } + + // FIXME: warn about unused matchers? + + printInfo("removed %d packages, kept %d packages", + oldManifest.elements.size() - newManifest.elements.size(), + newManifest.elements.size()); + + updateProfile(newManifest.build(store)); + } +}; + +struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "upgrade packages using their most recent flake"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade all packages that were installed using a mutable flake reference:", + "nix profile upgrade '.*'" + }, + Example{ + "To upgrade a specific package:", + "nix profile upgrade packages.x86_64-linux.hello" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + // FIXME: code duplication + std::vector<StorePathWithOutputs> pathsToBuild; + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (element.source + && !element.source->originalRef.input.isImmutable() + && matches(*store, element, i, matchers)) + { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags); + + auto [attrPath, resolvedRef, drv] = installable.toDerivation(); + + if (element.source->resolvedRef == resolvedRef) continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->resolvedRef, resolvedRef); + + if (!drv.outPath) + throw UnimplementedError("CA derivations are not yet supported by 'nix profile'"); + element.storePaths = {*drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable.flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + } + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "list installed packages"; + } + + Examples examples() override + { + return { + Example{ + "To show what packages are installed in the default profile:", + "nix profile info" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + logger->stdout("%d %s %s %s", i, + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", + concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } + } +}; + +struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "show the closure difference between each generation of a profile"; + } + + Examples examples() override + { + return { + Example{ + "To show what changed between each generation of the NixOS system profile:", + "nix profile diff-closure --profile /nix/var/nix/profiles/system" + }, + }; + } + + void run(ref<Store> store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional<Generation> prevGen; + bool first = true; + + for (auto & gen : gens) { + if (prevGen) { + if (!first) std::cout << "\n"; + first = false; + std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number); + printClosureDiff(store, + store->followLinksToStorePath(prevGen->path), + store->followLinksToStorePath(gen.path), + " "); + } + + prevGen = gen; + } + } +}; + +struct CmdProfile : NixMultiCommand +{ + CmdProfile() + : MultiCommand({ + {"install", []() { return make_ref<CmdProfileInstall>(); }}, + {"remove", []() { return make_ref<CmdProfileRemove>(); }}, + {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }}, + {"info", []() { return make_ref<CmdProfileInfo>(); }}, + {"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }}, + }) + { } + + std::string description() override + { + return "manage Nix profiles"; + } + + void run() override + { + if (!command) + throw UsageError("'nix profile' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdProfile = registerCommand<CmdProfile>("profile"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc new file mode 100644 index 000000000..8e8983ad0 --- /dev/null +++ b/src/nix/registry.cc @@ -0,0 +1,146 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "flake/flake.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "registry.hh" + +using namespace nix; +using namespace nix::flake; + +struct CmdRegistryList : StoreCommand +{ + std::string description() override + { + return "list available Nix flakes"; + } + + void run(nix::ref<nix::Store> store) override + { + using namespace fetchers; + + auto registries = getRegistries(store); + + for (auto & registry : registries) { + for (auto & entry : registry->entries) { + // FIXME: format nicely + logger->stdout("%s %s %s", + registry->type == Registry::Flag ? "flags " : + registry->type == Registry::User ? "user " : + registry->type == Registry::System ? "system" : + "global", + entry.from.toURLString(), + entry.to.toURLString(attrsToQuery(entry.extraAttrs))); + } + } + } +}; + +struct CmdRegistryAdd : MixEvalArgs, Command +{ + std::string fromUrl, toUrl; + + std::string description() override + { + return "add/replace flake in user flake registry"; + } + + CmdRegistryAdd() + { + expectArg("from-url", &fromUrl); + expectArg("to-url", &toUrl); + } + + void run() override + { + auto fromRef = parseFlakeRef(fromUrl); + auto toRef = parseFlakeRef(toUrl); + fetchers::Attrs extraAttrs; + if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(fromRef.input); + userRegistry->add(fromRef.input, toRef.input, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command +{ + std::string url; + + std::string description() override + { + return "remove flake from user flake registry"; + } + + CmdRegistryRemove() + { + expectArg("url", &url); + } + + void run() override + { + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(parseFlakeRef(url).input); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdRegistryPin : virtual Args, EvalCommand +{ + std::string url; + + std::string description() override + { + return "pin a flake to its current version in user flake registry"; + } + + CmdRegistryPin() + { + expectArg("url", &url); + } + + void run(nix::ref<nix::Store> store) override + { + auto ref = parseFlakeRef(url); + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(ref.input); + auto [tree, resolved] = ref.resolve(store).input.fetch(store); + fetchers::Attrs extraAttrs; + if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; + userRegistry->add(ref.input, resolved, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdRegistry : virtual NixMultiCommand +{ + CmdRegistry() + : MultiCommand({ + {"list", []() { return make_ref<CmdRegistryList>(); }}, + {"add", []() { return make_ref<CmdRegistryAdd>(); }}, + {"remove", []() { return make_ref<CmdRegistryRemove>(); }}, + {"pin", []() { return make_ref<CmdRegistryPin>(); }}, + }) + { + } + + std::string description() override + { + return "manage the flake registry"; + } + + Category category() override { return catSecondary; } + + void run() override + { + if (!command) + throw UsageError("'nix registry' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdRegistry = registerCommand<CmdRegistry>("registry"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fdacf604b..9ff386b1d 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -32,13 +32,19 @@ extern "C" { #include "globals.hh" #include "command.hh" #include "finally.hh" +#include "markdown.hh" +#if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include <gc/gc_cpp.h> +#endif namespace nix { -struct NixRepl : gc +struct NixRepl + #if HAVE_BOEHMGC + : gc + #endif { string curDir; std::unique_ptr<EvalState> state; @@ -59,7 +65,7 @@ struct NixRepl : gc void mainLoop(const std::vector<std::string> & files); StringSet completePrefix(string prefix); bool getLine(string & input, const std::string &prompt); - Path getDerivationPath(Value & v); + StorePath getDerivationPath(Value & v); bool processLine(string line); void loadFile(const Path & path); void initEnv(); @@ -370,13 +376,16 @@ bool isVarName(const string & s) } -Path NixRepl::getDerivationPath(Value & v) { +StorePath NixRepl::getDerivationPath(Value & v) { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); - Path drvPath = drvInfo->queryDrvPath(); - if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath))) - throw Error("expression did not evaluate to a valid derivation"); + Path drvPathRaw = drvInfo->queryDrvPath(); + if (drvPathRaw == "") + throw Error("expression did not evaluate to a valid derivation (no drv path)"); + StorePath drvPath = state->store->parseStorePath(drvPathRaw); + if (!state->store->isValidPath(drvPath)) + throw Error("expression did not evaluate to a valid derivation (invalid drv path)"); return drvPath; } @@ -411,7 +420,8 @@ bool NixRepl::processLine(string line) << " :r Reload all files\n" << " :s <expr> Build dependencies of derivation, then start nix-shell\n" << " :t <expr> Describe result of evaluation\n" - << " :u <expr> Build derivation, then start nix-shell\n"; + << " :u <expr> Build derivation, then start nix-shell\n" + << " :doc <expr> Show documentation of a builtin function\n"; } else if (command == ":a" || command == ":add") { @@ -469,29 +479,30 @@ bool NixRepl::processLine(string line) evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); state->callFunction(f, v, result, Pos()); - Path drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + StorePath drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { Value v; evalString(arg, v); - Path drvPath = getDerivationPath(v); + StorePath drvPath = getDerivationPath(v); + Path drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b") { /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - auto drv = readDerivation(*state->store, drvPath); + if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) { + auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; - for (auto & i : drv.outputs) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); + for (auto & i : drv.outputsAndOptPaths(*state->store)) + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second)); } } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw}); } } @@ -504,6 +515,29 @@ bool NixRepl::processLine(string line) else if (command == ":q" || command == ":quit") return false; + else if (command == ":doc") { + Value v; + evalString(arg, v); + if (auto doc = state->getDoc(v)) { + std::string markdown; + + if (!doc->args.empty() && doc->name) { + auto args = doc->args; + for (auto & arg : args) + arg = "*" + arg + "*"; + + markdown += + "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` " + + concatStringsSep(" ", args) + "\n\n"; + } + + markdown += trim(stripIndentation(doc->doc)); + + std::cout << renderMarkdownToTerminal(markdown); + } else + throw Error("value does not have documentation"); + } + else if (command != "") throw Error("unknown command '%1%'", command); @@ -760,7 +794,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs CmdRepl() { - expectArgs("files", &files); + expectArgs({ + .label = "files", + .handler = {&files}, + .completer = completePath + }); } std::string description() override @@ -773,19 +811,20 @@ struct CmdRepl : StoreCommand, MixEvalArgs return { Example{ "Display all special commands within the REPL:", - "nix repl\n nix-repl> :?" + "nix repl\nnix-repl> :?" } }; } void run(ref<Store> store) override { + evalSettings.pureEval = false; auto repl = std::make_unique<NixRepl>(searchPath, openStore()); repl->autoArgs = getAutoArgs(*repl->state); repl->mainLoop(files); } }; -static auto r1 = registerCommand<CmdRepl>("repl"); +static auto rCmdRepl = registerCommand<CmdRepl>("repl"); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 321ee1d11..790784382 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -84,31 +84,30 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment { return { Example{ - "To start a shell providing GNU Hello from NixOS 17.03:", - "nix shell -f channel:nixos-17.03 hello" + "To start a shell providing GNU Hello from NixOS 20.03:", + "nix shell nixpkgs/nixos-20.03#hello" }, Example{ "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix shell nixpkgs.youtube-dl" + "nix shell nixpkgs#youtube-dl" }, Example{ "To run GNU Hello:", - "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" + "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'" }, Example{ "To run GNU Hello in a chroot store:", - "nix shell --store ~/my-nix nixpkgs.hello -c hello" + "nix shell --store ~/my-nix nixpkgs#hello -c hello" }, }; } void run(ref<Store> store) override { - auto outPaths = toStorePaths(store, Build, installables); + auto outPaths = toStorePaths(store, Realise::Outputs, OperateOn::Output, installables); auto accessor = store->getFSAccessor(); - std::unordered_set<StorePath> done; std::queue<StorePath> todo; for (auto & path : outPaths) todo.push(path); @@ -141,7 +140,76 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment } }; -static auto r1 = registerCommand<CmdShell>("shell"); +static auto rCmdShell = registerCommand<CmdShell>("shell"); + +struct CmdRun : InstallableCommand, RunCommon +{ + std::vector<std::string> args; + + CmdRun() + { + expectArgs({ + .label = "args", + .handler = {&args}, + .completer = completePath + }); + } + + std::string description() override + { + return "run a Nix application"; + } + + Examples examples() override + { + return { + Example{ + "To run Blender:", + "nix run blender-bin" + }, + Example{ + "To run vim from nixpkgs:", + "nix run nixpkgs#vim" + }, + Example{ + "To run vim from nixpkgs with arguments:", + "nix run nixpkgs#vim -- --help" + }, + }; + } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + + auto app = installable->toApp(*state); + + state->store->buildPaths(app.context); + + Strings allArgs{app.program}; + for (auto & i : args) allArgs.push_back(i); + + runProgram(store, app.program, allArgs); + } +}; + +static auto rCmdRun = registerCommand<CmdRun>("run"); void chrootHelper(int argc, char * * argv) { diff --git a/src/nix/search.cc b/src/nix/search.cc index 93c3f3f83..d4326dc84 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -6,8 +6,9 @@ #include "get-drvs.hh" #include "common-args.hh" #include "json.hh" -#include "json-to-value.hh" #include "shared.hh" +#include "eval-cache.hh" +#include "attr-path.hh" #include <regex> #include <fstream> @@ -25,33 +26,17 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos m.empty() ? s : std::string(m.prefix()) - + ANSI_RED + std::string(m.str()) + postfix + + ANSI_GREEN + std::string(m.str()) + postfix + std::string(m.suffix()); } -struct CmdSearch : SourceExprCommand, MixJSON +struct CmdSearch : InstallableCommand, MixJSON { std::vector<std::string> res; - bool writeCache = true; - bool useCache = true; - CmdSearch() { expectArgs("regex", &res); - - addFlag({ - .longName = "update-cache", - .shortName = 'u', - .description = "update the package search cache", - .handler = {[&]() { writeCache = true; useCache = false; }} - }); - - addFlag({ - .longName = "no-cache", - .description = "do not use or update the package search cache", - .handler = {[&]() { writeCache = false; useCache = false; }} - }); } std::string description() override @@ -63,24 +48,32 @@ struct CmdSearch : SourceExprCommand, MixJSON { return { Example{ - "To show all available packages:", + "To show all packages in the flake in the current directory:", "nix search" }, Example{ - "To show any packages containing 'blender' in its name or description:", - "nix search blender" + "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:", + "nix search nixpkgs blender" }, Example{ "To search for Firefox or Chromium:", - "nix search 'firefox|chromium'" + "nix search nixpkgs 'firefox|chromium'" }, Example{ - "To search for git and frontend or gui:", - "nix search git 'frontend|gui'" + "To search for packages containing 'git' and either 'frontend' or 'gui':", + "nix search nixpkgs git 'frontend|gui'" } }; } + Strings getDefaultFlakeAttrPaths() override + { + return { + "packages." + settings.thisSystem.get() + ".", + "legacyPackages." + settings.thisSystem.get() + "." + }; + } + void run(ref<Store> store) override { settings.readOnlyMode = true; @@ -88,190 +81,108 @@ struct CmdSearch : SourceExprCommand, MixJSON // Empty search string should match all packages // Use "^" here instead of ".*" due to differences in resulting highlighting // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) - if (res.empty()) { + if (res.empty()) res.push_back("^"); - } std::vector<std::regex> regexes; regexes.reserve(res.size()); - for (auto &re : res) { + for (auto & re : res) regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); - } auto state = getEvalState(); auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; - auto sToplevel = state->symbols.create("_toplevel"); - auto sRecurse = state->symbols.create("recurseForDerivations"); - - bool fromCache = false; + uint64_t results = 0; - std::map<std::string, std::string> results; - - std::function<void(Value *, std::string, bool, JSONObject *)> doExpr; - - doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { - debug("at attribute '%s'", attrPath); + std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)> visit; + visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath) + { + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPath))); try { - uint found = 0; + auto recurse = [&]() + { + for (const auto & attr : cursor.getAttrs()) { + auto cursor2 = cursor.getAttr(attr); + auto attrPath2(attrPath); + attrPath2.push_back(attr); + visit(*cursor2, attrPath2); + } + }; - state->forceValue(*v); + if (cursor.isDerivation()) { + size_t found = 0; - if (v->type == tLambda && toplevel) { - Value * v2 = state->allocValue(); - state->autoCallFunction(*state->allocBindings(1), *v, *v2); - v = v2; - state->forceValue(*v); - } + DrvName name(cursor.getAttr("name")->getString()); - if (state->isDerivation(*v)) { + auto aMeta = cursor.maybeGetAttr("meta"); + auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; + auto description = aDescription ? aDescription->getString() : ""; + std::replace(description.begin(), description.end(), '\n', ' '); + auto attrPath2 = concatStringsSep(".", attrPath); - DrvInfo drv(*state, attrPath, v->attrs); - std::string description; std::smatch attrPathMatch; std::smatch descriptionMatch; std::smatch nameMatch; - std::string name; - - DrvName parsed(drv.queryName()); - - for (auto ®ex : regexes) { - std::regex_search(attrPath, attrPathMatch, regex); - name = parsed.name; - std::regex_search(name, nameMatch, regex); - - description = drv.queryMetaString("description"); - std::replace(description.begin(), description.end(), '\n', ' '); + for (auto & regex : regexes) { + std::regex_search(attrPath2, attrPathMatch, regex); + std::regex_search(name.name, nameMatch, regex); std::regex_search(description, descriptionMatch, regex); - if (!attrPathMatch.empty() || !nameMatch.empty() || !descriptionMatch.empty()) - { found++; - } } if (found == res.size()) { + results++; if (json) { - - auto jsonElem = jsonOut->object(attrPath); - - jsonElem.attr("pkgName", parsed.name); - jsonElem.attr("version", parsed.version); + auto jsonElem = jsonOut->object(attrPath2); + jsonElem.attr("pname", name.name); + jsonElem.attr("version", name.version); jsonElem.attr("description", description); - } else { - auto name = hilite(parsed.name, nameMatch, "\e[0;2m") - + std::string(parsed.fullName, parsed.name.length()); - results[attrPath] = fmt( - "* %s (%s)\n %s\n", - wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), - wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), - hilite(description, descriptionMatch, ANSI_NORMAL)); - } - } - - if (cache) { - cache->attr("type", "derivation"); - cache->attr("name", drv.queryName()); - cache->attr("system", drv.querySystem()); - if (description != "") { - auto meta(cache->object("meta")); - meta.attr("description", description); + auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); + if (results > 1) logger->stdout(""); + logger->stdout( + "* %s%s", + wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), + name.version != "" ? " (" + name.version + ")" : ""); + if (description != "") + logger->stdout( + " %s", hilite(description, descriptionMatch, ANSI_NORMAL)); } } } - else if (v->type == tAttrs) { + else if ( + attrPath.size() == 0 + || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2) + || (attrPath[0] == "packages" && attrPath.size() <= 2)) + recurse(); - if (!toplevel) { - auto attrs = v->attrs; - Bindings::iterator j = attrs->find(sRecurse); - if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { - debug("skip attribute '%s'", attrPath); - return; - } - } - - bool toplevel2 = false; - if (!fromCache) { - Bindings::iterator j = v->attrs->find(sToplevel); - toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); - } - - for (auto & i : *v->attrs) { - auto cache2 = - cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr; - doExpr(i.value, - attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, - toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); - } + else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { + auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); + if (attr && attr->getBool()) + recurse(); } - } catch (AssertionError & e) { - } catch (Error & e) { - if (!toplevel) { - e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath); + } catch (EvalError & e) { + if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) throw; - } } }; - Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; - - if (useCache && pathExists(jsonCacheFileName)) { - - warn("using cached results; pass '-u' to update the cache"); - - Value vRoot; - parseJSON(*state, readFile(jsonCacheFileName), vRoot); - - fromCache = true; - - doExpr(&vRoot, "", true, nullptr); - } + for (auto & [cursor, prefix] : installable->getCursors(*state)) + visit(*cursor, parseAttrPath(*state, prefix)); - else { - createDirs(dirOf(jsonCacheFileName)); - - Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); - - std::ofstream jsonCacheFile; - - try { - // iostream considered harmful - jsonCacheFile.exceptions(std::ofstream::failbit); - jsonCacheFile.open(tmpFile); - - auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr; - - doExpr(getSourceExpr(*state), "", true, cache.get()); - - } catch (std::exception &) { - /* Fun fact: catching std::ios::failure does not work - due to C++11 ABI shenanigans. - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ - if (!jsonCacheFile) - throw Error("error writing to %s", tmpFile); - throw; - } - - if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) - throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); - } - - if (!json && results.size() == 0) + if (!json && !results) throw Error("no results for the given search term(s)!"); - - RunPager pager; - for (auto el : results) std::cout << el.second << "\n"; - } }; -static auto r1 = registerCommand<CmdSearch>("search"); +static auto rCmdSearch = registerCommand<CmdSearch>("search"); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 4fd8886de..1ef54a33a 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -2,7 +2,8 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "json.hh" + +#include <nlohmann/json.hpp> using namespace nix; @@ -19,8 +20,7 @@ struct CmdShowConfig : Command, MixJSON { if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). - JSONObject jsonObj(std::cout); - globalConfig.toJSON(jsonObj); + logger->stdout("%s", globalConfig.toJSON().dump()); } else { std::map<std::string, Config::SettingInfo> settings; globalConfig.getSettings(settings); @@ -30,4 +30,4 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto r1 = registerCommand<CmdShowConfig>("show-config"); +static auto rShowConfig = registerCommand<CmdShowConfig>("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 5d77cfdca..2542537d3 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand return { Example{ "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs.hello" + "nix show-derivation nixpkgs#hello" }, Example{ "To show the full derivation graph (if available) that produced your NixOS system:", @@ -67,13 +67,22 @@ struct CmdShowDerivation : InstallablesCommand { auto outputsObj(drvObj.object("outputs")); - for (auto & output : drv.outputs) { - auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash) { - outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); - outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); - } + for (auto & [_outputName, output] : drv.outputs) { + auto & outputName = _outputName; // work around clang bug + auto outputObj { outputsObj.object(outputName) }; + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + outputObj.attr("path", store->printStorePath(doi.path)); + }, + [&](DerivationOutputCAFixed dof) { + outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); + outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); + outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); + }, + [&](DerivationOutputCAFloating dof) { + outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + }, + }, output.output); } } @@ -114,4 +123,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto r1 = registerCommand<CmdShowDerivation>("show-derivation"); +static auto rCmdShowDerivation = registerCommand<CmdShowDerivation>("show-derivation"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6c9b9a792..44916c77f 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,7 +92,7 @@ struct CmdCopySigs : StorePathsCommand } }; -static auto r1 = registerCommand<CmdCopySigs>("copy-sigs"); +static auto rCmdCopySigs = registerCommand<CmdCopySigs>("copy-sigs"); struct CmdSignPaths : StorePathsCommand { @@ -105,7 +105,8 @@ struct CmdSignPaths : StorePathsCommand .shortName = 'k', .description = "file containing the secret signing key", .labels = {"file"}, - .handler = {&secretKeyFile} + .handler = {&secretKeyFile}, + .completer = completePath }); } @@ -143,4 +144,4 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto r2 = registerCommand<CmdSignPaths>("sign-paths"); +static auto rCmdSignPaths = registerCommand<CmdSignPaths>("sign-paths"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a880bdae0..66ecc5b34 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -158,4 +158,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand<CmdUpgradeNix>("upgrade-nix"); +static auto rCmdUpgradeNix = registerCommand<CmdUpgradeNix>("upgrade-nix"); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index bb5e4529b..ec7333d03 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -77,20 +77,23 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath)); - MaintainCount<std::atomic<size_t>> mcActive(active); update(); auto info = store->queryPathInfo(store->parseStorePath(storePath)); + // Note: info->path can be different from storePath + // for binary cache stores when using --all (since we + // can't enumerate names efficiently). + Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path))); + if (!noContents) { std::unique_ptr<AbstractHashSink> hashSink; if (!info->ca) - hashSink = std::make_unique<HashSink>(*info->narHash.type); + hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); @@ -186,4 +189,4 @@ struct CmdVerify : StorePathsCommand } }; -static auto r1 = registerCommand<CmdVerify>("verify"); +static auto rCmdVerify = registerCommand<CmdVerify>("verify"); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 167c974ee..63bf087e6 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -55,15 +55,15 @@ struct CmdWhyDepends : SourceExprCommand return { Example{ "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs.hello nixpkgs.glibc" + "nix why-depends nixpkgs#hello nixpkgs#glibc" }, Example{ "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11" + "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11" }, Example{ "To show why Glibc depends on itself:", - "nix why-depends nixpkgs.glibc nixpkgs.glibc" + "nix why-depends nixpkgs#glibc nixpkgs#glibc" }, }; } @@ -72,17 +72,19 @@ struct CmdWhyDepends : SourceExprCommand void run(ref<Store> store) override { - auto package = parseInstallable(*this, store, _package, false); - auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); - auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto package = parseInstallable(store, _package); + auto packagePath = toStorePath(store, Realise::Outputs, operateOn, package); + auto dependency = parseInstallable(store, _dependency); + auto dependencyPath = toStorePath(store, Realise::Derivation, operateOn, dependency); auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); if (!closure.count(dependencyPath)) { - printError("'%s' does not depend on '%s'", package->what(), dependency->what()); + printError("'%s' does not depend on '%s'", + store->printStorePath(packagePath), + store->printStorePath(dependencyPath)); return; } @@ -106,7 +108,11 @@ struct CmdWhyDepends : SourceExprCommand std::map<StorePath, Node> graph; for (auto & path : closure) - graph.emplace(path, Node { .path = path, .refs = store->queryPathInfo(path)->references }); + graph.emplace(path, Node { + .path = path, + .refs = store->queryPathInfo(path)->references, + .dist = path == dependencyPath ? 0 : inf + }); // Transpose the graph. for (auto & node : graph) @@ -115,8 +121,6 @@ struct CmdWhyDepends : SourceExprCommand /* Run Dijkstra's shortest path algorithm to get the distance of every path in the closure to 'dependency'. */ - graph.emplace(dependencyPath, Node { .path = dependencyPath, .dist = 0 }); - std::priority_queue<Node *> queue; queue.push(&graph.at(dependencyPath)); @@ -259,4 +263,4 @@ struct CmdWhyDepends : SourceExprCommand } }; -static auto r1 = registerCommand<CmdWhyDepends>("why-depends"); +static auto rCmdWhyDepends = registerCommand<CmdWhyDepends>("why-depends"); diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 434ad80a6..d30227e4e 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -111,11 +111,7 @@ std::set<std::string> runResolver(const Path & filename) bool isSymlink(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError("getting attributes of path '%1%'", path); - - return S_ISLNK(st.st_mode); + return S_ISLNK(lstat(path).st_mode); } Path resolveSymlink(const Path & path) |