diff options
Diffstat (limited to 'src/libstore')
58 files changed, 4773 insertions, 2148 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..f6224d6a0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,17 +11,20 @@ #include "nar-accessor.hh" #include "json.hh" #include "thread-pool.hh" +#include "callback.hh" #include <chrono> #include <future> #include <regex> +#include <fstream> #include <nlohmann/json.hpp> namespace nix { BinaryCacheStore::BinaryCacheStore(const Params & params) - : Store(params) + : BinaryCacheStoreConfig(params) + , Store(params) { if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); @@ -57,6 +60,13 @@ void BinaryCacheStore::init() } } +void BinaryCacheStore::upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType) +{ + upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType); +} + void BinaryCacheStore::getFile(const std::string & path, Callback<std::shared_ptr<std::string>> callback) noexcept { @@ -113,13 +123,71 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) +AutoCloseFD openFile(const Path & path) +{ + auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return fd; +} + +struct FileSource : FdSource +{ + AutoCloseFD fd2; + + FileSource(const Path & path) + : fd2(openFile(path)) + { + fd = fd2.get(); + } +}; + +ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, + std::function<ValidPathInfo(HashResult)> mkInfo) { - // FIXME: See if we can use the original source to reduce memory usage. - auto nar = make_ref<std::string>(narSource.drain()); + auto [fdTemp, fnTemp] = createTempFile(); - if (!repair && isValidPath(info.path)) return; + AutoDelete autoDelete(fnTemp); + + auto now1 = std::chrono::steady_clock::now(); + + /* Read the NAR simultaneously into a CompressionSink+FileSink (to + write the compressed NAR to disk), into a HashSink (to get the + NAR hash), and into a NarAccessor (to get the NAR listing). */ + HashSink fileHashSink { htSHA256 }; + std::shared_ptr<FSAccessor> narAccessor; + HashSink narHashSink { htSHA256 }; + { + FdSink fileSink(fdTemp.get()); + TeeSink teeSinkCompressed { fileSink, fileHashSink }; + auto compressionSink = makeCompressionSink(compression, teeSinkCompressed); + TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; + TeeSource teeSource { narSource, teeSinkUncompressed }; + narAccessor = makeNarAccessor(teeSource); + compressionSink->finish(); + fileSink.flush(); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto info = mkInfo(narHashSink.finish()); + auto narInfo = make_ref<NarInfo>(info); + narInfo->compression = compression; + auto [fileHash, fileSize] = fileHashSink.finish(); + narInfo->fileHash = fileHash; + narInfo->fileSize = fileSize; + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), info.narSize, + ((1.0 - (double) fileSize / info.narSize) * 100.0), + duration); /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -132,23 +200,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource printStorePath(info.path), printStorePath(ref)); } - assert(nar->compare(0, narMagic.size(), narMagic) == 0); - - auto narInfo = make_ref<NarInfo>(info); - - narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); - - if (info.narHash && info.narHash != narInfo->narHash) - throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); - - auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); - - auto narAccessor = makeNarAccessor(nar); - - if (accessor_) - accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -160,33 +211,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource { auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); + listNar(res, ref<FSAccessor>(narAccessor), "", true); } } - upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json"); } - /* Compress the NAR. */ - narInfo->compression = compression; - auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); - auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); - narInfo->fileSize = narCompressed->size(); - - auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", - printStorePath(narInfo->path), narInfo->narSize, - ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), - duration); - - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/<build-id>' that specify the NAR file and member containing the debug info. */ @@ -247,12 +278,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + upsertFile(narInfo->url, + std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary), + "application/x-nix-nar"); } else stats.narWriteAverted++; - stats.narWriteBytes += nar->size(); - stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteBytes += info.narSize; + stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -261,6 +294,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource writeNarInfo(narInfo); stats.narInfoWrite++; + + return narInfo; +} + +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + if (!repair && isValidPath(info.path)) { + // FIXME: copyNAR -> null sink + narSource.drain(); + return; + } + + addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { + /* FIXME reinstate these, once we can correctly do hash modulo sink as + needed. We need to throw here in case we uploaded a corrupted store path. */ + // assert(info.narHash == nar.first); + // assert(info.narSize == nar.second); + return info; + }}); +} + +StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + unsupported("addToStoreFromDump"); + return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { + makeFixedOutputPath(method, nar.first, name), + nar.first, + }; + info.narSize = nar.second; + return info; + })->path; } bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) @@ -275,14 +343,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast<const NarInfo>(); - uint64_t narSize = 0; + LengthSink narSize; + TeeSink tee { sink, narSize }; - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - narSize += len; - }); - - auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + auto decompressor = makeDecompressionSink(info->compression, tee); try { getFile(info->url, *decompressor); @@ -294,7 +358,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME - stats.narReadBytes += narSize; + stats.narReadBytes += narSize.length; } void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, @@ -332,44 +396,52 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { - // FIXME: some cut&paste from LocalStore::addToStore(). + /* FIXME: Make BinaryCacheStore::addToStoreCommon support + non-recursive+sha256 so we can just use the default + implementation of this method in terms of addToStoreFromDump. */ - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - Hash h; + HashSink sink { hashAlgo }; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); - h = hashString(hashAlgo, *sink.s); } else { - auto s = readFile(srcPath); - dumpString(s, sink); - h = hashString(hashAlgo, s); + readFile(srcPath, sink); } + auto h = sink.finish().first; - ValidPathInfo info(makeFixedOutputPath(method, h, name)); - - auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); - - return std::move(info.path); + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink, filter); + }); + return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { + makeFixedOutputPath(method, h, name), + nar.first, + }; + info.narSize = nar.second; + info.ca = FixedOutputHash { + .method = method, + .hash = h, + }; + return info; + })->path; } StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info(computeStorePathForText(name, s, references)); - info.references = references; - - if (repair || !isValidPath(info.path)) { - StringSink sink; - dumpString(s, sink); - auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); - } - - return std::move(info.path); + auto textHash = hashString(htSHA256, s); + auto path = makeTextPath(name, textHash, references); + + if (!repair && isValidPath(path)) + return path; + + auto source = StringSource { s }; + return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { path, nar.first }; + info.narSize = nar.second; + info.ca = TextHash { textHash }; + info.references = references; + return info; + })->path; } ref<FSAccessor> BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 52ef8aa7a..5224d7ec8 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -11,17 +11,21 @@ namespace nix { struct NarInfo; -class BinaryCacheStore : public Store +struct BinaryCacheStoreConfig : virtual StoreConfig { -public: - - const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; - const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; - const Setting<bool> parallelCompression{this, false, "parallel-compression", + using StoreConfig::StoreConfig; + + const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; + const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", "enable multi-threading compression, available for xz only currently"}; +}; + +class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +{ private: @@ -36,9 +40,13 @@ public: virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) = 0; + void upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType); + /* Note: subclasses must implement at least one of the two following getFile() methods. */ @@ -54,7 +62,7 @@ public: public: - virtual void init(); + virtual void init() override; private: @@ -64,6 +72,10 @@ private: void writeNarInfo(ref<NarInfo> narInfo); + ref<const ValidPathInfo> addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, + std::function<ValidPathInfo(HashResult)> mkInfo); + public: bool isValidPathUncached(const StorePath & path) override; @@ -75,8 +87,10 @@ public: { unsupported("queryPathFromHashPart"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e927a65f0..4c88504c4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,6 +17,8 @@ #include "daemon.hh" #include "worker-protocol.hh" #include "user-lock.hh" +#include "topo-sort.hh" +#include "callback.hh" #include <algorithm> #include <iostream> @@ -295,10 +297,22 @@ public: ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); + + /* derivation goal */ +private: + std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( + const StorePath & drvPath, const StringSet & wantedOutputs, + std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); +public: + std::shared_ptr<DerivationGoal> makeDerivationGoal( + const StorePath & drvPath, + const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + + /* substitution goal */ + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -606,6 +620,33 @@ typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; +/* Unless we are repairing, we don't both to test validity and just assume it, + so the choices are `Absent` or `Valid`. */ +enum struct PathStatus { + Corrupt, + Absent, + Valid, +}; + +struct InitialOutputStatus { + StorePath path; + PathStatus status; + /* Valid in the store, and additionally non-corrupt if we are repairing */ + bool isValid() const { + return status == PathStatus::Valid; + } + /* Merely present, allowed to be corrupt */ + bool isPresent() const { + return status == PathStatus::Corrupt + || status == PathStatus::Valid; + } +}; + +struct InitialOutput { + bool wanted; + std::optional<InitialOutputStatus> known; +}; + class DerivationGoal : public Goal { private: @@ -633,19 +674,14 @@ private: /* The remainder is state held during the build. */ - /* Locks on the output paths. */ + /* Locks on (fixed) output paths. */ PathLocks outputLocks; /* All input paths (that is, the union of FS closures of the immediate input paths). */ StorePathSet inputPaths; - /* Outputs that are already valid. If we're repairing, these are - the outputs that are valid *and* not corrupt. */ - StorePathSet validPaths; - - /* Outputs that are corrupt or not valid. */ - StorePathSet missingPaths; + std::map<std::string, InitialOutput> initialOutputs; /* User selected for running the builder. */ std::unique_ptr<UserLock> buildUser; @@ -702,8 +738,8 @@ private: /* RAII object to delete the chroot directory. */ std::shared_ptr<AutoDelete> autoDelChroot; - /* Whether this is a fixed-output derivation. */ - bool fixedOutput; + /* The sort of derivation we are building. */ + DerivationType derivationType; /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; @@ -735,6 +771,31 @@ private: typedef map<StorePath, StorePath> RedirectedOutputs; RedirectedOutputs redirectedOutputs; + /* The outputs paths used during the build. + + - Input-addressed derivations or fixed content-addressed outputs are + sometimes built when some of their outputs already exist, and can not + be hidden via sandboxing. We use temporary locations instead and + rewrite after the build. Otherwise the regular predetermined paths are + put here. + + - Floating content-addressed derivations do not know their final build + output paths until the outputs are hashed, so random locations are + used, and then renamed. The randomness helps guard against hidden + self-references. + */ + OutputPathMap scratchOutputs; + + /* The final output paths of the build. + + - For input-addressed derivations, always the precomputed paths + + - For content-addressed derivations, calcuated from whatever the hash + ends up being. (Note that fixed outputs derivations that produce the + "wrong" output still install that data under its true content-address.) + */ + OutputPathMap finalOutputs; + BuildMode buildMode; /* If we're repairing without a chroot, there may be outputs that @@ -796,10 +857,12 @@ private: friend struct RestrictedStore; public: - DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode = bmNormal); + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); ~DerivationGoal(); /* Whether we need to perform hash rewriting if there are valid output paths. */ @@ -833,13 +896,16 @@ private: void getDerivation(); void loadDerivation(); void haveDerivation(); - void outputsSubstituted(); + void outputsSubstitutionTried(); + void gaveUpOnSubstitution(); void closureRepaired(); void inputsRealised(); void tryToBuild(); void tryLocalBuild(); void buildDone(); + void resolvedFinished(); + /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); @@ -894,13 +960,27 @@ private: void handleEOF(int fd) override; void flushLine(); + /* Wrappers around the corresponding Store methods that first consult the + derivation. This is currently needed because when there is no drv file + there also is no DB entry. */ + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); + OutputPathMap queryDerivationOutputMap(); + /* Return the set of (in)valid paths. */ - StorePathSet checkPathValidity(bool returnValid, bool checkHash); + void checkPathValidity(); /* Forcibly kill the child process, if any. */ void killChild(); - void addHashRewrite(const StorePath & path); + /* Create alternative path calculated from but distinct from the + input, so we can avoid overwriting outputs (or other store paths) + that already exist. */ + StorePath makeFallbackPath(const StorePath & path); + /* Make a path to another based on the output name along with the + derivation hash. */ + /* FIXME add option to randomize, so we can audit whether our + rewrites caught everything */ + StorePath makeFallbackPath(std::string_view outputName); void repairClosure(); @@ -917,8 +997,8 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode) +DerivationGoal::DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) , drvPath(drvPath) @@ -926,7 +1006,9 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath)); + name = fmt( + "building of '%s' from .drv file", + StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -935,15 +1017,18 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode) + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) , drvPath(drvPath) + , wantedOutputs(wantedOutputs) , buildMode(buildMode) { this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); + name = fmt( + "building of '%s' from in-memory derivation", + StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1075,43 +1160,60 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); + if (drv->type() == DerivationType::CAFloating) + settings.requireExperimentalFeature("ca-derivations"); + retrySubstitution = false; - for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + worker.store.addTempRoot(*i.second.second); /* Check what outputs paths are not already valid. */ - auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } /* If they are all valid, then we're done. */ - if (invalidOutputs.size() == 0 && buildMode == bmNormal) { + if (allValid && buildMode == bmNormal) { done(BuildResult::AlreadyValid); return; } parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); - if (parsedDrv->contentAddressed()) { - settings.requireExperimentalFeature("ca-derivations"); - throw Error("ca-derivations isn't implemented yet"); - } - /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known) { + warn("do not know how to query for unknown floating content-addressed derivation output yet"); + /* Nothing to wait for; tail call */ + return DerivationGoal::gaveUpOnSubstitution(); + } + addWaitee(worker.makeSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv))); + } if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstituted(); + outputsSubstitutionTried(); else - state = &DerivationGoal::outputsSubstituted; + state = &DerivationGoal::outputsSubstitutionTried; } -void DerivationGoal::outputsSubstituted() +void DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); @@ -1135,7 +1237,14 @@ void DerivationGoal::outputsSubstituted() return; } - auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + checkPathValidity(); + size_t nrInvalid = 0; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) + nrInvalid++; + } + if (buildMode == bmNormal && nrInvalid == 0) { done(BuildResult::Substituted); return; @@ -1148,9 +1257,14 @@ void DerivationGoal::outputsSubstituted() throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); - /* Otherwise, at least one of the output paths could not be - produced using a substitute. So we have to build instead. */ + /* Nothing to wait for; tail call */ + gaveUpOnSubstitution(); +} +/* At least one of the output paths could not be + produced using a substitute. So we have to build instead. */ +void DerivationGoal::gaveUpOnSubstitution() +{ /* Make sure checkPathValidity() from now on checks all outputs. */ wantedOutputs.clear(); @@ -1183,15 +1297,16 @@ void DerivationGoal::repairClosure() that produced those outputs. */ /* Get the output closure. */ + auto outputs = queryDerivationOutputMap(); StorePathSet outputClosure; - for (auto & i : drv->outputs) { + for (auto & i : outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); + worker.store.computeFSClosure(i.second, outputClosure); } /* Filter out our own outputs (which we have already checked). */ - for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); + for (auto & i : outputs) + outputClosure.erase(i.second); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1201,9 +1316,10 @@ void DerivationGoal::repairClosure() std::map<StorePath, StorePath> outputsToDrv; for (auto & i : inputClosure) if (i.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(i); - for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path, i); + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + for (auto & j : depOutputs) + if (j.second) + outputsToDrv.insert_or_assign(*j.second, i); } /* Check each path (slow!). */ @@ -1265,35 +1381,72 @@ void DerivationGoal::inputsRealised() /* Determine the full set of input paths. */ /* First, the input derivations. */ - if (useDerivation) - for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { + if (useDerivation) { + auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); + + if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { + /* We are be able to resolve this derivation based on the + now-known results of dependencies. If so, we become a stub goal + aliasing that resolved derivation goal */ + std::optional attempt = fullDrv.tryResolve(worker.store); + assert(attempt); + Derivation drvResolved { *std::move(attempt) }; + + auto pathResolved = writeDerivation(worker.store, drvResolved); + /* Add to memotable to speed up downstream goal's queries with the + original derivation. */ + drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); + + auto msg = fmt("Resolved derivation: '%s' -> '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved)); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg, + Logger::Fields { + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved), + }); + + auto resolvedGoal = worker.makeDerivationGoal( + pathResolved, wantedOutputs, buildMode); + addWaitee(resolvedGoal); + + state = &DerivationGoal::resolvedFinished; + return; + } + + for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.store.isValidPath(i.first)); - Derivation inDrv = worker.store.derivationFromPath(i.first); - for (auto & j : i.second) { - auto k = inDrv.outputs.find(j); - if (k != inDrv.outputs.end()) - worker.store.computeFSClosure(k->second.path, inputPaths); - else + assert(worker.store.isValidPath(drvPath)); + auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + for (auto & j : wantedDepOutputs) { + if (outputs.count(j) > 0) { + auto optRealizedInput = outputs.at(j); + if (!optRealizedInput) + throw Error( + "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.computeFSClosure(*optRealizedInput, inputPaths); + } else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); } } + } /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); debug("added input paths %s", worker.store.showPaths(inputPaths)); - /* Is this a fixed-output derivation? */ - fixedOutput = drv->isFixedOutput(); + /* What type of derivation are we building? */ + derivationType = drv->type(); /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; + nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1324,14 +1477,18 @@ void DerivationGoal::tryToBuild() { trace("trying to build"); - /* Obtain locks on all output paths. The locks are automatically - released when we exit this function or Nix crashes. If we - can't acquire the lock, then continue; hopefully some other - goal can start a build, and if not, the main loop will sleep a - few seconds and then retry this goal. */ + /* Obtain locks on all output paths, if the paths are known a priori. + + The locks are automatically released when we exit this function or Nix + crashes. If we can't acquire the lock, then continue; hopefully some + other goal can start a build, and if not, the main loop will sleep a few + seconds and then retry this goal. */ PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.Store::toRealPath(outPath)); + /* FIXME: Should lock something like the drv itself so we don't build same + CA drv concurrently */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) @@ -1350,32 +1507,36 @@ void DerivationGoal::tryToBuild() omitted, but that would be less efficient.) Note that since we now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ - validPaths = checkPathValidity(true, buildMode == bmRepair); - if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); done(BuildResult::AlreadyValid); return; } - missingPaths = drv->outputPaths(); - if (buildMode != bmCheck) - for (auto & i : validPaths) missingPaths.erase(i); - /* If any of the outputs already exist but are not valid, delete them. */ - for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path)) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.Store::toRealPath(i.second.path)); + for (auto & [_, status] : initialOutputs) { + if (!status.known || status.known->isValid()) continue; + auto storePath = status.known->path; + debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); + deletePath(worker.store.Store::toRealPath(storePath)); } /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { case rpAccept: @@ -1403,6 +1564,13 @@ void DerivationGoal::tryToBuild() actLock.reset(); + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1413,13 +1581,6 @@ void DerivationGoal::tryToBuild() return; } - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::tryLocalBuild() { - if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(); @@ -1460,7 +1621,34 @@ void DerivationGoal::tryLocalBuild() { } -void replaceValidPath(const Path & storePath, const Path tmpPath) +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + +void replaceValidPath(const Path & storePath, const Path & tmpPath) { /* We can't atomically replace storePath (the original) with tmpPath (the replacement), so we have to move it out of the @@ -1468,11 +1656,20 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) we're repairing (say) Glibc, we end up with a broken system. */ Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); if (pathExists(storePath)) - rename(storePath.c_str(), oldPath.c_str()); - if (rename(tmpPath.c_str(), storePath.c_str()) == -1) { - rename(oldPath.c_str(), storePath.c_str()); // attempt to recover - throw SysError("moving '%s' to '%s'", tmpPath, storePath); + movePath(storePath, oldPath); + + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreException(); + } + throw; } + deletePath(oldPath); } @@ -1538,13 +1735,13 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; #endif @@ -1553,8 +1750,10 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - for (auto & i : missingPaths) { - auto p = worker.store.printStorePath(i); + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) rename((chrootRootDir + p).c_str(), p.c_str()); } @@ -1584,7 +1783,10 @@ void DerivationGoal::buildDone() fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); + StorePathSet outputPaths; + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); + } std::map<std::string, std::string> hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); @@ -1675,7 +1877,7 @@ void DerivationGoal::buildDone() st = dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - fixedOutput || diskFull ? BuildResult::TransientFailure : + derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1686,6 +1888,9 @@ void DerivationGoal::buildDone() done(BuildResult::Built); } +void DerivationGoal::resolvedFinished() { + done(BuildResult::Built); +} HookReply DerivationGoal::tryBuildHook() { @@ -1756,11 +1961,19 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - writeStorePaths(worker.store, hook->sink, inputPaths); + worker_proto::write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - writeStorePaths(worker.store, hook->sink, missingPaths); + { + StorePathSet missingPaths; + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + missingPaths.insert(status.known->path); + } + worker_proto::write(worker.store, hook->sink, missingPaths); + } hook->sink = FdSink(); hook->toHook.writeSide = -1; @@ -1777,13 +1990,6 @@ HookReply DerivationGoal::tryBuildHook() } -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - int childEntry(void * arg) { ((DerivationGoal *) arg)->runChild(); @@ -1811,8 +2017,15 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) for (auto & j : paths2) { if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + for (auto & k : drv.outputsAndOptPaths(worker.store)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + worker.store.computeFSClosure(*k.second.second, paths); + } } } @@ -1843,7 +2056,7 @@ void linkOrCopy(const Path & from, const Path & to) file (e.g. 32000 of ext3), which is quite possible after a 'nix-store --optimise'. FIXME: actually, why don't we just bind-mount in this case? - + It can also fail with EPERM in BeegFS v7 and earlier versions which don't allow hard-links to other directories */ if (errno != EMLINK && errno != EPERM) @@ -1856,13 +2069,13 @@ void linkOrCopy(const Path & from, const Path & to) void DerivationGoal::startBuilder() { /* Right platform? */ - if (!parsedDrv->canBuildLocally()) + if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep<StringSet>(", ", settings.systemFeatures)); + concatStringsSep<StringSet>(", ", worker.store.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); @@ -1888,7 +2101,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && !noChroot; + useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -1905,9 +2118,64 @@ void DerivationGoal::startBuilder() chownToBuilder(tmpDir); - /* Substitute output placeholders with the actual output paths. */ - for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + for (auto & [outputName, status] : initialOutputs) { + /* Set scratch path we'll actually use during the build. + + If we're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + auto scratchPath = + !status.known + ? makeFallbackPath(outputName) + : !needsHashRewrite() + /* Can always use original path in sandbox */ + ? status.known->path + : !status.known->isPresent() + /* If path doesn't yet exist can just use it */ + ? status.known->path + : buildMode != bmRepair && !status.known->isValid() + /* If we aren't repairing we'll delete a corrupted path, so we + can use original path */ + ? status.known->path + : /* If we are repairing or the path is totally valid, we'll need + to use a temporary path */ + makeFallbackPath(status.known->path); + scratchOutputs.insert_or_assign(outputName, scratchPath); + + /* A non-removed corrupted path needs to be stored here, too */ + if (buildMode == bmRepair && !status.known->isValid()) + redirectedBadOutputs.insert(status.known->path); + + /* Substitute output placeholders with the scratch output paths. + We'll use during the build. */ + inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); + + /* Additional tasks if we know the final path a priori. */ + if (!status.known) continue; + auto fixedFinalPath = status.known->path; + + /* Additional tasks if the final and scratch are both known and + differ. */ + if (fixedFinalPath == scratchPath) continue; + + /* Ensure scratch path is ours to use. */ + deletePath(worker.store.printStorePath(scratchPath)); + + /* Rewrite and unrewrite paths */ + { + std::string h1 { fixedFinalPath.hashPart() }; + std::string h2 { scratchPath.hashPart() }; + inputRewrites[h1] = h2; + } + + redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); + } /* Construct the environment passed to the builder. */ initEnv(); @@ -1933,7 +2201,10 @@ void DerivationGoal::startBuilder() if (!std::regex_match(fileName, regex)) throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - auto storePath = worker.store.parseStorePath(*i++); + auto storePathS = *i++; + if (!worker.store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + auto storePath = worker.store.toStorePath(storePathS).first; /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, @@ -1975,10 +2246,11 @@ void DerivationGoal::startBuilder() for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error("while processing 'sandbox-paths': %s", e.what()); + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -2065,7 +2337,7 @@ void DerivationGoal::startBuilder() "nogroup:x:65534:\n") % sandboxGid).str()); /* Create /etc/hosts with localhost entry. */ - if (!fixedOutput) + if (!(derivationIsImpure(derivationType))) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -2085,10 +2357,7 @@ void DerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - struct stat st; - if (lstat(r.c_str(), &st)) - throw SysError("getting attributes of path '%s'", p); - if (S_ISDIR(st.st_mode)) + if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else linkOrCopy(r, chrootRootDir + p); @@ -2099,8 +2368,15 @@ void DerivationGoal::startBuilder() rebuilding a path that is in settings.dirsInChroot (typically the dependencies of /bin/sh). Throw them out. */ - for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path)); + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + /* If the name isn't known a priori (i.e. floating + content-addressed derivation), the temporary location we use + should be fresh. Freshness means it is impossible that the path + is already in the sandbox, so we don't need to worry about + removing it. */ + if (i.second.second) + dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + } if (useSystemdCgroup) { settings.requireExperimentalFeature("systemd-cgroup"); @@ -2130,33 +2406,8 @@ void DerivationGoal::startBuilder() throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); } - if (needsHashRewrite()) { - - if (pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - /* We're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - if (validPaths.size() > 0) - for (auto & i : validPaths) - addHashRewrite(i); - - /* If we're repairing, then we don't want to delete the - corrupt outputs in advance. So rewrite them as well. */ - if (buildMode == bmRepair) - for (auto & i : missingPaths) - if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { - addHashRewrite(i); - redirectedBadOutputs.insert(i); - } - } + if (needsHashRewrite() && pathExists(homeDir)) + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { printMsg(lvlChatty, format("executing pre-build hook '%1%'") @@ -2293,7 +2544,7 @@ void DerivationGoal::startBuilder() us. */ - if (!fixedOutput) + if (!(derivationIsImpure(derivationType))) privateNetwork = true; userNamespaceSync.create(); @@ -2358,6 +2609,12 @@ void DerivationGoal::startBuilder() userNamespaceSync.readSide = -1; + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + pid_t tmp; if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; @@ -2392,7 +2649,6 @@ void DerivationGoal::startBuilder() /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); - userNamespaceSync.writeSide = -1; } else #endif @@ -2412,9 +2668,12 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { - if (msg.size() == 1) break; - throw Error(string(msg, 1)); + FdSource source(builderOut.readSide.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; } debug(msg); } @@ -2502,7 +2761,7 @@ void DerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -2513,7 +2772,7 @@ void DerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (fixedOutput) { + if (derivationIsImpure(derivationType)) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -2540,8 +2799,11 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + for (auto & i : drv->outputs) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); + } json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2634,18 +2896,23 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } +struct RestrictedStoreConfig : LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + const std::string name() { return "Restricted Store"; } +}; /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore +struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig { ref<LocalStore> next; DerivationGoal & goal; RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal) - : Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) { } Path getRealStoreDir() override @@ -2684,8 +2951,12 @@ struct RestrictedStore : public LocalFSStore void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - OutputPathMap queryDerivationOutputMap(const StorePath & path) override - { throw Error("queryDerivationOutputMap"); } + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); + return next->queryPartialDerivationOutputMap(path); + } std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } @@ -2696,21 +2967,12 @@ struct RestrictedStore : public LocalFSStore { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) override + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override { - next->addToStore(info, narSource, repair, checkSigs, accessor); + next->addToStore(info, narSource, repair, checkSigs); goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override - { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); - goal.addDependency(path); - return path; - } - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) override { @@ -2740,19 +3002,20 @@ struct RestrictedStore : public LocalFSStore StorePathSet newPaths; for (auto & path : paths) { - if (path.path.isDerivation()) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - auto drv = derivationFromPath(path.path); - for (auto & output : drv.outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path); - } else if (!goal.isAllowed(path.path)) + if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } next->buildPaths(paths, buildMode); + for (auto & path : paths) { + if (!path.path.isDerivation()) continue; + auto outputs = next->queryDerivationOutputMap(path.path); + for (auto & output : outputs) + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second); + } + StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) @@ -2780,7 +3043,7 @@ struct RestrictedStore : public LocalFSStore void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override + uint64_t & downloadSize, uint64_t & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are @@ -2849,7 +3112,8 @@ void DerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + daemon::NotTrusted, daemon::Recursive, + [&](Store & store) { store.createUser("nobody", 65535); }); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -2903,9 +3167,7 @@ void DerivationGoal::addDependency(const StorePath & path) if (pathExists(target)) throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - struct stat st; - if (lstat(source.c_str(), &st)) - throw SysError("getting attributes of path '%s'", source); + auto st = lstat(source); if (S_ISDIR(st.st_mode)) { @@ -3108,7 +3370,7 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -3124,7 +3386,7 @@ void DerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (fixedOutput) { + if (derivationIsImpure(derivationType)) { ss.push_back("/etc/resolv.conf"); // Only use nss functions to resolve hosts and @@ -3378,17 +3640,13 @@ void DerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (fixedOutput) + if (derivationIsImpure(derivationType)) sandboxProfile += "(import \"sandbox-network.sb\")\n"; - /* Our rwx outputs */ + /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : missingPaths) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i)); - - /* Also add redirected outputs to the chroot */ - for (auto & i : redirectedOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second)); + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); sandboxProfile += ")\n"; @@ -3474,7 +3732,7 @@ void DerivationGoal::runChild() args.push_back(rewriteStrings(i, inputRewrites)); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\1\n")); + writeFull(STDERR_FILENO, string("\2\n")); /* Execute the program. This should not return. */ if (drv->isBuiltin()) { @@ -3495,7 +3753,7 @@ void DerivationGoal::runChild() throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { - writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, e.what() + std::string("\n")); _exit(1); } } @@ -3504,62 +3762,31 @@ void DerivationGoal::runChild() throw SysError("executing '%1%'", drv->builder); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); + } catch (Error & e) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); _exit(1); } } -/* Parse a list of reference specifiers. Each element must either be - a store path, or the symbolic name of the output of the derivation - (such as `out'). */ -StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) -{ - StorePathSet result; - for (auto & i : paths) { - if (store.isStorePath(i)) - result.insert(store.parseStorePath(i)); - else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - return result; -} - - -static void moveCheckToStore(const Path & src, const Path & dst) -{ - /* For the rename of directory to succeed, we must be running as root or - the directory must be made temporarily writable (to update the - directory's parent link ".."). */ - struct stat st; - if (lstat(src.c_str(), &st) == -1) { - throw SysError("getting attributes of path '%1%'", src); - } - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have - to do anything here. */ + to do anything here. + + We can only early return when the outputs are known a priori. For + floating content-addressed derivations this isn't the case. + */ if (hook) { bool allValid = true; - for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + if (!i.second.second || !worker.store.isValidPath(*i.second.second)) + allValid = false; + } if (allValid) return; } @@ -3580,47 +3807,51 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); for (auto & p : addedPaths) referenceablePaths.insert(p); - /* Check whether the output paths were created, and grep each - output path to determine what other paths it references. Also make all - output paths read-only. */ - for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); - if (!missingPaths.count(i.second.path)) continue; - - Path actualPath = path; - if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path); - if (r != redirectedOutputs.end()) { - auto redirected = worker.store.Store::toRealPath(r->second); - if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path) - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck) - actualPath = redirected; - } - } else if (useChroot) { - actualPath = chrootRootDir + path; - if (pathExists(actualPath)) { - /* Move output paths from the chroot to the Nix store. */ - if (buildMode == bmRepair) - replaceValidPath(path, actualPath); - else - if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); - } - if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); + /* FIXME `needsHashRewrite` should probably be removed and we get to the + real reason why we aren't using the chroot dir */ + auto toRealPathChroot = [&](const Path & p) -> Path { + return useChroot && !needsHashRewrite() + ? chrootRootDir + p + : worker.store.toRealPath(p); + }; + + /* Check whether the output paths were created, and make all + output paths read-only. Then get the references of each output (that we + might need to register), so we can topologically sort them. For the ones + that are most definitely already installed, we just store their final + name so we can also use it in rewrites. */ + StringSet outputsToSort; + struct AlreadyRegistered { StorePath path; }; + struct PerhapsNeedToRegister { StorePathSet refs; }; + std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered; + std::map<std::string, struct stat> outputStats; + for (auto & [outputName, _] : drv->outputs) { + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); + + outputsToSort.insert(outputName); + + /* Updated wanted info to remove the outputs we definitely don't need to register */ + auto & initialInfo = initialOutputs.at(outputName); + + /* Don't register if already valid, and not checking */ + initialInfo.wanted = buildMode == bmCheck + || !(initialInfo.known && initialInfo.known->isValid()); + if (!initialInfo.wanted) { + outputReferencesIfUnregistered.insert_or_assign( + outputName, + AlreadyRegistered { .path = initialInfo.known->path }); + continue; } struct stat st; if (lstat(actualPath.c_str(), &st) == -1) { if (errno == ENOENT) throw BuildError( - "builder for '%s' failed to produce output path '%s'", - worker.store.printStorePath(drvPath), path); + "builder for '%s' failed to produce output path for output '%s' at '%s'", + worker.store.printStorePath(drvPath), outputName, actualPath); throw SysError("getting attributes of path '%s'", actualPath); } @@ -3631,140 +3862,296 @@ void DerivationGoal::registerOutputs() user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); + throw BuildError( + "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", + actualPath, outputName); #endif - /* Apply hash rewriting if necessary. */ - bool rewritten = false; - if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) - }); + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, + inodesSeen); - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData( - actualPath, - buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, - inodesSeen); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - rewritten = true; - } + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + auto references = worker.store.parseStorePathSet( + scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - /* Check that fixed-output derivations produced the right - outputs (i.e., the content hash should match the specified - hash). */ - std::optional<ContentAddress> ca; + outputReferencesIfUnregistered.insert_or_assign( + outputName, + PerhapsNeedToRegister { .refs = references }); + outputStats.insert_or_assign(outputName, std::move(st)); + } - if (fixedOutput) { + auto sortedOutputNames = topoSort(outputsToSort, + {[&](const std::string & name) { + return std::visit(overloaded { + /* Since we'll use the already installed versions of these, we + can treat them as leaves and ignore any references they + have. */ + [&](AlreadyRegistered _) { return StringSet {}; }, + [&](PerhapsNeedToRegister refs) { + StringSet referencedOutputs; + /* FIXME build inverted map up front so no quadratic waste here */ + for (auto & r : refs.refs) + for (auto & [o, p] : scratchOutputs) + if (r == p) + referencedOutputs.insert(o); + return referencedOutputs; + }, + }, outputReferencesIfUnregistered.at(name)); + }}, + {[&](const std::string & path, const std::string & parent) { + // TODO with more -vvvv also show the temporary paths for manual inspection. + return BuildError( + "cycle detected in build of '%s' in the references of output '%s' from output '%s'", + worker.store.printStorePath(drvPath), path, parent); + }}); + + std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + + for (auto & outputName : sortedOutputNames) { + auto output = drv->outputs.at(outputName); + auto & scratchPath = scratchOutputs.at(outputName); + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); + + auto finish = [&](StorePath finalStorePath) { + /* Store the final path */ + finalOutputs.insert_or_assign(outputName, finalStorePath); + /* The rewrite rule will be used in downstream outputs that refer to + use. This is why the topological sort is essential to do first + before this for loop. */ + if (scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; + }; - if (i.second.hash->method == FileIngestionMethod::Flat) { + std::optional<StorePathSet> referencesOpt = std::visit(overloaded { + [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> { + finish(skippedFinalPath.path); + return std::nullopt; + }, + [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> { + return r.refs; + }, + }, outputReferencesIfUnregistered.at(outputName)); + + if (!referencesOpt) + continue; + auto references = *referencesOpt; + + auto rewriteOutput = [&]() { + /* Apply hash rewriting if necessary. */ + if (!outputRewrites.empty()) { + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), + }); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + } + }; + + auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { + /* In the CA case, we need the rewritten refs to calculate the + final path, therefore we look for a *non-rewritten + self-reference, and use a bool rather try to solve the + computationally intractable fixed point. */ + std::pair<bool, StorePathSet> res { + false, + {}, + }; + for (auto & r : references) { + auto name = r.name(); + auto origHash = std::string { r.hashPart() }; + if (r == scratchPath) + res.first = true; + else if (outputRewrites.count(origHash) == 0) + res.second.insert(r); + else { + std::string newRef = outputRewrites.at(origHash); + newRef += '-'; + newRef += name; + res.second.insert(StorePath { newRef }); + } + } + return res; + }; + + auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { + auto & st = outputStats.at(outputName); + if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " "since recursive hashing is not enabled (outputHashMode=flat)", - path); + actualPath); } + rewriteOutput(); + /* FIXME optimize and deduplicate with addToStore */ + std::string oldHashPart { scratchPath.hashPart() }; + HashModuloSink caSink { outputHash.hashType, oldHashPart }; + switch (outputHash.method) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + auto got = caSink.finish().first; + auto refs = rewriteRefs(); + HashModuloSink narSink { htSHA256, oldHashPart }; + dumpPath(actualPath, narSink); + auto narHashAndSize = narSink.finish(); + ValidPathInfo newInfo0 { + worker.store.makeFixedOutputPath( + outputHash.method, + got, + outputPathName(drv->name, outputName), + refs.second, + refs.first), + narHashAndSize.first, + }; + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); - /* Check the hash. In hash mode, move the path produced by - the derivation to its content-addressed location. */ - Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); - - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); - - if (i.second.hash->hash != h2) { - - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), - i.second.hash->hash.to_string(SRI, true), - h2.to_string(SRI, true))); - - Path actualDest = worker.store.Store::toRealPath(dest); + assert(newInfo0.ca); + return newInfo0; + }; - if (worker.store.isValidPath(dest)) - std::rethrow_exception(delayedException); + ValidPathInfo newInfo = std::visit(overloaded { + [&](DerivationOutputInputAddressed output) { + /* input-addressed case */ + auto requiredFinalPath = output.path; + /* Preemptively add rewrite rule for final hash, as that is + what the NAR hash will use rather than normalized-self references */ + if (scratchPath != requiredFinalPath) + outputRewrites.insert_or_assign( + std::string { scratchPath.hashPart() }, + std::string { requiredFinalPath.hashPart() }); + rewriteOutput(); + auto narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; + newInfo0.narSize = narHashAndSize.second; + auto refs = rewriteRefs(); + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + return newInfo0; + }, + [&](DerivationOutputCAFixed dof) { + auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { + .method = dof.hash.method, + .hashType = dof.hash.hash.type, + }); - if (actualPath != actualDest) { - PathLocks outputLocks({actualDest}); - deletePath(actualDest); - if (rename(actualPath.c_str(), actualDest.c_str()) == -1) - throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest)); + /* Check wanted hash */ + Hash & wanted = dof.hash.hash; + assert(newInfo0.ca); + auto got = getContentAddressHash(*newInfo0.ca); + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + worker.store.printStorePath(drvPath), + wanted.to_string(SRI, true), + got.to_string(SRI, true))); } + return newInfo0; + }, + [&](DerivationOutputCAFloating dof) { + return newInfoFromCA(dof); + }, + }, output.output); + + /* Calculate where we'll move the output files. In the checking case we + will leave leave them where they are, for now, rather than move to + their usual "final destination" */ + auto finalDestPath = worker.store.printStorePath(newInfo.path); + + /* Lock final output path, if not already locked. This happens with + floating CA derivations and hash-mismatching fixed-output + derivations. */ + PathLocks dynamicOutputLock; + auto optFixedPath = output.path(worker.store, drv->name, outputName); + if (!optFixedPath || + worker.store.printStorePath(*optFixedPath) != finalDestPath) + { + assert(newInfo.ca); + dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); + } - path = worker.store.printStorePath(dest); - actualPath = actualDest; + /* Move files, if needed */ + if (worker.store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); + actualPath = worker.store.toRealPath(finalDestPath); + } else if (buildMode == bmCheck) { + /* Path already exists, and we want to compare, so we leave out + new path in place. */ + } else if (worker.store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + } else { + auto destPath = worker.store.toRealPath(finalDestPath); + movePath(actualPath, destPath); + actualPath = destPath; } - else - assert(worker.store.parseStorePath(path) == dest); - - ca = FixedOutputHash { - .method = i.second.hash->method, - .hash = h2, - }; } - /* Get rid of all weird permissions. This also checks that - all files are owned by the build user, if applicable. */ - canonicalisePathMetaData(actualPath, - buildUser && !rewritten - ? std::optional(buildUser->getUIDRange()) - : std::nullopt, - inodesSeen); - - /* For this output path, find the references to other paths - contained in it. Compute the SHA-256 NAR hash at the same - time. The hash is stored in the database so that we can - verify later on whether nobody has messed with the store. */ - debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); - if (buildMode == bmCheck) { - if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; - ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); - if (hash.first != info.narHash) { + if (!worker.store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { - Path dst = worker.store.toRealPath(path + checkSuffix); + auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); - moveCheckToStore(actualPath, dst); + movePath(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - path, dst, worker.store.printStorePath(drvPath), tmpDir); + finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); } /* Since we verified the build, it's now ultimately trusted. */ - if (!info.ultimate) { - info.ultimate = true; - worker.store.signPathInfo(info); - ValidPathInfos infos; - infos.push_back(std::move(info)); - worker.store.registerValidPaths(infos); + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + worker.store.signPathInfo(oldInfo); + worker.store.registerValidPaths({ std::move(oldInfo) }); } continue; @@ -3781,24 +4168,22 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(worker.store.parseStorePath(path)); + worker.markContentsGood(newInfo.path); } - ValidPathInfo info(worker.store.parseStorePath(path)); - info.narHash = hash.first; - info.narSize = hash.second; - info.references = std::move(references); - info.deriver = drvPath; - info.ultimate = true; - info.ca = ca; - worker.store.signPathInfo(info); - - if (!info.references.empty()) { - // FIXME don't we have an experimental feature for fixed output with references? - info.ca = {}; - } + newInfo.deriver = drvPath; + newInfo.ultimate = true; + worker.store.signPathInfo(newInfo); + + finish(newInfo.path); + + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration */ + if (newInfo.ca) + worker.store.registerValidPaths({newInfo}); - infos.emplace(i.first, std::move(info)); + infos.emplace(outputName, std::move(newInfo)); } if (buildMode == bmCheck) return; @@ -3841,8 +4226,8 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); + for (auto & [_, outputStorePath] : finalOutputs) { + auto path = worker.store.printStorePath(outputStorePath); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3859,8 +4244,8 @@ void DerivationGoal::registerOutputs() /* Remove the .check directories if we're done. FIXME: keep them if the result was not determistic? */ if (curRound == nrRounds) { - for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; + for (auto & [_, outputStorePath] : finalOutputs) { + Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; deletePath(prev); } } @@ -3870,7 +4255,9 @@ void DerivationGoal::registerOutputs() outputs, this will fail. */ { ValidPathInfos infos2; - for (auto & i : infos) infos2.push_back(i.second); + for (auto & [outputName, newInfo] : infos) { + infos2.push_back(newInfo); + } worker.store.registerValidPaths(infos2); } @@ -3878,6 +4265,27 @@ void DerivationGoal::registerOutputs() exception now that we have registered the output as valid. */ if (delayedException) std::rethrow_exception(delayedException); + + /* If we made it this far, we are sure the output matches the derivation + (since the delayedException would be a fixed output CA mismatch). That + means it's safe to link the derivation to the output hash. We must do + that for floating CA derivations, which otherwise couldn't be cached, + but it's fine to do in all cases. */ + bool isCaFloating = drv->type() == DerivationType::CAFloating; + + auto drvPathResolved = drvPath; + if (!useDerivation && isCaFloating) { + /* Once a floating CA derivations reaches this point, it + must already be resolved, so we don't bother trying to + downcast drv to get would would just be an empty + inputDrvs field. */ + Derivation drv2 { *drv }; + drvPathResolved = writeDerivation(worker.store, drv2); + } + + if (useDerivation || isCaFloating) + for (auto & [outputName, newInfo] : infos) + worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); } @@ -3946,7 +4354,17 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) { if (!value) return; - auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : *value) { + if (worker.store.isStorePath(i)) + spec.insert(worker.store.parseStorePath(i)); + else if (finalOutputs.count(i)) + spec.insert(finalOutputs.at(i)); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); + } auto used = recursive ? getClosure(info.path).first @@ -4155,31 +4573,67 @@ void DerivationGoal::flushLine() } -StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap() { - StorePathSet result; - for (auto & i : drv->outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + std::map<std::string, std::optional<StorePath>> res; + for (auto & [name, output] : drv->outputs) + res.insert_or_assign(name, output.path(worker.store, drv->name, name)); + return res; + } else { + return worker.store.queryPartialDerivationOutputMap(drvPath); } - return result; +} + +OutputPathMap DerivationGoal::queryDerivationOutputMap() +{ + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + OutputPathMap res; + for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) + res.insert_or_assign(name, *output.second); + return res; + } else { + return worker.store.queryDerivationOutputMap(drvPath); + } +} + + +void DerivationGoal::checkPathValidity() +{ + bool checkHash = buildMode == bmRepair; + for (auto & i : queryPartialDerivationOutputMap()) { + InitialOutput info { + .wanted = wantOutput(i.first, wantedOutputs), + }; + if (i.second) { + auto outputPath = *i.second; + info.known = { + .path = outputPath, + .status = !worker.store.isValidPath(outputPath) + ? PathStatus::Absent + : !checkHash || worker.pathContentsGood(outputPath) + ? PathStatus::Valid + : PathStatus::Corrupt, + }; + } + initialOutputs.insert_or_assign(i.first, info); + } +} + + +StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) +{ + return worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), + Hash(htSHA256), outputPathName(drv->name, outputName)); } -void DerivationGoal::addHashRewrite(const StorePath & path) +StorePath DerivationGoal::makeFallbackPath(const StorePath & path) { - auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); - auto p = worker.store.makeStorePath( + return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), Hash(htSHA256), path.name()); - auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); - deletePath(worker.store.printStorePath(p)); - inputRewrites[h1] = h2; - outputRewrites[h2] = h1; - redirectedOutputs.insert_or_assign(path, std::move(p)); } @@ -4220,6 +4674,10 @@ private: /* The store path that should be realised through a substitute. */ StorePath storePath; + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional<StorePath> subPath; + /* The remaining substituters. */ std::list<ref<Store>> subs; @@ -4253,8 +4711,11 @@ private: typedef void (SubstitutionGoal::*GoalState)(); GoalState state; + /* Content address for recomputing store path */ + std::optional<ContentAddress> ca; + public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4284,10 +4745,11 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) : Goal(worker) , storePath(storePath) , repair(repair) + , ca(ca) { state = &SubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); @@ -4362,14 +4824,18 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - if (sub->storeDir != worker.store.storeDir) { + if (ca) { + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { tryNext(); return; } try { // FIXME: make async - info = sub->queryPathInfo(storePath); + info = sub->queryPathInfo(subPath ? *subPath : storePath); } catch (InvalidPath &) { tryNext(); return; @@ -4388,6 +4854,19 @@ void SubstitutionGoal::tryNext() throw; } + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared<ValidPathInfo>(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + tryNext(); + return; + } + } + /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); @@ -4477,7 +4956,7 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), - storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4587,35 +5066,52 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const StorePath & path, - const StringSet & wantedOutputs, BuildMode buildMode) +std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( + const StorePath & drvPath, + const StringSet & wantedOutputs, + std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) { - GoalPtr goal = derivationGoals[path].lock(); // FIXME - if (!goal) { - goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path, goal); + WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; + GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME + std::shared_ptr<DerivationGoal> goal; + if (!abstract_goal) { + goal = mkDrvGoal(); + abstract_goal_weak = goal; wakeUp(goal); - } else - (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); + } else { + goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal); + assert(goal); + goal->addWantedOutputs(wantedOutputs); + } return goal; } +std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, BuildMode buildMode) +{ + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { + return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode); + }); +} + + std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, BuildMode buildMode) + const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); - wakeUp(goal); - return goal; + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { + return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode); + }); } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca) { - GoalPtr goal = substitutionGoals[path].lock(); // FIXME + WeakGoalPtr & goal_weak = substitutionGoals[path]; + GoalPtr goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path, *this, repair); - substitutionGoals.insert_or_assign(path, goal); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca); + goal_weak = goal; wakeUp(goal); } return goal; @@ -4771,8 +5267,17 @@ void Worker::run(const Goals & _topGoals) waitForInput(); else { if (awake.empty() && 0 == settings.maxBuildJobs) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + { + if (getMachines().empty()) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + else + throw Error("unable to start any build; remote machines may not have " + "all required system features." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + + } assert(!awake.empty()); } } @@ -4866,7 +5371,7 @@ void Worker::waitForInput() std::vector<unsigned char> buffer(4096); for (auto & k : fds2) { if (pollStatus.at(fdToPollStatus.at(k)).revents) { - ssize_t rd = read(k, buffer.data(), buffer.size()); + ssize_t rd = ::read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { @@ -4956,7 +5461,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } @@ -4982,7 +5487,7 @@ void Worker::markContentsGood(const StorePath & path) static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths) { StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) @@ -5037,7 +5542,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); BuildResult result; diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index e630cf6f1..4fb5d8a06 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,17 +58,14 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; - /* We always have one output, and if it's a fixed-output derivation (as - checked below) it must be the only output */ - auto & output = drv.outputs.begin()->second; - /* Try the hashed mirrors first. */ - if (output.hash && output.hash->method == FileIngestionMethod::Flat) + if (getAttr("outputHashMode") == "flat") for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto & h = output.hash->hash; - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo")); + Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 9e5e937df..0ae45e46d 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -4,7 +4,10 @@ #include "util.hh" #include <chrono> +#include <cmath> +#include <regex> #include <unordered_set> +#include <thread> #include <dirent.h> diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..90a3ad1f5 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,12 +1,16 @@ +#include "args.hh" #include "content-address.hh" +#include "split.hh" namespace nix { -std::string FixedOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); +std::string FixedOutputHash::printMethodAlgo() const +{ + return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) { +std::string makeFileIngestionPrefix(const FileIngestionMethod m) +{ switch (m) { case FileIngestionMethod::Flat: return ""; @@ -24,11 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) + hash.to_string(Base32, true); } -// FIXME Put this somewhere? -template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; -template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; - -std::string renderContentAddress(ContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) +{ return std::visit(overloaded { [](TextHash th) { return "text:" + th.hash.to_string(Base32, true); @@ -39,47 +40,110 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -ContentAddress parseContentAddress(std::string_view rawCa) { - auto prefixSeparator = rawCa.find(':'); - if (prefixSeparator != string::npos) { - auto prefix = string(rawCa, 0, prefixSeparator); - if (prefix == "text") { - auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); - Hash hash = Hash(string(hashTypeAndHash)); - if (*hash.type != htSHA256) { - throw Error("parseContentAddress: the text hash should have type SHA256"); - } - return TextHash { hash }; - } else if (prefix == "fixed") { - // This has to be an inverse of makeFixedOutputCA - auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); - if (methodAndHash.substr(0,2) == "r:") { - std::string_view hashRaw = methodAndHash.substr(2,string::npos); - return FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = Hash(string(hashRaw)), - }; - } else { - std::string_view hashRaw = methodAndHash; - return FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = Hash(string(hashRaw)), - }; - } - } else { - throw Error("parseContentAddress: format not recognized; has to be text or fixed"); +std::string renderContentAddressMethod(ContentAddressMethod cam) +{ + return std::visit(overloaded { + [](TextHashMethod &th) { + return std::string{"text:"} + printHashType(htSHA256); + }, + [](FixedOutputHashMethod &fshm) { + return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - } else { - throw Error("Not a content address because it lacks an appropriate prefix"); + }, cam); +} + +/* + Parses content address strings up to the hash. + */ +static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +{ + std::string_view wholeInput { rest }; + + std::string_view prefix; + { + auto optPrefix = splitPrefixTo(rest, ':'); + if (!optPrefix) + throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput); + prefix = *optPrefix; } -}; -std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt); + auto parseHashType_ = [&](){ + auto hashTypeRaw = splitPrefixTo(rest, ':'); + if (!hashTypeRaw) + throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput); + HashType hashType = parseHashType(*hashTypeRaw); + return std::move(hashType); + }; + + // Switch on prefix + if (prefix == "text") { + // No parsing of the ingestion method, "text" only support flat. + HashType hashType = parseHashType_(); + if (hashType != htSHA256) + throw Error("text content address hash should use %s, but instead uses %s", + printHashType(htSHA256), printHashType(hashType)); + return TextHashMethod {}; + } else if (prefix == "fixed") { + // Parse method + auto method = FileIngestionMethod::Flat; + if (splitPrefix(rest, "r:")) + method = FileIngestionMethod::Recursive; + HashType hashType = parseHashType_(); + return FixedOutputHashMethod { + .fileIngestionMethod = method, + .hashType = std::move(hashType), + }; + } else + throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); +} + +ContentAddress parseContentAddress(std::string_view rawCa) { + auto rest = rawCa; + + ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + + return std::visit( + overloaded { + [&](TextHashMethod thm) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) + }); + }, + [&](FixedOutputHashMethod fohMethod) { + return ContentAddress(FixedOutputHash { + .method = fohMethod.fileIngestionMethod, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), + }); + }, + }, caMethod); +} + +ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +{ + std::string_view asPrefix {std::string{caMethod} + ":"}; + return parseContentAddressMethodPrefix(asPrefix); +} + +std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) +{ + return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt); }; -std::string renderContentAddress(std::optional<ContentAddress> ca) { +std::string renderContentAddress(std::optional<ContentAddress> ca) +{ return ca ? renderContentAddress(*ca) : ""; } +Hash getContentAddressHash(const ContentAddress & ca) +{ + return std::visit(overloaded { + [](TextHash th) { + return th.hash; + }, + [](FixedOutputHash fsh) { + return fsh.hash; + } + }, ca); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b..f6a6f5140 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -53,4 +53,25 @@ ContentAddress parseContentAddress(std::string_view rawCa); std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); +Hash getContentAddressHash(const ContentAddress & ca); + +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index a8fb62e0a..99d8add92 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,7 +2,6 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" -#include "local-store.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -81,12 +80,12 @@ struct TunnelLogger : public Logger showErrorInfo(oss, ei, false); StringSink buf; - buf << STDERR_NEXT << oss.str() << "\n"; + buf << STDERR_NEXT << oss.str(); enqueueMsg(*buf.s); } /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ + want to send out stderr to the client. */ void startWork() { auto state(state_.lock()); @@ -102,17 +101,20 @@ struct TunnelLogger : public Logger /* stopWork() means that we're done; stop sending stderr to the client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + void stopWork(const Error * ex = nullptr) { auto state(state_.lock()); state->canSendStderr = false; - if (success) + if (!ex) to << STDERR_LAST; else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; + if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { + to << STDERR_ERROR << *ex; + } else { + to << STDERR_ERROR << ex->what() << ex->status; + } } } @@ -173,31 +175,6 @@ struct TunnelSource : BufferedSource } }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - struct ClientSettings { bool keepFailed; @@ -257,7 +234,7 @@ struct ClientSettings else if (setSubstituters(settings.extraSubstituters)) ; else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } @@ -265,6 +242,23 @@ struct ClientSettings } }; +static void writeValidPathInfo( + ref<Store> store, + unsigned int clientVersion, + Sink & to, + std::shared_ptr<const ValidPathInfo> info) +{ + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + worker_proto::write(*store, to, info->references); + to << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << renderContentAddress(info->ca); + } +} + static void performOp(TunnelLogger * logger, ref<Store> store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -281,11 +275,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryValidPaths: { - auto paths = readStorePaths<StorePathSet>(*store, from); + auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto res = store->queryValidPaths(paths); logger->stopWork(); - writeStorePaths(*store, to, res); + worker_proto::write(*store, to, res); break; } @@ -301,11 +295,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQuerySubstitutablePaths: { - auto paths = readStorePaths<StorePathSet>(*store, from); + auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - writeStorePaths(*store, to, res); + worker_proto::write(*store, to, res); break; } @@ -334,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - writeStorePaths(*store, to, paths); + worker_proto::write(*store, to, paths); break; } @@ -350,9 +344,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryDerivationOutputMap: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - OutputPathMap outputs = store->queryDerivationOutputMap(path); + auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - writeOutputPathMap(*store, to, outputs); + worker_proto::write(*store, to, outputs); break; } @@ -375,52 +369,90 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - std::string s, baseName; - FileIngestionMethod method; - { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) - throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - method = FileIngestionMethod::Recursive; + if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { + auto name = readString(from); + auto camStr = readString(from); + auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + bool repairBool; + from >> repairBool; + auto repair = RepairFlag{repairBool}; + + logger->startWork(); + auto pathInfo = [&]() { + // NB: FramedSource must be out of scope before logger->stopWork(); + ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + FramedSource source(from); + // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. + return std::visit(overloaded { + [&](TextHashMethod &_) { + // We could stream this by changing Store + std::string contents = source.drain(); + auto path = store->addTextToStore(name, contents, refs, repair); + return store->queryPathInfo(path); + }, + [&](FixedOutputHashMethod &fohm) { + if (!refs.empty()) + throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + return store->queryPathInfo(path); + }, + }, contentAddressMethod); + }(); + logger->stopWork(); + + to << store->printStorePath(pathInfo->path); + writeValidPathInfo(store, clientVersion, to, pathInfo); + + } else { + HashType hashAlgo; + std::string baseName; + FileIngestionMethod method; + { + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + hashAlgoRaw = "sha256"; + method = FileIngestionMethod::Recursive; + } + hashAlgo = parseHashType(hashAlgoRaw); } - } - HashType hashAlgo = parseHashType(s); - TeeSource savedNAR(from); - RetrieveRegularNARSink savedRegular; + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); + if (method == FileIngestionMethod::Recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else + parseDump(savedRegular, from); - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, - baseName, - method, - hashAlgo); - logger->stopWork(); + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + logger->stopWork(); - to << store->printStorePath(path); + to << store->printStorePath(path); + } break; } case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - auto refs = readStorePaths<StorePathSet>(*store, from); + auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -442,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopImportPaths: { logger->startWork(); TunnelSource source(from, to); - auto paths = store->importPaths(source, nullptr, + auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; @@ -474,11 +506,63 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv); + readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); + + /* Content-addressed derivations are trustless because their output paths + are verified by their content alone, so any derivation is free to + try to produce such a path. + + Input-addressed derivation output paths, however, are calculated + from the derivation closure that produced them---even knowing the + root derivation is not enough. That the output data actually came + from those derivations is fundamentally unverifiable, but the daemon + trusts itself on that matter. The question instead is whether the + submitted plan has rights to the output paths it wants to fill, and + at least the derivation closure proves that. + + It would have been nice if input-address algorithm merely depended + on the build time closure, rather than depending on the derivation + closure. That would mean input-addressed paths used at build time + would just be trusted and not need their own evidence. This is in + fact fine as the same guarantees would hold *inductively*: either + the remote builder has those paths and already trusts them, or it + needs to build them too and thus their evidence must be provided in + turn. The advantage of this variant algorithm is that the evidence + for input-addressed paths which the remote builder already has + doesn't need to be sent again. + + That said, now that we have floating CA derivations, it is better + that people just migrate to those which also solve this problem, and + others. It's the same migration difficulty with strictly more + benefit. + + Lastly, do note that when we parse fixed-output content-addressed + derivations, we throw out the precomputed output paths and just + store the hashes, so there aren't two competing sources of truth an + attacker could exploit. */ + if (drv.type() == DerivationType::InputAddressed && !trusted) + throw Error("you are not privileged to build input-addressed derivations"); + + /* Make sure that the non-input-addressed derivations that got this far + are in fact content-addressed if we don't trust them. */ + assert(derivationIsCA(drv.type()) || trusted); + + /* Recompute the derivation path when we cannot trust the original. */ + if (!trusted) { + /* Recomputing the derivation path for input-address derivations + makes it harder to audit them after the fact, since we need the + original not-necessarily-resolved derivation to verify the drv + derivation as adequate claim to the input-addressed output + paths. */ + assert(derivationIsCA(drv.type())); + + Derivation drv2; + static_cast<BasicDerivation &>(drv2) = drv; + drvPath = writeDerivation(*store, Derivation { drv2 }); + } + auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; @@ -541,7 +625,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths<StorePathSet>(*store, from); + options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {}); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -602,7 +686,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -610,7 +694,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - writeStorePaths(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -618,16 +702,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQuerySubstitutablePathInfos: { - auto paths = readStorePaths<StorePathSet>(*store, from); - logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); + StorePathCAMap pathsMap = {}; + if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + } else + pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {}); + logger->startWork(); + store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - writeStorePaths(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -637,7 +727,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - writeStorePaths(*store, to, paths); + worker_proto::write(*store, to, paths); break; } @@ -654,15 +744,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); - to << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << renderContentAddress(info->ca); - } + writeValidPathInfo(store, clientVersion, to, info); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; @@ -705,18 +787,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(store->printStorePath(path), to); + dumpPath(store->toRealPath(path), to); break; } case wopAddToStoreNar: { bool repair, dontCheckSigs; - ValidPathInfo info(store->parseStorePath(readString(from))); + auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); + auto narHash = Hash::parseAny(readString(from), htSHA256); + ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths<StorePathSet>(*store, from); + info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = parseContentAddressOpt(readString(from)); @@ -726,24 +809,37 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (!trusted) info.ultimate = false; - std::string saved; - std::unique_ptr<Source> source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique<TunnelSource>(from, to); - else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); - source = std::make_unique<StringSource>(saved); + if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + logger->startWork(); + { + FramedSource source(from); + store->addToStore(info, source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } + logger->stopWork(); } - logger->startWork(); + else { + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique<TunnelSource>(from, to); + else { + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique<StringSource>(std::move(*saved.s)); + } - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + + logger->stopWork(); + } - logger->stopWork(); break; } @@ -753,12 +849,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, targets.push_back(store->parsePathWithOutputs(s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - writeStorePaths(*store, to, willBuild); - writeStorePaths(*store, to, willSubstitute); - writeStorePaths(*store, to, unknown); + worker_proto::write(*store, to, willBuild); + worker_proto::write(*store, to, willSubstitute); + worker_proto::write(*store, to, unknown); to << downloadSize << narSize; break; } @@ -774,8 +870,7 @@ void processConnection( FdSink & to, TrustedFlag trusted, RecursiveFlag recursive, - const std::string & userName, - uid_t userId) + std::function<void(Store &)> authHook) { auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr; @@ -816,15 +911,7 @@ void processConnection( /* If we can't accept clientVersion, then throw an error *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - - store->createUser(userName, userId); + authHook(*store); tunnelLogger->stopWork(); to.flush(); @@ -851,10 +938,11 @@ void processConnection( during addTextToStore() / importPath(). If that happens, just send the error message and exit. */ bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); + tunnelLogger->stopWork(&e); if (!errorAllowed) throw; } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + auto ex = Error("Nix daemon out of memory"); + tunnelLogger->stopWork(&ex); throw; } @@ -863,8 +951,13 @@ void processConnection( assert(!tunnelLogger->state_.lock()->canSendStderr); }; + } catch (Error & e) { + tunnelLogger->stopWork(&e); + to.flush(); + return; } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); + auto ex = Error(e.what()); + tunnelLogger->stopWork(&ex); to.flush(); return; } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 266932013..67755d54e 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,3 +1,5 @@ +#pragma once + #include "serialise.hh" #include "store-api.hh" @@ -12,7 +14,10 @@ void processConnection( FdSink & to, TrustedFlag trusted, RecursiveFlag recursive, - const std::string & userName, - uid_t userId); + /* Arbitrary hook to check authorization / initialize user data / whatever + after the protocol has been negotiated. The idea is that this function + and everything it calls doesn't know about this stuff, and the + `nix-daemon` handles that instead. */ + std::function<void(Store &)> authHook); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..07b4e772b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,16 +4,61 @@ #include "util.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" -#include "istringstream_nocopy.hh" namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const +std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const { - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error("derivation has no output '%s'", id); - return i->second.path; + return std::visit(overloaded { + [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> { + return { doi.path }; + }, + [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> { + return { + dof.path(store, drvName, outputName) + }; + }, + [](DerivationOutputCAFloating dof) -> std::optional<StorePath> { + return std::nullopt; + }, + }, output); +} + + +StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { + return store.makeFixedOutputPath( + hash.method, hash.hash, + outputPathName(drvName, outputName)); +} + + +bool derivationIsCA(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return false; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return true; + }; + // Since enums can have non-variant values, but making a `default:` would + // disable exhaustiveness warnings. + assert(false); +} + +bool derivationIsFixed(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return false; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; + }; + assert(false); +} + +bool derivationIsImpure(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return false; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; + }; + assert(false); } @@ -23,8 +68,8 @@ bool BasicDerivation::isBuiltin() const } -StorePath writeDerivation(ref<Store> store, - const Derivation & drv, std::string_view name, RepairFlag repair) +StorePath writeDerivation(Store & store, + const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) @@ -32,11 +77,11 @@ StorePath writeDerivation(ref<Store> store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - auto suffix = std::string(name) + drvExtension; - auto contents = drv.unparse(*store, false); - return settings.readOnlyMode - ? store->computeStorePathForText(suffix, contents, references) - : store->addTextToStore(suffix, contents, references, repair); + auto suffix = std::string(drv.name) + drvExtension; + auto contents = drv.unparse(store, false); + return readOnly || settings.readOnlyMode + ? store.computeStorePathForText(suffix, contents, references) + : store.addTextToStore(suffix, contents, references, repair); } @@ -68,12 +113,15 @@ static string parseString(std::istream & str) return res; } +static void validatePath(std::string_view s) { + if (s.size() == 0 || s[0] != '/') + throw FormatError("bad path '%1%' in derivation", s); +} static Path parsePath(std::istream & str) { - string s = parseString(str); - if (s.size() == 0 || s[0] != '/') - throw FormatError("bad path '%1%' in derivation", s); + auto s = parseString(str); + validatePath(s); return s; } @@ -101,38 +149,63 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +static DerivationOutput parseDerivationOutput(const Store & store, + std::string_view pathS, std::string_view hashAlgo, std::string_view hash) { - expect(str, ","); auto path = store.parseStorePath(parsePath(str)); - expect(str, ","); auto hashAlgo = parseString(str); - expect(str, ","); const auto hash = parseString(str); - expect(str, ")"); - - std::optional<FixedOutputHash> fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { method = FileIngestionMethod::Recursive; - hashAlgo = string(hashAlgo, 2); + hashAlgo = hashAlgo.substr(2); + } + const auto hashType = parseHashType(hashAlgo); + if (hash != "") { + validatePath(pathS); + return DerivationOutput { + .output = DerivationOutputCAFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed(hash, hashType), + }, + }, + }; + } else { + settings.requireExperimentalFeature("ca-derivations"); + assert(pathS == ""); + return DerivationOutput { + .output = DerivationOutputCAFloating { + .method = std::move(method), + .hashType = std::move(hashType), + }, + }; } - const HashType hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), + } else { + validatePath(pathS); + return DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = store.parseStorePath(pathS), + } }; } +} - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; +static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +{ + expect(str, ","); const auto pathS = parseString(str); + expect(str, ","); const auto hashAlgo = parseString(str); + expect(str, ","); const auto hash = parseString(str); + expect(str, ")"); + + return parseDerivationOutput(store, pathS, hashAlgo, hash); } -static Derivation parseDerivation(const Store & store, const string & s) +Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) { Derivation drv; - istringstream_nocopy str(s); + drv.name = name; + + std::istringstream str(std::move(s)); expect(str, "Derive(["); /* Parse the list of outputs. */ @@ -175,34 +248,6 @@ static Derivation parseDerivation(const Store & store, const string & s) } -Derivation readDerivation(const Store & store, const Path & drvPath) -{ - try { - return parseDerivation(store, readFile(drvPath)); - } catch (FormatError & e) { - throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); - } -} - - -Derivation Store::derivationFromPath(const StorePath & drvPath) -{ - ensurePath(drvPath); - return readDerivation(drvPath); -} - - -Derivation Store::readDerivation(const StorePath & drvPath) -{ - auto accessor = getFSAccessor(); - try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); - } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); - } -} - - static void printString(string & res, std::string_view s) { char buf[s.size() * 2 + 2]; @@ -264,10 +309,23 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); - s += ','; printUnquotedString(s, - i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path)); + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + }, + [&](DerivationOutputCAFixed dof) { + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); + s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + }, + [&](DerivationOutputCAFloating dof) { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, ""); + }, + }, i.second.output); s += ')'; } @@ -319,59 +377,144 @@ bool isDerivation(const string & fileName) } -bool BasicDerivation::isFixedOutput() const +std::string outputPathName(std::string_view drvName, std::string_view outputName) { + std::string res { drvName }; + if (outputName != "out") { + res += "-"; + res += outputName; + } + return res; +} + + +DerivationType BasicDerivation::type() const { - return outputs.size() == 1 && - outputs.begin()->first == "out" && - outputs.begin()->second.hash; + std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; + std::optional<HashType> floatingHashType; + for (auto & i : outputs) { + std::visit(overloaded { + [&](DerivationOutputInputAddressed _) { + inputAddressedOutputs.insert(i.first); + }, + [&](DerivationOutputCAFixed _) { + fixedCAOutputs.insert(i.first); + }, + [&](DerivationOutputCAFloating dof) { + floatingCAOutputs.insert(i.first); + if (!floatingHashType) { + floatingHashType = dof.hashType; + } else { + if (*floatingHashType != dof.hashType) + throw Error("All floating outputs must use the same hash type"); + } + }, + }, i.second.output); + } + + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + throw Error("Must have at least one output"); + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + return DerivationType::InputAddressed; + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + if (fixedCAOutputs.size() > 1) + // FIXME: Experimental feature? + throw Error("Only one fixed output is allowed for now"); + if (*fixedCAOutputs.begin() != "out") + throw Error("Single fixed output must be named \"out\""); + return DerivationType::CAFixed; + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) { + return DerivationType::CAFloating; + } else { + throw Error("Can't mix derivation output types"); + } } DrvHashes drvHashes; +/* pathDerivationModulo and hashDerivationModulo are mutually recursive + */ + +/* Look up the derivation by value and memoize the + `hashDerivationModulo` call. + */ +static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath) +{ + auto h = drvHashes.find(drvPath); + if (h == drvHashes.end()) { + assert(store.isValidPath(drvPath)); + // Cache it + h = drvHashes.insert_or_assign( + drvPath, + hashDerivationModulo( + store, + store.readDerivation(drvPath), + false)).first; + } + return h->second; +} -/* Returns the hash of a derivation modulo fixed-output - subderivations. A fixed-output derivation is a derivation with one - output (`out') for which an expected hash and hash algorithm are - specified (using the `outputHash' and `outputHashAlgo' - attributes). We don't want changes to such derivations to - propagate upwards through the dependency graph, changing output - paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it - (after all, (the hash of) the file being downloaded is unchanged). - So the *output paths* should not change. On the other hand, the - *derivation paths* should change to reflect the new dependency - graph. - - That's what this function does: it returns a hash which is just the - hash of the derivation ATerm, except that any input derivation - paths have been replaced by the result of a recursive call to this - function, and that for fixed-output derivations we return a hash of - its output path. */ -Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +/* See the header for interface details. These are the implementation details. + + For fixed-output derivations, each hash in the map is not the + corresponding output's content hash, but a hash of that hash along + with other constant data. The key point is that the value is a pure + function of the output's contents, and there are no preimage attacks + either spoofing an output's contents for a derivation, or + spoofing a derivation for an output's contents. + + For regular derivations, it looks up each subderivation from its hash + and recurs. If the subderivation is also regular, it simply + substitutes the derivation path with its hash. If the subderivation + is fixed-output, however, it takes each output hash and pretends it + is a derivation hash producing a single "out" output. This is so we + don't leak the provenance of fixed outputs, reducing pointless cache + misses as the build itself won't know this. + */ +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(htSHA256, "fixed:out:" - + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base16, false) + ":" - + store.printStorePath(i->second.path)); + switch (drv.type()) { + case DerivationType::CAFloating: + throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); + case DerivationType::CAFixed: { + std::map<std::string, Hash> outputHashes; + for (const auto & i : drv.outputs) { + auto & dof = std::get<DerivationOutputCAFixed>(i.second.output); + auto hash = hashString(htSHA256, "fixed:out:" + + dof.hash.printMethodAlgo() + ":" + + dof.hash.hash.to_string(Base16, false) + ":" + + store.printStorePath(dof.path(store, drv.name, i.first))); + outputHashes.insert_or_assign(i.first, std::move(hash)); + } + return outputHashes; + } + case DerivationType::InputAddressed: + break; } /* For other derivations, replace the inputs paths with recursive - calls to this function.*/ + calls to this function. */ std::map<std::string, StringSet> inputs2; for (auto & i : drv.inputDrvs) { - auto h = drvHashes.find(i.first); - if (h == drvHashes.end()) { - assert(store.isValidPath(i.first)); - h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, - store.readDerivation(i.first), false)).first; - } - inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); + const auto & res = pathDerivationModulo(store, i.first); + std::visit(overloaded { + // Regular non-CA derivation, replace derivation + [&](Hash drvHash) { + inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); + }, + // CA derivation's output hashes + [&](CaOutputHashes outputHashes) { + std::set<std::string> justOut = { "out" }; + for (auto & output : i.second) { + /* Put each one in with a single "out" output.. */ + const auto h = outputHashes.at(output); + inputs2.insert_or_assign( + h.to_string(Base16, false), + justOut); + } + }, + }, res); } return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); @@ -392,38 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -StorePathSet BasicDerivation::outputPaths() const -{ - StorePathSet paths; - for (auto & i : outputs) - paths.insert(i.second.path); - return paths; -} - static DerivationOutput readDerivationOutput(Source & in, const Store & store) { - auto path = store.parseStorePath(readString(in)); - auto hashAlgo = readString(in); - auto hash = readString(in); + const auto pathS = readString(in); + const auto hashAlgo = readString(in); + const auto hash = readString(in); - std::optional<FixedOutputHash> fsh; - if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (string(hashAlgo, 0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = string(hashAlgo, 2); - } - auto hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }; - } - - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; + return parseDerivationOutput(store, pathS, hashAlgo, hash); } StringSet BasicDerivation::outputNames() const @@ -434,9 +552,30 @@ StringSet BasicDerivation::outputNames() const return names; } +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { + DerivationOutputsAndOptPaths outsAndOptPaths; + for (auto output : outputs) + outsAndOptPaths.insert(std::make_pair( + output.first, + std::make_pair(output.second, output.second.path(store, name, output.first)) + ) + ); + return outsAndOptPaths; +} + +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + auto nameWithSuffix = drvPath.name(); + constexpr std::string_view extension = ".drv"; + assert(hasSuffix(nameWithSuffix, extension)); + nameWithSuffix.remove_suffix(extension.size()); + return nameWithSuffix; +} + -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { + drv.name = name; + drv.outputs.clear(); auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { @@ -445,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = readStorePaths<StorePathSet>(store, in); + drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {}); in >> drv.platform >> drv.builder; drv.args = readStrings<Strings>(in); @@ -464,16 +603,26 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr { out << drv.outputs.size(); for (auto & i : drv.outputs) { - out << i.first - << store.printStorePath(i.second.path); - if (i.second.hash) { - out << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); - } else { - out << "" << ""; - } + out << i.first; + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + out << store.printStorePath(doi.path) + << "" + << ""; + }, + [&](DerivationOutputCAFixed dof) { + out << store.printStorePath(dof.path(store, drv.name, i.first)) + << dof.hash.printMethodAlgo() + << dof.hash.hash.to_string(Base16, false); + }, + [&](DerivationOutputCAFloating dof) { + out << "" + << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << ""; + }, + }, i.second.output); } - writeStorePaths(store, out, drv.inputSrcs); + worker_proto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) @@ -487,5 +636,65 @@ std::string hashPlaceholder(const std::string & outputName) return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); } +std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) +{ + auto drvNameWithExtension = drvPath.name(); + auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); + auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); + return "/" + hashString(htSHA256, clearText).to_string(Base32, false); +} + + +// N.B. Outputs are left unchanged +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + + debug("Rewriting the derivation"); + + for (auto &rewrite: rewrites) { + debug("rewriting %s as %s", rewrite.first, rewrite.second); + } + + drv.builder = rewriteStrings(drv.builder, rewrites); + for (auto & arg: drv.args) { + arg = rewriteStrings(arg, rewrites); + } + + StringPairs newEnv; + for (auto & envVar: drv.env) { + auto envName = rewriteStrings(envVar.first, rewrites); + auto envValue = rewriteStrings(envVar.second, rewrites); + newEnv.emplace(envName, envValue); + } + drv.env = newEnv; +} + + +Sync<DrvPathResolutions> drvPathResolutions; + +std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { + BasicDerivation resolved { *this }; + + // Input paths that we'll want to rewrite in the derivation + StringMap inputRewrites; + + for (auto & input : inputDrvs) { + auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); + StringSet newOutputNames; + for (auto & outputName : input.second) { + auto actualPathOpt = inputDrvOutputs.at(outputName); + if (!actualPathOpt) + return std::nullopt; + auto actualPath = *actualPathOpt; + inputRewrites.emplace( + downstreamPlaceholder(store, input.first, outputName), + store.printStorePath(actualPath)); + resolved.inputSrcs.insert(std::move(actualPath)); + } + } + + rewriteDerivation(store, resolved, inputRewrites); + + return resolved; +} } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 68c53c1ff..d48266774 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -4,8 +4,10 @@ #include "types.hh" #include "hash.hh" #include "content-address.hh" +#include "sync.hh" #include <map> +#include <variant> namespace nix { @@ -13,20 +15,82 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutput +/* The traditional non-fixed-output derivation type. */ +struct DerivationOutputInputAddressed { + /* Will need to become `std::optional<StorePath>` once input-addressed + derivations are allowed to depend on cont-addressed derivations */ StorePath path; - std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */ +}; + +/* Fixed-output derivations, whose output paths are content addressed + according to that fixed output. */ +struct DerivationOutputCAFixed +{ + FixedOutputHash hash; /* hash used for expected hash computation */ + StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; +}; + +/* Floating-output derivations, whose output paths are content addressed, but + not fixed, and so are dynamically calculated from whatever the output ends + up being. */ +struct DerivationOutputCAFloating +{ + /* information used for expected hash computation */ + FileIngestionMethod method; + HashType hashType; +}; + +struct DerivationOutput +{ + std::variant< + DerivationOutputInputAddressed, + DerivationOutputCAFixed, + DerivationOutputCAFloating + > output; + std::optional<HashType> hashAlgoOpt(const Store & store) const; + /* Note, when you use this function you should make sure that you're passing + the right derivation name. When in doubt, you should use the safer + interface provided by BasicDerivation::outputsAndOptPaths */ + std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; +/* These are analogues to the previous DerivationOutputs data type, but they + also contains, for each output, the (optional) store path in which it would + be written. To calculate values of these types, see the corresponding + functions in BasicDerivation */ +typedef std::map<string, std::pair<DerivationOutput, StorePath>> + DerivationOutputsAndPaths; +typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>> + DerivationOutputsAndOptPaths; + /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ typedef std::map<StorePath, StringSet> DerivationInputs; typedef std::map<string, string> StringPairs; +enum struct DerivationType : uint8_t { + InputAddressed, + CAFixed, + CAFloating, +}; + +/* Do the outputs of the derivation have paths calculated from their content, + or from the derivation itself? */ +bool derivationIsCA(DerivationType); + +/* Is the content of the outputs fixed a-priori via a hash? Never true for + non-CA derivations. */ +bool derivationIsFixed(DerivationType); + +/* Is the derivation impure and needs to access non-deterministic resources, or + pure and can be sandboxed? Note that whether or not we actually sandbox the + derivation is controlled separately. Never true for non-CA derivations. */ +bool derivationIsImpure(DerivationType); + struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ @@ -35,24 +99,25 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; + std::string name; - BasicDerivation() { } + BasicDerivation() = default; virtual ~BasicDerivation() { }; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - const StorePath & findOutput(const std::string & id) const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ - bool isFixedOutput() const; - - /* Return the output paths of a derivation. */ - StorePathSet outputPaths() const; + DerivationType type() const; /* Return the output names of a derivation. */ StringSet outputNames() const; + + /* Calculates the maps that contains all the DerivationOutputs, but + augmented with knowledge of the Store paths they would be written + into. */ + DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + + static std::string_view nameFromPath(const StorePath & storePath); }; struct Derivation : BasicDerivation @@ -63,7 +128,17 @@ struct Derivation : BasicDerivation std::string unparse(const Store & store, bool maskOutputs, std::map<std::string, StringSet> * actualInputs = nullptr) const; - Derivation() { } + /* Return the underlying basic derivation but with these changes: + + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. + + 2. Input placeholders are replaced with realized input store paths. */ + std::optional<BasicDerivation> tryResolve(Store & store); + + Derivation() = default; + Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } + Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } }; @@ -72,30 +147,96 @@ class Store; enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ -StorePath writeDerivation(ref<Store> store, - const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); +StorePath writeDerivation(Store & store, + const Derivation & drv, + RepairFlag repair = NoRepair, + bool readOnly = false); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath); +Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove bool isDerivation(const string & fileName); -Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +/* Calculate the name that will be used for the store path for this + output. + + This is usually <drv-name>-<output-name>, but is just <drv-name> when + the output name is "out". */ +std::string outputPathName(std::string_view drvName, std::string_view outputName); + +// known CA drv's output hashes, current just for fixed-output derivations +// whose output hashes are always known since they are fixed up-front. +typedef std::map<std::string, Hash> CaOutputHashes; + +typedef std::variant< + Hash, // regular DRV normalized hash + CaOutputHashes +> DrvHashModulo; + +/* Returns hashes with the details of fixed-output subderivations + expunged. + + A fixed-output derivation is a derivation whose outputs have a + specified content hash and hash algorithm. (Currently they must have + exactly one output (`out'), which is specified using the `outputHash' + and `outputHashAlgo' attributes, but the algorithm doesn't assume + this.) We don't want changes to such derivations to propagate upwards + through the dependency graph, changing output paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it---after + all, (the hash of) the file being downloaded is unchanged. So the + *output paths* should not change. On the other hand, the *derivation + paths* should change to reflect the new dependency graph. + + For fixed-output derivations, this returns a map from the name of + each output to its hash, unique up to the output's contents. + + For regular derivations, it returns a single hash of the derivation + ATerm, after subderivations have been likewise expunged from that + derivation. + */ +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Memoisation of hashDerivationModulo(). */ -typedef std::map<StorePath, Hash> DrvHashes; +typedef std::map<StorePath, DrvHashModulo> DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe +/* Memoisation of `readDerivation(..).resove()`. */ +typedef std::map< + StorePath, + std::optional<StorePath> +> DrvPathResolutions; + +// FIXME: global, though at least thread-safe. +// FIXME: arguably overlaps with hashDerivationModulo memo table. +extern Sync<DrvPathResolutions> drvPathResolutions; + bool wantOutput(const string & output, const std::set<string> & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +/* This creates an opaque and almost certainly unique string + deterministically from the output name. + + It is used as a placeholder to allow derivations to refer to their + own outputs without needing to use the hash of a derivation in + itself, making the hash near-impossible to calculate. */ std::string hashPlaceholder(const std::string & outputName); +/* This creates an opaque and almost certainly unique string + deterministically from a derivation path and output name. + + It is used as a placeholder to allow derivations to refer to + content-addressed paths whose content --- and thus the path + themselves --- isn't yet known. This occurs when a derivation has a + dependency which is a CA derivation. */ +std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); + } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc new file mode 100644 index 000000000..98b745c3a --- /dev/null +++ b/src/libstore/dummy-store.cc @@ -0,0 +1,67 @@ +#include "store-api.hh" +#include "callback.hh" + +namespace nix { + +struct DummyStoreConfig : virtual StoreConfig { + using StoreConfig::StoreConfig; + + const std::string name() override { return "Dummy Store"; } +}; + +struct DummyStore : public Store, public virtual DummyStoreConfig +{ + DummyStore(const std::string scheme, const std::string uri, const Params & params) + : DummyStore(params) + { } + + DummyStore(const Params & params) + : StoreConfig(params) + , Store(params) + { } + + string getUri() override + { + return *uriSchemes().begin(); + } + + void queryPathInfoUncached(const StorePath & path, + Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override + { + callback(nullptr); + } + + static std::set<std::string> uriSchemes() { + return {"dummy"}; + } + + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override + { unsupported("addToStore"); } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override + { unsupported("addToStore"); } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override + { unsupported("addTextToStore"); } + + void narFromPath(const StorePath & path, Sink & sink) override + { unsupported("narFromPath"); } + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { unsupported("buildDerivation"); } +}; + +static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; + +} diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..02c839520 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -7,24 +7,6 @@ namespace nix { -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - void Store::exportPaths(const StorePathSet & paths, Sink & sink) { auto sorted = topoSortPaths(paths); @@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashAndWriteSink hashAndWriteSink(sink); + HashSink hashSink(htSHA256); + TeeSink teeSink(sink, hashSink); - narFromPath(path, hashAndWriteSink); + narFromPath(path, teeSink); /* Refuse to export paths that have changed. This prevents filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + Hash hash = hashSink.currentHash().first; + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); - hashAndWriteSink + teeSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); - hashAndWriteSink + worker_proto::write(*this, teeSink, info->references); + teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; } -StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) +StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) { StorePaths res; while (true) { @@ -77,33 +60,36 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info(parseStorePath(readString(source))); + auto path = parseStorePath(readString(source)); //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths<StorePathSet>(*this, source); - + auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {}); auto deriver = readString(source); + auto narHash = hashString(htSHA256, *saved.s); + + ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.references = references; + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; - addToStore(info, source, NoRepair, checkSigs, accessor); + auto source = StringSource { *saved.s }; + addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 531b85af8..c2c65af05 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,6 +5,7 @@ #include "s3.hh" #include "compression.hh" #include "finally.hh" +#include "callback.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -22,6 +23,7 @@ #include <queue> #include <random> #include <thread> +#include <regex> using namespace std::string_literals; @@ -29,7 +31,7 @@ namespace nix { FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&fileTransferSettings); +static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -56,7 +58,7 @@ struct curlFileTransfer : public FileTransfer Callback<FileTransferResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; + std::string statusMsg; unsigned int attempt = 0; @@ -111,6 +113,9 @@ struct curlFileTransfer : public FileTransfer requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + for (auto it = request.headers.begin(); it != request.headers.end(); ++it){ + requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str()); + } } ~TransferItem() @@ -123,7 +128,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -144,6 +149,7 @@ struct curlFileTransfer : public FileTransfer LambdaSink finalSink; std::shared_ptr<CompressionSink> decompressionSink; + std::optional<StringSink> errorSink; std::exception_ptr writeException; @@ -153,9 +159,19 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + errorSink = StringSink { }; + } + } + if (errorSink) + (*errorSink)((unsigned char *) contents, realSize); (*decompressionSink)((unsigned char *) contents, realSize); return realSize; @@ -175,12 +191,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); + std::smatch match; + if (std::regex_match(line, match, statusLine)) { result.etag = ""; - auto ss = tokenizeString<vector<string>>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); result.bodySize = 0; + statusMsg = trim(match[1]); acceptRanges = false; encoding = ""; } else { @@ -194,7 +211,9 @@ struct curlFileTransfer : public FileTransfer the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - if (result.etag == request.expectedETag && status == "200") { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + if (result.etag == request.expectedETag && httpStatus == 200) { debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -408,16 +427,21 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr<std::string> response; + if (errorSink) + response = errorSink->s; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) + response, + fmt("unable to %s '%s': HTTP error %d ('%s')", + request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, + response, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -675,7 +699,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else @@ -820,6 +844,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } } +template<typename... Args> +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how + // to print different messages for different verbosity levels. For now + // we add some heuristics for detecting when we want to show the response. + if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0..c89c51a21 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -17,15 +17,30 @@ struct FileTransferSettings : Config Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", "String appended to the user agent in HTTP requests."}; - Setting<size_t> httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", + Setting<size_t> httpConnections{ + this, 25, "http-connections", + R"( + The maximum number of parallel TCP connections used to fetch + files from binary caches and by other downloads. It defaults + to 25. 0 means no limit. + )", {"binary-caches-parallel-connections"}}; - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - - Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", - "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + Setting<unsigned long> connectTimeout{ + this, 0, "connect-timeout", + R"( + The timeout (in seconds) for establishing connections in the + binary cache substituter. It corresponds to `curl`’s + `--connect-timeout` option. + )"}; + + Setting<unsigned long> stalledDownloadTimeout{ + this, 300, "stalled-download-timeout", + R"( + The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this + timeout's duration. + )"}; Setting<unsigned int> tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; @@ -36,6 +51,7 @@ extern FileTransferSettings fileTransferSettings; struct FileTransferRequest { std::string uri; + Headers headers; std::string expectedETag; bool verifyTLS = true; bool head = false; @@ -103,10 +119,12 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr<string> response; // intentionally optional + template<typename... Args> - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 57fb20845..518a357ef 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -85,8 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const StorePath & storePath, - const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) +Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); @@ -95,47 +94,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", gcRoot); - if (indirect) { - /* Don't clobber the link if it already exists and doesn't - point to the Nix store. */ - if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error("cannot create symlink '%1%'; already exists", gcRoot); - makeSymlink(gcRoot, printStorePath(storePath)); - addIndirectRoot(gcRoot); - } - - else { - if (!allowOutsideRootsDir) { - Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); - - if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error( - "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'", - gcRoot, rootsDir); - } - - if (baseNameOf(gcRoot) == std::string(storePath.to_string())) - writeFile(gcRoot, ""); - else - makeSymlink(gcRoot, printStorePath(storePath)); - } - - /* Check that the root can be found by the garbage collector. - !!! This can be very slow on machines that have many roots. - Instead of reading all the roots, it would be more efficient to - check if the root is in a directory in or linked from the - gcroots directory. */ - if (settings.checkRootReachability) { - auto roots = findRoots(false); - if (roots[storePath].count(gcRoot) == 0) - logWarning({ - .name = "GC root", - .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)) - }); - } + /* Don't clobber the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error("cannot create symlink '%1%'; already exists", gcRoot); + makeSymlink(gcRoot, printStorePath(storePath)); + addIndirectRoot(gcRoot); /* Grab the global GC root, causing us to block while a GC is in progress. This prevents the set of permanent roots from @@ -262,11 +226,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { - auto storePath = maybeParseStorePath(toStorePath(target)); - if (storePath && isValidPath(*storePath)) - roots[std::move(*storePath)].emplace(path); - else - printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + try { + auto storePath = toStorePath(target).first; + if (isValidPath(storePath)) + roots[std::move(storePath)].emplace(path); + else + printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + } catch (BadStorePath &) { } }; try { @@ -472,15 +438,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) for (auto & [target, links] : unchecked) { if (!isInStore(target)) continue; - Path pathS = toStorePath(target); - if (!isStorePath(pathS)) continue; - auto path = parseStorePath(pathS); - if (!isValidPath(path)) continue; - debug("got additional root '%1%'", pathS); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); + try { + auto path = toStorePath(target).first; + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", printStorePath(path)); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); + } catch (BadStorePath &) { } } } @@ -498,7 +464,7 @@ struct LocalStore::GCState StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; - unsigned long long bytesInvalidated; + uint64_t bytesInvalidated; bool moveToTrash = true; bool shouldDelete; GCState(const GCOptions & options, GCResults & results) @@ -516,7 +482,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, void LocalStore::deleteGarbage(GCState & state, const Path & path) { - unsigned long long bytesFreed; + uint64_t bytesFreed; deletePath(path, bytesFreed); state.results.bytesFreed += bytesFreed; } @@ -526,7 +492,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) { checkInterrupt(); - unsigned long long size = 0; + uint64_t size = 0; auto storePath = maybeParseStorePath(path); if (storePath && isValidPath(*storePath)) { @@ -608,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto /* If keep-derivations is set and this is a derivation, then don't delete the derivation if any of the outputs are alive. */ if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == path + ) + incoming.insert(*maybeOutPath); } /* If keep-outputs is set, then don't delete this path if there @@ -685,7 +654,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", linksDir); - long long actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { @@ -694,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (name == "." || name == "..") continue; Path path = linksDir + "/" + name; - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError("statting '%1%'", path); + auto st = lstat(path); if (st.st_nlink != 1) { actualSize += st.st_size; @@ -715,10 +682,10 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - long long overhead = st.st_blocks * 512ULL; + auto overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 683fa5196..1238dc530 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "abstract-setting-to-json.hh" #include <algorithm> #include <map> @@ -9,6 +10,8 @@ #include <dlfcn.h> #include <sys/utsname.h> +#include <nlohmann/json.hpp> + namespace nix { @@ -22,7 +25,7 @@ namespace nix { Settings settings; -static GlobalConfig::Register r1(&settings); +static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) @@ -39,6 +42,7 @@ Settings::Settings() { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; + allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (caFile == "") { @@ -144,6 +148,12 @@ bool Settings::isWSL1() const string nixVersion = PACKAGE_VERSION; +NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { + {SandboxMode::smEnabled, true}, + {SandboxMode::smRelaxed, "relaxed"}, + {SandboxMode::smDisabled, false}, +}); + template<> void BaseSetting<SandboxMode>::set(const std::string & str) { if (str == "true") value = smEnabled; @@ -160,11 +170,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const else abort(); } -template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) -{ - AbstractSetting::toJSON(out); -} - template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) { args.addFlag({ diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5cf73c7b4..aa1fbdeb7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -80,73 +80,178 @@ public: Setting<bool> keepGoing{this, false, "keep-going", "Whether to keep building derivations when another build fails."}; - Setting<bool> tryFallback{this, false, "fallback", - "Whether to fall back to building when substitution fails.", + Setting<bool> tryFallback{ + this, false, "fallback", + R"( + If set to `true`, Nix will fall back to building from source if a + binary substitute fails. This is equivalent to the `--fallback` + flag. The default is `false`. + )", {"build-fallback"}}; /* Whether to show build log output in real time. */ bool verboseBuild = true; Setting<size_t> logLines{this, 10, "log-lines", - "If verbose-build is false, the number of lines of the tail of " + "If `verbose-build` is false, the number of lines of the tail of " "the log to show if a build fails."}; - MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", - "Maximum number of parallel build jobs. \"auto\" means use number of cores.", + MaxBuildJobsSetting maxBuildJobs{ + this, 1, "max-jobs", + R"( + This option defines the maximum number of jobs that Nix will try to + build in parallel. The default is `1`. The special value `auto` + causes Nix to use the number of CPUs in your system. `0` is useful + when using remote builders to prevent any local builds (except for + `preferLocalBuild` derivation attribute which executes locally + regardless). It can be overridden using the `--max-jobs` (`-j`) + command line switch. + )", {"build-max-jobs"}}; - Setting<unsigned int> buildCores{this, getDefaultCores(), "cores", - "Number of CPU cores to utilize in parallel within a build, " - "i.e. by passing this number to Make via '-j'. 0 means that the " - "number of actual CPU cores on the local host ought to be " - "auto-detected.", {"build-cores"}}; + Setting<unsigned int> buildCores{ + this, getDefaultCores(), "cores", + R"( + Sets the value of the `NIX_BUILD_CORES` environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + `enableParallelBuilding` is set to `true`, the builder passes the + `-jN` flag to GNU Make. It can be overridden using the `--cores` + command line switch and defaults to `1`. The value `0` means that + the builder should use all available CPU cores in the system. + )", + {"build-cores"}}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ bool readOnlyMode = false; - Setting<std::string> thisSystem{this, SYSTEM, "system", - "The canonical Nix system name."}; - - Setting<time_t> maxSilentTime{this, 0, "max-silent-time", - "The maximum time in seconds that a builer can go without " - "producing any output on stdout/stderr before it is killed. " - "0 means infinity.", + Setting<std::string> thisSystem{ + this, SYSTEM, "system", + R"( + This option specifies the canonical Nix system name of the current + installation, such as `i686-linux` or `x86_64-darwin`. Nix can only + build derivations whose `system` attribute equals the value + specified here. In general, it never makes sense to modify this + value from its default, since you can use it to ‘lie’ about the + platform you are building on (e.g., perform a Mac OS build on a + Linux machine; the result would obviously be wrong). It only makes + sense if the Nix binaries can run on multiple platforms, e.g., + ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + + It defaults to the canonical Nix system name detected by `configure` + at build time. + )"}; + + Setting<time_t> maxSilentTime{ + this, 0, "max-silent-time", + R"( + This option defines the maximum number of seconds that a builder can + go without producing any data on standard output or standard error. + This is useful (for instance in an automated build system) to catch + builds that are stuck in an infinite loop, or to catch remote builds + that are hanging due to network problems. It can be overridden using + the `--max-silent-time` command line switch. + + The value `0` means that there is no timeout. This is also the + default. + )", {"build-max-silent-time"}}; - Setting<time_t> buildTimeout{this, 0, "timeout", - "The maximum duration in seconds that a builder can run. " - "0 means infinity.", {"build-timeout"}}; + Setting<time_t> buildTimeout{ + this, 0, "timeout", + R"( + This option defines the maximum number of seconds that a builder can + run. This is useful (for instance in an automated build system) to + catch builds that are stuck in an infinite loop but keep writing to + their standard output or standard error. It can be overridden using + the `--timeout` command line switch. + + The value `0` means that there is no timeout. This is also the + default. + )", + {"build-timeout"}}; PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", "The path of the helper program that executes builds to remote machines."}; - Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", - "A semicolon-separated list of build machines, in the format of nix.machines."}; - - Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes", - "Whether build machines should use their own substitutes for obtaining " - "build dependencies if possible, rather than waiting for this host to " - "upload them."}; + Setting<std::string> builders{ + this, "@" + nixConfDir + "/machines", "builders", + "A semicolon-separated list of build machines, in the format of `nix.machines`."}; + + Setting<bool> buildersUseSubstitutes{ + this, false, "builders-use-substitutes", + R"( + If set to `true`, Nix will instruct remote build machines to use + their own binary substitutes if available. In practical terms, this + means that remote hosts will fetch as many build dependencies as + possible from their own substitutes (e.g, from `cache.nixos.org`), + instead of waiting for this host to upload them all. This can + drastically reduce build times if the network connection between + this computer and the remote build host is slow. + )"}; Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; - Setting<bool> fsyncMetadata{this, true, "fsync-metadata", - "Whether SQLite should use fsync()."}; + Setting<bool> fsyncMetadata{ + this, true, "fsync-metadata", + R"( + If set to `true`, changes to the Nix store metadata (in + `/nix/var/nix/db`) are synchronously flushed to disk. This improves + robustness in case of system crashes, but reduces performance. The + default is `true`. + )"}; Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", - "Whether to call sync() before registering a path as valid."}; - - Setting<bool> useSubstitutes{this, true, "substitute", - "Whether to use substitutes.", + "Whether to call `sync()` before registering a path as valid."}; + + Setting<bool> useSubstitutes{ + this, true, "substitute", + R"( + If set to `true` (default), Nix will use binary substitutes if + available. This option can be disabled to force building from + source. + )", {"build-use-substitutes"}}; - Setting<std::string> buildUsersGroup{this, "", "build-users-group", - "The Unix group that contains the build users."}; + Setting<std::string> buildUsersGroup{ + this, "", "build-users-group", + R"( + This options specifies the Unix group containing the Nix build user + accounts. In multi-user Nix installations, builds should not be + performed by the Nix account since that would allow users to + arbitrarily modify the Nix store and database by supplying specially + crafted builders; and they cannot be performed by the calling user + since that would allow him/her to influence the build result. + + Therefore, if this option is non-empty and specifies a valid group, + builds will be performed under the user accounts that are a member + of the group specified here (as listed in `/etc/group`). Those user + accounts should not be used for any other purpose\! + + Nix will never run two builds under the same user account at the + same time. This is to prevent an obvious security hole: a malicious + user writing a Nix expression that modifies the build result of a + legitimate Nix expression being built by another user. Therefore it + is good to have as many Nix build user accounts as you can spare. + (Remember: uids are cheap.) + + The build users should have permission to create files in the Nix + store, but not delete them. Therefore, `/nix/store` should be owned + by the Nix account, its group should be the group specified here, + and its mode should be `1775`. + + If the build users group is empty, builds will be performed under + the uid of the Nix process (that is, the uid of the caller if + `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if + `NIX_REMOTE` is `daemon`). Obviously, this should not be used in + multi-user settings with untrusted users. + )"}; #if __linux__ Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids", @@ -165,17 +270,32 @@ public: "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; - Setting<bool> keepLog{this, true, "keep-build-log", - "Whether to store build logs.", + Setting<bool> keepLog{ + this, true, "keep-build-log", + R"( + If set to `true` (the default), Nix will write the build log of a + derivation (i.e. the standard output and error of its builder) to + the directory `/nix/var/log/nix/drvs`. The build log can be + retrieved using the command `nix-store -l path`. + )", {"build-keep-log"}}; - Setting<bool> compressLog{this, true, "compress-build-log", - "Whether to compress logs.", + Setting<bool> compressLog{ + this, true, "compress-build-log", + R"( + If set to `true` (the default), build logs written to + `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2. + Otherwise, they will not be compressed. + )", {"build-compress-log"}}; - Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size", - "Maximum number of bytes a builder can write to stdout/stderr " - "before being killed (0 means no limit).", + Setting<unsigned long> maxLogSize{ + this, 0, "max-build-log-size", + R"( + This option defines the maximum number of bytes that a builder can + write to its stdout/stderr. If the builder exceeds this limit, it’s + killed. A value of `0` (the default) means that there is no limit. + )", {"build-max-log-size"}}; /* When buildRepeat > 0 and verboseBuild == true, whether to print @@ -186,57 +306,156 @@ public: Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; - Setting<bool> checkRootReachability{this, false, "gc-check-reachability", - "Whether to check if new GC roots can in fact be found by the " - "garbage collector."}; - - Setting<bool> gcKeepOutputs{this, false, "keep-outputs", - "Whether the garbage collector should keep outputs of live derivations.", + Setting<bool> gcKeepOutputs{ + this, false, "keep-outputs", + R"( + If `true`, the garbage collector will keep the outputs of + non-garbage derivations. If `false` (default), outputs will be + deleted unless they are GC roots themselves (or reachable from other + roots). + + In general, outputs must be registered as roots separately. However, + even if the output of a derivation is registered as a root, the + collector will still delete store paths that are used only at build + time (e.g., the C compiler, or source tarballs downloaded from the + network). To prevent it from doing so, set this option to `true`. + )", {"gc-keep-outputs"}}; - Setting<bool> gcKeepDerivations{this, true, "keep-derivations", - "Whether the garbage collector should keep derivers of live paths.", + Setting<bool> gcKeepDerivations{ + this, true, "keep-derivations", + R"( + If `true` (default), the garbage collector will keep the derivations + from which non-garbage store paths were built. If `false`, they will + be deleted unless explicitly registered as a root (or reachable from + other roots). + + Keeping derivation around is useful for querying and traceability + (e.g., it allows you to ask with what dependencies or options a + store path was built), so by default this option is on. Turn it off + to save a bit of disk space (or a lot if `keep-outputs` is also + turned on). + )", {"gc-keep-derivations"}}; - Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", - "Whether to automatically replace files with identical contents with hard links."}; - - Setting<bool> envKeepDerivations{this, false, "keep-env-derivations", - "Whether to add derivations as a dependency of user environments " - "(to prevent them from being GCed).", + Setting<bool> autoOptimiseStore{ + this, false, "auto-optimise-store", + R"( + If set to `true`, Nix automatically detects files in the store + that have identical contents, and replaces them with hard links to + a single copy. This saves disk space. If set to `false` (the + default), you can still run `nix-store --optimise` to get rid of + duplicate files. + )"}; + + Setting<bool> envKeepDerivations{ + this, false, "keep-env-derivations", + R"( + If `false` (default), derivations are not stored in Nix user + environments. That is, the derivations of any build-time-only + dependencies may be garbage-collected. + + If `true`, when you add a Nix derivation to a user environment, the + path of the derivation is stored in the user environment. Thus, the + derivation will not be garbage-collected until the user environment + generation is deleted (`nix-env --delete-generations`). To prevent + build-time-only dependencies from being collected, you should also + turn on `keep-outputs`. + + The difference between this option and `keep-derivations` is that + this one is “sticky”: it applies to any user environment created + while this option was enabled, while `keep-derivations` only applies + at the moment the garbage collector is run. + )", {"env-keep-derivations"}}; /* Whether to lock the Nix client and worker to the same CPU. */ bool lockCPU; - Setting<SandboxMode> sandboxMode{this, + Setting<SandboxMode> sandboxMode{ + this, #if __linux__ smEnabled #else smDisabled #endif , "sandbox", - "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", + R"( + If set to `true`, builds will be performed in a *sandboxed + environment*, i.e., they’re isolated from the normal file system + hierarchy and will only see their dependencies in the Nix store, + the temporary build directory, private versions of `/proc`, + `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths + configured with the `sandbox-paths` option. This is useful to + prevent undeclared dependencies on files in directories such as + `/usr/bin`. In addition, on Linux, builds run in private PID, + mount, network, IPC and UTS namespaces to isolate them from other + processes in the system (except that fixed-output derivations do + not run in private network namespace to ensure they can access the + network). + + Currently, sandboxing only work on Linux and macOS. The use of a + sandbox requires that Nix is run as root (so you should use the + “build users” feature to perform the actual builds under different + users than root). + + If this option is set to `relaxed`, then fixed-output derivations + and derivations that have the `__noChroot` attribute set to `true` + do not run in sandboxes. + + The default is `true` on Linux and `false` on all other platforms. + )", {"build-use-chroot", "build-use-sandbox"}}; - Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths", - "The paths to make available inside the build sandbox.", + Setting<PathSet> sandboxPaths{ + this, {}, "sandbox-paths", + R"( + A list of paths bind-mounted into Nix sandbox environments. You can + use the syntax `target=source` to mount a path in a different + location in the sandbox; for instance, `/bin=/nix-bin` will mount + the path `/nix-bin` as `/bin` inside the sandbox. If *source* is + followed by `?`, then it is not an error if *source* does not exist; + for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will + only be mounted in the sandbox if it exists in the host filesystem. + + Depending on how Nix was built, the default value for this option + may be empty or provide `/bin/sh` as a bind-mount of `bash`. + )", {"build-chroot-dirs", "build-sandbox-paths"}}; Setting<bool> sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", - "Additional paths to make available inside the build sandbox.", + Setting<PathSet> extraSandboxPaths{ + this, {}, "extra-sandbox-paths", + R"( + A list of additional paths appended to `sandbox-paths`. Useful if + you want to extend its default value. + )", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting<size_t> buildRepeat{this, 0, "repeat", - "The number of times to repeat a build in order to verify determinism.", + Setting<size_t> buildRepeat{ + this, 0, "repeat", + R"( + How many times to repeat builds to check whether they are + deterministic. The default value is 0. If the value is non-zero, + every build is repeated the specified number of times. If the + contents of any of the runs differs from the previous ones and + `enforce-determinism` is true, the build is rejected and the + resulting store paths are not registered as “valid” in Nix’s + database. + )", {"build-repeat"}}; #if __linux__ - Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", - "The size of /dev/shm in the build sandbox."}; + Setting<std::string> sandboxShmSize{ + this, "50%", "sandbox-dev-shm-size", + R"( + This option determines the maximum size of the `tmpfs` filesystem + mounted on `/dev/shm` in Linux sandboxes. For the format, see the + description of the `size` option of `tmpfs` in mount8. The default + is `50%`. + )"}; Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", "The build directory inside the sandbox."}; @@ -250,121 +469,457 @@ public: "Whether to log Darwin sandbox access violations to the system log."}; #endif - Setting<bool> runDiffHook{this, false, "run-diff-hook", - "Whether to run the program specified by the diff-hook setting " - "repeated builds produce a different result. Typically used to " - "plug in diffoscope."}; + Setting<bool> runDiffHook{ + this, false, "run-diff-hook", + R"( + If true, enable the execution of the `diff-hook` program. - PathSetting diffHook{this, true, "", "diff-hook", - "A program that prints out the differences between the two paths " - "specified on its command line."}; + When using the Nix daemon, `run-diff-hook` must be set in the + `nix.conf` configuration file, and cannot be passed at the command + line. + )"}; - Setting<bool> enforceDeterminism{this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output."}; + PathSetting diffHook{ + this, true, "", "diff-hook", + R"( + Absolute path to an executable capable of diffing build + results. The hook is executed if `run-diff-hook` is true, and the + output of a build is known to not be the same. This program is not + executed to determine if two results are the same. - Setting<Strings> trustedPublicKeys{this, - {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, - "trusted-public-keys", - "Trusted public keys for secure substitution.", - {"binary-cache-public-keys"}}; + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + + The diff hook program receives three parameters: + + 1. A path to the previous build's results + + 2. A path to the current build's results + + 3. The path to the build's derivation - Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", - "Secret keys with which to sign local builds."}; + 4. The path to the build's scratch directory. This directory will + exist only if the build was run with `--keep-failed`. - Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", - "How long downloaded files are considered up-to-date."}; + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. - Setting<bool> requireSigs{this, true, "require-sigs", - "Whether to check that any non-content-addressed path added to the " - "Nix store has a valid signature (that is, one signed using a key " - "listed in 'trusted-public-keys'."}; + When using the Nix daemon, `diff-hook` must be set in the `nix.conf` + configuration file, and cannot be passed at the command line. + )"}; - Setting<StringSet> extraPlatforms{this, + Setting<bool> enforceDeterminism{ + this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output. See `repeat`."}; + + Setting<Strings> trustedPublicKeys{ + this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "trusted-public-keys", + R"( + A whitespace-separated list of public keys. When paths are copied + from another Nix store (such as a binary cache), they must be + signed with one of these keys. For example: + `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + )", + {"binary-cache-public-keys"}}; + + Setting<Strings> secretKeyFiles{ + this, {}, "secret-key-files", + R"( + A whitespace-separated list of files containing secret (private) + keys. These are used to sign locally-built paths. They can be + generated using `nix-store --generate-binary-cache-key`. The + corresponding public key can be distributed to other users, who + can add it to `trusted-public-keys` in their `nix.conf`. + )"}; + + Setting<unsigned int> tarballTtl{ + this, 60 * 60, "tarball-ttl", + R"( + The number of seconds a downloaded tarball is considered fresh. If + the cached tarball is stale, Nix will check whether it is still up + to date using the ETag header. Nix will download a new version if + the ETag header is unsupported, or the cached ETag doesn't match. + + Setting the TTL to `0` forces Nix to always check if the tarball is + up to date. + + Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`. + + Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`, + `fetchTarball`, and `fetchurl` respect this TTL. + )"}; + + Setting<bool> requireSigs{ + this, true, "require-sigs", + R"( + If set to `true` (the default), any non-content-addressed path added + or copied to the Nix store (e.g. when substituting from a binary + cache) must have a valid signature, that is, be signed using one of + the keys listed in `trusted-public-keys` or `secret-key-files`. Set + to `false` to disable signature checking. + )"}; + + Setting<StringSet> extraPlatforms{ + this, std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", - "Additional platforms that can be built on the local system. " - "These may be supported natively (e.g. armv7 on some aarch64 CPUs " - "or using hacks like qemu-user."}; - - Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), + R"( + Platforms other than the native one which this machine is capable of + building for. This can be useful for supporting additional + architectures on compatible machines: i686-linux can be built on + x86\_64-linux machines (and the default for this setting reflects + this); armv7 is backwards-compatible with armv6 and armv5tel; some + aarch64 machines can also natively run 32-bit ARM code; and + qemu-user may be used to support non-native platforms (though this + may be slow and buggy). Most values for this are not enabled by + default because build systems will often misdetect the target + platform and generate incompatible code, so you may wish to + cross-check the results of using this option against proper + natively-built versions of your derivations. + )"}; + + Setting<StringSet> systemFeatures{ + this, getDefaultSystemFeatures(), "system-features", - "Optional features that this system implements (like \"kvm\")."}; + R"( + A set of system “features” supported by this machine, e.g. `kvm`. + Derivations can express a dependency on such features through the + derivation attribute `requiredSystemFeatures`. For example, the + attribute - Setting<Strings> substituters{this, + requiredSystemFeatures = [ "kvm" ]; + + ensures that the derivation can only be built on a machine with the + `kvm` feature. + + This setting by default includes `kvm` if `/dev/kvm` is accessible, + and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` + that are used in Nixpkgs to route builds to specific machines. + )"}; + + Setting<Strings> substituters{ + this, nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), "substituters", - "The URIs of substituters (such as https://cache.nixos.org/).", + R"( + A list of URLs of substituters, separated by whitespace. The default + is `https://cache.nixos.org`. + )", {"binary-caches"}}; // FIXME: provide a way to add to option values. - Setting<Strings> extraSubstituters{this, {}, "extra-substituters", - "Additional URIs of substituters.", + Setting<Strings> extraSubstituters{ + this, {}, "extra-substituters", + R"( + Additional binary caches appended to those specified in + `substituters`. When used by unprivileged users, untrusted + substituters (i.e. those not listed in `trusted-substituters`) are + silently ignored. + )", {"extra-binary-caches"}}; - Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", - "Disabled substituters that may be enabled via the substituters option by untrusted users.", + Setting<StringSet> trustedSubstituters{ + this, {}, "trusted-substituters", + R"( + A list of URLs of substituters, separated by whitespace. These are + not used by default, but can be enabled by users of the Nix daemon + by specifying `--option substituters urls` on the command + line. Unprivileged users are only allowed to pass a subset of the + URLs listed in `substituters` and `trusted-substituters`. + )", {"trusted-binary-caches"}}; - Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", - "Which users or groups are trusted to ask the daemon to do unsafe things."}; - - Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", - "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " - "return an invalid path result"}; - - Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", - "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " - "return a valid path result."}; + Setting<Strings> trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + Setting<unsigned int> ttlNegativeNarInfoCache{ + this, 3600, "narinfo-cache-negative-ttl", + R"( + The TTL in seconds for negative lookups. If a store path is queried + from a substituter but was not found, there will be a negative + lookup cached in the local disk cache database for the specified + duration. + )"}; + + Setting<unsigned int> ttlPositiveNarInfoCache{ + this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + R"( + The TTL in seconds for positive lookups. If a store path is queried + from a substituter, the result of the query will be cached in the + local disk cache database including some of the NAR metadata. The + default TTL is a month, setting a shorter TTL for positive lookups + can be useful for binary caches that have frequent garbage + collection, in which case having a more frequent cache invalidation + would prevent trying to pull the path again and failing with a hash + mismatch if the build isn't reproducible. + )"}; /* ?Who we trust to use the daemon in safe ways */ - Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", - "Which users or groups are allowed to connect to the daemon."}; + Setting<Strings> allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; Setting<bool> printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; - Setting<std::string> preBuildHook{this, "", - "pre-build-hook", - "A program to run just before a build to set derivation-specific build settings."}; + Setting<std::string> preBuildHook{ + this, "", "pre-build-hook", + R"( + If set, the path to a program that can set extra derivation-specific + settings for this system. This is used for settings that can't be + captured by the derivation model itself and are too variable between + different versions of the same system to be hard-coded into nix. - Setting<std::string> postBuildHook{this, "", "post-build-hook", - "A program to run just after each successful build."}; + The hook is passed the derivation path and, if sandboxes are + enabled, the sandbox directory. It can then modify the sandbox and + send a series of commands to modify various settings to stdout. The + currently recognized commands are: - Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", - "Path to the netrc file used to obtain usernames/passwords for downloads."}; + - `extra-sandbox-paths` + Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an + empty line. Entries have the same format as `sandbox-paths`. + )"}; + + Setting<std::string> postBuildHook{ + this, "", "post-build-hook", + R"( + Optional. The path to a program to execute after each build. + + This option is only settable in the global `nix.conf`, or on the + command line by trusted users. + + When using the nix-daemon, the daemon executes the hook as `root`. + If the nix-daemon is not involved, the hook runs as the user + executing the nix-build. + + - The hook executes after an evaluation-time build. + + - The hook does not execute on substituted paths. + + - The hook's output always goes to the user's terminal. + + - If the hook fails, the build succeeds but no further builds + execute. + + - The hook executes synchronously, and blocks other builds from + progressing while it runs. + + The program executes with no arguments. The program's environment + contains the following environment variables: + + - `DRV_PATH` + The derivation for the built paths. + + Example: + `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` + + - `OUT_PATHS` + Output paths of the built derivation, separated by a space + character. + + Example: + `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. + )"}; + + Setting<std::string> netrcFile{ + this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", + R"( + If set to an absolute path to a `netrc` file, Nix will use the HTTP + authentication credentials in this file when trying to download from + a remote host through HTTP or HTTPS. Defaults to + `$NIX_CONF_DIR/netrc`. + + The `netrc` file consists of a list of accounts in the following + format: + + machine my-machine + login my-username + password my-password + + For the exact syntax, see [the `curl` + documentation](https://ec.haxx.se/usingcurl-netrc.html). + + > **Note** + > + > This must be an absolute path, and `~` is not resolved. For + > example, `~/.netrc` won't resolve to your home directory's + > `.netrc`. + )"}; /* Path to the SSL CA file used */ Path caFile; #if __linux__ - Setting<bool> filterSyscalls{this, true, "filter-syscalls", - "Whether to prevent certain dangerous system calls, such as " - "creation of setuid/setgid files or adding ACLs or extended " - "attributes. Only disable this if you're aware of the " - "security implications."}; - - Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges", - "Whether builders can acquire new privileges by calling programs with " - "setuid/setgid bits or with file capabilities."}; + Setting<bool> filterSyscalls{ + this, true, "filter-syscalls", + R"( + Whether to prevent certain dangerous system calls, such as + creation of setuid/setgid files or adding ACLs or extended + attributes. Only disable this if you're aware of the + security implications. + )"}; + + Setting<bool> allowNewPrivileges{ + this, false, "allow-new-privileges", + R"( + (Linux-specific.) By default, builders on Linux cannot acquire new + privileges by calling setuid/setgid programs or programs that have + file capabilities. For example, programs such as `sudo` or `ping` + will fail. (Note that in sandbox builds, no such programs are + available unless you bind-mount them into the sandbox via the + `sandbox-paths` option.) You can allow the use of such programs by + enabling this option. This is impure and usually undesirable, but + may be useful in certain scenarios (e.g. to spin up containers or + set up userspace network interfaces in tests). + )"}; #endif - Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", - "A list of servers used by builtins.fetchurl to fetch files by hash."}; - - Setting<uint64_t> minFree{this, 0, "min-free", - "Automatically run the garbage collector when free disk space drops below the specified amount."}; - - Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", - "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<Strings> hashedMirrors{ + this, {}, "hashed-mirrors", + R"( + A list of web servers used by `builtins.fetchurl` to obtain files by + hash. The default is `http://tarballs.nixos.org/`. Given a hash type + *ht* and a base-16 hash *h*, Nix will try to download the file from + *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if + they have disappeared from their original URI. For example, given + the default mirror `http://tarballs.nixos.org/`, when building the + derivation + + ```nix + builtins.fetchurl { + url = "https://example.org/foo-1.2.3.tar.xz"; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; + } + ``` + + Nix will attempt to download this file from + `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae` + first. If it is not available there, if will try the original URI. + )"}; + + Setting<uint64_t> minFree{ + this, 0, "min-free", + R"( + When free disk space in `/nix/store` drops below `min-free` during a + build, Nix performs a garbage-collection until `max-free` bytes are + available or there is no more garbage. A value of `0` (the default) + disables this feature. + )"}; + + Setting<uint64_t> maxFree{ + this, std::numeric_limits<uint64_t>::max(), "max-free", + R"( + When a garbage collection is triggered by the `min-free` option, it + stops as soon as `max-free` bytes are available. The default is + infinity (i.e. delete all garbage). + )"}; Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - Setting<Paths> pluginFiles{this, {}, "plugin-files", - "Plugins to dynamically load at nix initialization time."}; - - Setting<std::string> githubAccessToken{this, "", "github-access-token", - "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting<Paths> pluginFiles{ + this, {}, "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix, allowing them to affect execution through static + initialization. In particular, these plugins may construct static + instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; + + Setting<StringMap> accessTokens{this, {}, "access-tokens", + R"( + Access tokens used to access protected GitHub, GitLab, or + other locations requiring token-based authentication. + + Access tokens are specified as a string made up of + space-separated `host=token` values. The specific token + used is selected by matching the `host` portion against the + "host" specification of the input. The actual use of the + `token` value is determined by the type of resource being + accessed: + + * Github: the token value is the OAUTH-TOKEN string obtained + as the Personal Access Token from the Github server (see + https://docs.github.com/en/developers/apps/authorizing-oath-apps). + + * Gitlab: the token value is either the OAuth2 token or the + Personal Access Token (these are different types tokens + for gitlab, see + https://docs.gitlab.com/12.10/ee/api/README.html#authentication). + The `token` value should be `type:tokenstring` where + `type` is either `OAuth2` or `PAT` to indicate which type + of token is being specified. + + Example `~/.config/nix/nix.conf`: + + ``` + access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + ``` + + Example `~/code/flake.nix`: + + ```nix + input.foo = { + type = "gitlab"; + host = "gitlab.mycompany.com"; + owner = "mycompany"; + repo = "pro"; + }; + ``` + + This example specifies three tokens, one each for accessing + github.com, gitlab.mycompany.com, and sourceforge.net. + + The `input.foo` uses the "gitlab" fetcher, which might + requires specifying the token type along with the token + value. + )"}; Setting<Strings> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; @@ -378,6 +933,25 @@ public: Setting<bool> warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + + Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", + "Maximum size of NARs before spilling them to disk."}; + + Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; + + Setting<bool> allowSymlinkedStore{ + this, false, "allow-symlinked-store", + R"( + If set to `true`, Nix will stop complaining if the store directory + (typically /nix/store) contains symlink components. + + This risks making some builds "impure" because builders sometimes + "canonicalise" paths by resolving all symlink components. Problems + occur if those builds are then deployed to machines where /nix/store + resolves to a different location from that of the build machine. You + can enable this setting if you are sure you're not going to do that. + )"}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 451a64785..9d2a89f96 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,12 +2,20 @@ #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "callback.hh" namespace nix { MakeError(UploadToHTTP, Error); -class HttpBinaryCacheStore : public BinaryCacheStore +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Http Binary Cache Store"; } +}; + +class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig { private: @@ -24,9 +32,12 @@ private: public: HttpBinaryCacheStore( - const Params & params, const Path & _cacheUri) - : BinaryCacheStore(params) - , cacheUri(_cacheUri) + const std::string & scheme, + const Path & _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) + , cacheUri(scheme + "://" + _cacheUri) { if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -55,6 +66,14 @@ public: } } + static std::set<std::string> uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set<std::string>({"http", "https"}); + if (forceHttp) ret.insert("file"); + return ret; + } + protected: void maybeDisable() @@ -85,7 +104,7 @@ protected: checkEnabled(); try { - FileTransferRequest request(cacheUri + "/" + path); + FileTransferRequest request(makeRequest(path)); request.head = true; getFileTransfer()->download(request); return true; @@ -100,11 +119,11 @@ protected: } void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) override { - auto req = FileTransferRequest(cacheUri + "/" + path); - req.data = std::make_shared<string>(data); // FIXME: inefficient + auto req = makeRequest(path); + req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain()); req.mimeType = mimeType; try { getFileTransfer()->upload(req); @@ -115,8 +134,11 @@ protected: FileTransferRequest makeRequest(const std::string & path) { - FileTransferRequest request(cacheUri + "/" + path); - return request; + return FileTransferRequest( + hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") + ? path + : cacheUri + "/" + path); + } void getFile(const std::string & path, Sink & sink) override @@ -159,18 +181,6 @@ protected: }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - if (std::string(uri, 0, 7) != "http://" && - std::string(uri, 0, 8) != "https://" && - (!forceHttp || std::string(uri, 0, 7) != "file://")) - return 0; - auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); - store->init(); - return store; -}); +static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..467169ce8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,21 +6,28 @@ #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" +#include "callback.hh" namespace nix { -static std::string uriScheme = "ssh://"; - -struct LegacySSHStore : public Store +struct LegacySSHStoreConfig : virtual StoreConfig { - const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; + using StoreConfig::StoreConfig; + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "Legacy SSH Store"; } +}; +struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +{ // Hack for getting remote build log output. - const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { @@ -37,8 +44,11 @@ struct LegacySSHStore : public Store SSHMaster master; - LegacySSHStore(const string & host, const Params & params) - : Store(params) + static std::set<std::string> uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const string & scheme, const string & host, const Params & params) + : StoreConfig(params) + , Store(params) , host(host) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), @@ -84,7 +94,7 @@ struct LegacySSHStore : public Store string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } void queryPathInfoUncached(const StorePath & path, @@ -93,6 +103,9 @@ struct LegacySSHStore : public Store try { auto conn(connections->get()); + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; @@ -100,23 +113,27 @@ struct LegacySSHStore : public Store auto p = readString(conn->from); if (p.empty()) return callback(nullptr); - auto info = std::make_shared<ValidPathInfo>(parseStorePath(p)); - assert(path == info->path); + auto path2 = parseStorePath(p); + assert(path == path2); + /* Hash will be set below. FIXME construct ValidPathInfo at end. */ + auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy); PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths<StorePathSet>(*this, conn->from); + info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); - info->ca = parseContentAddressOpt(readString(conn->from)); - info->sigs = readStrings<StringSet>(conn->from); + if (s == "") + throw Error("NAR hash is now mandatory"); + info->narHash = Hash::parseAnyPrefixed(s); } + info->ca = parseContentAddressOpt(readString(conn->from)); + info->sigs = readStrings<StringSet>(conn->from); auto s = readString(conn->from); assert(s == ""); @@ -126,8 +143,7 @@ struct LegacySSHStore : public Store } void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override + RepairFlag repair, CheckSigsFlag checkSigs) override { debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); @@ -140,7 +156,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -169,7 +185,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - writeStorePaths(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -203,6 +219,24 @@ struct LegacySSHStore : public Store const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } +private: + + void putBuildSettings(Connection & conn) + { + conn.to + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) + conn.to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) + conn.to + << settings.buildRepeat + << settings.enforceDeterminism; + } + +public: + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { @@ -212,16 +246,8 @@ struct LegacySSHStore : public Store << cmdBuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); - conn->to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) - conn->to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->to - << settings.buildRepeat - << settings.enforceDeterminism; + + putBuildSettings(*conn); conn->to.flush(); @@ -235,6 +261,29 @@ struct LegacySSHStore : public Store return status; } + void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override + { + auto conn(connections->get()); + + conn->to << cmdBuildPaths; + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + + putBuildSettings(*conn); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } + } + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } @@ -252,10 +301,10 @@ struct LegacySSHStore : public Store conn->to << cmdQueryClosure << includeOutputs; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})) out.insert(i); } @@ -268,10 +317,10 @@ struct LegacySSHStore : public Store << cmdQueryValidPaths << false // lock << maybeSubstitute; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn->to.flush(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } void connect() override @@ -286,12 +335,6 @@ struct LegacySSHStore : public Store } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 48aca478c..7d979c5c2 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -4,7 +4,14 @@ namespace nix { -class LocalBinaryCacheStore : public BinaryCacheStore +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Local Binary Cache Store"; } +}; + +class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig { private: @@ -13,8 +20,11 @@ private: public: LocalBinaryCacheStore( - const Params & params, const Path & binaryCacheDir) - : BinaryCacheStore(params) + const std::string scheme, + const Path & binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { } @@ -26,13 +36,25 @@ public: return "file://" + binaryCacheDir; } + static std::set<std::string> uriSchemes(); + protected: bool fileExists(const std::string & path) override; void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) override; + std::shared_ptr<std::basic_iostream<char>> istream, + const std::string & mimeType) override + { + auto path2 = binaryCacheDir + "/" + path; + Path tmp = path2 + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + StreamToSourceAdapter source(istream); + writeFile(tmp, source); + if (rename(tmp.c_str(), path2.c_str())) + throw SysError("renaming '%1%' to '%2%'", tmp, path2); + del.cancel(); + } void getFile(const std::string & path, Sink & sink) override { @@ -52,7 +74,9 @@ protected: if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); + paths.insert(parseStorePath( + storeDir + "/" + entry.name.substr(0, entry.name.size() - 8) + + "-" + MissingName)); } return paths; @@ -68,38 +92,19 @@ void LocalBinaryCacheStore::init() BinaryCacheStore::init(); } -static void atomicWrite(const Path & path, const std::string & s) -{ - Path tmp = path + ".tmp." + std::to_string(getpid()); - AutoDelete del(tmp, false); - writeFile(tmp, s); - if (rename(tmp.c_str(), path.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path); - del.cancel(); -} - bool LocalBinaryCacheStore::fileExists(const std::string & path) { return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) +std::set<std::string> LocalBinaryCacheStore::uriSchemes() { - atomicWrite(binaryCacheDir + "/" + path, data); + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1") + return {}; + else + return {"file"}; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || - std::string(uri, 0, 7) != "file://") - return 0; - auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); - store->init(); - return store; -}); +static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regLocalBinaryCacheStore; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index dd96d2578..2f1d9663a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { - Path storePath = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + auto storePath = store->toStorePath(path).first; + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80ebe903f..6a5f13996 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" +#include "callback.hh" #include <iostream> #include <algorithm> @@ -42,7 +43,8 @@ namespace nix { LocalStore::LocalStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", "physical path to the Nix store"} @@ -108,12 +110,11 @@ LocalStore::LocalStore(const Params & params) } /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + if (!settings.allowSymlinkedStore) { Path path = realStoreDir; struct stat st; while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); + st = lstat(path); if (S_ISLNK(st.st_mode)) throw Error( "the path '%1%' is a symlink; " @@ -417,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct void canonicaliseTimestampAndPermissions(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); - canonicaliseTimestampAndPermissions(path, st); + canonicaliseTimestampAndPermissions(path, lstat(path)); } @@ -441,9 +439,7 @@ static void canonicalisePathMetaData_( } #endif - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) @@ -479,8 +475,7 @@ static void canonicalisePathMetaData_( ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - assert(!S_ISDIR(st.st_mode)); - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); @@ -526,9 +521,7 @@ void canonicalisePathMetaData( /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); @@ -551,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) { - if (actual != expected) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(actual), printStorePath(expected)); auto j = drv.env.find(varName); if (j == drv.env.end() || parseStorePath(j->second) != actual) throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", @@ -563,24 +553,54 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat }; - if (drv.isFixedOutput()) { - DerivationOutputs::const_iterator out = drv.outputs.find("out"); - if (out == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + drv.type(); - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); + std::optional<Hash> h; + for (auto & i : drv.outputs) { + std::visit(overloaded { + [&](DerivationOutputInputAddressed doia) { + if (!h) { + // somewhat expensive so we do lazily + auto temp = hashDerivationModulo(*this, drv, true); + h = std::get<Hash>(temp); + } + StorePath recomputed = makeOutputPath(i.first, *h, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](DerivationOutputCAFixed dof) { + StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + envHasRightPath(path, i.first); + }, + [&](DerivationOutputCAFloating _) { + /* Nothing to check */ + }, + }, i.second.output); } +} + + +void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) +{ + auto state(_state.lock()); + return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); +} + +void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +{ + retrySQLite<void>([&]() { + state.stmtAddDerivationOutput.use() + (deriver) + (outputName) + (printStorePath(output)) + .exec(); + }); - else { - Hash h = hashDerivationModulo(*this, drv, true); - for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); - } } @@ -601,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state, (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); + uint64_t id = state.db.getLastInsertedRowId(); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -617,12 +637,11 @@ uint64_t LocalStore::addValidPath(State & state, registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - for (auto & i : drv.outputs) { - state.stmtAddDerivationOutput.use() - (id) - (i.first) - (printStorePath(i.second.path)) - .exec(); + for (auto & i : drv.outputsAndOptPaths(*this)) { + /* Floating CA derivations have indeterminate output paths until + they are built, so don't register anything in that case */ + if (i.second.second) + linkDeriverToPath(state, id, i.first, *i.second.second); } } @@ -640,25 +659,28 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(path); - callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); if (!useQueryPathInfo.next()) return std::shared_ptr<ValidPathInfo>(); - info->id = useQueryPathInfo.getInt(0); + auto id = useQueryPathInfo.getInt(0); + auto narHash = Hash::dummy; try { - info->narHash = Hash(useQueryPathInfo.getStr(1)); + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); } + auto info = std::make_shared<ValidPathInfo>(path, narHash); + + info->id = id; + info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); @@ -706,7 +728,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); if (!use.next()) - throw Error("path '%s' is not valid", printStorePath(path)); + throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); } @@ -781,17 +803,61 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) } -OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) +std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) { - return retrySQLite<OutputPathMap>([&]() { + auto path = path_; + std::map<std::string, std::optional<StorePath>> outputs; + Derivation drv = readDerivation(path); + for (auto & [outName, _] : drv.outputs) { + outputs.insert_or_assign(outName, std::nullopt); + } + bool haveCached = false; + { + auto resolutions = drvPathResolutions.lock(); + auto resolvedPathOptIter = resolutions->find(path); + if (resolvedPathOptIter != resolutions->end()) { + auto & [_, resolvedPathOpt] = *resolvedPathOptIter; + if (resolvedPathOpt) + path = *resolvedPathOpt; + haveCached = true; + } + } + /* can't just use else-if instead of `!haveCached` because we need to unlock + `drvPathResolutions` before it is locked in `Derivation::resolve`. */ + if (!haveCached && drv.type() == DerivationType::CAFloating) { + /* Try resolve drv and use that path instead. */ + auto attempt = drv.tryResolve(*this); + if (!attempt) + /* If we cannot resolve the derivation, we cannot have any path + assigned so we return the map of all std::nullopts. */ + return outputs; + /* Just compute store path */ + auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true); + /* Store in memo table. */ + /* FIXME: memo logic should not be local-store specific, should have + wrapper-method instead. */ + drvPathResolutions.lock()->insert_or_assign(path, pathResolved); + path = std::move(pathResolved); + } + return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() { auto state(_state.lock()); - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); + uint64_t drvId; + try { + drvId = queryValidPathId(*state, path); + } catch (InvalidPath &) { + /* FIXME? if the derivation doesn't exist, we cannot have a mapping + for it. */ + return outputs; + } + + auto useQueryDerivationOutputs { + state->stmtQueryDerivationOutputs.use() + (drvId) + }; - OutputPathMap outputs; while (useQueryDerivationOutputs.next()) - outputs.emplace( + outputs.insert_or_assign( useQueryDerivationOutputs.getStr(0), parseStorePath(useQueryDerivationOutputs.getStr(1)) ); @@ -853,20 +919,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; for (auto & path : paths) { - if (infos.count(path)) continue; - debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); + auto subPath(path.first); + + // recompute store path so that we can use a different store root + if (path.second) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); try { - auto info = sub->queryPathInfo(path); + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ + infos.insert_or_assign(path.first, SubstitutablePathInfo{ info->deriver, info->references, narInfo ? narInfo->fileSize : 0, @@ -969,11 +1047,8 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { - if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); @@ -983,7 +1058,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); + auto realPath = Store::toRealPath(info.path); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a @@ -1008,11 +1083,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, else hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); - LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { - size_t n = source.read(data, len); - (*hashSink)(data, n); - return n; - }); + TeeSource wrapperSource { source, *hashSink }; restorePath(realPath, wrapperSource); @@ -1040,12 +1111,64 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(hashAlgo, dump); + /* For computing the store path. */ + auto hashSink = std::make_unique<HashSink>(hashAlgo); + TeeSource source { source0, *hashSink }; + + /* Read the source path into memory, but only if it's up to + narBufferSize bytes. If it's larger, write it to a temporary + location in the Nix store. If the subsequently computed + destination store path is already valid, we just delete the + temporary path. Otherwise, we move it to the destination store + path. */ + bool inMemory = false; + + std::string dump; + + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 65536; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } + + std::unique_ptr<AutoDelete> delTempDir; + Path tempPath; + + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; + + auto tempDir = createTempDir(realStoreDir, "add"); + delTempDir = std::make_unique<AutoDelete>(tempDir); + tempPath = tempDir + "/x"; + + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); + + dump.clear(); + } - auto dstPath = makeFixedOutputPath(method, h, name); + auto [hash, size] = hashSink->finish(); + + auto dstPath = makeFixedOutputPath(method, hash, name); addTempRoot(dstPath); @@ -1054,8 +1177,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1065,31 +1187,35 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); + if (inMemory) { + StringSource dumpSource { dump }; + /* Restore from the NAR in memory. */ + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); + } else { + /* Move the temporary path we restored above. */ + if (rename(tempPath.c_str(), realPath.c_str())) + throw Error("renaming '%s' to '%s'", tempPath, realPath); + } - canonicalisePathMetaData(realPath, {}); + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath - optimisePath(realPath); // FIXME: combine with hashPath() + optimisePath(realPath); - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; + ValidPathInfo info { dstPath, narHash.first }; + info.narSize = narHash.second; + info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1100,24 +1226,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } -StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - Path srcPath(absPath(_srcPath)); - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref<std::string>(readFile(srcPath)); - - return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); -} - - StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { @@ -1128,8 +1236,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1149,8 +1256,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info(dstPath); - info.narHash = narHash; + ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s->size(); info.references = references; info.ca = TextHash { .hash = hash }; @@ -1265,9 +1371,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr<AbstractHashSink> hashSink; if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique<HashSink>(*info->narHash.type); + hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1387,7 +1493,7 @@ static void makeMutable(const Path & path) { checkInterrupt(); - struct stat st = lstat(path); + auto st = lstat(path); if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 79b415875..118c7149b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -23,18 +23,26 @@ namespace nix { const int nixSchemaVersion = 10; -struct Derivation; - - struct OptimiseStats { unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; + uint64_t bytesFreed = 0; + uint64_t blocksFreed = 0; +}; + +struct LocalStoreConfig : virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + + Setting<bool> requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + + const std::string name() override { return "Local Store"; } }; -class LocalStore : public LocalFSStore +class LocalStore : public LocalFSStore, public virtual LocalStoreConfig { private: @@ -98,10 +106,6 @@ public: private: - Setting<bool> requireSigs{(Store*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const PublicKeys & getPublicKeys(); public: @@ -133,29 +137,20 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - OutputPathMap queryDerivationOutputMap(const StorePath & path) override; + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, + void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -291,6 +286,11 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); + /* Register the store path 'output' as the output named 'outputName' of + derivation 'deriver'. */ + void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output); + void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index aec4ed493..d266c8efe 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -61,3 +61,6 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libstore/builtins/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index f848582da..7db2556f4 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,7 @@ #include "machines.hh" #include "util.hh" #include "globals.hh" +#include "store-api.hh" #include <algorithm> @@ -48,6 +49,29 @@ bool Machine::mandatoryMet(const std::set<string> & features) const { }); } +ref<Store> Machine::openStore() const { + Store::Params storeParams; + if (hasPrefix(storeUri, "ssh://")) { + storeParams["max-connections"] = "1"; + storeParams["log-fd"] = "4"; + if (sshKey != "") + storeParams["ssh-key"] = sshKey; + } + { + auto & fs = storeParams["system-features"]; + auto append = [&](auto feats) { + for (auto & f : feats) { + if (fs.size() > 0) fs += ' '; + fs += f; + } + }; + append(supportedFeatures); + append(mandatoryFeatures); + } + + return nix::openStore(storeUri, storeParams); +} + void parseMachines(const std::string & s, Machines & machines) { for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index de92eb924..341d9bd97 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -4,6 +4,8 @@ namespace nix { +class Store; + struct Machine { const string storeUri; @@ -28,6 +30,8 @@ struct Machine { decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + + ref<Store> openStore() const; }; typedef std::vector<Machine> Machines; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,7 +4,8 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" - +#include "topo-sort.hh" +#include "callback.hh" namespace nix { @@ -108,9 +109,19 @@ void Store::computeFSClosure(const StorePath & startPath, } +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) +{ + auto out = drv.outputs.find("out"); + if (out != drv.outputs.end()) { + if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output)) + return v->hash; + } + return std::nullopt; +} + void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) + uint64_t & downloadSize_, uint64_t & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -122,8 +133,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, { std::unordered_set<std::string> done; StorePathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; + uint64_t & downloadSize; + uint64_t & narSize; }; struct DrvState @@ -157,7 +168,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); + querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -192,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, return; } + PathSet invalid; + /* true for regular derivations, and CA derivations for which we + have a trust mapping for all wanted outputs. */ + auto knownOutputPaths = true; + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { + if (!pathOpt) { + knownOutputPaths = false; + break; + } + if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) + invalid.insert(printStorePath(*pathOpt)); + } + if (knownOutputPaths && invalid.empty()) return; + auto drv = make_ref<Derivation>(derivationFromPath(path.path)); ParsedDerivation parsedDrv(StorePath(path.path), *drv); - PathSet invalid; - for (auto & j : drv->outputs) - if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path)) - invalid.insert(printStorePath(j.second.path)); - if (invalid.empty()) return; - - if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); @@ -214,7 +232,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path.path}, infos); + querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); @@ -246,41 +264,21 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePaths Store::topoSortPaths(const StorePathSet & paths) { - StorePaths sorted; - StorePathSet visited, parents; - - std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit; - - dfsVisit = [&](const StorePath & path, const StorePath * parent) { - if (parents.count(path)) - throw BuildError("cycle detected in the references of '%s' from '%s'", - printStorePath(path), printStorePath(*parent)); - - if (!visited.insert(path).second) return; - parents.insert(path); - - StorePathSet references; - try { - references = queryPathInfo(path)->references; - } catch (InvalidPath &) { - } - - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) - dfsVisit(i, &path); - - sorted.push_back(path); - parents.erase(path); - }; - - for (auto & i : paths) - dfsVisit(i, nullptr); - - std::reverse(sorted.begin(), sorted.end()); - - return sorted; + return topoSort(paths, + {[&](const StorePath & path) { + StorePathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath &) { + } + return references; + }}, + {[&](const StorePath & path, const StorePath & parent) { + return BuildError( + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); + }}); } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index d1c8a6101..41e28dc99 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,10 +1,18 @@ #include "names.hh" #include "util.hh" +#include <regex> + namespace nix { +struct Regex +{ + std::regex regex; +}; + + DrvName::DrvName() { name = ""; @@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0) } +DrvName::~DrvName() +{ } + + bool DrvName::matches(DrvName & n) { if (name != "*") { - if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); - if (!std::regex_match(n.name, *regex)) return false; + if (!regex) { + regex = std::make_unique<Regex>(); + regex->regex = std::regex(name, std::regex::extended); + } + if (!std::regex_match(n.name, regex->regex)) return false; } if (version != "" && version != n.version) return false; return true; @@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs) { DrvNames result; for (auto & i : opArgs) - result.push_back(DrvName(i)); + result.emplace_back(i); return result; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 00e14b8c7..bc62aac93 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -3,10 +3,11 @@ #include <memory> #include "types.hh" -#include <regex> namespace nix { +struct Regex; + struct DrvName { string fullName; @@ -16,10 +17,12 @@ struct DrvName DrvName(); DrvName(std::string_view s); + ~DrvName(); + bool matches(DrvName & n); private: - std::unique_ptr<std::regex> regex; + std::unique_ptr<Regex> regex; }; typedef list<DrvName> DrvNames; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ca663d837..a9efdd0b6 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -18,7 +18,7 @@ struct NarMember /* If this is a regular file, position of the contents of this file in the NAR. */ - size_t start = 0, size = 0; + uint64_t start = 0, size = 0; std::string target; @@ -34,20 +34,23 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, StringSource + struct NarIndexer : ParseSink, Source { NarAccessor & acc; + Source & source; std::stack<NarMember *> parents; - std::string currentStart; bool isExec = false; - NarIndexer(NarAccessor & acc, const std::string & nar) - : StringSource(nar), acc(acc) + uint64_t pos = 0; + + NarIndexer(NarAccessor & acc, Source & source) + : acc(acc), source(source) { } - void createMember(const Path & path, NarMember member) { + void createMember(const Path & path, NarMember member) + { size_t level = std::count(path.begin(), path.end(), '/'); while (parents.size() > level) parents.pop(); @@ -77,33 +80,40 @@ struct NarAccessor : public FSAccessor parents.top()->isExecutable = true; } - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits<size_t>::max()); - parents.top()->size = (size_t)size; + assert(size <= std::numeric_limits<uint64_t>::max()); + parents.top()->size = (uint64_t) size; parents.top()->start = pos; } - void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); - } - } + void receiveContents(unsigned char * data, size_t len) override + { } void createSymlink(const Path & path, const string & target) override { createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } + + size_t read(unsigned char * data, size_t len) override + { + auto n = source.read(data, len); + pos += n; + return n; + } }; NarAccessor(ref<const std::string> nar) : nar(nar) { - NarIndexer indexer(*this, *nar); + StringSource source(*nar); + NarIndexer indexer(*this, source); + parseDump(indexer, indexer); + } + + NarAccessor(Source & source) + { + NarIndexer indexer(*this, source); parseDump(indexer, indexer); } @@ -219,6 +229,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) return make_ref<NarAccessor>(nar); } +ref<FSAccessor> makeNarAccessor(Source & source) +{ + return make_ref<NarAccessor>(source); +} + ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 2871199de..8af1272f6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -6,10 +6,14 @@ namespace nix { +struct Source; + /* Return an object that provides access to the contents of a NAR file. */ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); +ref<FSAccessor> makeNarAccessor(Source & source); + /* Create a NAR accessor from a NAR listing (in the format produced by listNar()). The callback getNarBytes(offset, length) is used by the readFile() method of the accessor to get the contents of files diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..8541cc51f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -189,13 +189,14 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart)); + auto narInfo = make_ref<NarInfo>( + StorePath(hashPart + "-" + namePart), + Hash::parseAnyPrefixed(queryNAR.getStr(6))); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) - narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4)); narInfo->fileSize = queryNAR.getInt(5); - narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) narInfo->references.insert(StorePath(r)); @@ -230,7 +231,7 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) (info->narHash.to_string(Base32, true)) (info->narSize) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 04550ed97..3454f34bb 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,36 +1,37 @@ #include "globals.hh" #include "nar-info.hh" +#include "store-api.hh" namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { - return Hash(s); + return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; bool havePath = false; + bool haveNarHash = false; size_t pos = 0; while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } - else if (name == "NarHash") + else if (name == "NarHash") { narHash = parseHashField(value); + haveNarHash = true; + } else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString<Strings>(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } @@ -77,7 +80,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -87,8 +90,8 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..39ced76e5 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -2,20 +2,22 @@ #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "path-info.hh" namespace nix { +class Store; + struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional<Hash> fileHash; uint64_t fileSize = 0; std::string system; NarInfo() = delete; - NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } + NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b2b2412a3..a0d482ddf 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -17,9 +17,7 @@ namespace nix { static void makeWritable(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError("changing writability of '%1%'", path); } @@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Yes! We've seen a file with the same contents. Replace the current file with a hard link to that file. */ - struct stat stLink; - if (lstat(linkPath.c_str(), &stLink)) - throw SysError("getting attributes of path '%1%'", linkPath); + auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); @@ -282,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(unsigned long long bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - void LocalStore::optimiseStore() { OptimiseStats stats; optimiseStore(stats); - printInfo( - format("%1% freed by hard-linking %2% files") - % showBytes(stats.bytesFreed) - % stats.filesLinked); + printInfo("%s freed by hard-linking %d files", + showBytes(stats.bytesFreed), + stats.filesLinked); } void LocalStore::optimisePath(const Path & path) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c7797b730..e7b7202d4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -94,7 +94,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const return res; } -bool ParsedDerivation::canBuildLocally() const +bool ParsedDerivation::canBuildLocally(Store & localStore) const { if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform) @@ -102,14 +102,14 @@ bool ParsedDerivation::canBuildLocally() const return false; for (auto & feature : getRequiredSystemFeatures()) - if (!settings.systemFeatures.get().count(feature)) return false; + if (!localStore.systemFeatures.get().count(feature)) return false; return true; } -bool ParsedDerivation::willBuildLocally() const +bool ParsedDerivation::willBuildLocally(Store & localStore) const { - return getBoolAttr("preferLocalBuild") && canBuildLocally(); + return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore); } bool ParsedDerivation::substitutesAllowed() const @@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } -bool ParsedDerivation::contentAddressed() const -{ - return getBoolAttr("__contentAddressed", false); -} - } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0b8e8d031..c9fbe68c4 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,3 +1,5 @@ +#pragma once + #include "store-api.hh" #include <nlohmann/json_fwd.hpp> @@ -29,13 +31,11 @@ public: StringSet getRequiredSystemFeatures() const; - bool canBuildLocally() const; + bool canBuildLocally(Store & localStore) const; - bool willBuildLocally() const; + bool willBuildLocally(Store & localStore) const; bool substitutesAllowed() const; - - bool contentAddressed() const; }; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh new file mode 100644 index 000000000..8ff5c466e --- /dev/null +++ b/src/libstore/path-info.hh @@ -0,0 +1,112 @@ +#pragma once + +#include "crypto.hh" +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" + +#include <string> +#include <optional> + +namespace nix { + + +class Store; + + +struct SubstitutablePathInfo +{ + std::optional<StorePath> deriver; + StorePathSet references; + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ +}; + +typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; + + +struct ValidPathInfo +{ + StorePath path; + std::optional<StorePath> deriver; + // TODO document this + Hash narHash; + StorePathSet references; + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + /* If non-empty, an assertion that the path is content-addressed, + i.e., that the store path is computed from a cryptographic hash + of the contents of the path, plus some other bits of data like + the "name" part of the path. Such a path doesn't need + signatures, since we don't have to trust anybody's claim that + the path is the output of a particular derivation. (In the + extensional store model, we have to trust that the *contents* + of an output path of a derivation were actually produced by + that derivation. In the intensional model, we have to trust + that a particular output path was produced by a derivation; the + path then implies the contents.) + + Ideally, the content-addressability assertion would just be a Boolean, + and the store path would be computed from the name component, ‘narHash’ + and ‘references’. However, we support many types of content addresses. + */ + std::optional<ContentAddress> ca; + + bool operator == (const ValidPathInfo & i) const + { + return + path == i.path + && narHash == i.narHash + && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint(const Store & store) const; + + void sign(const Store & store, const SecretKey & secretKey); + + /* Return true iff the path is verifiably content-addressed. */ + bool isContentAddressed(const Store & store) const; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + + static const size_t maxSigs = std::numeric_limits<size_t>::max(); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys, or maxSigs if the path + is content-addressed. */ + size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + + Strings shortRefs() const; + + ValidPathInfo(const ValidPathInfo & other) = default; + + ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; + ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + + virtual ~ValidPathInfo() { } +}; + +typedef list<ValidPathInfo> ValidPathInfos; + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index b3d8ce95c..dc9dc3897 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,8 +2,6 @@ namespace nix { -MakeError(BadStorePath, Error); - static void checkName(std::string_view path, std::string_view name) { if (name.empty()) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index e43a8b50c..b03a0f69d 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -64,6 +64,8 @@ typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; typedef std::map<string, StorePath> OutputPathMap; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 6cfe393a4..c3809bad7 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -12,30 +12,24 @@ namespace nix { -static bool cmpGensByNumber(const Generation & a, const Generation & b) -{ - return a.number < b.number; -} - - /* Parse a generation name of the format `<profilename>-<number>-link'. */ -static int parseName(const string & profileName, const string & name) +static std::optional<GenerationNumber> parseName(const string & profileName, const string & name) { - if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + if (string(name, 0, profileName.size() + 1) != profileName + "-") return {}; string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); - if (p == string::npos) return -1; - int n; + if (p == string::npos) return {}; + unsigned int n; if (string2Int(string(s, 0, p), n) && n >= 0) return n; else - return -1; + return {}; } -Generations findGenerations(Path profile, int & curGen) +std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile) { Generations gens; @@ -43,30 +37,31 @@ Generations findGenerations(Path profile, int & curGen) auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { - int n; - if ((n = parseName(profileName, i.name)) != -1) { - Generation gen; - gen.path = profileDir + "/" + i.name; - gen.number = n; - struct stat st; - if (lstat(gen.path.c_str(), &st) != 0) - throw SysError("statting '%1%'", gen.path); - gen.creationTime = st.st_mtime; - gens.push_back(gen); + if (auto n = parseName(profileName, i.name)) { + auto path = profileDir + "/" + i.name; + gens.push_back({ + .number = *n, + .path = path, + .creationTime = lstat(path).st_mtime + }); } } - gens.sort(cmpGensByNumber); + gens.sort([](const Generation & a, const Generation & b) + { + return a.number < b.number; + }); - curGen = pathExists(profile) + return { + gens, + pathExists(profile) ? parseName(profileName, readLink(profile)) - : -1; - - return gens; + : std::nullopt + }; } -static void makeName(const Path & profile, unsigned int num, +static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { Path prefix = (format("%1%-%2%") % profile % num).str(); @@ -74,18 +69,17 @@ static void makeName(const Path & profile, unsigned int num, } -Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) +Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath) { /* The new generation number should be higher than old the previous ones. */ - int dummy; - Generations gens = findGenerations(profile, dummy); + auto [gens, dummy] = findGenerations(profile); - unsigned int num; + GenerationNumber num; if (gens.size() > 0) { Generation last = gens.back(); - if (readLink(last.path) == outPath) { + if (readLink(last.path) == store->printStorePath(outPath)) { /* We only create a new generation symlink if it differs from the last one. @@ -108,7 +102,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - store->addPermRoot(store->parseStorePath(outPath), generation, false, true); + store->addPermRoot(outPath, generation); return generation; } @@ -121,7 +115,7 @@ static void removeFile(const Path & path) } -void deleteGeneration(const Path & profile, unsigned int gen) +void deleteGeneration(const Path & profile, GenerationNumber gen) { Path generation; makeName(profile, gen, generation); @@ -129,7 +123,7 @@ void deleteGeneration(const Path & profile, unsigned int gen) } -static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) +static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) printInfo(format("would remove generation %1%") % gen); @@ -140,31 +134,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu } -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun) +void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); - if (gensToDelete.find(curGen) != gensToDelete.end()) + if (gensToDelete.count(*curGen)) throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { - if (gensToDelete.find(i.number) == gensToDelete.end()) continue; + if (!gensToDelete.count(i.number)) continue; deleteGeneration2(profile, i.number, dryRun); } } -void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; bool fromCurGen = false; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto i = gens.rbegin(); i != gens.rend(); ++i) { if (i->number == curGen) { fromCurGen = true; @@ -186,8 +178,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto & i : gens) if (i.number != curGen) @@ -200,8 +191,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); bool canDelete = false; for (auto i = gens.rbegin(); i != gens.rend(); ++i) diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 78645d8b6..be55a65d4 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -8,38 +8,35 @@ namespace nix { +class StorePath; + + +typedef unsigned int GenerationNumber; struct Generation { - int number; + GenerationNumber number; Path path; time_t creationTime; - Generation() - { - number = -1; - } - operator bool() const - { - return number != -1; - } }; -typedef list<Generation> Generations; +typedef std::list<Generation> Generations; /* Returns the list of currently present generations for the specified - profile, sorted by generation number. */ -Generations findGenerations(Path profile, int & curGen); + profile, sorted by generation number. Also returns the number of + the current generation. */ +std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); class LocalFSStore; -Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath); +Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath); -void deleteGeneration(const Path & profile, unsigned int gen); +void deleteGeneration(const Path & profile, GenerationNumber gen); -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); +void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun); -void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); void deleteOldGenerations(const Path & profile, bool dryRun); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..d2096cb49 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len, struct RefScanSink : Sink { - HashSink hashSink; StringSet hashes; StringSet seen; string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() { } void operator () (const unsigned char * data, size_t len); }; @@ -62,8 +61,6 @@ struct RefScanSink : Sink void RefScanSink::operator () (const unsigned char * data, size_t len) { - hashSink(data, len); - /* It's possible that a reference spans the previous and current fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ @@ -79,10 +76,20 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair<PathSet, HashResult> scanForReferences(const string & path, + const PathSet & refs) { - RefScanSink sink; + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair<PathSet, HashResult>(found, hash); +} + +PathSet scanForReferences(Sink & toTee, + const string & path, const PathSet & refs) +{ + RefScanSink refsSink; + TeeSink sink { refsSink, toTee }; std::map<string, Path> backMap; /* For efficiency (and a higher hit rate), just search for the @@ -97,7 +104,7 @@ PathSet scanForReferences(const string & path, assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); - sink.hashes.insert(s); + refsSink.hashes.insert(s); backMap[s] = i; } @@ -106,14 +113,12 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - for (auto & i : sink.seen) { + for (auto & i : refsSink.seen) { std::map<string, Path>::iterator j; if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } - hash = sink.hashSink.finish(); - return found; } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..c2efd095c 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,9 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs); + +PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index bd698d781..2d02a181b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) createDirs(cacheDir); } -Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) +Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); + return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, +void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar, ref<FSAccessor> narAccessor) { - nars.emplace(storePath, narAccessor); + nars.emplace(hashPart, narAccessor); if (cacheDir != "") { try { std::ostringstream str; JSONPlaceholder jsonRoot(str); listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(storePath, "ls"), str.str()); + writeFile(makeCacheFile(hashPart, "ls"), str.str()); /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(storePath, "nar"), nar); + writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { ignoreException(); @@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) { auto path = canonPath(path_); - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); + auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - auto i = nars.find(storePath); + auto i = nars.find(std::string(storePath.hashPart())); if (i != nars.end()) return {i->second, restPath}; StringSink sink; std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { - listing = nix::readFile(makeCacheFile(storePath, "ls")); + listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { @@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) return buf; }); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } @@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) *sink.s = nix::readFile(cacheFile); auto narAccessor = makeNarAccessor(sink.s); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } } - store->narFromPath(store->parseStorePath(storePath), sink); + store->narFromPath(storePath, sink); auto narAccessor = makeNarAccessor(sink.s); - addToCache(storePath, *sink.s, narAccessor); + addToCache(storePath.hashPart(), *sink.s, narAccessor); return {narAccessor, restPath}; } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 4afb3be95..347cf5764 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor { ref<Store> store; - std::map<Path, ref<FSAccessor>> nars; + std::map<std::string, ref<FSAccessor>> nars; Path cacheDir; @@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor friend class BinaryCacheStore; - Path makeCacheFile(const Path & storePath, const std::string & ext); + Path makeCacheFile(std::string_view hashPart, const std::string & ext); - void addToCache(const Path & storePath, const std::string & nar, + void addToCache(std::string_view hashPart, const std::string & nar, ref<FSAccessor> narAccessor); public: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 118aadf7e..23b1942ce 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" @@ -9,6 +10,7 @@ #include "pool.hh" #include "finally.hh" #include "logging.hh" +#include "callback.hh" #include <sys/types.h> #include <sys/stat.h> @@ -22,53 +24,82 @@ namespace nix { +namespace worker_proto { -template<> StorePathSet readStorePaths(const Store & store, Source & from) +std::string read(const Store & store, Source & from, Phantom<std::string> _) { - StorePathSet paths; - for (auto & i : readStrings<Strings>(from)) - paths.insert(store.parseStorePath(i)); - return paths; + return readString(from); } +void write(const Store & store, Sink & out, const std::string & str) +{ + out << str; +} -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) + +StorePath read(const Store & store, Source & from, Phantom<StorePath> _) { - out << paths.size(); - for (auto & i : paths) - out << store.printStorePath(i); + return store.parseStorePath(readString(from)); } -std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from) +void write(const Store & store, Sink & out, const StorePath & storePath) { - std::map<string, StorePath> pathMap; - auto rawInput = readStrings<Strings>(from); - if (rawInput.size() % 2) - throw Error("got an odd number of elements from the daemon when trying to read a output path map"); - auto curInput = rawInput.begin(); - while (curInput != rawInput.end()) { - auto thisKey = *curInput++; - auto thisValue = *curInput++; - pathMap.emplace(thisKey, store.parseStorePath(thisValue)); - } - return pathMap; + out << store.printStorePath(storePath); } -void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap) + +ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) { - out << 2*pathMap.size(); - for (auto & i : pathMap) { - out << i.first; - out << store.printStorePath(i.second); - } + return parseContentAddress(readString(from)); } +void write(const Store & store, Sink & out, const ContentAddress & ca) +{ + out << renderContentAddress(ca); +} + + +std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) +{ + auto s = readString(from); + return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s); +} + +void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt) +{ + out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) +{ + return parseContentAddressOpt(readString(from)); +} + +void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) +{ + out << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} + + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : Store(params) + , RemoteStoreConfig(params) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), - [this]() { return openConnectionWrapper(); }, + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, [this](const ref<Connection> & r) { return r->to.good() @@ -95,19 +126,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , RemoteStore(params) { } -UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) - , path(socket_path) +UDSRemoteStore::UDSRemoteStore( + const std::string scheme, + std::string socket_path, + const Params & params) + : UDSRemoteStore(params) { + path.emplace(socket_path); } @@ -151,8 +184,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() conn->startTime = std::chrono::steady_clock::now(); - initConnection(*conn); - return conn; } @@ -260,14 +291,16 @@ struct ConnectionHandle RemoteStore::Connection * operator -> () { return &*handle; } - void processStderr(Sink * sink = 0, Source * source = 0) + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) { - auto ex = handle->processStderr(sink, source); + auto ex = handle->processStderr(sink, source, flush); if (ex) { daemonException = true; std::rethrow_exception(ex); } } + + void withFramedSink(std::function<void(Sink & sink)> fun); }; @@ -296,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } } @@ -308,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } @@ -325,41 +358,46 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - writeStorePaths(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) { - if (paths.empty()) return; + if (pathsMap.empty()) return; auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - for (auto & i : paths) { + for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths<StorePathSet>(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i, std::move(info)); + infos.insert_or_assign(i.first, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; - writeStorePaths(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + worker_proto::write(*this, conn->to, paths); + } else + worker_proto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { @@ -367,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths<StorePathSet>(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -376,11 +414,28 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, } +ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref<ValidPathInfo>(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings<StringSet>(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - std::shared_ptr<ValidPathInfo> info; + std::shared_ptr<const ValidPathInfo> info; { auto conn(getConnection()); conn->to << wopQueryPathInfo << printStorePath(path); @@ -396,17 +451,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(StorePath(path)); - auto deriver = readString(conn->from); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<StorePathSet>(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings<StringSet>(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } + info = readValidPathInfo(conn, path); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -419,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})) referrers.insert(i); } @@ -429,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } @@ -441,17 +486,32 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return readStorePaths<StorePathSet>(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); } -OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path) +std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) { - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return readOutputPathMap(*this, conn->from); - + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {}); + } else { + // Fallback for old daemon versions. + // For floating-CA derivations (and their co-dependencies) this is an + // under-approximation as it only returns the paths that can be inferred + // from the derivation itself (and not the ones that are known because + // the have been built), but as old stores don't handle floating-CA + // derivations this shouldn't matter + auto derivation = readDerivation(path); + auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); + std::map<std::string, std::optional<StorePath>> ret; + for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { + ret.emplace(outputName, outputAndPath.second); + } + return ret; + } } std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) @@ -465,8 +525,95 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & } +ref<const ValidPathInfo> RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) +{ + std::optional<ConnectionHandle> conn_(getConnection()); + auto & conn = *conn_; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(caMethod); + worker_proto::write(*this, conn->to, references); + conn->to << repair; + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + worker_proto::write(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + } + }, caMethod); + auto path = parseStorePath(readString(conn->from)); + // Release our connection to prevent a deadlock in queryPathInfo(). + conn_.reset(); + return queryPathInfo(path); + } +} + + +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; +} + + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { auto conn(getConnection()); @@ -480,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -490,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from); + auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); assert(importedPaths.size() <= 1); } @@ -499,68 +646,30 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); - } -} - -StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - - Path srcPath(absPath(_srcPath)); - - conn->to - << wopAddToStore - << name - << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashAlgo); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + conn.withFramedSink([&](Sink & sink) { + copyNAR(source, sink); + }); + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; } - - return parseStorePath(readString(conn->from)); } StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); - - conn.processStderr(); - return parseStorePath(readString(conn->from)); + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } @@ -659,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - writeStorePaths(*this, conn->to, options.pathsToDelete); + worker_proto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -707,7 +816,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); @@ -721,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = readStorePaths<StorePathSet>(*this, conn->from); - willSubstitute = readStorePaths<StorePathSet>(*this, conn->from); - unknown = readStorePaths<StorePathSet>(*this, conn->from); + willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); conn->from >> downloadSize >> narSize; return; } @@ -762,6 +871,18 @@ RemoteStore::Connection::~Connection() } } +void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << printStorePath(path); + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref<FSAccessor> RemoteStore::getFSAccessor() +{ + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); +} static Logger::Fields readFields(Source & from) { @@ -780,9 +901,10 @@ static Logger::Fields readFields(Source & from) } -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) { - to.flush(); + if (flush) + to.flush(); while (true) { @@ -803,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } } else if (msg == STDERR_NEXT) @@ -843,14 +969,49 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> +void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) { - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params); -}); + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink((*this)->to, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + +static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fb2052752..ec04be985 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -16,18 +16,22 @@ struct FdSource; template<typename T> class Pool; struct ConnectionHandle; - -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ -class RemoteStore : public virtual Store +struct RemoteStoreConfig : virtual StoreConfig { -public: + using StoreConfig::StoreConfig; - const Setting<int> maxConnections{(Store*) this, 1, + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(), + const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(), "max-connection-age", "number of seconds to reuse a connection"}; +}; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +{ +public: virtual bool sameMachine() = 0; @@ -51,21 +55,28 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - OutputPathMap queryDerivationOutputMap(const StorePath & path) override; + std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, + void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + /* Add a content-addressable store path. `dump` will be drained. */ + ref<const ValidPathInfo> addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair); - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; + /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + + void addToStore(const ValidPathInfo & info, Source & nar, + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -95,7 +106,7 @@ public: void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + uint64_t & downloadSize, uint64_t & narSize) override; void connect() override; @@ -103,8 +114,6 @@ public: void flushBadConnections(); -protected: - struct Connection { AutoCloseFD fd; @@ -115,11 +124,13 @@ protected: virtual ~Connection(); - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; ref<Connection> openConnectionWrapper(); +protected: + virtual ref<Connection> openConnection() = 0; void initConnection(Connection & conn); @@ -132,24 +143,56 @@ protected: friend struct ConnectionHandle; + virtual ref<FSAccessor> getFSAccessor() override; + + virtual void narFromPath(const StorePath & path, Sink & sink) override; + + ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path); + private: std::atomic_bool failed{false}; }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore +struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +{ + UDSRemoteStoreConfig(const Store::Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + { + } + + UDSRemoteStoreConfig() + : UDSRemoteStoreConfig(Store::Params({})) + { + } + + const std::string name() override { return "Local Daemon Store"; } +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig { public: UDSRemoteStore(const Params & params); - UDSRemoteStore(std::string path, const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, const Params & params); std::string getUri() override; + static std::set<std::string> uriSchemes() + { return {"unix"}; } + bool sameMachine() override { return true; } + ref<FSAccessor> getFSAccessor() override + { return LocalFSStore::getFSAccessor(); } + + void narFromPath(const StorePath & path, Sink & sink) override + { LocalFSStore::narFromPath(path, sink); } + private: ref<RemoteStore::Connection> openConnection() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 427dd48ce..552c4aac7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,7 +7,6 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" -#include "istringstream_nocopy.hh" #include <aws/core/Aws.h> #include <aws/core/VersionConfig.h> @@ -173,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; - const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; + const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; + const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; const Setting<bool> multipartUpload{ - this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting<uint64_t> bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + const std::string name() override { return "S3 Binary Cache Store"; } +}; + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +{ std::string bucketName; Stats stats; @@ -194,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore S3Helper s3Helper; S3BinaryCacheStoreImpl( - const Params & params, const std::string & bucketName) - : S3BinaryCacheStore(params) + const std::string & scheme, + const std::string & bucketName, + const Params & params) + : StoreConfig(params) + , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { @@ -262,11 +270,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr<TransferManager> transferManager; std::once_flag transferManagerCreated; - void uploadFile(const std::string & path, const std::string & data, + void uploadFile(const std::string & path, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType, const std::string & contentEncoding) { - auto stream = std::make_shared<istringstream_nocopy>(data); + istream->seekg(0, istream->end); + auto size = istream->tellg(); + istream->seekg(0, istream->beg); auto maxThreads = std::thread::hardware_concurrency(); @@ -307,7 +318,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr<TransferHandle> transferHandle = transferManager->UploadFile( - stream, bucketName, path, mimeType, + istream, bucketName, path, mimeType, Aws::Map<Aws::String, Aws::String>(), nullptr /*, contentEncoding */); @@ -333,9 +344,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore if (contentEncoding != "") request.SetContentEncoding(contentEncoding); - auto stream = std::make_shared<istringstream_nocopy>(data); - - request.SetBody(stream); + request.SetBody(istream); auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request)); @@ -347,25 +356,32 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) .count(); - printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % - bucketName % path % data.size() % duration); + printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, size, duration); stats.putTimeMs += duration; - stats.putBytes += data.size(); + stats.putBytes += std::max(size, (decltype(size)) 0); stats.put++; } - void upsertFile(const std::string & path, const std::string & data, + void upsertFile(const std::string & path, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) override { + auto compress = [&](std::string compression) + { + auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain()); + return std::make_shared<std::stringstream>(std::move(*compressed)); + }; + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + uploadFile(path, compress(lsCompression), mimeType, lsCompression); else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + uploadFile(path, compress(logCompression), mimeType, logCompression); else - uploadFile(path, data, mimeType, ""); + uploadFile(path, istream, mimeType, ""); } void getFile(const std::string & path, Sink & sink) override @@ -410,7 +426,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); } marker = res.GetNextMarker(); @@ -419,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return paths; } + static std::set<std::string> uriSchemes() { return {"s3"}; } + }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, 5) != "s3://") return 0; - auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); - store->init(); - return store; -}); +static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore; } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 76c822c4e..31a1f0cac 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt) }); } +uint64_t SQLite::getLastInsertedRowId() +{ + return sqlite3_last_insert_rowid(db); +} + void SQLiteStmt::create(sqlite3 * db, const string & sql) { checkInterrupt(); @@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use() sqlite3_reset(stmt); } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throwSQLiteError(stmt.db, "binding argument"); } else bind(); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index dd81ab051..99f0d56ce 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -26,6 +26,8 @@ struct SQLite void isCache(); void exec(const std::string & stmt); + + uint64_t getLastInsertedRowId(); }; /* RAII wrapper to create and destroy SQLite prepared statements. */ @@ -54,7 +56,7 @@ struct SQLiteStmt ~Use(); /* Bind the next parameter. */ - Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596..08d0bd565 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -8,19 +8,25 @@ namespace nix { -static std::string uriScheme = "ssh-ng://"; +struct SSHStoreConfig : virtual RemoteStoreConfig +{ + using RemoteStoreConfig::RemoteStoreConfig; + + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "SSH Store"; } +}; -class SSHStore : public RemoteStore +class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig { public: - const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"}; - - SSHStore(const std::string & host, const Params & params) - : Store(params) + SSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , Store(params) , RemoteStore(params) , host(host) , master( @@ -32,18 +38,16 @@ public: { } + static std::set<std::string> uriSchemes() { return {"ssh-ng"}; } + std::string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } bool sameMachine() override { return false; } - void narFromPath(const StorePath & path, Sink & sink) override; - - ref<FSAccessor> getFSAccessor() override; - private: struct Connection : RemoteStore::Connection @@ -68,19 +72,6 @@ private: }; }; -void SSHStore::narFromPath(const StorePath & path, Sink & sink) -{ - auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); - conn->processStderr(); - copyNAR(conn->from, sink); -} - -ref<FSAccessor> SSHStore::getFSAccessor() -{ - return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); -} - ref<RemoteStore::Connection> SSHStore::openConnection() { auto conn = make_ref<Connection>(); @@ -89,16 +80,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection() + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - initConnection(*conn); return conn; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0a8bc9f6..1bbc74db8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,15 +1,14 @@ #include "crypto.hh" +#include "fs-accessor.hh" #include "globals.hh" #include "store-api.hh" #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "json.hh" -#include "derivations.hh" #include "url.hh" - -#include <future> - +#include "archive.hh" +#include "callback.hh" namespace nix { @@ -20,15 +19,15 @@ bool Store::isInStore(const Path & path) const } -Path Store::toStorePath(const Path & path) const +std::pair<StorePath, Path> Store::toStorePath(const Path & path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) - return path; + return {parseStorePath(path), ""}; else - return Path(path, 0, slash); + return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; } @@ -41,14 +40,14 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw NotInStore("path '%1%' is not in the Nix store", path); + throw BadStorePath("path '%1%' is not in the Nix store", path); return path; } StorePath Store::followLinksToStorePath(std::string_view path) const { - return parseStorePath(toStorePath(followLinksToStore(path))); + return toStorePath(followLinksToStore(path)).first; } @@ -139,21 +138,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p */ -StorePath Store::makeStorePath(const string & type, - const Hash & hash, std::string_view name) const +StorePath Store::makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); + string s = std::string { type } + ":" + std::string { hash } + + ":" + storeDir + ":" + std::string { name }; auto h = compressHash(hashString(htSHA256, s), 20); return StorePath(h, name); } -StorePath Store::makeOutputPath(const string & id, +StorePath Store::makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const +{ + return makeStorePath(type, hash.to_string(Base16, true), name); +} + + +StorePath Store::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { - return makeStorePath("output:" + id, hash, - std::string(name) + (id == "out" ? "" : "-" + id)); + return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); } @@ -192,6 +198,19 @@ StorePath Store::makeFixedOutputPath( } } +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references, bool hasSelfReference) const +{ + // New template + return std::visit(overloaded { + [&](TextHash th) { + return makeTextPath(name, th.hash, references); + }, + [&](FixedOutputHash fsh) { + return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + } + }, ca); +} StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const @@ -221,8 +240,111 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +StorePath Store::addToStore(const string & name, const Path & _srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} + + +/* +The aim of this function is to compute in one pass the correct ValidPathInfo for +the files that we are trying to add to the store. To accomplish that in one +pass, given the different kind of inputs that we can take (normal nar archives, +nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks +and aliases. Also, since the dataflow is obfuscated by this, we include here a +graphviz diagram: + +digraph graphname { + node [shape=box] + fileSource -> narSink + narSink [style=dashed] + narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"] + narSink -> narHashSink [style = dashed, label = "else"] + unsualHashTee -> narHashSink + unsualHashTee -> caHashSink + fileSource -> parseSink + parseSink [style=dashed] + parseSink-> fileSink [style = dashed, label = "Flat"] + parseSink -> blank [style = dashed, label = "Recursive"] + fileSink -> caHashSink +} +*/ +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional<Hash> expectedCAHash) +{ + HashSink narHashSink { htSHA256 }; + HashSink caHashSink { hashAlgo }; + + /* Note that fileSink and unusualHashTee must be mutually exclusive, since + they both write to caHashSink. Note that that requisite is currently true + because the former is only used in the flat case. */ + RetrieveRegularNARSink fileSink { caHashSink }; + TeeSink unusualHashTee { narHashSink, caHashSink }; + + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + ? static_cast<Sink &>(unusualHashTee) + : narHashSink; + + /* Functionally, this means that fileSource will yield the content of + srcPath. The fact that we use scratchpadSink as a temporary buffer here + is an implementation detail. */ + auto fileSource = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + + /* tapped provides the same data as fileSource, but we also write all the + information to narSink. */ + TeeSource tapped { *fileSource, narSink }; + + ParseSink blank; + auto & parseSink = method == FileIngestionMethod::Flat + ? fileSink + : blank; + + /* The information that flows from tapped (besides being replicated in + narSink), is now put in parseSink. */ + parseDump(parseSink, tapped); + + /* We extract the result of the computation from the sink by calling + finish. */ + auto [narHash, narSize] = narHashSink.finish(); + + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? narHash + : caHashSink.finish().first; + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + ValidPathInfo info { + makeFixedOutputPath(method, hash, name), + narHash, + }; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) - : Config(params) + : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { } @@ -242,6 +364,17 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { + auto resp = queryPartialDerivationOutputMap(path); + OutputPathMap result; + for (auto & [outName, optOutPath] : resp) { + if (!optOutPath) + throw Error("output '%s' has no store path mapped to it", outName); + result.insert_or_assign(outName, *optOutPath); + } + return result; +} + StorePathSet Store::queryDerivationOutputs(const StorePath & path) { auto outputMap = this->queryDerivationOutputMap(path); @@ -316,6 +449,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath) } +static bool goodStorePath(const StorePath & expected, const StorePath & actual) +{ + return + expected.hashPart() == actual.hashPart() + && (expected.name() == Store::MissingName || expected.name() == actual.name()); +} + + void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { @@ -343,7 +484,7 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || - res.second->path != storePath) + !goodStorePath(storePath, res.second->path)) throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref<const ValidPathInfo>(res.second)); @@ -355,7 +496,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -368,9 +509,11 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - if (!info || info->path != parseStorePath(storePath)) { + auto storePath = parseStorePath(storePathS); + + if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); + throw InvalidPath("path '%s' is not valid", storePathS); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -515,7 +658,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -591,18 +734,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); + // recompute store path on the chance dstStore does it differently + if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; + info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(info->path == info2->path); info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; } if (info->ultimate) { @@ -612,12 +750,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, } auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); + LambdaSink progressSink([&](const unsigned char * data, size_t len) { total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath(storePath, wrapperSink); + TeeSink tee { sink, progressSink }; + srcStore->narFromPath(storePath, tee); }, [&]() { throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); @@ -626,16 +764,20 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, } -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, +std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); - PathSet missing; + StorePathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); + if (!valid.count(path)) missing.insert(path); - if (missing.empty()) return; + std::map<StorePath, StorePath> pathsMap; + for (auto & path : storePaths) + pathsMap.insert_or_assign(path, path); + + if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -650,30 +792,49 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st ThreadPool pool; - processGraph<Path>(pool, - PathSet(missing.begin(), missing.end()), + processGraph<StorePath>(pool, + StorePathSet(missing.begin(), missing.end()), + + [&](const StorePath & storePath) { + auto info = srcStore->queryPathInfo(storePath); + auto storePathForDst = storePath; + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); + if (storePathForDst != storePath) + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + } + pathsMap.insert_or_assign(storePath, storePathForDst); - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + if (dstStore->isValidPath(storePath)) { nrDone++; showProgress(); - return PathSet(); + return StorePathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); - bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return srcStore->printStorePathSet(info->references); + return info->references; }, - [&](const Path & storePathS) { + [&](const StorePath & storePath) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); + auto info = srcStore->queryPathInfo(storePath); + + auto storePathForDst = storePath; + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); + if (storePathForDst != storePath) + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + } + pathsMap.insert_or_assign(storePath, storePathForDst); - if (!dstStore->isValidPath(storePath)) { + if (!dstStore->isValidPath(storePathForDst)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); try { @@ -682,7 +843,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; } @@ -691,6 +852,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrDone++; showProgress(); }); + + return pathsMap; } @@ -704,19 +867,22 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, } -std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) +std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven) { std::string path; getline(str, path); if (str.eof()) { return {}; } - ValidPathInfo info(store.parseStorePath(path)); - if (hashGiven) { + if (!hashGiven) { string s; getline(str, s); - info.narHash = Hash(s, htSHA256); + auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); + uint64_t narSize; + if (!string2Int(s, narSize)) throw Error("number expected"); + hashGiven = { narHash, narSize }; } + ValidPathInfo info(store.parseStorePath(path), hashGiven->first); + info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); @@ -751,8 +917,8 @@ string showPaths(const PathSet & paths) std::string ValidPathInfo::fingerprint(const Store & store) const { - if (narSize == 0 || !narHash) - throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known", + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" @@ -767,10 +933,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -// FIXME Put this somewhere? -template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; -template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; - bool ValidPathInfo::isContentAddressed(const Store & store) const { if (! ca) return false; @@ -826,6 +988,25 @@ Strings ValidPathInfo::shortRefs() const } +Derivation Store::derivationFromPath(const StorePath & drvPath) +{ + ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ + auto accessor = getFSAccessor(); + try { + return parseDerivation(*this, + accessor->readFile(printStorePath(drvPath)), + Derivation::nameFromPath(drvPath)); + } catch (FormatError & e) { + throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); + } +} + } @@ -835,9 +1016,6 @@ Strings ValidPathInfo::shortRefs() const namespace nix { - -RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) { @@ -851,62 +1029,71 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_ return {uri, params}; } -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) -{ - auto [uri, uriParams] = splitUriAndParams(uri_); - auto params = extraParams; - params.insert(uriParams.begin(), uriParams.end()); - - for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri, params); - if (store) { - store->warnUnknownSettings(); - return ref<Store>(store); - } - } - - throw Error("don't know how to open Nix store '%s'", uri); +static bool isNonUriPath(const std::string & spec) { + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; } - -StoreType getStoreType(const std::string & uri, const std::string & stateDir) +std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params) { - if (uri == "daemon") { - return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { - return tLocal; - } else if (uri == "" || uri == "auto") { + if (uri == "" || uri == "auto") { + auto stateDir = get(params, "state").value_or(settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return tLocal; + return std::make_shared<LocalStore>(params); else if (pathExists(settings.nixDaemonSocketFile)) - return tDaemon; + return std::make_shared<UDSRemoteStore>(params); else - return tLocal; + return std::make_shared<LocalStore>(params); + } else if (uri == "daemon") { + return std::make_shared<UDSRemoteStore>(params); + } else if (uri == "local") { + return std::make_shared<LocalStore>(params); + } else if (isNonUriPath(uri)) { + Store::Params params2 = params; + params2["root"] = absPath(uri); + return std::make_shared<LocalStore>(params2); } else { - return tOther; + return nullptr; } } - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) { - switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { - case tDaemon: - return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); - case tLocal: { - Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; - return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); + auto params = extraParams; + try { + auto parsedUri = parseURL(uri_); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + for (auto implem : *Implementations::registered) { + if (implem.uriSchemes.count(parsedUri.scheme)) { + auto store = implem.create(parsedUri.scheme, baseURI, params); + if (store) { + store->init(); + store->warnUnknownSettings(); + return ref<Store>(store); + } + } } - default: - return nullptr; } -}); + catch (BadURL &) { + auto [uri, uriParams] = splitUriAndParams(uri_); + params.insert(uriParams.begin(), uriParams.end()); + if (auto store = openFromNonUri(uri, params)) { + store->warnUnknownSettings(); + return ref<Store>(store); + } + } + + throw Error("don't know how to open Nix store '%s'", uri_); +} std::list<ref<Store>> getDefaultSubstituters() { @@ -940,5 +1127,6 @@ std::list<ref<Store>> getDefaultSubstituters() return stores; } +std::vector<StoreFactory> * Implementations::registered = 0; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b122e05d6..450c0f554 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -4,12 +4,12 @@ #include "hash.hh" #include "content-address.hh" #include "serialise.hh" -#include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" #include "config.hh" #include "derivations.hh" +#include "path-info.hh" #include <atomic> #include <limits> @@ -24,6 +24,31 @@ namespace nix { +/** + * About the class hierarchy of the store implementations: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting<T>` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation<Foo, FooConfig> regStore; + * ``` + */ MakeError(SubstError, Error); MakeError(BuildError, Error); // denotes a permanent build failure @@ -31,8 +56,9 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(NotInStore, Error); +MakeError(BadStorePath, Error); +MakeError(InvalidStoreURI, Error); class FSAccessor; class NarInfoDiskCache; @@ -85,7 +111,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; + uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; }; @@ -97,98 +123,10 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + uint64_t bytesFreed = 0; }; -struct SubstitutablePathInfo -{ - std::optional<StorePath> deriver; - StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - -struct ValidPathInfo -{ - StorePath path; - std::optional<StorePath> deriver; - Hash narHash; - StorePathSet references; - time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only - - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ - bool ultimate = false; - - StringSet sigs; // note: not necessarily verified - - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ - std::optional<ContentAddress> ca; - - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint(const Store & store) const; - - void sign(const Store & store, const SecretKey & secretKey); - - /* Return true iff the path is verifiably content-addressed. */ - bool isContentAddressed(const Store & store) const; - - static const size_t maxSigs = std::numeric_limits<size_t>::max(); - - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ - size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - - /* Verify a single signature. */ - bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; - - Strings shortRefs() const; - - ValidPathInfo(const ValidPathInfo & other) = default; - - ValidPathInfo(StorePath && path) : path(std::move(path)) { }; - ValidPathInfo(const StorePath & path) : path(path) { }; - - virtual ~ValidPathInfo() { } -}; - -typedef list<ValidPathInfo> ValidPathInfos; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; @@ -232,12 +170,33 @@ struct BuildResult } }; - -class Store : public std::enable_shared_from_this<Store>, public Config +struct StoreConfig : public Config { -public: - - typedef std::map<std::string, std::string> Params; + using Config::Config; + + /** + * When constructing a store implementation, we pass in a map `params` of + * parameters that's supposed to initialize the associated config. + * To do that, we must use the `StoreConfig(StringMap & params)` + * constructor, so we'd like to `delete` its default constructor to enforce + * it. + * + * However, actually deleting it means that all the subclasses of + * `StoreConfig` will have their default constructor deleted (because it's + * supposed to call the deleted default constructor of `StoreConfig`). But + * because we're always using virtual inheritance, the constructors of + * child classes will never implicitely call this one, so deleting it will + * be more painful than anything else. + * + * So we `assert(false)` here to ensure at runtime that the right + * constructor is always called without having to redefine a custom + * constructor for each `*Config` class. + */ + StoreConfig() { assert(false); } + + virtual ~StoreConfig() { } + + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, "store", "path to the Nix store"}; @@ -251,6 +210,18 @@ public: Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting<StringSet> systemFeatures{this, settings.systemFeatures, + "system-features", + "Optional features that the system this store builds on implements (like \"kvm\")."}; + +}; + +class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig +{ +public: + + typedef std::map<std::string, std::string> Params; + protected: struct PathInfoCacheValue { @@ -284,6 +255,11 @@ protected: Store(const Params & params); public: + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init() {}; virtual ~Store() { } @@ -317,9 +293,9 @@ public: the Nix store. */ bool isStorePath(std::string_view path) const; - /* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ - Path toStorePath(const Path & path) const; + /* Split a path like /nix/store/<hash>-<name>/<bla> into + /nix/store/<hash>-<name> and /<bla>. */ + std::pair<StorePath, Path> toStorePath(const Path & path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; @@ -331,10 +307,12 @@ public: StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; /* Constructs a unique store path name. */ - StorePath makeStorePath(const string & type, + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const; - StorePath makeOutputPath(const string & id, + StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; StorePath makeFixedOutputPath(FileIngestionMethod method, @@ -343,7 +321,11 @@ public: bool hasSelfReference = false) const; StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const; + const StorePathSet & references = {}) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -384,13 +366,16 @@ public: SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be omitted - (i.e. you'll get /nix/store/<hash> rather than + backends, the name part of store paths may be replaced by 'x' + (i.e. you'll get /nix/store/<hash>-x rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the - full store path. */ + full store path. FIXME: should return a set of + std::variant<StorePath, HashPart> to get rid of this hack. */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + constexpr static const char * MissingName = "x"; + /* Query information about a valid path. It is permitted to omit the name part of the store path. */ ref<const ValidPathInfo> queryPathInfo(const StorePath & path); @@ -420,9 +405,15 @@ public: /* Query the outputs of the derivation denoted by `path'. */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation */ - virtual OutputPathMap queryDerivationOutputMap(const StorePath & path) - { unsupported("queryDerivationOutputMap"); } + /* Query the mapping outputName => outputPath for the given derivation. All + outputs are mentioned so ones mising the mapping are mapped to + `std::nullopt`. */ + virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) + { unsupported("queryPartialDerivationOutputMap"); } + + /* Query the mapping outputName=>outputPath for the given derivation. + Assume every output has a mapping and throw an exception otherwise. */ + OutputPathMap queryDerivationOutputMap(const StorePath & path); /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ @@ -432,15 +423,15 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathSet & paths, + sizes) of a map of paths to their optional ca values. If a path + does not have substitute info, it's omitted from the resulting + ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) = 0; + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. @@ -448,14 +439,24 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional<Hash> expectedCAHash = {}); + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). + `dump` may be drained */ // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) - { - throw Error("addToStoreFromDump() is not supported by this store"); - } + { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ @@ -478,8 +479,38 @@ public: BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). Note that ‘drvPath’ is only used for - informational purposes. */ + on-disk .drv file). + + ‘drvPath’ is used to deduplicate worker goals so it is imperative that + is correct. That said, it doesn't literally need to be store path that + would be calculated from writing this derivation to the store: it is OK + if it instead is that of a Derivation which would resolve to this (by + taking the outputs of it's input derivations and adding them as input + sources) such that the build time referenceable-paths are the same. + + In the input-addressed case, we usually *do* use an "original" + unresolved derivations's path, as that is what will be used in the + `buildPaths` case. Also, the input-addressed output paths are verified + only by that contents of that specific unresolved derivation, so it is + nice to keep that information around so if the original derivation is + ever obtained later, it can be verified whether the trusted user in fact + used the proper output path. + + In the content-addressed case, we want to always use the + resolved drv path calculated from the provided derivation. This serves + two purposes: + + - It keeps the operation trustless, by ruling out a maliciously + invalid drv path corresponding to a non-resolution-equivalent + derivation. + + - For the floating case in particular, it ensures that the derivation + to output mapping respects the resolution equivalence relation, so + one cannot choose different resolution-equivalent derivations to + subvert dependency coherence (i.e. the property that one doesn't end + up with multiple different versions of dependencies without + explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; @@ -516,7 +547,7 @@ public: - The collector isn't running, or it's just started but hasn't acquired the GC lock yet. In that case we get and release the lock right away, then exit. The collector scans the - permanent root and sees our's. + permanent root and sees ours. In either case the permanent root is seen by the collector. */ virtual void syncWithGC() { }; @@ -600,7 +631,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ @@ -616,8 +647,7 @@ public: the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, - CheckSigsFlag checkSigs = CheckSigs); + StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats { @@ -685,22 +715,25 @@ protected: }; - -class LocalFSStore : public virtual Store +struct LocalFSStoreConfig : virtual StoreConfig { -public: - - // FIXME: the (Store*) cast works around a bug in gcc that causes + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(Store*) this, true, "", + const PathSetting rootDir{(StoreConfig*) this, true, "", "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(Store*) this, false, + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", "directory where Nix will store state"}; - const PathSetting logDir{(Store*) this, false, + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: const static string drvsLogDir; @@ -710,8 +743,7 @@ public: ref<FSAccessor> getFSAccessor() override; /* Register a permanent GC root. */ - Path addPermRoot(const StorePath & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return storeDir; } @@ -731,11 +763,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, /* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order - (i.e. if A is a reference of B, then A is copied before B), but - the set of store paths is not automatically closed; use - copyClosure() for that. */ -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, + in parallel. They are copied in a topologically sorted order (i.e. + if A is a reference of B, then A is copied before B), but the set + of store paths is not automatically closed; use copyClosure() for + that. Returns a map of what each path was copied to the dstStore + as. */ +std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -788,37 +822,48 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -enum StoreType { - tDaemon, - tLocal, - tOther -}; - - -StoreType getStoreType(const std::string & uri = settings.storeUri.get(), - const std::string & stateDir = settings.nixStateDir); - /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); +struct StoreFactory +{ + std::set<std::string> uriSchemes; + std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + std::function<std::shared_ptr<StoreConfig> ()> getConfig; +}; -/* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>( - const std::string & uri, const Store::Params & params)> OpenStore; - -struct RegisterStoreImplementation +struct Implementations { - typedef std::vector<OpenStore> Implementations; - static Implementations * implementations; + static std::vector<StoreFactory> * registered; - RegisterStoreImplementation(OpenStore fun) + template<typename T, typename TConfig> + static void add() { - if (!implementations) implementations = new Implementations; - implementations->push_back(fun); + if (!registered) registered = new std::vector<StoreFactory>(); + StoreFactory factory{ + .uriSchemes = T::uriSchemes(), + .create = + ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> + { return std::make_shared<T>(scheme, uri, params); }), + .getConfig = + ([]() + -> std::shared_ptr<StoreConfig> + { return std::make_shared<TConfig>(StringMap({})); }) + }; + registered->push_back(factory); } }; +template<typename T, typename TConfig> +struct RegisterStoreImplementation +{ + RegisterStoreImplementation() + { + Implementations::add<T, TConfig>(); + } +}; /* Display a set of paths in human-readable form (i.e., between quotes @@ -829,9 +874,11 @@ string showPaths(const PathSet & paths); std::optional<ValidPathInfo> decodeValidPathInfo( const Store & store, std::istream & str, - bool hashGiven = false); + std::optional<HashResult> hashGiven = std::nullopt); /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); + } diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc index fb2a45f48..704bb4ad0 100644 --- a/src/libstore/user-lock.cc +++ b/src/libstore/user-lock.cc @@ -3,6 +3,9 @@ #include "pathlocks.hh" #include "cgroup.hh" +#include <pwd.h> +#include <grp.h> + namespace nix { struct SimpleUserLock : UserLock diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 8b538f6da..b3705578e 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x116 +#define PROTOCOL_VERSION 0x11a #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,7 +18,7 @@ typedef enum { wopQueryReferences = 5, // obsolete wopQueryReferrers = 6, wopAddToStore = 7, - wopAddTextToStore = 8, + wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore wopBuildPaths = 9, wopEnsurePath = 10, wopAddTempRoot = 11, @@ -66,10 +66,89 @@ typedef enum { class Store; struct Source; -template<class T> T readStorePaths(const Store & store, Source & from); +/* To guide overloading */ +template<typename T> +struct Phantom {}; -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); +namespace worker_proto { +/* FIXME maybe move more stuff inside here */ + +#define MAKE_WORKER_PROTO(TEMPLATE, T) \ + TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ + TEMPLATE void write(const Store & store, Sink & out, const T & str) + +MAKE_WORKER_PROTO(, std::string); +MAKE_WORKER_PROTO(, StorePath); +MAKE_WORKER_PROTO(, ContentAddress); + +MAKE_WORKER_PROTO(template<typename T>, std::set<T>); + +#define X_ template<typename K, typename V> +#define Y_ std::map<K, V> +MAKE_WORKER_PROTO(X_, Y_); +#undef X_ +#undef Y_ + +/* These use the empty string for the null case, relying on the fact + that the underlying types never serialize to the empty string. + + We do this instead of a generic std::optional<T> instance because + ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + the same reason, we don't have a std::variant<T..> instances (ordinal + tags 0...n). + + We could the generic instances and then these as specializations for + compatability, but that's proven a bit finnicky, and also makes the + worker protocol harder to implement in other languages where such + specializations may not be allowed. + */ +MAKE_WORKER_PROTO(, std::optional<StorePath>); +MAKE_WORKER_PROTO(, std::optional<ContentAddress>); + +template<typename T> +std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _) +{ + std::set<T> resSet; + auto size = readNum<size_t>(from); + while (size--) { + resSet.insert(read(store, from, Phantom<T> {})); + } + return resSet; +} + +template<typename T> +void write(const Store & store, Sink & out, const std::set<T> & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + +template<typename K, typename V> +std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _) +{ + std::map<K, V> resMap; + auto size = readNum<size_t>(from); + while (size--) { + auto k = read(store, from, Phantom<K> {}); + auto v = read(store, from, Phantom<V> {}); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template<typename K, typename V> +void write(const Store & store, Sink & out, const std::map<K, V> & resMap) +{ + out << resMap.size(); + for (auto & i : resMap) { + write(store, out, i.first); + write(store, out, i.second); + } +} + +} } |