diff options
Diffstat (limited to 'src/libstore')
47 files changed, 1091 insertions, 626 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..30150eeba 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -15,6 +15,7 @@ #include <chrono> #include <future> #include <regex> +#include <fstream> #include <nlohmann/json.hpp> @@ -57,6 +58,13 @@ void BinaryCacheStore::init() } } +void BinaryCacheStore::upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType) +{ + upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType); +} + void BinaryCacheStore::getFile(const std::string & path, Callback<std::shared_ptr<std::string>> callback) noexcept { @@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); } +AutoCloseFD openFile(const Path & path) +{ + auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return fd; +} + +struct FileSource : FdSource +{ + AutoCloseFD fd2; + + FileSource(const Path & path) + : fd2(openFile(path)) + { + fd = fd2.get(); + } +}; + void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { - // FIXME: See if we can use the original source to reduce memory usage. - auto nar = make_ref<std::string>(narSource.drain()); + assert(info.narHash && info.narSize); + + if (!repair && isValidPath(info.path)) { + // FIXME: copyNAR -> null sink + narSource.drain(); + return; + } + + auto [fdTemp, fnTemp] = createTempFile(); + + auto now1 = std::chrono::steady_clock::now(); + + /* Read the NAR simultaneously into a CompressionSink+FileSink (to + write the compressed NAR to disk), into a HashSink (to get the + NAR hash), and into a NarAccessor (to get the NAR listing). */ + HashSink fileHashSink(htSHA256); + std::shared_ptr<FSAccessor> narAccessor; + { + FdSink fileSink(fdTemp.get()); + TeeSink teeSink(fileSink, fileHashSink); + auto compressionSink = makeCompressionSink(compression, teeSink); + TeeSource teeSource(narSource, *compressionSink); + narAccessor = makeNarAccessor(teeSource); + compressionSink->finish(); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto narInfo = make_ref<NarInfo>(info); + narInfo->narSize = info.narSize; + narInfo->narHash = info.narHash; + narInfo->compression = compression; + auto [fileHash, fileSize] = fileHashSink.finish(); + narInfo->fileHash = fileHash; + narInfo->fileSize = fileSize; + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); - if (!repair && isValidPath(info.path)) return; + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), info.narSize, + ((1.0 - (double) fileSize / info.narSize) * 100.0), + duration); /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource printStorePath(info.path), printStorePath(ref)); } - assert(nar->compare(0, narMagic.size(), narMagic) == 0); - - auto narInfo = make_ref<NarInfo>(info); - - narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); - - if (info.narHash && info.narHash != narInfo->narHash) - throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); - - auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); - - auto narAccessor = makeNarAccessor(nar); - - if (accessor_) - accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource { auto res = jsonRoot.placeholder("root"); - listNar(res, narAccessor, "", true); + listNar(res, ref<FSAccessor>(narAccessor), "", true); } } upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } - /* Compress the NAR. */ - narInfo->compression = compression; - auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar, parallelCompression); - auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); - narInfo->fileSize = narCompressed->size(); - - auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", - printStorePath(narInfo->path), narInfo->narSize, - ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), - duration); - - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); - /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/<build-id>' that specify the NAR file and member containing the debug info. */ @@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + upsertFile(narInfo->url, + std::make_shared<std::fstream>(fnTemp, std::ios_base::in), + "application/x-nix-nar"); } else stats.narWriteAverted++; - stats.narWriteBytes += nar->size(); - stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteBytes += info.narSize; + stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -338,7 +372,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional<Hash> h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -348,10 +382,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); + addToStore(info, source, repair, CheckSigs); return std::move(info.path); } @@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s StringSink sink; dumpString(s, sink); auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs, nullptr); + addToStore(info, source, repair, CheckSigs); } return std::move(info.path); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 52ef8aa7a..9bcdf5901 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -36,9 +36,13 @@ public: virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) = 0; + void upsertFile(const std::string & path, + std::string && data, + const std::string & mimeType); + /* Note: subclasses must implement at least one of the two following getFile() methods. */ @@ -75,8 +79,7 @@ public: { unsupported("queryPathFromHashPart"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9ac5fd923..05235510d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store))); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation() retrySubstitution = false; for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + worker.store.addTempRoot(i.second.path(worker.store, drv->name)); /* Check what outputs paths are not already valid. */ auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); @@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure() StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); + worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure); } /* Filter out our own outputs (which we have already checked). */ for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); + outputClosure.erase(i.second.path(worker.store, drv->name)); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path, i); + outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i); } /* Check each path (slow!). */ @@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised() for (auto & j : i.second) { auto k = inDrv.outputs.find(j); if (k != inDrv.outputs.end()) - worker.store.computeFSClosure(k->second.path, inputPaths); + worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths); else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", @@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild() goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) + for (auto & outPath : drv->outputPaths(worker.store)) lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { @@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = drv->outputPaths(); + missingPaths = drv->outputPaths(worker.store); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path)) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.Store::toRealPath(i.second.path)); + if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name))); + deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name))); } /* Don't do a remote build if the derivation has the attribute @@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; #endif @@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone() fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); + auto outputPaths = drv->outputPaths(worker.store); std::map<std::string, std::string> hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); @@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths); } } @@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder() /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name)); /* Construct the environment passed to the builder. */ initEnv(); @@ -2041,7 +2041,10 @@ void DerivationGoal::startBuilder() if (!std::regex_match(fileName, regex)) throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - auto storePath = worker.store.parseStorePath(*i++); + auto storePathS = *i++; + if (!worker.store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + auto storePath = worker.store.toStorePath(storePathS).first; /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, @@ -2080,7 +2083,7 @@ void DerivationGoal::startBuilder() for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { throw Error("while processing 'sandbox-paths': %s", e.what()); @@ -2197,7 +2200,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path)); + dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name))); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2610,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2753,8 +2756,8 @@ struct RestrictedStore : public LocalFSStore void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - StorePathSet queryDerivationOutputs(const StorePath & path) override - { throw Error("queryDerivationOutputs"); } + OutputPathMap queryDerivationOutputMap(const StorePath & path) override + { throw Error("queryDerivationOutputMap"); } std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } @@ -2765,14 +2768,13 @@ struct RestrictedStore : public LocalFSStore { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) override + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override { - next->addToStore(info, narSource, repair, checkSigs, accessor); + next->addToStore(info, narSource, repair, checkSigs); goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); @@ -2815,7 +2817,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path); + newPaths.insert(output.second.path(*this, drv.name)); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -2849,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override + uint64_t & downloadSize, uint64_t & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are @@ -3577,7 +3579,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path); + result.insert(drv.outputs.find(i)->second.path(store, drv.name)); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; @@ -3615,7 +3617,7 @@ void DerivationGoal::registerOutputs() if (hook) { bool allValid = true; for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; + if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false; if (allValid) return; } @@ -3636,23 +3638,23 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name)); for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); - if (!missingPaths.count(i.second.path)) continue; + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); + if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue; Path actualPath = path; if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path); + auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name)); if (r != redirectedOutputs.end()) { auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path) + && redirectedBadOutputs.count(i.second.path(worker.store, drv->name)) && pathExists(redirected)) replaceValidPath(path, redirected); if (buildMode == bmCheck) @@ -3721,7 +3723,9 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - if (i.second.hash->method == FileIngestionMethod::Flat) { + FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash; + + if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3732,13 +3736,13 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + Hash h2 = outputHash.method == FileIngestionMethod::Recursive + ? hashPath(outputHash.hash.type, actualPath).first + : hashFile(outputHash.hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); - if (i.second.hash->hash != h2) { + if (outputHash.hash != h2) { /* Throw an error after registering the path as valid. */ @@ -3746,7 +3750,7 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), - i.second.hash->hash.to_string(SRI, true), + outputHash.hash.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3768,7 +3772,7 @@ void DerivationGoal::registerOutputs() assert(worker.store.parseStorePath(path) == dest); ca = FixedOutputHash { - .method = i.second.hash->method, + .method = outputHash.method, .hash = h2, }; } @@ -3783,8 +3787,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; @@ -3892,7 +3898,7 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3910,7 +3916,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix; deletePath(prev); } } @@ -4211,9 +4217,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + worker.store.isValidPath(i.second.path(worker.store, drv->name)) && + (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name))); + if (good == returnValid) result.insert(i.second.path(worker.store, drv->name)); } return result; } @@ -5006,7 +5012,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } @@ -5032,7 +5038,7 @@ void Worker::markContentsGood(const StorePath & path) static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths) { StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index e630cf6f1..03bb77488 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,16 +63,19 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto & output = drv.outputs.begin()->second; /* Try the hashed mirrors first. */ - if (output.hash && output.hash->method == FileIngestionMethod::Flat) - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto & h = output.hash->hash; - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); + if (auto hash = std::get_if<DerivationOutputFixed>(&output.output)) { + if (hash->hash.method == FileIngestionMethod::Flat) { + for (auto hashedMirror : settings.hashedMirrors.get()) { + try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + fetch(hashedMirror + printHashType(hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false)); + return; + } catch (Error & e) { + debug(e.what()); + } } + } + } /* Otherwise try the specified URL. */ fetch(mainUrl); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..f83b98a98 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -3,7 +3,7 @@ namespace nix { std::string FixedOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } std::string makeFileIngestionPrefix(const FileIngestionMethod m) { @@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { if (prefix == "text") { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); - if (*hash.type != htSHA256) { + if (hash.type != htSHA256) { throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; @@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) { return ca ? renderContentAddress(*ca) : ""; } +Hash getContentAddressHash(const ContentAddress & ca) +{ + return std::visit(overloaded { + [](TextHash th) { + return th.hash; + }, + [](FixedOutputHash fsh) { + return fsh.hash; + } + }, ca); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b..22a039242 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa); std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); +Hash getContentAddressHash(const ContentAddress & ca); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 842aef20c..9ac7e7cca 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -78,15 +78,15 @@ struct TunnelLogger : public Logger if (ei.level > verbosity) return; std::stringstream oss; - oss << ei; + showErrorInfo(oss, ei, false); StringSink buf; - buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n"); + buf << STDERR_NEXT << oss.str(); enqueueMsg(*buf.s); } /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ + want to send out stderr to the client. */ void startWork() { auto state(state_.lock()); @@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource } }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - struct ClientSettings { bool keepFailed; @@ -314,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -347,6 +322,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + case wopQueryDerivationOutputMap: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + OutputPathMap outputs = store->queryDerivationOutputMap(path); + logger->stopWork(); + writeOutputPathMap(*store, to, outputs); + break; + } + case wopQueryDeriver: { auto path = store->parseStorePath(readString(from)); logger->startWork(); @@ -366,42 +350,44 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - std::string s, baseName; + HashType hashAlgo; + std::string baseName; FileIngestionMethod method; { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > (uint8_t) FileIngestionMethod::Recursive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { - s = "sha256"; + hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } + hashAlgo = parseHashType(hashAlgoRaw); } - HashType hashAlgo = parseHashType(s); - TeeSource savedNAR(from); - RetrieveRegularNARSink savedRegular; + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); + parseDump(sink, savedNARSource); } else parseDump(savedRegular, from); logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, - baseName, - method, - hashAlgo); + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -433,7 +419,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopImportPaths: { logger->startWork(); TunnelSource source(from, to); - auto paths = store->importPaths(source, nullptr, + auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; @@ -465,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv); + readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); if (!trusted) @@ -646,7 +632,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { @@ -717,24 +703,84 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (!trusted) info.ultimate = false; - std::string saved; - std::unique_ptr<Source> source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique<TunnelSource>(from, to); - else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); - source = std::make_unique<StringSource>(saved); + if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + + struct FramedSource : Source + { + Source & from; + bool eof = false; + std::vector<unsigned char> pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector<unsigned char> data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector<unsigned char>(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } + }; + + logger->startWork(); + + { + FramedSource source(from); + store->addToStore(info, source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } + + logger->stopWork(); } - logger->startWork(); + else { + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique<TunnelSource>(from, to); + else { + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique<StringSource>(std::move(*saved.s)); + } - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + + logger->stopWork(); + } - logger->stopWork(); break; } @@ -744,7 +790,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, targets.push_back(store->parsePathWithOutputs(s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); writeStorePaths(*store, to, willBuild); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42551ef6b..7d0a5abeb 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,16 +4,23 @@ #include "util.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" -#include "istringstream_nocopy.hh" namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const +// FIXME Put this somewhere? +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + +StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error("derivation has no output '%s'", id); - return i->second.path; + return std::visit(overloaded { + [](DerivationOutputInputAddressed doi) { + return doi.path; + }, + [&](DerivationOutputFixed dof) { + return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + } + }, output); } @@ -101,14 +108,13 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) { expect(str, ","); auto path = store.parseStorePath(parsePath(str)); expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - std::optional<FixedOutputHash> fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -116,23 +122,30 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream hashAlgo = string(hashAlgo, 2); } const HashType hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }; - } - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(path), + } + }; } -static Derivation parseDerivation(const Store & store, const string & s) +static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) { Derivation drv; - istringstream_nocopy str(s); + drv.name = name; + + std::istringstream str(std::move(s)); expect(str, "Derive(["); /* Parse the list of outputs. */ @@ -175,10 +188,10 @@ static Derivation parseDerivation(const Store & store, const string & s) } -Derivation readDerivation(const Store & store, const Path & drvPath) +Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name) { try { - return parseDerivation(store, readFile(drvPath)); + return parseDerivation(store, readFile(drvPath), name); } catch (FormatError & e) { throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } @@ -196,7 +209,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) { auto accessor = getFSAccessor(); try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } @@ -264,10 +277,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); - s += ','; printUnquotedString(s, - i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name))); + if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) { + s += ','; printUnquotedString(s, hash->hash.printMethodAlgo()); + s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false)); + } else { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } s += ')'; } @@ -323,7 +340,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash; + std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output); } @@ -355,10 +372,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); + auto hash = std::get<DerivationOutputFixed>(i->second.output); return hashString(htSHA256, "fixed:out:" - + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base16, false) + ":" - + store.printStorePath(i->second.path)); + + hash.hash.printMethodAlgo() + ":" + + hash.hash.hash.to_string(Base16, false) + ":" + + store.printStorePath(i->second.path(store, drv.name))); } /* For other derivations, replace the inputs paths with recursive @@ -392,11 +410,11 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -StorePathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths(const Store & store) const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path(store, name)); return paths; } @@ -406,7 +424,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashAlgo = readString(in); auto hash = readString(in); - std::optional<FixedOutputHash> fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -414,16 +431,20 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) hashAlgo = string(hashAlgo, 2); } auto hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(path), + } }; - } - - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; } StringSet BasicDerivation::outputNames() const @@ -435,8 +456,19 @@ StringSet BasicDerivation::outputNames() const } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + auto nameWithSuffix = drvPath.name(); + constexpr std::string_view extension = ".drv"; + assert(hasSuffix(nameWithSuffix, extension)); + nameWithSuffix.remove_suffix(extension.size()); + return nameWithSuffix; +} + + +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { + drv.name = name; + drv.outputs.clear(); auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { @@ -465,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << drv.outputs.size(); for (auto & i : drv.outputs) { out << i.first - << store.printStorePath(i.second.path); - if (i.second.hash) { - out << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); + << store.printStorePath(i.second.path(store, drv.name)); + if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) { + out << hash->hash.printMethodAlgo() + << hash->hash.hash.to_string(Base16, false); } else { out << "" << ""; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 68c53c1ff..133ffe50e 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,10 +13,20 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutput +struct DerivationOutputInputAddressed { StorePath path; - std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutputFixed +{ + FixedOutputHash hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutput +{ + std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output; + StorePath path(const Store & store, std::string_view drvName) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -35,24 +45,23 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; + std::string name; BasicDerivation() { } virtual ~BasicDerivation() { }; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - const StorePath & findOutput(const std::string & id) const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ bool isFixedOutput() const; /* Return the output paths of a derivation. */ - StorePathSet outputPaths() const; + StorePathSet outputPaths(const Store & store) const; /* Return the output names of a derivation. */ StringSet outputNames() const; + + static std::string_view nameFromPath(const StorePath & storePath); }; struct Derivation : BasicDerivation @@ -76,7 +85,7 @@ StorePath writeDerivation(ref<Store> store, const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Store & store, const Path & drvPath); +Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name); // FIXME: remove bool isDerivation(const string & fileName); @@ -93,7 +102,7 @@ bool wantOutput(const string & output, const std::set<string> & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..a0fc22264 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -7,24 +7,6 @@ namespace nix { -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - void Store::exportPaths(const StorePathSet & paths, Sink & sink) { auto sorted = topoSortPaths(paths); @@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashAndWriteSink hashAndWriteSink(sink); + HashSink hashSink(htSHA256); + TeeSink teeSink(sink, hashSink); - narFromPath(path, hashAndWriteSink); + narFromPath(path, teeSink); /* Refuse to export paths that have changed. This prevents filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + Hash hash = hashSink.currentHash().first; + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); - hashAndWriteSink + teeSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); - hashAndWriteSink + writeStorePaths(*this, teeSink, info->references); + teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; } -StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) +StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) { StorePaths res; while (true) { @@ -77,8 +60,10 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -94,16 +79,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); - info.narSize = tee.source.data->size(); + info.narHash = hashString(htSHA256, *saved.s); + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.source.data }; - addToStore(info, source, NoRepair, checkSigs, accessor); + auto source = StringSource { *saved.s }; + addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 531b85af8..4149f8155 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -22,6 +22,7 @@ #include <queue> #include <random> #include <thread> +#include <regex> using namespace std::string_literals; @@ -56,7 +57,7 @@ struct curlFileTransfer : public FileTransfer Callback<FileTransferResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; + std::string statusMsg; unsigned int attempt = 0; @@ -123,7 +124,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -144,6 +145,7 @@ struct curlFileTransfer : public FileTransfer LambdaSink finalSink; std::shared_ptr<CompressionSink> decompressionSink; + std::optional<StringSink> errorSink; std::exception_ptr writeException; @@ -153,9 +155,19 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + errorSink = StringSink { }; + } + } + if (errorSink) + (*errorSink)((unsigned char *) contents, realSize); (*decompressionSink)((unsigned char *) contents, realSize); return realSize; @@ -175,12 +187,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); + std::smatch match; + if (std::regex_match(line, match, statusLine)) { result.etag = ""; - auto ss = tokenizeString<vector<string>>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); result.bodySize = 0; + statusMsg = trim(match[1]); acceptRanges = false; encoding = ""; } else { @@ -194,7 +207,9 @@ struct curlFileTransfer : public FileTransfer the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - if (result.etag == request.expectedETag && status == "200") { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + if (result.etag == request.expectedETag && httpStatus == 200) { debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -408,16 +423,21 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr<std::string> response; + if (errorSink) + response = errorSink->s; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) + response, + fmt("unable to %s '%s': HTTP error %d ('%s')", + request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, + response, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -675,7 +695,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else @@ -820,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } } +template<typename... Args> +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how + // to print different messages for different verbosity levels. For now + // we add some heuristics for detecting when we want to show the response. + if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0..25ade0add 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,10 +103,12 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr<string> response; // intentionally optional + template<typename... Args> - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 57fb20845..e74382ed2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { - auto storePath = maybeParseStorePath(toStorePath(target)); - if (storePath && isValidPath(*storePath)) - roots[std::move(*storePath)].emplace(path); - else - printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + try { + auto storePath = toStorePath(target).first; + if (isValidPath(storePath)) + roots[std::move(storePath)].emplace(path); + else + printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + } catch (BadStorePath &) { } }; try { @@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) for (auto & [target, links] : unchecked) { if (!isInStore(target)) continue; - Path pathS = toStorePath(target); - if (!isStorePath(pathS)) continue; - auto path = parseStorePath(pathS); - if (!isValidPath(path)) continue; - debug("got additional root '%1%'", pathS); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); + try { + auto path = toStorePath(target).first; + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", printStorePath(path)); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); + } catch (BadStorePath &) { } } } @@ -498,7 +500,7 @@ struct LocalStore::GCState StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; - unsigned long long bytesInvalidated; + uint64_t bytesInvalidated; bool moveToTrash = true; bool shouldDelete; GCState(const GCOptions & options, GCResults & results) @@ -516,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, void LocalStore::deleteGarbage(GCState & state, const Path & path) { - unsigned long long bytesFreed; + uint64_t bytesFreed; deletePath(path, bytesFreed); state.results.bytesFreed += bytesFreed; } @@ -526,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) { checkInterrupt(); - unsigned long long size = 0; + uint64_t size = 0; auto storePath = maybeParseStorePath(path); if (storePath && isValidPath(*storePath)) { @@ -685,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", linksDir); - long long actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { @@ -715,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - long long overhead = st.st_blocks * 512ULL; + auto overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index fa8799314..7aec6d1c0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -35,7 +35,7 @@ Settings::Settings() , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) + , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 2fbcafff8..d47e0b6b5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -196,10 +196,6 @@ public: /* Whether to lock the Nix client and worker to the same CPU. */ bool lockCPU; - /* Whether to show a stack trace if Nix evaluation fails. */ - Setting<bool> showTrace{this, false, "show-trace", - "Whether to show a stack trace on evaluation errors."}; - Setting<SandboxMode> sandboxMode{this, #if __linux__ smEnabled @@ -369,6 +365,12 @@ public: Setting<bool> warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + + Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", + "Maximum size of NARs before spilling them to disk."}; + + Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 451a64785..c1ceb08cf 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -100,11 +100,11 @@ protected: } void upsertFile(const std::string & path, - const std::string & data, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) override { auto req = FileTransferRequest(cacheUri + "/" + path); - req.data = std::make_shared<string>(data); // FIXME: inefficient + req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain()); req.mimeType = mimeType; try { getFileTransfer()->upload(req); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..5d7566121 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s}; info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings<StringSet>(conn->from); } @@ -126,8 +126,7 @@ struct LegacySSHStore : public Store } void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override + RepairFlag repair, CheckSigsFlag checkSigs) override { debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); @@ -139,7 +138,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 48aca478c..87d8334d7 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -31,8 +31,18 @@ protected: bool fileExists(const std::string & path) override; void upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) override; + std::shared_ptr<std::basic_iostream<char>> istream, + const std::string & mimeType) override + { + auto path2 = binaryCacheDir + "/" + path; + Path tmp = path2 + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + StreamToSourceAdapter source(istream); + writeFile(tmp, source); + if (rename(tmp.c_str(), path2.c_str())) + throw SysError("renaming '%1%' to '%2%'", tmp, path2); + del.cancel(); + } void getFile(const std::string & path, Sink & sink) override { @@ -52,7 +62,9 @@ protected: if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); + paths.insert(parseStorePath( + storeDir + "/" + entry.name.substr(0, entry.name.size() - 8) + + "-" + MissingName)); } return paths; @@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init() BinaryCacheStore::init(); } -static void atomicWrite(const Path & path, const std::string & s) -{ - Path tmp = path + ".tmp." + std::to_string(getpid()); - AutoDelete del(tmp, false); - writeFile(tmp, s); - if (rename(tmp.c_str(), path.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path); - del.cancel(); -} - bool LocalBinaryCacheStore::fileExists(const std::string & path) { return pathExists(binaryCacheDir + "/" + path); } -void LocalBinaryCacheStore::upsertFile(const std::string & path, - const std::string & data, - const std::string & mimeType) -{ - atomicWrite(binaryCacheDir + "/" + path, data); -} - static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index dd96d2578..2f1d9663a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { - Path storePath = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + auto storePath = store->toStorePath(path).first; + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0dfbed9fc..eb367d778 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); } else { Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); + check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first); } } @@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -594,7 +587,7 @@ uint64_t LocalStore::addValidPath(State & state, (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); + uint64_t id = state.db.getLastInsertedRowId(); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (printStorePath(i.second.path)) + (printStorePath(i.second.path(*this, drv.name))) .exec(); } } @@ -686,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -774,17 +767,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) } -StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) +OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) { - return retrySQLite<StorePathSet>([&]() { + return retrySQLite<OutputPathMap>([&]() { auto state(_state.lock()); auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() (queryValidPathId(*state, path))); - StorePathSet outputs; + OutputPathMap outputs; while (useQueryDerivationOutputs.next()) - outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1))); + outputs.emplace( + useQueryDerivationOutputs.getStr(0), + parseStorePath(useQueryDerivationOutputs.getStr(1)) + ); return outputs; }); @@ -897,7 +893,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash && i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -959,7 +955,7 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { if (!info.narHash) throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); @@ -973,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); + auto realPath = Store::toRealPath(info.path); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a @@ -1010,7 +1006,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1030,12 +1026,78 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, +StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} + + +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { - Hash h = hashString(hashAlgo, dump); + /* For computing the store path. */ + auto hashSink = std::make_unique<HashSink>(hashAlgo); + TeeSource source { source0, *hashSink }; + + /* Read the source path into memory, but only if it's up to + narBufferSize bytes. If it's larger, write it to a temporary + location in the Nix store. If the subsequently computed + destination store path is already valid, we just delete the + temporary path. Otherwise, we move it to the destination store + path. */ + bool inMemory = false; + + std::string dump; + + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 65536; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } + + std::unique_ptr<AutoDelete> delTempDir; + Path tempPath; + + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; + + auto tempDir = createTempDir(realStoreDir, "add"); + delTempDir = std::make_unique<AutoDelete>(tempDir); + tempPath = tempDir + "/x"; + + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); + + dump.clear(); + } + + auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, h, name); + auto dstPath = makeFixedOutputPath(method, hash, name); addTempRoot(dstPath); @@ -1044,8 +1106,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1055,31 +1116,36 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); + if (inMemory) { + StringSource dumpSource { dump }; + /* Restore from the NAR in memory. */ + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); + } else { + /* Move the temporary path we restored above. */ + if (rename(tempPath.c_str(), realPath.c_str())) + throw Error("renaming '%s' to '%s'", tempPath, realPath); + } - canonicalisePathMetaData(realPath, -1); + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath - optimisePath(realPath); // FIXME: combine with hashPath() + optimisePath(realPath); ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; + info.narHash = narHash.first; + info.narSize = narHash.second; + info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1090,24 +1156,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } -StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - Path srcPath(absPath(_srcPath)); - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref<std::string>(readFile(srcPath)); - - return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); -} - - StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { @@ -1118,8 +1166,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1255,9 +1302,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr<AbstractHashSink> hashSink; if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique<HashSink>(*info->narHash.type); + hashSink = std::make_unique<HashSink>(info->narHash->type); else - hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1266,7 +1313,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e17cc45ae..547fe589c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -29,8 +29,8 @@ struct Derivation; struct OptimiseStats { unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; + uint64_t bytesFreed = 0; + uint64_t blocksFreed = 0; }; @@ -133,7 +133,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - StorePathSet queryDerivationOutputs(const StorePath & path) override; + OutputPathMap queryDerivationOutputMap(const StorePath & path) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; @@ -143,8 +143,7 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, @@ -154,7 +153,7 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/local.mk b/src/libstore/local.mk index aec4ed493..d266c8efe 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -61,3 +61,6 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libstore/builtins/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..a542a259d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -110,7 +110,7 @@ void Store::computeFSClosure(const StorePath & startPath, void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) + uint64_t & downloadSize_, uint64_t & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -122,8 +122,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, { std::unordered_set<std::string> done; StorePathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; + uint64_t & downloadSize; + uint64_t & narSize; }; struct DrvState @@ -198,8 +198,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, PathSet invalid; for (auto & j : drv->outputs) if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path)) - invalid.insert(printStorePath(j.second.path)); + && !isValidPath(j.second.path(*this, drv->name))) + invalid.insert(printStorePath(j.second.path(*this, drv->name))); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ca663d837..59ec164b6 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -18,7 +18,7 @@ struct NarMember /* If this is a regular file, position of the contents of this file in the NAR. */ - size_t start = 0, size = 0; + uint64_t start = 0, size = 0; std::string target; @@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, StringSource + struct NarIndexer : ParseSink, Source { NarAccessor & acc; + Source & source; std::stack<NarMember *> parents; - std::string currentStart; bool isExec = false; - NarIndexer(NarAccessor & acc, const std::string & nar) - : StringSource(nar), acc(acc) + uint64_t pos = 0; + + NarIndexer(NarAccessor & acc, Source & source) + : acc(acc), source(source) { } void createMember(const Path & path, NarMember member) { @@ -77,33 +79,40 @@ struct NarAccessor : public FSAccessor parents.top()->isExecutable = true; } - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { - currentStart = string(s, pos, 16); - assert(size <= std::numeric_limits<size_t>::max()); - parents.top()->size = (size_t)size; + assert(size <= std::numeric_limits<uint64_t>::max()); + parents.top()->size = (uint64_t) size; parents.top()->start = pos; } - void receiveContents(unsigned char * data, unsigned int len) override - { - // Sanity check - if (!currentStart.empty()) { - assert(len < 16 || currentStart == string((char *) data, 16)); - currentStart.clear(); - } - } + void receiveContents(unsigned char * data, size_t len) override + { } void createSymlink(const Path & path, const string & target) override { createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } + + size_t read(unsigned char * data, size_t len) override + { + auto n = source.read(data, len); + pos += n; + return n; + } }; NarAccessor(ref<const std::string> nar) : nar(nar) { - NarIndexer indexer(*this, *nar); + StringSource source(*nar); + NarIndexer indexer(*this, source); + parseDump(indexer, indexer); + } + + NarAccessor(Source & source) + { + NarIndexer indexer(*this, source); parseDump(indexer, indexer); } @@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) return make_ref<NarAccessor>(nar); } +ref<FSAccessor> makeNarAccessor(Source & source) +{ + return make_ref<NarAccessor>(source); +} + ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 2871199de..8af1272f6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -6,10 +6,14 @@ namespace nix { +struct Source; + /* Return an object that provides access to the contents of a NAR file. */ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); +ref<FSAccessor> makeNarAccessor(Source & source); + /* Create a NAR accessor from a NAR listing (in the format produced by listNar()). The callback getNarBytes(offset, length) is used by the readFile() method of the accessor to get the contents of files diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..9ddb9957f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash->to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 04550ed97..ca471463c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { return Hash(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; @@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } else if (name == "NarHash") narHash = parseHashField(value); else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString<Strings>(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } @@ -77,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -87,11 +86,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash && narHash->type == htSHA256); + res += "NarHash: " + narHash->to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional<Hash> fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b2b2412a3..e4b4b6213 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(unsigned long long bytes) +static string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index b3d8ce95c..dc9dc3897 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,8 +2,6 @@ namespace nix { -MakeError(BadStorePath, Error); - static void checkName(std::string_view path, std::string_view name) { if (name.empty()) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4f79843fe..e43a8b50c 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -62,6 +62,7 @@ public: typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; +typedef std::map<string, StorePath> OutputPathMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 6cfe393a4..6862b42f0 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -12,30 +12,24 @@ namespace nix { -static bool cmpGensByNumber(const Generation & a, const Generation & b) -{ - return a.number < b.number; -} - - /* Parse a generation name of the format `<profilename>-<number>-link'. */ -static int parseName(const string & profileName, const string & name) +static std::optional<GenerationNumber> parseName(const string & profileName, const string & name) { - if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + if (string(name, 0, profileName.size() + 1) != profileName + "-") return {}; string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); - if (p == string::npos) return -1; - int n; + if (p == string::npos) return {}; + unsigned int n; if (string2Int(string(s, 0, p), n) && n >= 0) return n; else - return -1; + return {}; } -Generations findGenerations(Path profile, int & curGen) +std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile) { Generations gens; @@ -43,30 +37,34 @@ Generations findGenerations(Path profile, int & curGen) auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { - int n; - if ((n = parseName(profileName, i.name)) != -1) { - Generation gen; - gen.path = profileDir + "/" + i.name; - gen.number = n; + if (auto n = parseName(profileName, i.name)) { + auto path = profileDir + "/" + i.name; struct stat st; - if (lstat(gen.path.c_str(), &st) != 0) - throw SysError("statting '%1%'", gen.path); - gen.creationTime = st.st_mtime; - gens.push_back(gen); + if (lstat(path.c_str(), &st) != 0) + throw SysError("statting '%1%'", path); + gens.push_back({ + .number = *n, + .path = path, + .creationTime = st.st_mtime + }); } } - gens.sort(cmpGensByNumber); + gens.sort([](const Generation & a, const Generation & b) + { + return a.number < b.number; + }); - curGen = pathExists(profile) + return { + gens, + pathExists(profile) ? parseName(profileName, readLink(profile)) - : -1; - - return gens; + : std::nullopt + }; } -static void makeName(const Path & profile, unsigned int num, +static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { Path prefix = (format("%1%-%2%") % profile % num).str(); @@ -78,10 +76,9 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) { /* The new generation number should be higher than old the previous ones. */ - int dummy; - Generations gens = findGenerations(profile, dummy); + auto [gens, dummy] = findGenerations(profile); - unsigned int num; + GenerationNumber num; if (gens.size() > 0) { Generation last = gens.back(); @@ -121,7 +118,7 @@ static void removeFile(const Path & path) } -void deleteGeneration(const Path & profile, unsigned int gen) +void deleteGeneration(const Path & profile, GenerationNumber gen) { Path generation; makeName(profile, gen, generation); @@ -129,7 +126,7 @@ void deleteGeneration(const Path & profile, unsigned int gen) } -static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) +static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) printInfo(format("would remove generation %1%") % gen); @@ -140,31 +137,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu } -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun) +void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); - if (gensToDelete.find(curGen) != gensToDelete.end()) + if (gensToDelete.count(*curGen)) throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { - if (gensToDelete.find(i.number) == gensToDelete.end()) continue; + if (!gensToDelete.count(i.number)) continue; deleteGeneration2(profile, i.number, dryRun); } } -void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; bool fromCurGen = false; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto i = gens.rbegin(); i != gens.rend(); ++i) { if (i->number == curGen) { fromCurGen = true; @@ -186,8 +181,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto & i : gens) if (i.number != curGen) @@ -200,8 +194,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); bool canDelete = false; for (auto i = gens.rbegin(); i != gens.rend(); ++i) diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 78645d8b6..abe507f0e 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -9,37 +9,32 @@ namespace nix { +typedef unsigned int GenerationNumber; + struct Generation { - int number; + GenerationNumber number; Path path; time_t creationTime; - Generation() - { - number = -1; - } - operator bool() const - { - return number != -1; - } }; -typedef list<Generation> Generations; +typedef std::list<Generation> Generations; /* Returns the list of currently present generations for the specified - profile, sorted by generation number. */ -Generations findGenerations(Path profile, int & curGen); + profile, sorted by generation number. Also returns the number of + the current generation. */ +std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); class LocalFSStore; Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath); -void deleteGeneration(const Path & profile, unsigned int gen); +void deleteGeneration(const Path & profile, GenerationNumber gen); -void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); +void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun); -void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); void deleteOldGenerations(const Path & profile, bool dryRun); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..62a3cda61 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len, struct RefScanSink : Sink { - HashSink hashSink; StringSet hashes; StringSet seen; string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() { } void operator () (const unsigned char * data, size_t len); }; @@ -62,8 +61,6 @@ struct RefScanSink : Sink void RefScanSink::operator () (const unsigned char * data, size_t len) { - hashSink(data, len); - /* It's possible that a reference spans the previous and current fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ @@ -79,10 +76,12 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair<PathSet, HashResult> scanForReferences(const string & path, + const PathSet & refs) { - RefScanSink sink; + RefScanSink refsSink; + HashSink hashSink { htSHA256 }; + TeeSink sink { refsSink, hashSink }; std::map<string, Path> backMap; /* For efficiency (and a higher hit rate), just search for the @@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path, assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); - sink.hashes.insert(s); + refsSink.hashes.insert(s); backMap[s] = i; } @@ -106,15 +105,15 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - for (auto & i : sink.seen) { + for (auto & i : refsSink.seen) { std::map<string, Path>::iterator j; if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } - hash = sink.hashSink.finish(); + auto hash = hashSink.finish(); - return found; + return std::pair<PathSet, HashResult>(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index bd698d781..2d02a181b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) createDirs(cacheDir); } -Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) +Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); + return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, +void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar, ref<FSAccessor> narAccessor) { - nars.emplace(storePath, narAccessor); + nars.emplace(hashPart, narAccessor); if (cacheDir != "") { try { std::ostringstream str; JSONPlaceholder jsonRoot(str); listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(storePath, "ls"), str.str()); + writeFile(makeCacheFile(hashPart, "ls"), str.str()); /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(storePath, "nar"), nar); + writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { ignoreException(); @@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) { auto path = canonPath(path_); - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); + auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath("path '%1%' is not a valid store path", storePath); + if (!store->isValidPath(storePath)) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - auto i = nars.find(storePath); + auto i = nars.find(std::string(storePath.hashPart())); if (i != nars.end()) return {i->second, restPath}; StringSink sink; std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { - listing = nix::readFile(makeCacheFile(storePath, "ls")); + listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { @@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) return buf; }); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } @@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) *sink.s = nix::readFile(cacheFile); auto narAccessor = makeNarAccessor(sink.s); - nars.emplace(storePath, narAccessor); + nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; } catch (SysError &) { } } - store->narFromPath(store->parseStorePath(storePath), sink); + store->narFromPath(storePath, sink); auto narAccessor = makeNarAccessor(sink.s); - addToCache(storePath, *sink.s, narAccessor); + addToCache(storePath.hashPart(), *sink.s, narAccessor); return {narAccessor, restPath}; } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 4afb3be95..347cf5764 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor { ref<Store> store; - std::map<Path, ref<FSAccessor>> nars; + std::map<std::string, ref<FSAccessor>> nars; Path cacheDir; @@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor friend class BinaryCacheStore; - Path makeCacheFile(const Path & storePath, const std::string & ext); + Path makeCacheFile(std::string_view hashPart, const std::string & ext); - void addToCache(const Path & storePath, const std::string & nar, + void addToCache(std::string_view hashPart, const std::string & nar, ref<FSAccessor> narAccessor); public: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7cc7a5fc..611f70783 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -8,6 +8,7 @@ #include "derivations.hh" #include "pool.hh" #include "finally.hh" +#include "logging.hh" #include <sys/types.h> #include <sys/stat.h> @@ -38,6 +39,29 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } +std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from) +{ + std::map<string, StorePath> pathMap; + auto rawInput = readStrings<Strings>(from); + if (rawInput.size() % 2) + throw Error("got an odd number of elements from the daemon when trying to read a output path map"); + auto curInput = rawInput.begin(); + while (curInput != rawInput.end()) { + auto thisKey = *curInput++; + auto thisValue = *curInput++; + pathMap.emplace(thisKey, store.parseStorePath(thisValue)); + } + return pathMap; +} + +void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap) +{ + out << 2*pathMap.size(); + for (auto & i : pathMap) { + out << i.first; + out << store.printStorePath(i.second); + } +} /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -197,7 +221,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.maxSilentTime.name); overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); - overrides.erase(settings.showTrace.name); + overrides.erase(loggerSettings.showTrace.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -412,12 +436,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + return Store::queryDerivationOutputs(path); + } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); return readStorePaths<StorePathSet>(*this, conn->from); } +OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return readOutputPathMap(*this, conn->from); + +} + std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); @@ -430,7 +466,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { auto conn(getConnection()); @@ -462,14 +498,89 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + + std::exception_ptr ex; + + struct FramedSink : BufferedSink + { + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; + }; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + conn.processStderr(); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(conn, ex); + copyNAR(source, sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } } } @@ -671,7 +782,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 80c8e9f11..eb6f225b1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,6 +51,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; + OutputPathMap queryDerivationOutputMap(const StorePath & path) override; std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -59,8 +60,7 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) override; + RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, @@ -94,7 +94,7 @@ public: void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + uint64_t & downloadSize, uint64_t & narSize) override; void connect() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 427dd48ce..67935f3ba 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,7 +7,6 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" -#include "istringstream_nocopy.hh" #include <aws/core/Aws.h> #include <aws/core/VersionConfig.h> @@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr<TransferManager> transferManager; std::once_flag transferManagerCreated; - void uploadFile(const std::string & path, const std::string & data, + void uploadFile(const std::string & path, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType, const std::string & contentEncoding) { - auto stream = std::make_shared<istringstream_nocopy>(data); - auto maxThreads = std::thread::hardware_concurrency(); static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> @@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::shared_ptr<TransferHandle> transferHandle = transferManager->UploadFile( - stream, bucketName, path, mimeType, + istream, bucketName, path, mimeType, Aws::Map<Aws::String, Aws::String>(), nullptr /*, contentEncoding */); @@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore if (contentEncoding != "") request.SetContentEncoding(contentEncoding); - auto stream = std::make_shared<istringstream_nocopy>(data); - - request.SetBody(stream); + request.SetBody(istream); auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request)); @@ -347,25 +343,31 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) .count(); - printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % - bucketName % path % data.size() % duration); + printInfo("uploaded 's3://%s/%s' in %d ms", + bucketName, path, duration); stats.putTimeMs += duration; - stats.putBytes += data.size(); stats.put++; } - void upsertFile(const std::string & path, const std::string & data, + void upsertFile(const std::string & path, + std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) override { + auto compress = [&](std::string compression) + { + auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain()); + return std::make_shared<std::stringstream>(std::move(*compressed)); + }; + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + uploadFile(path, compress(lsCompression), mimeType, lsCompression); else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + uploadFile(path, compress(logCompression), mimeType, logCompression); else - uploadFile(path, data, mimeType, ""); + uploadFile(path, istream, mimeType, ""); } void getFile(const std::string & path, Sink & sink) override @@ -410,7 +412,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); } marker = res.GetNextMarker(); diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..b2b75d498 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -19,7 +19,6 @@ public: struct Stats { std::atomic<uint64_t> put{0}; - std::atomic<uint64_t> putBytes{0}; std::atomic<uint64_t> putTimeMs{0}; std::atomic<uint64_t> get{0}; std::atomic<uint64_t> getBytes{0}; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 76c822c4e..31a1f0cac 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt) }); } +uint64_t SQLite::getLastInsertedRowId() +{ + return sqlite3_last_insert_rowid(db); +} + void SQLiteStmt::create(sqlite3 * db, const string & sql) { checkInterrupt(); @@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use() sqlite3_reset(stmt); } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throwSQLiteError(stmt.db, "binding argument"); } else bind(); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index dd81ab051..99f0d56ce 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -26,6 +26,8 @@ struct SQLite void isCache(); void exec(const std::string & stmt); + + uint64_t getLastInsertedRowId(); }; /* RAII wrapper to create and destroy SQLite prepared statements. */ @@ -54,7 +56,7 @@ struct SQLiteStmt ~Use(); /* Bind the next parameter. */ - Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4a4ae11e..c804399d2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "archive.hh" #include <future> @@ -20,15 +21,15 @@ bool Store::isInStore(const Path & path) const } -Path Store::toStorePath(const Path & path) const +std::pair<StorePath, Path> Store::toStorePath(const Path & path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) - return path; + return {parseStorePath(path), ""}; else - return Path(path, 0, slash); + return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; } @@ -41,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw NotInStore("path '%1%' is not in the Nix store", path); + throw BadStorePath("path '%1%' is not in the Nix store", path); return path; } StorePath Store::followLinksToStorePath(std::string_view path) const { - return parseStorePath(toStorePath(followLinksToStore(path))); + return toStorePath(followLinksToStore(path)).first; } @@ -221,6 +222,93 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +/* +The aim of this function is to compute in one pass the correct ValidPathInfo for +the files that we are trying to add to the store. To accomplish that in one +pass, given the different kind of inputs that we can take (normal nar archives, +nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks +and aliases. Also, since the dataflow is obfuscated by this, we include here a +graphviz diagram: + +digraph graphname { + node [shape=box] + fileSource -> narSink + narSink [style=dashed] + narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"] + narSink -> narHashSink [style = dashed, label = "else"] + unsualHashTee -> narHashSink + unsualHashTee -> caHashSink + fileSource -> parseSink + parseSink [style=dashed] + parseSink-> fileSink [style = dashed, label = "Flat"] + parseSink -> blank [style = dashed, label = "Recursive"] + fileSink -> caHashSink +} +*/ +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional<Hash> expectedCAHash) +{ + HashSink narHashSink { htSHA256 }; + HashSink caHashSink { hashAlgo }; + + /* Note that fileSink and unusualHashTee must be mutually exclusive, since + they both write to caHashSink. Note that that requisite is currently true + because the former is only used in the flat case. */ + RetrieveRegularNARSink fileSink { caHashSink }; + TeeSink unusualHashTee { narHashSink, caHashSink }; + + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + ? static_cast<Sink &>(unusualHashTee) + : narHashSink; + + /* Functionally, this means that fileSource will yield the content of + srcPath. The fact that we use scratchpadSink as a temporary buffer here + is an implementation detail. */ + auto fileSource = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + + /* tapped provides the same data as fileSource, but we also write all the + information to narSink. */ + TeeSource tapped { *fileSource, narSink }; + + ParseSink blank; + auto & parseSink = method == FileIngestionMethod::Flat + ? fileSink + : blank; + + /* The information that flows from tapped (besides being replicated in + narSink), is now put in parseSink. */ + parseDump(parseSink, tapped); + + /* We extract the result of the computation from the sink by calling + finish. */ + auto [narHash, narSize] = narHashSink.finish(); + + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? narHash + : caHashSink.finish().first; + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + ValidPathInfo info(makeFixedOutputPath(method, hash, name)); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) : Config(params) , state({(size_t) pathInfoCacheSize}) @@ -242,6 +330,16 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +StorePathSet Store::queryDerivationOutputs(const StorePath & path) +{ + auto outputMap = this->queryDerivationOutputMap(path); + StorePathSet outputPaths; + for (auto & i: outputMap) { + outputPaths.emplace(std::move(i.second)); + } + return outputPaths; +} + bool Store::isValidPath(const StorePath & storePath) { std::string hashPart(storePath.hashPart()); @@ -306,6 +404,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath) } +static bool goodStorePath(const StorePath & expected, const StorePath & actual) +{ + return + expected.hashPart() == actual.hashPart() + && (expected.name() == Store::MissingName || expected.name() == actual.name()); +} + + void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { @@ -333,7 +439,7 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || - res.second->path != storePath) + !goodStorePath(storePath, res.second->path)) throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref<const ValidPathInfo>(res.second)); @@ -345,7 +451,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -358,9 +464,11 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - if (!info || info->path != parseStorePath(storePath)) { + auto storePath = parseStorePath(storePathS); + + if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); + throw InvalidPath("path '%s' is not valid", storePathS); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -430,7 +538,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash->to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -462,7 +570,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) + .attr("narHash", info->narHash->to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -505,7 +613,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -621,9 +729,9 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st { auto valid = dstStore->queryValidPaths(storePaths, substitute); - PathSet missing; + StorePathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); + if (!valid.count(path)) missing.insert(path); if (missing.empty()) return; @@ -640,29 +748,27 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st ThreadPool pool; - processGraph<Path>(pool, - PathSet(missing.begin(), missing.end()), + processGraph<StorePath>(pool, + StorePathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const StorePath & storePath) { + if (dstStore->isValidPath(storePath)) { nrDone++; showProgress(); - return PathSet(); + return StorePathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); + auto info = srcStore->queryPathInfo(storePath); bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return srcStore->printStorePathSet(info->references); + return info->references; }, - [&](const Path & storePathS) { + [&](const StorePath & storePath) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); - if (!dstStore->isValidPath(storePath)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); @@ -672,7 +778,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; } @@ -746,7 +852,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash->to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -859,12 +965,20 @@ ref<Store> openStore(const std::string & uri_, throw Error("don't know how to open Nix store '%s'", uri); } +static bool isNonUriPath(const std::string & spec) { + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; +} StoreType getStoreType(const std::string & uri, const std::string & stateDir) { if (uri == "daemon") { return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { + } else if (uri == "local" || isNonUriPath(uri)) { return tLocal; } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -888,8 +1002,9 @@ static RegisterStoreImplementation regStore([]( return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); case tLocal: { Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; + if (isNonUriPath(uri)) { + params2["root"] = absPath(uri); + } return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); } default: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 25d78c297..667dc4117 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -31,7 +31,7 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(NotInStore, Error); +MakeError(BadStorePath, Error); class FSAccessor; @@ -85,7 +85,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; + uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; }; @@ -97,7 +97,7 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + uint64_t bytesFreed = 0; }; @@ -105,8 +105,8 @@ struct SubstitutablePathInfo { std::optional<StorePath> deriver; StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; @@ -115,7 +115,8 @@ struct ValidPathInfo { StorePath path; std::optional<StorePath> deriver; - Hash narHash; + // TODO document this + std::optional<Hash> narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -317,9 +318,9 @@ public: the Nix store. */ bool isStorePath(std::string_view path) const; - /* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ - Path toStorePath(const Path & path) const; + /* Split a path like /nix/store/<hash>-<name>/<bla> into + /nix/store/<hash>-<name> and /<bla>. */ + std::pair<StorePath, Path> toStorePath(const Path & path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; @@ -384,13 +385,16 @@ public: SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be omitted - (i.e. you'll get /nix/store/<hash> rather than + backends, the name part of store paths may be replaced by 'x' + (i.e. you'll get /nix/store/<hash>-x rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the - full store path. */ + full store path. FIXME: should return a set of + std::variant<StorePath, HashPart> to get rid of this hack. */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + constexpr static const char * MissingName = "x"; + /* Query information about a valid path. It is permitted to omit the name part of the store path. */ ref<const ValidPathInfo> queryPathInfo(const StorePath & path); @@ -418,8 +422,11 @@ public: virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual StorePathSet queryDerivationOutputs(const StorePath & path) - { unsupported("queryDerivationOutputs"); } + virtual StorePathSet queryDerivationOutputs(const StorePath & path); + + /* Query the mapping outputName=>outputPath for the given derivation */ + virtual OutputPathMap queryDerivationOutputMap(const StorePath & path) + { unsupported("queryDerivationOutputMap"); } /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ @@ -436,8 +443,7 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) = 0; + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. @@ -447,8 +453,15 @@ public: FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional<Hash> expectedCAHash = {}); + // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); @@ -597,7 +610,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ @@ -613,8 +626,7 @@ public: the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, - CheckSigsFlag checkSigs = CheckSigs); + StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ac42457fc..dcba73116 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x115 +#define PROTOCOL_VERSION 0x117 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -30,7 +30,7 @@ typedef enum { wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, + wopQueryDerivationOutputs = 22, // obsolete wopQueryAllValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, @@ -49,6 +49,7 @@ typedef enum { wopNarFromPath = 38, wopAddToStoreNar = 39, wopQueryMissing = 40, + wopQueryDerivationOutputMap = 41, } WorkerOp; @@ -69,5 +70,6 @@ template<class T> T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); +void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); } |