Merge remote-tracking branch 'upstream/master' into path-info

1 files changed, 72 insertions, 59 deletions

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 319b06269..9c022f9e9 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -31,7 +31,7 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
 
     StringSink sink;
     sink << narVersionMagic1;
-    narMagic = *sink.s;
+    narMagic = sink.s;
 }
 
 void BinaryCacheStore::init()
@@ -68,7 +68,7 @@ void BinaryCacheStore::upsertFile(const std::string & path,
 }
 
 void BinaryCacheStore::getFile(const std::string & path,
-    Callback<std::shared_ptr<std::string>> callback) noexcept
+    Callback<std::optional<std::string>> callback) noexcept
 {
     try {
         callback(getFile(path));
@@ -77,9 +77,9 @@ void BinaryCacheStore::getFile(const std::string & path,
 
 void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
 {
-    std::promise<std::shared_ptr<std::string>> promise;
+    std::promise<std::optional<std::string>> promise;
     getFile(path,
-        {[&](std::future<std::shared_ptr<std::string>> result) {
+        {[&](std::future<std::optional<std::string>> result) {
             try {
                 promise.set_value(result.get());
             } catch (...) {
@@ -89,15 +89,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
     sink(*promise.get_future().get());
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
+std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
 {
     StringSink sink;
     try {
         getFile(path, sink);
     } catch (NoSuchBinaryCacheFile &) {
-        return nullptr;
+        return std::nullopt;
     }
-    return sink.s;
+    return std::move(sink.s);
 }
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    std::string hashPart(narInfo->path.hashPart());
-
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
+        state_->pathInfoCache.upsert(
+            std::string(narInfo->path.to_string()),
+            PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
     }
 
     if (diskCache)
-        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+        diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
 }
 
 AutoCloseFD openFile(const Path & path)
@@ -149,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     {
     FdSink fileSink(fdTemp.get());
     TeeSink teeSinkCompressed { fileSink, fileHashSink };
-    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
+    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
     TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
     TeeSource teeSource { narSource, teeSinkUncompressed };
     narAccessor = makeNarAccessor(teeSource);
@@ -306,8 +306,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     }});
 }
 
-StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
+StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
     if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
         unsupported("addToStoreFromDump");
@@ -315,13 +315,16 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
         ValidPathInfo info {
             *this,
             {
-                .name = name,
+                .name = std::string { name },
                 .info = FixedOutputInfo {
                     {
                         .method = method,
                         .hash = nar.first,
                     },
-                    {},
+                    {
+                        .references = references,
+                        .hasSelfReference = false,
+                    },
                 },
             },
             nar.first,
@@ -375,11 +378,11 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     getFile(narInfoFile,
-        {[=](std::future<std::shared_ptr<std::string>> fut) {
+        {[=](std::future<std::optional<std::string>> fut) {
             try {
                 auto data = fut.get();
 
-                if (!data) return (*callbackPtr)(nullptr);
+                if (!data) return (*callbackPtr)({});
 
                 stats.narInfoRead++;
 
@@ -393,8 +396,14 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
         }});
 }
 
-StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+StorePath BinaryCacheStore::addToStore(
+    std::string_view name,
+    const Path & srcPath,
+    FileIngestionMethod method,
+    HashType hashAlgo,
+    PathFilter & filter,
+    RepairFlag repair,
+    const StorePathSet & references)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -415,13 +424,16 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         ValidPathInfo info {
             *this,
             {
-                .name = name,
+                .name = std::string { name },
                 .info = FixedOutputInfo {
                     {
                         .method = method,
                         .hash = h,
                     },
-                    {},
+                    {
+                        .references = references,
+                        .hasSelfReference = false,
+                    },
                 },
             },
             nar.first,
@@ -431,8 +443,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     })->path;
 }
 
-StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
-    const StorePathSet & references, RepairFlag repair)
+StorePath BinaryCacheStore::addTextToStore(
+    std::string_view name,
+    std::string_view s,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     auto textHash = hashString(htSHA256, s);
     auto path = makeTextPath(name, TextInfo { textHash, references });
@@ -442,12 +457,12 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
 
     StringSink sink;
     dumpString(s, sink);
-    auto source = StringSource { *sink.s };
+    StringSource source(sink.s);
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             {
-                .name = name,
+                .name = std::string { name },
                 .info = TextInfo {
                     { .hash = textHash },
                     references,
@@ -456,45 +471,33 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
             nar.first,
         };
         info.narSize = nar.second;
-        info.ca = TextHash { textHash };
         return info;
     })->path;
 }
 
-std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
+void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
+    Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
-    if (diskCache) {
-        auto [cacheOutcome, maybeCachedRealisation] =
-            diskCache->lookupRealisation(getUri(), id);
-        switch (cacheOutcome) {
-            case NarInfoDiskCache::oValid:
-                debug("Returning a cached realisation for %s", id.to_string());
-                return *maybeCachedRealisation;
-            case NarInfoDiskCache::oInvalid:
-                debug("Returning a cached missing realisation for %s", id.to_string());
-                return {};
-            case NarInfoDiskCache::oUnknown:
-                break;
-        }
-    }
-
     auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
-    auto rawOutputInfo = getFile(outputInfoFilePath);
 
-    if (rawOutputInfo) {
-        auto realisation = Realisation::fromJSON(
-            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
+    Callback<std::optional<std::string>> newCallback = {
+        [=](std::future<std::optional<std::string>> fut) {
+            try {
+                auto data = fut.get();
+                if (!data) return (*callbackPtr)({});
 
-        if (diskCache)
-            diskCache->upsertRealisation(
-                getUri(), realisation);
+                auto realisation = Realisation::fromJSON(
+                    nlohmann::json::parse(*data), outputInfoFilePath);
+                return (*callbackPtr)(std::make_shared<const Realisation>(realisation));
+            } catch (...) {
+                callbackPtr->rethrow();
+            }
+        }
+    };
 
-        return {realisation};
-    } else {
-        if (diskCache)
-            diskCache->upsertAbsentRealisation(getUri(), id);
-        return std::nullopt;
-    }
+    getFile(outputInfoFilePath, std::move(newCallback));
 }
 
 void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
@@ -523,7 +526,7 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
     writeNarInfo(narInfo);
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
+std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
     auto drvPath = path;
 
@@ -531,10 +534,10 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
-            if (!info->deriver) return nullptr;
+            if (!info->deriver) return std::nullopt;
             drvPath = *info->deriver;
         } catch (InvalidPath &) {
-            return nullptr;
+            return std::nullopt;
         }
     }
 
@@ -545,4 +548,14 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
     return getFile(logPath);
 }
 
+void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    upsertFile(
+        "log/" + std::string(drvPath.to_string()),
+        (std::string) log, // FIXME: don't copy
+        "text/plain; charset=utf-8");
+}
+
 }