aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatthew Bauer <mjbauer95@gmail.com>2020-06-22 13:08:11 -0400
committerMatthew Bauer <mjbauer95@gmail.com>2020-06-22 13:08:11 -0400
commit66a62b3189c8c9b0965850e6b3c9b0fda0b50fd8 (patch)
tree70a25a2cb1973a05559eca3134979fc8be873aca
parentf2a6cee334255ced72a70f527cf3c283b4586e42 (diff)
parent965b80347e97169f266466603e29a57359c4083c (diff)
Merge remote-tracking branch 'origin/master' into substitute-other-storedir
-rw-r--r--src/libexpr/get-drvs.cc2
-rw-r--r--src/libexpr/primops.cc6
-rw-r--r--src/libexpr/primops/context.cc2
-rw-r--r--src/libfetchers/tarball.cc5
-rw-r--r--src/libstore/build.cc26
-rw-r--r--src/libstore/content-address.cc85
-rw-r--r--src/libstore/content-address.hh56
-rw-r--r--src/libstore/daemon.cc5
-rw-r--r--src/libstore/derivations.cc13
-rw-r--r--src/libstore/derivations.hh14
-rw-r--r--src/libstore/legacy-ssh-store.cc4
-rw-r--r--src/libstore/local-store.cc24
-rw-r--r--src/libstore/misc.cc25
-rw-r--r--src/libstore/nar-info-disk-cache.cc4
-rw-r--r--src/libstore/nar-info.cc9
-rw-r--r--src/libstore/parsed-derivations.hh2
-rw-r--r--src/libstore/path.hh8
-rw-r--r--src/libstore/remote-store.cc15
-rw-r--r--src/libstore/store-api.cc109
-rw-r--r--src/libstore/store-api.hh38
-rw-r--r--src/libstore/worker-protocol.hh2
-rw-r--r--src/libutil/hash.cc2
-rw-r--r--src/nix-store/nix-store.cc4
-rw-r--r--src/nix/add-to-store.cc5
-rw-r--r--src/nix/develop.cc2
-rw-r--r--src/nix/hash.cc1
-rw-r--r--src/nix/make-content-addressable.cc5
-rw-r--r--src/nix/path-info.cc2
-rw-r--r--src/nix/verify.cc2
29 files changed, 278 insertions, 199 deletions
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index a4937e722..9055f59a1 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -1,7 +1,7 @@
#include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
#include <cstring>
#include <regex>
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index f5fbd3fa6..3830d8107 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -776,7 +776,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
.path = std::move(outPath),
- .hash = DerivationOutputHash {
+ .hash = FixedOutputHash {
.method = ingestionMethod,
.hash = std::move(h),
},
@@ -795,7 +795,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
drv.outputs.insert_or_assign(i,
DerivationOutput {
.path = StorePath::dummy,
- .hash = std::optional<DerivationOutputHash> {},
+ .hash = std::optional<FixedOutputHash> {},
});
}
@@ -807,7 +807,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
drv.outputs.insert_or_assign(i,
DerivationOutput {
.path = std::move(outPath),
- .hash = std::optional<DerivationOutputHash>(),
+ .hash = std::optional<FixedOutputHash>(),
});
}
}
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index efa2e9576..301e8c5dd 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -1,6 +1,6 @@
#include "primops.hh"
#include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
namespace nix {
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index ac83d52b9..f5356f0af 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -70,7 +70,10 @@ DownloadFileResult downloadFile(
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
- info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
+ info.ca = FixedOutputHash {
+ .method = FileIngestionMethod::Flat,
+ .hash = hash,
+ };
auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b80b5c251..697c3fcff 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<std::string> ca = std::nullopt);
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
@@ -3714,7 +3714,7 @@ void DerivationGoal::registerOutputs()
/* Check that fixed-output derivations produced the right
outputs (i.e., the content hash should match the specified
hash). */
- std::string ca;
+ std::optional<ContentAddress> ca;
if (fixedOutput) {
@@ -3764,7 +3764,10 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);
- ca = makeFixedOutputCA(i.second.hash->method, h2);
+ ca = FixedOutputHash {
+ .method = i.second.hash->method,
+ .hash = h2,
+ };
}
/* Get rid of all weird permissions. This also checks that
@@ -3837,7 +3840,10 @@ void DerivationGoal::registerOutputs()
info.ca = ca;
worker.store.signPathInfo(info);
- if (!info.references.empty()) info.ca.clear();
+ if (!info.references.empty()) {
+ // FIXME don't we have an experimental feature for fixed output with references?
+ info.ca = {};
+ }
infos.emplace(i.first, std::move(info));
}
@@ -4295,10 +4301,10 @@ private:
GoalState state;
/* Content address for recomputing store path */
- std::optional<std::string> ca;
+ std::optional<ContentAddress> ca;
public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<std::string> ca = std::nullopt);
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@@ -4328,7 +4334,7 @@ public:
};
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<std::string> ca)
+SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
@@ -4408,7 +4414,7 @@ void SubstitutionGoal::tryNext()
subs.pop_front();
auto subPath = storePath;
- if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) {
+ if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
@@ -4537,7 +4543,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id);
auto subPath = storePath;
- if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) {
+ if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
@@ -4677,7 +4683,7 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<std::string> ca)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) {
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
new file mode 100644
index 000000000..3d753836f
--- /dev/null
+++ b/src/libstore/content-address.cc
@@ -0,0 +1,85 @@
+#include "content-address.hh"
+
+namespace nix {
+
+std::string FixedOutputHash::printMethodAlgo() const {
+ return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+}
+
+std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+ switch (m) {
+ case FileIngestionMethod::Flat:
+ return "";
+ case FileIngestionMethod::Recursive:
+ return "r:";
+ default:
+ throw Error("impossible, caught both cases");
+ }
+}
+
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+{
+ return "fixed:"
+ + makeFileIngestionPrefix(method)
+ + hash.to_string(Base32, true);
+}
+
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+std::string renderContentAddress(ContentAddress ca) {
+ return std::visit(overloaded {
+ [](TextHash th) {
+ return "text:" + th.hash.to_string(Base32, true);
+ },
+ [](FixedOutputHash fsh) {
+ return makeFixedOutputCA(fsh.method, fsh.hash);
+ }
+ }, ca);
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+ auto prefixSeparator = rawCa.find(':');
+ if (prefixSeparator != string::npos) {
+ auto prefix = string(rawCa, 0, prefixSeparator);
+ if (prefix == "text") {
+ auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+ Hash hash = Hash(string(hashTypeAndHash));
+ if (*hash.type != htSHA256) {
+ throw Error("parseContentAddress: the text hash should have type SHA256");
+ }
+ return TextHash { hash };
+ } else if (prefix == "fixed") {
+ // This has to be an inverse of makeFixedOutputCA
+ auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+ if (methodAndHash.substr(0,2) == "r:") {
+ std::string_view hashRaw = methodAndHash.substr(2,string::npos);
+ return FixedOutputHash {
+ .method = FileIngestionMethod::Recursive,
+ .hash = Hash(string(hashRaw)),
+ };
+ } else {
+ std::string_view hashRaw = methodAndHash;
+ return FixedOutputHash {
+ .method = FileIngestionMethod::Flat,
+ .hash = Hash(string(hashRaw)),
+ };
+ }
+ } else {
+ throw Error("parseContentAddress: format not recognized; has to be text or fixed");
+ }
+ } else {
+ throw Error("Not a content address because it lacks an appropriate prefix");
+ }
+};
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
+ return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
+};
+
+std::string renderContentAddress(std::optional<ContentAddress> ca) {
+ return ca ? renderContentAddress(*ca) : "";
+}
+
+}
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
new file mode 100644
index 000000000..ba4797f5b
--- /dev/null
+++ b/src/libstore/content-address.hh
@@ -0,0 +1,56 @@
+#pragma once
+
+#include <variant>
+#include "hash.hh"
+
+namespace nix {
+
+enum struct FileIngestionMethod : uint8_t {
+ Flat = false,
+ Recursive = true
+};
+
+struct TextHash {
+ Hash hash;
+};
+
+/// Pair of a hash, and how the file system was ingested
+struct FixedOutputHash {
+ FileIngestionMethod method;
+ Hash hash;
+ std::string printMethodAlgo() const;
+};
+
+/*
+ We've accumulated several types of content-addressed paths over the years;
+ fixed-output derivations support multiple hash algorithms and serialisation
+ methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
+
+ * ‘text:sha256:<sha256 hash of file contents>’: For paths
+ computed by makeTextPath() / addTextToStore().
+
+ * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
+ makeFixedOutputPath() / addToStore().
+*/
+typedef std::variant<
+ TextHash, // for paths computed by makeTextPath() / addTextToStore
+ FixedOutputHash // for path computed by makeFixedOutputPath
+> ContentAddress;
+
+/* Compute the prefix to the hash algorithm which indicates how the files were
+ ingested. */
+std::string makeFileIngestionPrefix(const FileIngestionMethod m);
+
+/* Compute the content-addressability assertion (ValidPathInfo::ca)
+ for paths created by makeFixedOutputPath() / addToStore(). */
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
+
+std::string renderContentAddress(ContentAddress ca);
+
+std::string renderContentAddress(std::optional<ContentAddress> ca);
+
+ContentAddress parseContentAddress(std::string_view rawCa);
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+
+}
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index c613ab47d..c2c8612d1 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -658,7 +658,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
- << info->ca;
+ << renderContentAddress(info->ca);
}
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -716,7 +716,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
- from >> info.ca >> repair >> dontCheckSigs;
+ info.ca = parseContentAddressOpt(readString(from));
+ from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs)
dontCheckSigs = false;
if (!trusted)
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index a79b78db6..6c49075ba 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -8,11 +8,6 @@
namespace nix {
-std::string DerivationOutputHash::printMethodAlgo() const {
- return makeFileIngestionPrefix(method) + printHashType(*hash.type);
-}
-
-
const StorePath & BasicDerivation::findOutput(const string & id) const
{
auto i = outputs.find(id);
@@ -113,7 +108,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
- std::optional<DerivationOutputHash> fsh;
+ std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@@ -121,7 +116,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
- fsh = DerivationOutputHash {
+ fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
@@ -411,7 +406,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
auto hashAlgo = readString(in);
const auto hash = readString(in);
- std::optional<DerivationOutputHash> fsh;
+ std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@@ -419,7 +414,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
- fsh = DerivationOutputHash {
+ fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 7b677ca49..68c53c1ff 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -1,8 +1,9 @@
#pragma once
+#include "path.hh"
#include "types.hh"
#include "hash.hh"
-#include "store-api.hh"
+#include "content-address.hh"
#include <map>
@@ -12,18 +13,10 @@ namespace nix {
/* Abstract syntax of derivations. */
-/// Pair of a hash, and how the file system was ingested
-struct DerivationOutputHash {
- FileIngestionMethod method;
- Hash hash;
- std::string printMethodAlgo() const;
-};
-
struct DerivationOutput
{
StorePath path;
- std::optional<DerivationOutputHash> hash; /* hash used for expected hash computation */
- void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
+ std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -76,6 +69,7 @@ struct Derivation : BasicDerivation
class Store;
+enum RepairFlag : bool { NoRepair = false, Repair = true };
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store,
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 45c70fad6..5657aa593 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
- conn->from >> info->ca;
+ info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}
@@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
<< info.narSize
<< info.ultimate
<< info.sigs
- << info.ca;
+ << renderContentAddress(info.ca);
try {
copyNAR(source, conn->to);
} catch (...) {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 39065e600..659f80e77 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -580,7 +580,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
- if (info.ca != "" && !info.isContentAddressed(*this))
+ if (info.ca.has_value() && !info.isContentAddressed(*this))
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));
@@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state,
(info.narSize, info.narSize != 0)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
- (info.ca, !info.ca.empty())
+ (renderContentAddress(info.ca), (bool) info.ca)
.exec();
uint64_t id = sqlite3_last_insert_rowid(state.db);
@@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
- if (s) info->ca = s;
+ if (s) info->ca = parseContentAddressOpt(s);
/* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -689,7 +689,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
(info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
- (info.ca, !info.ca.empty())
+ (renderContentAddress(info.ca), (bool) info.ca)
(printStorePath(info.path))
.exec();
}
@@ -851,7 +851,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
auto subPath(path.first);
// recompute store path so that we can use a different store root
- if (path.second && (hasPrefix(*path.second, "fixed:") || hasPrefix(*path.second, "text:"))) {
+ if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
if (sub->storeDir == storeDir)
assert(subPath == path.first);
@@ -997,15 +997,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
deletePath(realPath);
- if (info.ca != "" &&
- !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
- || info.references.empty()))
+ // text hashing has long been allowed to have non-self-references because it is used for drv files.
+ bool refersToSelf = info.references.count(info.path) > 0;
+ if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
settings.requireExperimentalFeature("ca-references");
/* While restoring the path from the NAR, compute the hash
of the NAR. */
std::unique_ptr<AbstractHashSink> hashSink;
- if (info.ca == "" || !info.references.count(info.path))
+ if (!info.ca.has_value() || !info.references.count(info.path))
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
@@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
ValidPathInfo info(dstPath);
info.narHash = hash.first;
info.narSize = hash.second;
- info.ca = makeFixedOutputCA(method, h);
+ info.ca = FixedOutputHash { .method = method, .hash = h };
registerValidPath(info);
}
@@ -1155,7 +1155,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
info.narHash = narHash;
info.narSize = sink.s->size();
info.references = references;
- info.ca = "text:" + hash.to_string(Base32, true);
+ info.ca = TextHash { .hash = hash };
registerValidPath(info);
}
@@ -1266,7 +1266,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
std::unique_ptr<AbstractHashSink> hashSink;
- if (info->ca == "" || !info->references.count(info->path))
+ if (!info->ca || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index a01e81b78..3256a48e0 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -108,28 +108,11 @@ void Store::computeFSClosure(const StorePath & startPath,
}
-std::optional<std::string> getDerivationCA(const BasicDerivation & drv)
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
{
- auto outputHashMode = drv.env.find("outputHashMode");
- auto outputHash = drv.env.find("outputHash");
-
- std::optional<HashType> ht = std::nullopt;
- auto outputHashAlgo = drv.env.find("outputHashAlgo");
- if (outputHashAlgo != drv.env.end())
- ht = parseHashTypeOpt(outputHashAlgo->second);
-
- if (outputHashMode != drv.env.end() && outputHash != drv.env.end()) {
- auto h = Hash(outputHash->second, ht);
- FileIngestionMethod ingestionMethod;
- if (outputHashMode->second == "recursive")
- ingestionMethod = FileIngestionMethod::Recursive;
- else if (outputHashMode->second == "flat")
- ingestionMethod = FileIngestionMethod::Flat;
- else
- throw Error("unknown outputHashMode: '%s'", outputHashMode->second);
- return makeFixedOutputCA(ingestionMethod, h);
- }
-
+ auto out = drv.outputs.find("out");
+ if (out != drv.outputs.end())
+ return out->second.hash;
return std::nullopt;
}
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 552970248..012dea6ea 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -203,7 +203,7 @@ public:
narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
- narInfo->ca = queryNAR.getStr(11);
+ narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11));
return {oValid, narInfo};
});
@@ -237,7 +237,7 @@ public:
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
(concatStringsSep(" ", info->sigs))
- (info->ca)
+ (renderContentAddress(info->ca))
(time(0)).exec();
} else {
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index bb4448c90..04550ed97 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
- if (!ca.empty()) corrupt();
- ca = value;
+ if (ca) corrupt();
+ // FIXME: allow blank ca or require skipping field?
+ ca = parseContentAddressOpt(value);
}
pos = eol + 1;
@@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const
for (auto sig : sigs)
res += "Sig: " + sig + "\n";
- if (!ca.empty())
- res += "CA: " + ca + "\n";
+ if (ca)
+ res += "CA: " + renderContentAddress(*ca) + "\n";
return res;
}
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index d24d1eb4f..0b8e8d031 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -1,4 +1,4 @@
-#include "derivations.hh"
+#include "store-api.hh"
#include <nlohmann/json_fwd.hpp>
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index 91962d114..a07186323 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -1,5 +1,6 @@
#pragma once
+#include "content-address.hh"
#include "types.hh"
namespace nix {
@@ -62,16 +63,11 @@ public:
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
-typedef std::map<StorePath, std::optional<std::string>> StorePathCAMap;
+typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
-enum struct FileIngestionMethod : uint8_t {
- Flat = false,
- Recursive = true
-};
-
struct StorePathWithOutputs
{
StorePath path;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f6158685c..32c736d08 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -38,15 +38,12 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
-std::map<StorePath, std::optional<std::string>> readStorePathCAMap(const Store & store, Source & from)
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
{
StorePathCAMap paths;
auto count = readNum<size_t>(from);
- while (count--) {
- auto path = readString(from);
- auto ca = readString(from);
- paths.insert_or_assign(store.parseStorePath(path), !ca.empty() ? std::optional(ca) : std::nullopt);
- }
+ while (count--)
+ paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
return paths;
}
@@ -55,7 +52,7 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
out << paths.size();
for (auto & i : paths) {
out << store.printStorePath(i.first);
- out << (i.second ? *i.second : "");
+ out << renderContentAddress(i.second);
}
}
@@ -407,7 +404,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
conn->from >> info->ultimate;
info->sigs = readStrings<StringSet>(conn->from);
- conn->from >> info->ca;
+ info->ca = parseContentAddressOpt(readString(conn->from));
}
}
callback(std::move(info));
@@ -491,7 +488,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< info.narHash.to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
- << info.ultimate << info.sigs << info.ca
+ << info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
if (!tunnel) copyNAR(source, conn->to);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0458fd9a6..c4fc1c861 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -192,18 +192,22 @@ StorePath Store::makeFixedOutputPath(
}
}
-StorePath Store::makeFixedOutputPathFromCA(std::string_view name, std::string ca,
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const
{
- if (hasPrefix(ca, "fixed:")) {
- FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 };
- Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6));
- return makeFixedOutputPath(ingestionMethod, hash, name, references, hasSelfReference);
- } else if (hasPrefix(ca, "text:")) {
- Hash hash(std::string(ca, 5));
- return makeTextPath(name, hash, references);
- } else
- throw Error("'%s' is not a valid ca", ca);
+ // New template
+ return std::visit(overloaded {
+ [&](TextHash th) {
+ return makeTextPath(name, th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
+ }
+ }, ca);
}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
@@ -484,8 +488,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
jsonRefs.elem(printStorePath(ref));
}
- if (info->ca != "")
- jsonPath.attr("ca", info->ca);
+ if (info->ca)
+ jsonPath.attr("ca", renderContentAddress(info->ca));
std::pair<uint64_t, uint64_t> closureSizes;
@@ -595,9 +599,9 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0;
// recompute store path on the chance dstStore does it differently
- if (info->isContentAddressed(*srcStore) && info->references.empty()) {
+ if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info);
- info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca);
+ info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(info->path == info2->path);
info = info2;
@@ -674,8 +678,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath;
- if (info->isContentAddressed(*srcStore) && info->references.empty()) {
- storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
@@ -702,8 +706,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto info = srcStore->queryPathInfo(storePath);
auto storePathForDst = storePath;
- if (info->isContentAddressed(*srcStore) && info->references.empty()) {
- storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca);
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore->storeDir == srcStore->storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
@@ -807,41 +811,31 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
-
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
- auto warn = [&]() {
- logWarning(
- ErrorInfo{
- .name = "Path not content-addressed",
- .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
- });
- };
+ if (! ca) return false;
- if (hasPrefix(ca, "text:")) {
- Hash hash(ca.substr(5));
- if (store.makeTextPath(path.name(), hash, references) == path)
- return true;
- else
- warn();
- }
-
- else if (hasPrefix(ca, "fixed:")) {
- FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
- Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
- auto refs = references;
- bool hasSelfReference = false;
- if (refs.count(path)) {
- hasSelfReference = true;
- refs.erase(path);
+ auto caPath = std::visit(overloaded {
+ [&](TextHash th) {
+ return store.makeTextPath(path.name(), th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ auto refs = references;
+ bool hasSelfReference = false;
+ if (refs.count(path)) {
+ hasSelfReference = true;
+ refs.erase(path);
+ }
+ return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
}
- if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
- return true;
- else
- warn();
- }
+ }, *ca);
+
+ bool res = caPath == path;
+
+ if (!res)
+ printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
- return false;
+ return res;
}
@@ -872,25 +866,6 @@ Strings ValidPathInfo::shortRefs() const
}
-std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
- switch (m) {
- case FileIngestionMethod::Flat:
- return "";
- case FileIngestionMethod::Recursive:
- return "r:";
- default:
- throw Error("impossible, caught both cases");
- }
-}
-
-std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
-{
- return "fixed:"
- + makeFileIngestionPrefix(method)
- + hash.to_string(Base32, true);
-}
-
-
}
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 2b12879e3..c4a8d3890 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -2,12 +2,14 @@
#include "path.hh"
#include "hash.hh"
+#include "content-address.hh"
#include "serialise.hh"
#include "crypto.hh"
#include "lru-cache.hh"
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
+#include "derivations.hh"
#include <atomic>
#include <limits>
@@ -17,6 +19,7 @@
#include <memory>
#include <string>
#include <chrono>
+#include <variant>
namespace nix {
@@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
-struct BasicDerivation;
-struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
class JSONPlaceholder;
-enum RepairFlag : bool { NoRepair = false, Repair = true };
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@@ -111,7 +111,6 @@ struct SubstitutablePathInfo
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
-
struct ValidPathInfo
{
StorePath path;
@@ -140,21 +139,11 @@ struct ValidPathInfo
that a particular output path was produced by a derivation; the
path then implies the contents.)
- Ideally, the content-addressability assertion would just be a
- Boolean, and the store path would be computed from
- the name component, ‘narHash’ and ‘references’. However,
- 1) we've accumulated several types of content-addressed paths
- over the years; and 2) fixed-output derivations support
- multiple hash algorithms and serialisation methods (flat file
- vs NAR). Thus, ‘ca’ has one of the following forms:
-
- * ‘text:sha256:<sha256 hash of file contents>’: For paths
- computed by makeTextPath() / addTextToStore().
-
- * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
- makeFixedOutputPath() / addToStore().
+ Ideally, the content-addressability assertion would just be a Boolean,
+ and the store path would be computed from the name component, ‘narHash’
+ and ‘references’. However, we support many types of content addresses.
*/
- std::string ca;
+ std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
@@ -356,7 +345,7 @@ public:
StorePath makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references = {}) const;
- StorePath makeFixedOutputPathFromCA(std::string_view name, std::string ca,
+ StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
@@ -846,18 +835,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
bool hashGiven = false);
-/* Compute the prefix to the hash algorithm which indicates how the files were
- ingested. */
-std::string makeFileIngestionPrefix(const FileIngestionMethod m);
-
-/* Compute the content-addressability assertion (ValidPathInfo::ca)
- for paths created by makeFixedOutputPath() / addToStore(). */
-std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
-
-
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
-std::optional<std::string> getDerivationCA(const BasicDerivation & drv);
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
}
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index d2944674f..5a3e33c08 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -69,7 +69,7 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
-std::map<StorePath, std::optional<std::string>> readStorePathCAMap(const Store & store, Source & from);
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index e49eb4569..c8fcdfed0 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -223,7 +223,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
if (!ht)
throw BadHash("empty hash requires explicit hash type");
Hash h(*ht);
- warn("found empty hash, assuming '%s'", h.to_string(Base::SRI, true));
+ warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
return h;
} else
return Hash(hashStr, ht);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 4e02aa2bf..7d81bf54f 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs)
out << info->narSize // downloadSize
<< info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
- out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs;
+ out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs;
} catch (InvalidPath &) {
}
}
@@ -952,7 +952,7 @@ static void opServe(Strings opFlags, Strings opArgs)
info.references = readStorePaths<StorePathSet>(*store, in);
in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in);
- in >> info.ca;
+ info.ca = parseContentAddressOpt(readString(in));
if (info.narSize == 0)
throw Error("narInfo is too old and missing the narSize field");
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 7abb82556..48a4722ae 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -70,7 +70,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash;
info.narSize = sink.s->size();
- info.ca = makeFixedOutputCA(ingestionMethod, hash);
+ info.ca = std::optional { FixedOutputHash {
+ .method = ingestionMethod,
+ .hash = hash,
+ } };
if (!dryRun) {
auto source = StringSource { *sink.s };
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 8b85caf82..037987313 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -137,7 +137,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput {
.path = shellOutPath,
- .hash = DerivationOutputHash {
+ .hash = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash { },
},
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index f435192fc..b97c6d21f 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -1,5 +1,6 @@
#include "command.hh"
#include "hash.hh"
+#include "content-address.hh"
#include "legacy.hh"
#include "shared.hh"
#include "references.hh"
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 0ebb8f13b..fb36fc410 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -82,7 +82,10 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
if (hasSelfReference) info.references.insert(info.path);
info.narHash = narHash;
info.narSize = sink.s->size();
- info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
+ info.ca = FixedOutputHash {
+ .method = FileIngestionMethod::Recursive,
+ .hash = info.narHash,
+ };
if (!json)
printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index fb7bacc4c..b89a44f83 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
std::cout << '\t';
Strings ss;
if (info->ultimate) ss.push_back("ultimate");
- if (info->ca != "") ss.push_back("ca:" + info->ca);
+ if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca));
for (auto & sig : info->sigs) ss.push_back(sig);
std::cout << concatStringsSep(" ", ss);
}
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index d1aba08e3..bb5e4529b 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -87,7 +87,7 @@ struct CmdVerify : StorePathsCommand
if (!noContents) {
std::unique_ptr<AbstractHashSink> hashSink;
- if (info->ca == "")
+ if (!info->ca)
hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));