aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc2
-rw-r--r--src/libstore/build.cc41
-rw-r--r--src/libstore/builtins/fetchurl.cc4
-rw-r--r--src/libstore/content-address.cc85
-rw-r--r--src/libstore/content-address.hh56
-rw-r--r--src/libstore/daemon.cc5
-rw-r--r--src/libstore/derivations.cc116
-rw-r--r--src/libstore/derivations.hh8
-rw-r--r--src/libstore/export-import.cc2
-rw-r--r--src/libstore/filetransfer.cc6
-rw-r--r--src/libstore/legacy-ssh-store.cc4
-rw-r--r--src/libstore/local-store.cc36
-rw-r--r--src/libstore/nar-info-disk-cache.cc4
-rw-r--r--src/libstore/nar-info.cc9
-rw-r--r--src/libstore/parsed-derivations.cc5
-rw-r--r--src/libstore/parsed-derivations.hh4
-rw-r--r--src/libstore/path.hh6
-rw-r--r--src/libstore/remote-store.cc6
-rw-r--r--src/libstore/store-api.cc89
-rw-r--r--src/libstore/store-api.hh36
20 files changed, 345 insertions, 179 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f8eff508c..9f52ddafa 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
narInfo->sigs.insert(sigs.begin(), sigs.end());
- auto narInfoFile = narInfoFileFor(narInfo->path);
-
writeNarInfo(narInfo);
}
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 53a0958aa..0c25897f8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -86,7 +86,7 @@ struct HookInstance;
/* A pointer to a goal. */
-class Goal;
+struct Goal;
class DerivationGoal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
@@ -1195,6 +1195,12 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+ if (parsedDrv->contentAddressed()) {
+ settings.requireExperimentalFeature("ca-derivations");
+ throw Error("ca-derivations isn't implemented yet");
+ }
+
+
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@@ -3708,14 +3714,11 @@ void DerivationGoal::registerOutputs()
/* Check that fixed-output derivations produced the right
outputs (i.e., the content hash should match the specified
hash). */
- std::string ca;
+ std::optional<ContentAddress> ca;
if (fixedOutput) {
- FileIngestionMethod outputHashMode; Hash h;
- i.second.parseHashInfo(outputHashMode, h);
-
- if (outputHashMode == FileIngestionMethod::Flat) {
+ if (i.second.hash->method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@@ -3726,20 +3729,22 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
- Hash h2 = outputHashMode == FileIngestionMethod::Recursive
- ? hashPath(h.type, actualPath).first
- : hashFile(h.type, actualPath);
+ Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
+ ? hashPath(*i.second.hash->hash.type, actualPath).first
+ : hashFile(*i.second.hash->hash.type, actualPath);
- auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
+ auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
- if (h != h2) {
+ if (i.second.hash->hash != h2) {
/* Throw an error after registering the path as
valid. */
worker.hashMismatch = true;
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
- worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
+ worker.store.printStorePath(dest),
+ i.second.hash->hash.to_string(SRI, true),
+ h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@@ -3759,7 +3764,10 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);
- ca = makeFixedOutputCA(outputHashMode, h2);
+ ca = FixedOutputHash {
+ .method = i.second.hash->method,
+ .hash = h2,
+ };
}
/* Get rid of all weird permissions. This also checks that
@@ -3832,7 +3840,10 @@ void DerivationGoal::registerOutputs()
info.ca = ca;
worker.store.signPathInfo(info);
- if (!info.references.empty()) info.ca.clear();
+ if (!info.references.empty()) {
+ // FIXME don't we have an experimental feature for fixed output with references?
+ info.ca = {};
+ }
infos.emplace(i.first, std::move(info));
}
@@ -4992,7 +5003,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
- HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
+ HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 2048f8f87..1cfe4a46a 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
- auto ht = parseHashType(getAttr("outputHashAlgo"));
+ auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
- fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+ fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
new file mode 100644
index 000000000..3d753836f
--- /dev/null
+++ b/src/libstore/content-address.cc
@@ -0,0 +1,85 @@
+#include "content-address.hh"
+
+namespace nix {
+
+std::string FixedOutputHash::printMethodAlgo() const {
+ return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+}
+
+std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+ switch (m) {
+ case FileIngestionMethod::Flat:
+ return "";
+ case FileIngestionMethod::Recursive:
+ return "r:";
+ default:
+ throw Error("impossible, caught both cases");
+ }
+}
+
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+{
+ return "fixed:"
+ + makeFileIngestionPrefix(method)
+ + hash.to_string(Base32, true);
+}
+
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+std::string renderContentAddress(ContentAddress ca) {
+ return std::visit(overloaded {
+ [](TextHash th) {
+ return "text:" + th.hash.to_string(Base32, true);
+ },
+ [](FixedOutputHash fsh) {
+ return makeFixedOutputCA(fsh.method, fsh.hash);
+ }
+ }, ca);
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+ auto prefixSeparator = rawCa.find(':');
+ if (prefixSeparator != string::npos) {
+ auto prefix = string(rawCa, 0, prefixSeparator);
+ if (prefix == "text") {
+ auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+ Hash hash = Hash(string(hashTypeAndHash));
+ if (*hash.type != htSHA256) {
+ throw Error("parseContentAddress: the text hash should have type SHA256");
+ }
+ return TextHash { hash };
+ } else if (prefix == "fixed") {
+ // This has to be an inverse of makeFixedOutputCA
+ auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+ if (methodAndHash.substr(0,2) == "r:") {
+ std::string_view hashRaw = methodAndHash.substr(2,string::npos);
+ return FixedOutputHash {
+ .method = FileIngestionMethod::Recursive,
+ .hash = Hash(string(hashRaw)),
+ };
+ } else {
+ std::string_view hashRaw = methodAndHash;
+ return FixedOutputHash {
+ .method = FileIngestionMethod::Flat,
+ .hash = Hash(string(hashRaw)),
+ };
+ }
+ } else {
+ throw Error("parseContentAddress: format not recognized; has to be text or fixed");
+ }
+ } else {
+ throw Error("Not a content address because it lacks an appropriate prefix");
+ }
+};
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
+ return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
+};
+
+std::string renderContentAddress(std::optional<ContentAddress> ca) {
+ return ca ? renderContentAddress(*ca) : "";
+}
+
+}
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
new file mode 100644
index 000000000..ba4797f5b
--- /dev/null
+++ b/src/libstore/content-address.hh
@@ -0,0 +1,56 @@
+#pragma once
+
+#include <variant>
+#include "hash.hh"
+
+namespace nix {
+
+enum struct FileIngestionMethod : uint8_t {
+ Flat = false,
+ Recursive = true
+};
+
+struct TextHash {
+ Hash hash;
+};
+
+/// Pair of a hash, and how the file system was ingested
+struct FixedOutputHash {
+ FileIngestionMethod method;
+ Hash hash;
+ std::string printMethodAlgo() const;
+};
+
+/*
+ We've accumulated several types of content-addressed paths over the years;
+ fixed-output derivations support multiple hash algorithms and serialisation
+ methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
+
+ * ‘text:sha256:<sha256 hash of file contents>’: For paths
+ computed by makeTextPath() / addTextToStore().
+
+ * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
+ makeFixedOutputPath() / addToStore().
+*/
+typedef std::variant<
+ TextHash, // for paths computed by makeTextPath() / addTextToStore
+ FixedOutputHash // for path computed by makeFixedOutputPath
+> ContentAddress;
+
+/* Compute the prefix to the hash algorithm which indicates how the files were
+ ingested. */
+std::string makeFileIngestionPrefix(const FileIngestionMethod m);
+
+/* Compute the content-addressability assertion (ValidPathInfo::ca)
+ for paths created by makeFixedOutputPath() / addToStore(). */
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
+
+std::string renderContentAddress(ContentAddress ca);
+
+std::string renderContentAddress(std::optional<ContentAddress> ca);
+
+ContentAddress parseContentAddress(std::string_view rawCa);
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+
+}
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index e370e278c..842aef20c 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -652,7 +652,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
- << info->ca;
+ << renderContentAddress(info->ca);
}
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -710,7 +710,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
- from >> info.ca >> repair >> dontCheckSigs;
+ info.ca = parseContentAddressOpt(readString(from));
+ from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs)
dontCheckSigs = false;
if (!trusted)
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index b95f7bfdc..42551ef6b 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -8,25 +8,6 @@
namespace nix {
-
-void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
-{
- recursive = FileIngestionMethod::Flat;
- string algo = hashAlgo;
-
- if (string(algo, 0, 2) == "r:") {
- recursive = FileIngestionMethod::Recursive;
- algo = string(algo, 2);
- }
-
- HashType hashType = parseHashType(algo);
- if (hashType == htUnknown)
- throw Error("unknown hash algorithm '%s'", algo);
-
- hash = Hash(this->hash, hashType);
-}
-
-
const StorePath & BasicDerivation::findOutput(const string & id) const
{
auto i = outputs.find(id);
@@ -120,6 +101,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
+static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
+{
+ expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+ expect(str, ","); auto hashAlgo = parseString(str);
+ expect(str, ","); const auto hash = parseString(str);
+ expect(str, ")");
+
+ std::optional<FixedOutputHash> fsh;
+ if (hashAlgo != "") {
+ auto method = FileIngestionMethod::Flat;
+ if (string(hashAlgo, 0, 2) == "r:") {
+ method = FileIngestionMethod::Recursive;
+ hashAlgo = string(hashAlgo, 2);
+ }
+ const HashType hashType = parseHashType(hashAlgo);
+ fsh = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash(hash, hashType),
+ };
+ }
+
+ return DerivationOutput {
+ .path = std::move(path),
+ .hash = std::move(fsh),
+ };
+}
+
+
static Derivation parseDerivation(const Store & store, const string & s)
{
Derivation drv;
@@ -129,15 +138,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str);
- expect(str, ","); auto path = store.parseStorePath(parsePath(str));
- expect(str, ","); auto hashAlgo = parseString(str);
- expect(str, ","); auto hash = parseString(str);
- expect(str, ")");
- drv.outputs.emplace(id, DerivationOutput {
- .path = std::move(path),
- .hashAlgo = std::move(hashAlgo),
- .hash = std::move(hash)
- });
+ auto output = parseDerivationOutput(store, str);
+ drv.outputs.emplace(std::move(id), std::move(output));
}
/* Parse the list of input derivations. */
@@ -263,8 +265,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
- s += ','; printUnquotedString(s, i.second.hashAlgo);
- s += ','; printUnquotedString(s, i.second.hash);
+ s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
+ s += ','; printUnquotedString(s,
+ i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ')';
}
@@ -320,7 +323,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
- outputs.begin()->second.hash != "";
+ outputs.begin()->second.hash;
}
@@ -353,8 +356,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:"
- + i->second.hashAlgo + ":"
- + i->second.hash + ":"
+ + i->second.hash->printMethodAlgo() + ":"
+ + i->second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path));
}
@@ -397,6 +400,31 @@ StorePathSet BasicDerivation::outputPaths() const
return paths;
}
+static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+{
+ auto path = store.parseStorePath(readString(in));
+ auto hashAlgo = readString(in);
+ auto hash = readString(in);
+
+ std::optional<FixedOutputHash> fsh;
+ if (hashAlgo != "") {
+ auto method = FileIngestionMethod::Flat;
+ if (string(hashAlgo, 0, 2) == "r:") {
+ method = FileIngestionMethod::Recursive;
+ hashAlgo = string(hashAlgo, 2);
+ }
+ auto hashType = parseHashType(hashAlgo);
+ fsh = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash(hash, hashType),
+ };
+ }
+
+ return DerivationOutput {
+ .path = std::move(path),
+ .hash = std::move(fsh),
+ };
+}
StringSet BasicDerivation::outputNames() const
{
@@ -413,14 +441,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
- auto path = store.parseStorePath(readString(in));
- auto hashAlgo = readString(in);
- auto hash = readString(in);
- drv.outputs.emplace(name, DerivationOutput {
- .path = std::move(path),
- .hashAlgo = std::move(hashAlgo),
- .hash = std::move(hash)
- });
+ auto output = readDerivationOutput(in, store);
+ drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@@ -441,8 +463,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
- for (auto & i : drv.outputs)
- out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
+ for (auto & i : drv.outputs) {
+ out << i.first
+ << store.printStorePath(i.second.path);
+ if (i.second.hash) {
+ out << i.second.hash->printMethodAlgo()
+ << i.second.hash->hash.to_string(Base16, false);
+ } else {
+ out << "" << "";
+ }
+ }
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index d349c6d4d..68c53c1ff 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -1,8 +1,9 @@
#pragma once
+#include "path.hh"
#include "types.hh"
#include "hash.hh"
-#include "store-api.hh"
+#include "content-address.hh"
#include <map>
@@ -15,9 +16,7 @@ namespace nix {
struct DerivationOutput
{
StorePath path;
- std::string hashAlgo; /* hash used for expected hash computation */
- std::string hash; /* expected hash, may be null */
- void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
+ std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -70,6 +69,7 @@ struct Derivation : BasicDerivation
class Store;
+enum RepairFlag : bool { NoRepair = false, Repair = true };
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store,
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index cb9da027d..57b7e9590 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -55,7 +55,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
- if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
+ if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index cd15cde62..beb508e67 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -73,6 +73,7 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0;
+ inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
@@ -99,7 +100,7 @@ struct curlFileTransfer : public FileTransfer
/* Only write data to the sink if this is a
successful response. */
- if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+ if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
}
@@ -356,8 +357,7 @@ struct curlFileTransfer : public FileTransfer
if (writeException)
failEx(writeException);
- else if (code == CURLE_OK &&
- (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
+ else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
{
result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 45c70fad6..5657aa593 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
- conn->from >> info->ca;
+ info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}
@@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
<< info.narSize
<< info.ultimate
<< info.sigs
- << info.ca;
+ << renderContentAddress(info.ca);
try {
copyNAR(source, conn->to);
} catch (...) {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2ddb43cb7..5b462c5b3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
- FileIngestionMethod method; Hash h;
- out->second.parseHashInfo(method, h);
-
- check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
+ check(
+ makeFixedOutputPath(
+ out->second.hash->method,
+ out->second.hash->hash,
+ drvName),
+ out->second.path, "out");
}
else {
@@ -578,7 +580,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
- if (info.ca != "" && !info.isContentAddressed(*this))
+ if (info.ca.has_value() && !info.isContentAddressed(*this))
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));
@@ -590,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state,
(info.narSize, info.narSize != 0)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
- (info.ca, !info.ca.empty())
+ (renderContentAddress(info.ca), (bool) info.ca)
.exec();
uint64_t id = state.db.getLastInsertedRowId();
@@ -664,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
- if (s) info->ca = s;
+ if (s) info->ca = parseContentAddressOpt(s);
/* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -687,7 +689,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
(info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
- (info.ca, !info.ca.empty())
+ (renderContentAddress(info.ca), (bool) info.ca)
(printStorePath(info.path))
.exec();
}
@@ -983,15 +985,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
deletePath(realPath);
- if (info.ca != "" &&
- !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
- || info.references.empty()))
+ // text hashing has long been allowed to have non-self-references because it is used for drv files.
+ bool refersToSelf = info.references.count(info.path) > 0;
+ if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
settings.requireExperimentalFeature("ca-references");
/* While restoring the path from the NAR, compute the hash
of the NAR. */
std::unique_ptr<AbstractHashSink> hashSink;
- if (info.ca == "" || !info.references.count(info.path))
+ if (!info.ca.has_value() || !info.references.count(info.path))
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
@@ -1077,7 +1079,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
ValidPathInfo info(dstPath);
info.narHash = hash.first;
info.narSize = hash.second;
- info.ca = makeFixedOutputCA(method, h);
+ info.ca = FixedOutputHash { .method = method, .hash = h };
registerValidPath(info);
}
@@ -1141,7 +1143,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
info.narHash = narHash;
info.narSize = sink.s->size();
info.references = references;
- info.ca = "text:" + hash.to_string(Base32, true);
+ info.ca = TextHash { .hash = hash };
registerValidPath(info);
}
@@ -1252,10 +1254,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
std::unique_ptr<AbstractHashSink> hashSink;
- if (info->ca == "" || !info->references.count(info->path))
- hashSink = std::make_unique<HashSink>(info->narHash.type);
+ if (!info->ca || !info->references.count(info->path))
+ hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
- hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
+ hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 552970248..012dea6ea 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -203,7 +203,7 @@ public:
narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
- narInfo->ca = queryNAR.getStr(11);
+ narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11));
return {oValid, narInfo};
});
@@ -237,7 +237,7 @@ public:
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
(concatStringsSep(" ", info->sigs))
- (info->ca)
+ (renderContentAddress(info->ca))
(time(0)).exec();
} else {
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index bb4448c90..04550ed97 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
- if (!ca.empty()) corrupt();
- ca = value;
+ if (ca) corrupt();
+ // FIXME: allow blank ca or require skipping field?
+ ca = parseContentAddressOpt(value);
}
pos = eol + 1;
@@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const
for (auto sig : sigs)
res += "Sig: " + sig + "\n";
- if (!ca.empty())
- res += "CA: " + ca + "\n";
+ if (ca)
+ res += "CA: " + renderContentAddress(*ca) + "\n";
return res;
}
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 24f848e46..c7797b730 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -117,4 +117,9 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true);
}
+bool ParsedDerivation::contentAddressed() const
+{
+ return getBoolAttr("__contentAddressed", false);
+}
+
}
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 7621342d7..0b8e8d031 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -1,4 +1,4 @@
-#include "derivations.hh"
+#include "store-api.hh"
#include <nlohmann/json_fwd.hpp>
@@ -34,6 +34,8 @@ public:
bool willBuildLocally() const;
bool substitutesAllowed() const;
+
+ bool contentAddressed() const;
};
}
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index aaebd3ec3..4f79843fe 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -1,5 +1,6 @@
#pragma once
+#include "content-address.hh"
#include "types.hh"
namespace nix {
@@ -65,11 +66,6 @@ typedef std::vector<StorePath> StorePaths;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
-enum struct FileIngestionMethod : uint8_t {
- Flat = false,
- Recursive = true
-};
-
struct StorePathWithOutputs
{
StorePath path;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index fc5ab5865..b7cc7a5fc 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -228,7 +228,7 @@ struct ConnectionHandle
~ConnectionHandle()
{
- if (!daemonException && std::uncaught_exception()) {
+ if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
debug("closing daemon connection because of an exception");
}
@@ -381,7 +381,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
conn->from >> info->ultimate;
info->sigs = readStrings<StringSet>(conn->from);
- conn->from >> info->ca;
+ info->ca = parseContentAddressOpt(readString(conn->from));
}
}
callback(std::move(info));
@@ -465,7 +465,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< info.narHash.to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
- << info.ultimate << info.sigs << info.ca
+ << info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
if (!tunnel) copyNAR(source, conn->to);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index aae227bae..e4a4ae11e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -61,7 +61,7 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
/* Store paths have the following form:
- <store>/<h>-<name>
+ <realized-path> = <store>/<h>-<name>
where
@@ -85,11 +85,14 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
<type> = one of:
"text:<r1>:<r2>:...<rN>"
for plain text files written to the store using
- addTextToStore(); <r1> ... <rN> are the references of the
- path.
- "source"
+ addTextToStore(); <r1> ... <rN> are the store paths referenced
+ by this path, in the form described by <realized-path>
+ "source:<r1>:<r2>:...:<rN>:self"
for paths copied to the store using addToStore() when recursive
- = true and hashAlgo = "sha256"
+ = true and hashAlgo = "sha256". Just like in the text case, we
+ can have the store paths referenced by the path.
+ Additionally, we can have an optional :self label to denote self
+ reference.
"output:<id>"
for either the outputs created by derivations, OR paths copied
to the store using addToStore() with recursive != true or
@@ -117,6 +120,12 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
the contents of the path (or expected contents of the
path for fixed-output derivations)
+ Note that since an output derivation has always type output, while
+ something added by addToStore can have type output or source depending
+ on the hash, this means that the same input can be hashed differently
+ if added to the store via addToStore or via a derivation, in the sha256
+ recursive case.
+
It would have been nicer to handle fixed-output derivations under
"source", e.g. have something like "source:<rec><algo>", but we're
stuck with this for now...
@@ -164,20 +173,20 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(
- FileIngestionMethod recursive,
+ FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
- if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
+ if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
- + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+ + makeFileIngestionPrefix(method)
+ hash.to_string(Base16, true) + ":"),
name);
}
@@ -462,8 +471,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
jsonRefs.elem(printStorePath(ref));
}
- if (info->ca != "")
- jsonPath.attr("ca", info->ca);
+ if (info->ca)
+ jsonPath.attr("ca", renderContentAddress(info->ca));
std::pair<uint64_t, uint64_t> closureSizes;
@@ -748,41 +757,35 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
- auto warn = [&]() {
- logWarning(
- ErrorInfo{
- .name = "Path not content-addressed",
- .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
- });
- };
-
- if (hasPrefix(ca, "text:")) {
- Hash hash(ca.substr(5));
- if (store.makeTextPath(path.name(), hash, references) == path)
- return true;
- else
- warn();
- }
+ if (! ca) return false;
- else if (hasPrefix(ca, "fixed:")) {
- FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
- Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
- auto refs = references;
- bool hasSelfReference = false;
- if (refs.count(path)) {
- hasSelfReference = true;
- refs.erase(path);
+ auto caPath = std::visit(overloaded {
+ [&](TextHash th) {
+ return store.makeTextPath(path.name(), th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ auto refs = references;
+ bool hasSelfReference = false;
+ if (refs.count(path)) {
+ hasSelfReference = true;
+ refs.erase(path);
+ }
+ return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
}
- if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
- return true;
- else
- warn();
- }
+ }, *ca);
+
+ bool res = caPath == path;
- return false;
+ if (!res)
+ printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
+
+ return res;
}
@@ -813,14 +816,6 @@ Strings ValidPathInfo::shortRefs() const
}
-std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
-{
- return "fixed:"
- + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
- + hash.to_string(Base32, true);
-}
-
-
}
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6f4dd959c..25d78c297 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -2,12 +2,14 @@
#include "path.hh"
#include "hash.hh"
+#include "content-address.hh"
#include "serialise.hh"
#include "crypto.hh"
#include "lru-cache.hh"
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
+#include "derivations.hh"
#include <atomic>
#include <limits>
@@ -17,6 +19,7 @@
#include <memory>
#include <string>
#include <chrono>
+#include <variant>
namespace nix {
@@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
-struct BasicDerivation;
-struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
class JSONPlaceholder;
-enum RepairFlag : bool { NoRepair = false, Repair = true };
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@@ -111,7 +111,6 @@ struct SubstitutablePathInfo
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
-
struct ValidPathInfo
{
StorePath path;
@@ -140,21 +139,11 @@ struct ValidPathInfo
that a particular output path was produced by a derivation; the
path then implies the contents.)
- Ideally, the content-addressability assertion would just be a
- Boolean, and the store path would be computed from
- the name component, ‘narHash’ and ‘references’. However,
- 1) we've accumulated several types of content-addressed paths
- over the years; and 2) fixed-output derivations support
- multiple hash algorithms and serialisation methods (flat file
- vs NAR). Thus, ‘ca’ has one of the following forms:
-
- * ‘text:sha256:<sha256 hash of file contents>’: For paths
- computed by makeTextPath() / addTextToStore().
-
- * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
- makeFixedOutputPath() / addToStore().
+ Ideally, the content-addressability assertion would just be a Boolean,
+ and the store path would be computed from the name component, ‘narHash’
+ and ‘references’. However, we support many types of content addresses.
*/
- std::string ca;
+ std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
@@ -189,9 +178,10 @@ struct ValidPathInfo
Strings shortRefs() const;
- ValidPathInfo(const StorePath & path) : path(path) { }
+ ValidPathInfo(const ValidPathInfo & other) = default;
- ValidPathInfo(StorePath && path) : path(std::move(path)) { }
+ ValidPathInfo(StorePath && path) : path(std::move(path)) { };
+ ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
@@ -838,12 +828,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
bool hashGiven = false);
-
-/* Compute the content-addressability assertion (ValidPathInfo::ca)
- for paths created by makeFixedOutputPath() / addToStore(). */
-std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
-
-
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 19:29:58 +0000