aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/manual/command-ref/conf-file.xml28
-rw-r--r--src/libstore/build.cc43
-rw-r--r--src/libstore/builtins/fetchurl.cc17
-rw-r--r--src/libstore/daemon.cc14
-rw-r--r--src/libstore/globals.hh3
-rw-r--r--src/libstore/local-store.cc26
-rw-r--r--src/libstore/local-store.hh2
-rw-r--r--src/libstore/misc.cc12
-rw-r--r--src/libstore/path.hh2
-rw-r--r--src/libstore/remote-store.cc37
-rw-r--r--src/libstore/remote-store.hh2
-rw-r--r--src/libstore/store-api.cc74
-rw-r--r--src/libstore/store-api.hh27
-rw-r--r--src/libstore/worker-protocol.hh4
-rw-r--r--src/nix/add-to-store.cc28
-rw-r--r--tests/check.sh14
-rw-r--r--tests/fetchurl.sh24
17 files changed, 234 insertions, 123 deletions
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1fa74a143..9c55526a3 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,34 +370,6 @@ false</literal>.</para>
</varlistentry>
- <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
-
- <listitem><para>A list of web servers used by
- <function>builtins.fetchurl</function> to obtain files by
- hash. The default is
- <literal>http://tarballs.nixos.org/</literal>. Given a hash type
- <replaceable>ht</replaceable> and a base-16 hash
- <replaceable>h</replaceable>, Nix will try to download the file
- from
- <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
- This allows files to be downloaded even if they have disappeared
- from their original URI. For example, given the default mirror
- <literal>http://tarballs.nixos.org/</literal>, when building the derivation
-
-<programlisting>
-builtins.fetchurl {
- url = "https://example.org/foo-1.2.3.tar.xz";
- sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-</programlisting>
-
- Nix will attempt to download this file from
- <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
- first. If it is not available there, if will try the original URI.</para></listitem>
-
- </varlistentry>
-
-
<varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
<listitem><para>The maximum number of parallel TCP connections
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 347fe1b99..55fc43cfb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
@@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation()
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & i : invalidOutputs)
- addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
+ addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted();
@@ -4270,6 +4270,10 @@ private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
+ /* The path the substituter refers to the path as. This will be
+ * different when the stores have different names. */
+ std::optional<StorePath> subPath;
+
/* The remaining substituters. */
std::list<ref<Store>> subs;
@@ -4303,8 +4307,11 @@ private:
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
+ /* Content address for recomputing store path */
+ std::optional<ContentAddress> ca;
+
public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair);
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@@ -4334,10 +4341,11 @@ public:
};
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair)
+SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
+ , ca(ca)
{
state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@@ -4412,14 +4420,18 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
- if (sub->storeDir != worker.store.storeDir) {
+ if (ca) {
+ subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+ if (sub->storeDir == worker.store.storeDir)
+ assert(subPath == storePath);
+ } else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
- info = sub->queryPathInfo(storePath);
+ info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
@@ -4438,6 +4450,19 @@ void SubstitutionGoal::tryNext()
throw;
}
+ if (info->path != storePath) {
+ if (info->isContentAddressed(*sub) && info->references.empty()) {
+ auto info2 = std::make_shared<ValidPathInfo>(*info);
+ info2->path = storePath;
+ info = info2;
+ } else {
+ printError("asked '%s' for '%s' but got '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+ tryNext();
+ return;
+ }
+ }
+
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
@@ -4527,7 +4552,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
- storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+ subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
@@ -4660,11 +4685,11 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) {
- goal = std::make_shared<SubstitutionGoal>(path, *this, repair);
+ goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
substitutionGoals.insert_or_assign(path, goal);
wakeUp(goal);
}
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index e630cf6f1..6585a480d 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,23 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
- /* We always have one output, and if it's a fixed-output derivation (as
- checked below) it must be the only output */
- auto & output = drv.outputs.begin()->second;
-
- /* Try the hashed mirrors first. */
- if (output.hash && output.hash->method == FileIngestionMethod::Flat)
- for (auto hashedMirror : settings.hashedMirrors.get())
- try {
- if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
- auto & h = output.hash->hash;
- fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
- return;
- } catch (Error & e) {
- debug(e.what());
- }
-
- /* Otherwise try the specified URL. */
fetch(mainUrl);
}
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 6042bd610..843ec8ca8 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -602,7 +602,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos({path}, infos);
+ store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
logger->stopWork();
auto i = infos.find(path);
if (i == infos.end())
@@ -618,10 +618,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePathInfos: {
- auto paths = readStorePaths<StorePathSet>(*store, from);
- logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos(paths, infos);
+ StorePathCAMap pathsMap = {};
+ if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
+ auto paths = readStorePaths<StorePathSet>(*store, from);
+ for (auto & path : paths)
+ pathsMap.emplace(path, std::nullopt);
+ } else
+ pathsMap = readStorePathCAMap(*store, from);
+ logger->startWork();
+ store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 2fbcafff8..aa626ce1b 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -339,9 +339,6 @@ public:
"setuid/setgid bits or with file capabilities."};
#endif
- Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
- "A list of servers used by builtins.fetchurl to fetch files by hash."};
-
Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index eed225349..84b47273d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -846,20 +846,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
- if (sub->storeDir != storeDir) continue;
for (auto & path : paths) {
- if (infos.count(path)) continue;
- debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
+ auto subPath(path.first);
+
+ // recompute store path so that we can use a different store root
+ if (path.second) {
+ subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
+ if (sub->storeDir == storeDir)
+ assert(subPath == path.first);
+ if (subPath != path.first)
+ debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
+ } else if (sub->storeDir != storeDir) continue;
+
+ debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
- auto info = sub->queryPathInfo(path);
+ auto info = sub->queryPathInfo(subPath);
+
+ if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
+ continue;
+
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
- infos.insert_or_assign(path, SubstitutablePathInfo{
+ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index ff36cb00e..17f6202bb 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -139,7 +139,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index e68edb38c..3256a48e0 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -108,6 +108,14 @@ void Store::computeFSClosure(const StorePath & startPath,
}
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
+{
+ auto out = drv.outputs.find("out");
+ if (out != drv.outputs.end())
+ return out->second.hash;
+ return std::nullopt;
+}
+
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
unsigned long long & downloadSize_, unsigned long long & narSize_)
@@ -157,7 +165,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({outPath}, infos);
+ querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@@ -214,7 +222,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({path.path}, infos);
+ querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index e43a8b50c..b03a0f69d 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -64,6 +64,8 @@ typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
+typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
+
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8c648d671..d636dd765 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -38,6 +38,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
+{
+ StorePathCAMap paths;
+ auto count = readNum<size_t>(from);
+ while (count--)
+ paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
+ return paths;
+}
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
+{
+ out << paths.size();
+ for (auto & i : paths) {
+ out << store.printStorePath(i.first);
+ out << renderContentAddress(i.second);
+ }
+}
+
std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
{
std::map<string, StorePath> pathMap;
@@ -331,18 +349,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
{
- if (paths.empty()) return;
+ if (pathsMap.empty()) return;
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
- for (auto & i : paths) {
+ for (auto & i : pathsMap) {
SubstitutablePathInfo info;
- conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
+ conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
conn.processStderr();
unsigned int reply = readInt(conn->from);
if (reply == 0) continue;
@@ -352,13 +369,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
- infos.insert_or_assign(i, std::move(info));
+ infos.insert_or_assign(i.first, std::move(info));
}
} else {
conn->to << wopQuerySubstitutablePathInfos;
- writeStorePaths(*this, conn->to, paths);
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+ StorePathSet paths;
+ for (auto & path : pathsMap)
+ paths.insert(path.first);
+ writeStorePaths(*this, conn->to, paths);
+ } else
+ writeStorePathCAMap(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index fb2052752..290ffbfc8 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -56,7 +56,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c0a8bc9f6..189cf54bb 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -192,6 +192,23 @@ StorePath Store::makeFixedOutputPath(
}
}
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references, bool hasSelfReference) const
+{
+ // New template
+ return std::visit(overloaded {
+ [&](TextHash th) {
+ return makeTextPath(name, th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
+ }
+ }, ca);
+}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
@@ -591,6 +608,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0;
+ // recompute store path on the chance dstStore does it differently
+ if (info->ca && info->references.empty()) {
+ auto info2 = make_ref<ValidPathInfo>(*info);
+ info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(info->path == info2->path);
+ info = info2;
+ }
+
if (!info->narHash) {
StringSink sink;
srcStore->narFromPath({storePath}, sink);
@@ -626,7 +652,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
auto valid = dstStore->queryValidPaths(storePaths, substitute);
@@ -635,7 +661,11 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
- if (missing.empty()) return;
+ std::map<StorePath, StorePath> pathsMap;
+ for (auto & path : storePaths)
+ pathsMap.insert_or_assign(path, path);
+
+ if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@@ -653,15 +683,26 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
processGraph<Path>(pool,
PathSet(missing.begin(), missing.end()),
- [&](const Path & storePath) {
- if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
+ [&](const Path & storePathS) {
+ auto storePath = srcStore->parseStorePath(storePathS);
+
+ auto info = srcStore->queryPathInfo(storePath);
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (dstStore->isValidPath(storePathForDst)) {
nrDone++;
showProgress();
return PathSet();
}
- auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
-
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
@@ -671,9 +712,20 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
[&](const Path & storePathS) {
checkInterrupt();
- auto storePath = dstStore->parseStorePath(storePathS);
+ auto storePath = srcStore->parseStorePath(storePathS);
+ auto info = srcStore->queryPathInfo(storePath);
- if (!dstStore->isValidPath(storePath)) {
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
@@ -691,6 +743,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrDone++;
showProgress();
});
+
+ return pathsMap;
}
@@ -767,10 +821,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b122e05d6..d9d3fe185 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -343,7 +343,11 @@ public:
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
- const StorePathSet & references) const;
+ const StorePathSet & references = {}) const;
+
+ StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references = {},
+ bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
@@ -432,9 +436,10 @@ public:
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
- sizes) of a set of paths. If a path does not have substitute
- info, it's omitted from the resulting ‘infos’ map. */
- virtual void querySubstitutablePathInfos(const StorePathSet & paths,
+ sizes) of a map of paths to their optional ca values. If a path
+ does not have substitute info, it's omitted from the resulting
+ ‘infos’ map. */
+ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; };
/* Import a path into the store. */
@@ -731,11 +736,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
/* Copy store paths from one store to another. The paths may be copied
- in parallel. They are copied in a topologically sorted order
- (i.e. if A is a reference of B, then A is copied before B), but
- the set of store paths is not automatically closed; use
- copyClosure() for that. */
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+ in parallel. They are copied in a topologically sorted order (i.e.
+ if A is a reference of B, then A is copied before B), but the set
+ of store paths is not automatically closed; use copyClosure() for
+ that. Returns a map of what each path was copied to the dstStore
+ as. */
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
+ const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@@ -834,4 +841,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
+
}
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 8b538f6da..48fbb481f 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -70,6 +70,10 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
+
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
}
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f9d6de16e..48a4722ae 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
std::optional<std::string> namePart;
+ FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
CmdAddToStore()
{
@@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.labels = {"name"},
.handler = {&namePart},
});
+
+ addFlag({
+ .longName = "flat",
+ .shortName = 0,
+ .description = "add flat file to the Nix store",
+ .handler = {&ingestionMethod, FileIngestionMethod::Flat},
+ });
}
std::string description() override
@@ -45,12 +53,26 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
+ Hash hash;
+ switch (ingestionMethod) {
+ case FileIngestionMethod::Recursive: {
+ hash = narHash;
+ break;
+ }
+ case FileIngestionMethod::Flat: {
+ HashSink hsink(htSHA256);
+ readFile(path, hsink);
+ hash = hsink.finish().first;
+ break;
+ }
+ }
+
+ ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash;
info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = info.narHash,
+ .method = ingestionMethod,
+ .hash = hash,
} };
if (!dryRun) {
diff --git a/tests/check.sh b/tests/check.sh
index 5f25d04cb..5f4997e28 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
[ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log
-path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
+path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path
echo foo > $path
chmod -w $path
-nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --check
# Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
[[ $(cat $path) = foo ]]
-nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]]
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ]
echo -n > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
+nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ]
# Multiple failures with --keep-going
nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
+nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
[ "$status" = "110" ]
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 2535651b0..0f2044342 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
# Test fetching a flat file.
hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -14,7 +14,7 @@ clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh)
[[ $hash =~ ^sha256- ]]
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
cmp $outPath fetchurl.sh
-# Test the hashed mirror feature.
+# Test that we can substitute from a different store dir.
clearStore
-hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
+other_store=file://$TEST_ROOT/other_store?store=/fnord/store
+
+hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
-mirror=$TEST_ROOT/hashed-mirror
-rm -rf $mirror
-mkdir -p $mirror/sha512
-ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
+storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
# Test hashed mirrors with an SRI hash.
-nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
- --argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
+nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
+ --no-out-link --substituters $other_store
# Test unpacking a NAR.
rm -rf $TEST_ROOT/archive
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 23:33:16 +0000