aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/libexpr/common-eval-args.cc8
-rw-r--r--src/libexpr/parser.y6
-rw-r--r--src/libexpr/primops.cc67
-rw-r--r--src/libexpr/primops/fetchTree.cc77
-rw-r--r--src/libstore/download.cc146
-rw-r--r--src/libstore/download.hh34
-rw-r--r--src/libstore/fetchers/fetchers.hh1
-rw-r--r--src/libstore/fetchers/tarball.cc17
-rwxr-xr-xsrc/nix-channel/nix-channel.cc17
9 files changed, 113 insertions, 260 deletions
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 59a2b9c89..9ebb6a835 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -4,7 +4,9 @@
#include "util.hh"
#include "eval.hh"
#include "fetchers/registry.hh"
+#include "fetchers/fetchers.hh"
#include "flake/flakeref.hh"
+#include "store-api.hh"
namespace nix {
@@ -68,9 +70,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
Path lookupFileArg(EvalState & state, string s)
{
if (isUri(s)) {
- CachedDownloadRequest request(s);
- request.unpack = true;
- return getDownloader()->downloadCached(state.store, request).path;
+ return state.store->toRealPath(
+ fetchers::downloadTarball(
+ state.store, resolveUri(s), "source", false).storePath);
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2);
return state.findFile(p);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 9c769e803..dd5436c04 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -545,6 +545,7 @@ formal
#include "eval.hh"
#include "download.hh"
+#include "fetchers/fetchers.hh"
#include "store-api.hh"
@@ -687,9 +688,8 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) {
try {
- CachedDownloadRequest request(elem.second);
- request.unpack = true;
- res = { true, getDownloader()->downloadCached(store, request).path };
+ res = { true, store->toRealPath(fetchers::downloadTarball(
+ store, resolveUri(elem.second), "source", false).storePath) };
} catch (DownloadError & e) {
printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
res = { false, "" };
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index b0f8b4414..fb7d5497b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1,6 +1,5 @@
#include "archive.hh"
#include "derivations.hh"
-#include "download.hh"
#include "eval-inline.hh"
#include "eval.hh"
#include "globals.hh"
@@ -2047,68 +2046,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
/*************************************************************
- * Networking
- *************************************************************/
-
-
-void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
- const string & who, bool unpack, const std::string & defaultName)
-{
- CachedDownloadRequest request("");
- request.unpack = unpack;
- request.name = defaultName;
-
- state.forceValue(*args[0]);
-
- if (args[0]->type == tAttrs) {
-
- state.forceAttrs(*args[0], pos);
-
- for (auto & attr : *args[0]->attrs) {
- string n(attr.name);
- if (n == "url")
- request.uri = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "sha256")
- request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
- else if (n == "name")
- request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
- else
- throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
- }
-
- if (request.uri.empty())
- throw EvalError(format("'url' argument required, at %1%") % pos);
-
- } else
- request.uri = state.forceStringNoCtx(*args[0], pos);
-
- state.checkURI(request.uri);
-
- if (evalSettings.pureEval && !request.expectedHash)
- throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
-
- auto res = getDownloader()->downloadCached(state.store, request);
-
- if (state.allowedPaths)
- state.allowedPaths->insert(res.path);
-
- mkString(v, res.storePath, PathSet({res.storePath}));
-}
-
-
-static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
- fetch(state, pos, args, v, "fetchurl", false, "");
-}
-
-
-static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
- fetch(state, pos, args, v, "fetchTarball", true, "source");
-}
-
-
-/*************************************************************
* Primop registration
*************************************************************/
@@ -2290,10 +2227,6 @@ void EvalState::createBaseEnv()
addPrimOp("derivationStrict", 1, prim_derivationStrict);
addPrimOp("placeholder", 1, prim_placeholder);
- // Networking
- addPrimOp("__fetchurl", 1, prim_fetchurl);
- addPrimOp("fetchTarball", 1, prim_fetchTarball);
-
/* Add a wrapper around the derivation primop that computes the
`drvPath' and `outPath' attributes lazily. */
string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 2b3f1b76d..8e8b48fc8 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -3,6 +3,7 @@
#include "store-api.hh"
#include "fetchers/fetchers.hh"
#include "fetchers/registry.hh"
+#include "download.hh"
#include <ctime>
#include <iomanip>
@@ -89,4 +90,80 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+ const string & who, bool unpack, std::string name)
+{
+ std::optional<std::string> url;
+ std::optional<Hash> expectedHash;
+
+ state.forceValue(*args[0]);
+
+ if (args[0]->type == tAttrs) {
+
+ state.forceAttrs(*args[0], pos);
+
+ for (auto & attr : *args[0]->attrs) {
+ string n(attr.name);
+ if (n == "url")
+ url = state.forceStringNoCtx(*attr.value, *attr.pos);
+ else if (n == "sha256")
+ expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+ else if (n == "name")
+ name = state.forceStringNoCtx(*attr.value, *attr.pos);
+ else
+ throw EvalError("unsupported argument '%s' to '%s', at %s",
+ attr.name, who, attr.pos);
+ }
+
+ if (!url)
+ throw EvalError("'url' argument required, at %s", pos);
+
+ } else
+ url = state.forceStringNoCtx(*args[0], pos);
+
+ url = resolveUri(*url);
+
+ state.checkURI(*url);
+
+ if (name == "")
+ name = baseNameOf(*url);
+
+ if (evalSettings.pureEval && !expectedHash)
+ throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
+ auto storePath =
+ unpack
+ ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+ : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+
+ auto path = state.store->toRealPath(storePath);
+
+ if (expectedHash) {
+ auto hash = unpack
+ ? state.store->queryPathInfo(storePath)->narHash
+ : hashFile(htSHA256, path);
+ if (hash != *expectedHash)
+ throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
+ *url, expectedHash->to_string(), hash.to_string());
+ }
+
+ if (state.allowedPaths)
+ state.allowedPaths->insert(path);
+
+ mkString(v, path, PathSet({path}));
+}
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ fetch(state, pos, args, v, "fetchurl", false, "");
+}
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ fetch(state, pos, args, v, "fetchTarball", true, "source");
+}
+
+static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
+static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+
}
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 8e0d7d42a..35a35cd78 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -35,10 +35,6 @@ DownloadSettings downloadSettings;
static GlobalConfig::Register r1(&downloadSettings);
-CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
- : uri(uri), ttl(settings.tarballTtl)
-{ }
-
std::string resolveUri(const std::string & uri)
{
if (uri.compare(0, 8, "channel:") == 0)
@@ -801,148 +797,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
}
}
-CachedDownloadResult Downloader::downloadCached(
- ref<Store> store, const CachedDownloadRequest & request)
-{
- auto url = resolveUri(request.uri);
-
- auto name = request.name;
- if (name == "") {
- auto p = url.rfind('/');
- if (p != string::npos) name = string(url, p + 1);
- }
-
- std::optional<StorePath> expectedStorePath;
- if (request.expectedHash) {
- expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
- if (store->isValidPath(*expectedStorePath)) {
- CachedDownloadResult result;
- result.storePath = store->printStorePath(*expectedStorePath);
- result.path = store->toRealPath(result.storePath);
- assert(!request.getLastModified); // FIXME
- return result;
- }
- }
-
- Path cacheDir = getCacheDir() + "/nix/tarballs";
- createDirs(cacheDir);
-
- string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
-
- Path dataFile = cacheDir + "/" + urlHash + ".info";
- Path fileLink = cacheDir + "/" + urlHash + "-file";
-
- PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
-
- std::optional<StorePath> storePath;
-
- string expectedETag;
-
- bool skip = false;
-
- CachedDownloadResult result;
-
- if (pathExists(fileLink) && pathExists(dataFile)) {
- storePath = store->parseStorePath(readLink(fileLink));
- // FIXME
- store->addTempRoot(*storePath);
- if (store->isValidPath(*storePath)) {
- auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
- if (ss.size() >= 3 && ss[0] == url) {
- time_t lastChecked;
- if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
- skip = true;
- result.effectiveUri = request.uri;
- result.etag = ss[1];
- } else if (!ss[1].empty()) {
- debug(format("verifying previous ETag '%1%'") % ss[1]);
- expectedETag = ss[1];
- }
- }
- } else
- storePath.reset();
- }
-
- if (!skip) {
-
- try {
- DownloadRequest request2(url);
- request2.expectedETag = expectedETag;
- auto res = download(request2);
- result.effectiveUri = res.effectiveUri;
- result.etag = res.etag;
-
- if (!res.cached) {
- StringSink sink;
- dumpString(*res.data, sink);
- Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
- ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
- info.narHash = hashString(htSHA256, *sink.s);
- info.narSize = sink.s->size();
- info.ca = makeFixedOutputCA(false, hash);
- store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
- storePath = info.path.clone();
- }
-
- assert(storePath);
- replaceSymlink(store->printStorePath(*storePath), fileLink);
-
- writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
- } catch (DownloadError & e) {
- if (!storePath) throw;
- warn("warning: %s; using cached result", e.msg());
- result.etag = expectedETag;
- }
- }
-
- if (request.unpack) {
- Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
- PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
- std::optional<StorePath> unpackedStorePath;
- if (pathExists(unpackedLink)) {
- unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
- store->addTempRoot(*unpackedStorePath);
- if (!store->isValidPath(*unpackedStorePath))
- unpackedStorePath.reset();
- else
- result.lastModified = lstat(unpackedLink).st_mtime;
- }
- if (!unpackedStorePath) {
- printInfo("unpacking '%s'...", url);
- Path tmpDir = createTempDir();
- AutoDelete autoDelete(tmpDir, true);
- unpackTarfile(store->toRealPath(*storePath), tmpDir);
- auto members = readDirectory(tmpDir);
- if (members.size() != 1)
- throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
- auto topDir = tmpDir + "/" + members.begin()->name;
- result.lastModified = lstat(topDir).st_mtime;
- unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
- }
- // Store the last-modified date of the tarball in the symlink
- // mtime. This saves us from having to store it somewhere
- // else.
- replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified);
- storePath = std::move(*unpackedStorePath);
- }
-
- if (expectedStorePath && *storePath != *expectedStorePath) {
- unsigned int statusCode = 102;
- Hash gotHash = request.unpack
- ? hashPath(request.expectedHash.type, store->toRealPath(*storePath)).first
- : hashFile(request.expectedHash.type, store->toRealPath(*storePath));
- throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
- url, request.expectedHash.to_string(), gotHash.to_string());
- }
-
- if (request.gcRoot)
- store->addIndirectRoot(fileLink);
-
- result.storePath = store->printStorePath(*storePath);
- result.path = store->toRealPath(result.storePath);
- return result;
-}
-
bool isUri(const string & s)
{
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 487036833..28c8a9162 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -65,31 +65,6 @@ struct DownloadResult
uint64_t bodySize = 0;
};
-struct CachedDownloadRequest
-{
- std::string uri;
- bool unpack = false;
- std::string name;
- Hash expectedHash;
- unsigned int ttl;
- bool gcRoot = false;
- bool getLastModified = false;
-
- CachedDownloadRequest(const std::string & uri);
- CachedDownloadRequest() = delete;
-};
-
-struct CachedDownloadResult
-{
- // Note: 'storePath' may be different from 'path' when using a
- // chroot store.
- Path storePath;
- Path path;
- std::optional<std::string> etag;
- std::string effectiveUri;
- std::optional<time_t> lastModified;
-};
-
class Store;
struct Downloader
@@ -111,12 +86,6 @@ struct Downloader
invoked on the thread of the caller. */
void download(DownloadRequest && request, Sink & sink);
- /* Check if the specified file is already in ~/.cache/nix/tarballs
- and is more recent than ‘tarball-ttl’ seconds. Otherwise,
- use the recorded ETag to verify if the server has a more
- recent version, and if so, download it to the Nix store. */
- CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
-
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
};
@@ -138,4 +107,7 @@ public:
bool isUri(const string & s);
+/* Resolve deprecated 'channel:<foo>' URLs. */
+std::string resolveUri(const std::string & uri);
+
}
diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh
index 9c931dfa1..deb1c9c59 100644
--- a/src/libstore/fetchers/fetchers.hh
+++ b/src/libstore/fetchers/fetchers.hh
@@ -97,6 +97,7 @@ struct DownloadFileResult
{
StorePath storePath;
std::string etag;
+ std::string effectiveUrl;
};
DownloadFileResult downloadFile(
diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc
index 53971ec2b..54b59babc 100644
--- a/src/libstore/fetchers/tarball.cc
+++ b/src/libstore/fetchers/tarball.cc
@@ -28,7 +28,8 @@ DownloadFileResult downloadFile(
if (cached && !cached->expired)
return {
.storePath = std::move(cached->storePath),
- .etag = getStrAttr(cached->infoAttrs, "etag")
+ .etag = getStrAttr(cached->infoAttrs, "etag"),
+ .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
};
DownloadRequest request(url);
@@ -40,6 +41,7 @@ DownloadFileResult downloadFile(
Attrs infoAttrs({
{"etag", res.etag},
+ {"url", res.effectiveUri},
});
std::optional<StorePath> storePath;
@@ -67,9 +69,22 @@ DownloadFileResult downloadFile(
*storePath,
immutable);
+ if (url != res.effectiveUri)
+ getCache()->add(
+ store,
+ {
+ {"type", "file"},
+ {"url", res.effectiveUri},
+ {"name", name},
+ },
+ infoAttrs,
+ *storePath,
+ immutable);
+
return {
.storePath = std::move(*storePath),
.etag = res.etag,
+ .effectiveUrl = res.effectiveUri,
};
}
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 1b337a712..8b0937c53 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -3,6 +3,7 @@
#include "download.hh"
#include "store-api.hh"
#include "legacy.hh"
+#include "fetchers/fetchers.hh"
#include <fcntl.h>
#include <regex>
@@ -86,12 +87,9 @@ static void update(const StringSet & channelNames)
// We want to download the url to a file to see if it's a tarball while also checking if we
// got redirected in the process, so that we can grab the various parts of a nix channel
// definition from a consistent location if the redirect changes mid-download.
- CachedDownloadRequest request(url);
- request.ttl = 0;
- auto dl = getDownloader();
- auto result = dl->downloadCached(store, request);
- auto filename = result.path;
- url = chomp(result.effectiveUri);
+ auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
+ auto filename = store->toRealPath(result.storePath);
+ url = result.effectiveUrl;
// If the URL contains a version number, append it to the name
// attribute (so that "nix-env -q" on the channels profile
@@ -114,11 +112,10 @@ static void update(const StringSet & channelNames)
if (!unpacked) {
// Download the channel tarball.
try {
- filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path;
+ filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
} catch (DownloadError & e) {
- filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path;
+ filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
}
- chomp(filename);
}
// Regardless of where it came from, add the expression representing this channel to accumulated expression
@@ -185,6 +182,8 @@ static int _main(int argc, char ** argv)
} else if (*arg == "--rollback") {
cmd = cRollback;
} else {
+ if (hasPrefix(*arg, "-"))
+ throw UsageError("unsupported argument '%s'", *arg);
args.push_back(std::move(*arg));
}
return true;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 20:58:21 +0000