From cca4a8dc1a622ab086639e5c09347303c062922e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 15:20:10 +0100 Subject: importPaths(): Optionally add NARs to binary cache accessor This enables an optimisation in hydra-queue-runner, preventing a download of a NAR it just uploaded to the cache when reading files like hydra-build-products. --- src/libstore/binary-cache-store.cc | 66 +++++++++++++++++++++----------------- src/libstore/binary-cache-store.hh | 5 +-- src/libstore/local-store.cc | 3 +- src/libstore/local-store.hh | 3 +- src/libstore/remote-store.cc | 3 +- src/libstore/remote-store.hh | 3 +- src/libstore/store-api.hh | 7 ++-- 7 files changed, 52 insertions(+), 38 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6a4e3a560..02e73d2ce 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink sink << exportMagic << storePath << res.references << res.deriver << 0; } -Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) +Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { assert(!requireSignature); Paths res; @@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) unsigned long long n = readLongLong(source); if (n == 0) break; if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); - res.push_back(importPath(source)); + res.push_back(importPath(source, accessor)); } return res; } @@ -215,34 +216,6 @@ struct NopSink : ParseSink { }; -Path BinaryCacheStore::importPath(Source & source) -{ - /* FIXME: some cut&paste of LocalStore::importPath(). */ - - /* Extract the NAR from the source. */ - TeeSource tee(source); - NopSink sink; - parseDump(sink, tee); - - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); - - ValidPathInfo info; - info.path = readStorePath(source); - - info.references = readStorePaths(source); - - readString(source); // deriver, don't care - - bool haveSignature = readInt(source) == 1; - assert(!haveSignature); - - addToCache(info, tee.data); - - return info.path; -} - ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) { return ValidPathInfo(readNarInfo(storePath)); @@ -416,4 +389,37 @@ ref BinaryCacheStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } +Path BinaryCacheStore::importPath(Source & source, std::shared_ptr accessor) +{ + /* FIXME: some cut&paste of LocalStore::importPath(). */ + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + info.path = readStorePath(source); + + info.references = readStorePaths(source); + + readString(source); // deriver, don't care + + bool haveSignature = readInt(source) == 1; + assert(!haveSignature); + + addToCache(info, tee.data); + + auto accessor_ = std::dynamic_pointer_cast(accessor); + if (accessor_) + // FIXME: more gratuitous string copying + accessor_->nars.emplace(info.path, makeNarAccessor(make_ref(tee.data))); + + return info.path; +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c6f319d1b..6feb84cd2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -127,9 +127,10 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; - Path importPath(Source & source); + Path importPath(Source & source, std::shared_ptr accessor); void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 33f256912..9a5706681 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } -Paths LocalStore::importPaths(bool requireSignature, Source & source) +Paths LocalStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { Paths res; while (true) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fded63ab3..c7ea9e503 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -147,7 +147,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a1ac169c2..82b7cfd7c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Paths RemoteStore::importPaths(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { auto conn(connections->get()); conn->to << wopImportPaths; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 0019cd8f9..85c8292c7 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 347387f03..488f32e16 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -227,8 +227,11 @@ public: void exportPaths(const Paths & paths, bool sign, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + virtual Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If -- cgit v1.2.3 From 0a62d9b3d73afb8a5111cdecb23a0a4d6954493d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 21:43:59 +0100 Subject: Remove bad assertion --- src/libstore/nar-accessor.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 88b027ff3..ff7890af8 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -51,7 +51,6 @@ struct NarIndexer : ParseSink, StringSource void preallocateContents(unsigned long long size) override { - assert(currentPath != ""); currentStart = string(s, pos, 16); members.emplace(currentPath, NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); -- cgit v1.2.3 From 68a541498258949231e35544501e11c6003f2dce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Feb 2016 16:11:11 +0100 Subject: Make store implementations pluggable This for instance allows hydra-queue-runner to add the S3 backend at runtime. --- src/libstore/local-binary-cache-store.cc | 9 +++++++ src/libstore/store-api.cc | 43 ++++++++++++++++++-------------- src/libstore/store-api.hh | 17 +++++++++++++ 3 files changed, 50 insertions(+), 19 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 5714688e0..9fae80870 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -41,4 +41,13 @@ std::string LocalBinaryCacheStore::getFile(const std::string & path) return readFile(binaryCacheDir + "/" + path); } +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { + if (std::string(uri, 0, 7) != "file://") return 0; + auto store = std::make_shared(std::shared_ptr(0), + "", "", // FIXME: allow the signing key to be set + std::string(uri, 7)); + store->init(); + return store; +}); + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 24c05b8b4..378233654 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -314,27 +314,38 @@ void Store::exportPaths(const Paths & paths, #include "local-store.hh" #include "remote-store.hh" -#include "local-binary-cache-store.hh" namespace nix { +RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; + + ref openStoreAt(const std::string & uri) { - if (std::string(uri, 0, 7) == "file://") { - auto store = make_ref(std::shared_ptr(0), - "", "", // FIXME: allow the signing key to be set - std::string(uri, 7)); - store->init(); - return store; + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri); + if (store) return ref(store); } + throw Error(format("don't know how to open Nix store ‘%s’") % uri); +} + + +ref openStore() +{ + return openStoreAt(getEnv("NIX_REMOTE")); +} + + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { enum { mDaemon, mLocal, mAuto } mode; - mode = - uri == "daemon" ? mDaemon : - uri == "local" ? mLocal : mAuto; + if (uri == "daemon") mode = mDaemon; + else if (uri == "local") mode = mLocal; + else if (uri == "") mode = mAuto; + else return 0; if (mode == mAuto) { if (LocalStore::haveWriteAccess()) @@ -346,15 +357,9 @@ ref openStoreAt(const std::string & uri) } return mode == mDaemon - ? (ref) make_ref() - : (ref) make_ref(); -} - - -ref openStore() -{ - return openStoreAt(getEnv("NIX_REMOTE")); -} + ? std::shared_ptr(std::make_shared()) + : std::shared_ptr(std::make_shared()); +}); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 488f32e16..97e834ed2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -453,6 +453,23 @@ ref openStoreAt(const std::string & uri); ref openStore(); +/* Store implementation registration. */ +typedef std::function(const std::string & uri)> OpenStore; + +struct RegisterStoreImplementation +{ + typedef std::vector Implementations; + static Implementations * implementations; + + RegisterStoreImplementation(OpenStore fun) + { + if (!implementations) implementations = new Implementations; + implementations->push_back(fun); + } +}; + + + /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ string showPaths(const PathSet & paths); -- cgit v1.2.3 From 0402b6398d879889a621d60d049f8d81ac4ed86a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Feb 2016 16:14:39 +0100 Subject: Eliminate local-binary-cache-store.hh --- src/libstore/local-binary-cache-store.cc | 26 +++++++++++++++++++++++++- src/libstore/local-binary-cache-store.hh | 31 ------------------------------- 2 files changed, 25 insertions(+), 32 deletions(-) delete mode 100644 src/libstore/local-binary-cache-store.hh (limited to 'src/libstore') diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 9fae80870..a10c9d106 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,7 +1,31 @@ -#include "local-binary-cache-store.hh" +#include "binary-cache-store.hh" namespace nix { +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir); + + void init() override; + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::string getFile(const std::string & path) override; + +}; + LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr localStore, const Path & secretKeyFile, const Path & publicKeyFile, const Path & binaryCacheDir) diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh deleted file mode 100644 index 0303ebe73..000000000 --- a/src/libstore/local-binary-cache-store.hh +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include "binary-cache-store.hh" - -namespace nix { - -class LocalBinaryCacheStore : public BinaryCacheStore -{ -private: - - Path binaryCacheDir; - -public: - - LocalBinaryCacheStore(std::shared_ptr localStore, - const Path & secretKeyFile, const Path & publicKeyFile, - const Path & binaryCacheDir); - - void init() override; - -protected: - - bool fileExists(const std::string & path) override; - - void upsertFile(const std::string & path, const std::string & data) override; - - std::string getFile(const std::string & path) override; - -}; - -} -- cgit v1.2.3 From 201b48de60751979835037a4b4f78128ba3fb7b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Feb 2016 18:15:20 +0100 Subject: Add an HTTP binary cache store Allowing stuff like NIX_REMOTE=https://cache.nixos.org nix-store -qR /nix/store/x1p1gl3a4kkz5ci0nfbayjqlqmczp1kq-geeqie-1.1 or NIX_REMOTE=https://cache.nixos.org nix-store --export /nix/store/x1p1gl3a4kkz5ci0nfbayjqlqmczp1kq-geeqie-1.1 | nix-store --import --- src/libstore/builtins.cc | 4 +- src/libstore/download.cc | 124 ++++++++++++++++++++++++++++------------------- src/libstore/download.hh | 23 +++++++-- 3 files changed, 96 insertions(+), 55 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index a1c4b48bf..c22c44f3c 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -17,9 +17,9 @@ void builtinFetchurl(const BasicDerivation & drv) options.verifyTLS = false; /* Show a progress indicator, even though stderr is not a tty. */ - options.forceProgress = true; + options.showProgress = DownloadOptions::yes; - auto data = downloadFile(url->second, options); + auto data = makeDownloader()->download(url->second, options); auto out = drv.env.find("out"); if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e754e82fb..4776d0091 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -18,7 +18,7 @@ double getTime() return tv.tv_sec + (tv.tv_usec / 1000000.0); } -struct Curl +struct CurlDownloader : public Downloader { CURL * curl; string data; @@ -30,37 +30,40 @@ struct Curl double prevProgressTime{0}, startTime{0}; unsigned int moveBack{1}; - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + size_t writeCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); + data.append((char *) contents, realSize); return realSize; } - static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; string line = string((char *) contents, realSize); printMsg(lvlVomit, format("got header: %1%") % trim(line)); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - c.etag = ""; + etag = ""; auto ss = tokenizeString>(line, " "); - c.status = ss.size() >= 2 ? ss[1] : ""; + status = ss.size() >= 2 ? ss[1] : ""; } else { auto i = line.find(':'); if (i != string::npos) { string name = trim(string(line, 0, i)); if (name == "ETag") { // FIXME: case - c.etag = trim(string(line, i + 1)); + etag = trim(string(line, i + 1)); /* Hack to work around a GitHub bug: it sends ETags, but ignores If-None-Match. So if we get the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - printMsg(lvlDebug, format("got ETag: %1%") % c.etag); - if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("got ETag: %1%") % etag); + if (etag == expectedETag && status == "200") { printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -70,6 +73,11 @@ struct Curl return realSize; } + static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->headerCallback(contents, size, nmemb); + } + int progressCallback(double dltotal, double dlnow) { if (showProgress) { @@ -88,45 +96,48 @@ struct Curl return _isInterrupted; } - static int progressCallback_(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) + static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - Curl & c(* (Curl *) userp); - return c.progressCallback(dltotal, dlnow); + return ((CurlDownloader *) userp)->progressCallback(dltotal, dlnow); } - Curl() + CurlDownloader() { requestHeaders = 0; curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); + if (!curl) throw nix::Error("unable to initialize curl"); + } + + ~CurlDownloader() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const DownloadOptions & options) + { + showProgress = + options.showProgress == DownloadOptions::yes || + (options.showProgress == DownloadOptions::automatic && isatty(STDERR_FILENO)); + + curl_easy_reset(curl); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback_); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - } - - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - if (requestHeaders) curl_slist_free_all(requestHeaders); - } - - bool fetch(const string & url, const DownloadOptions & options) - { - showProgress = options.forceProgress || isatty(STDERR_FILENO); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); @@ -151,6 +162,9 @@ struct Curl curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + if (options.head) + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + if (showProgress) { std::cerr << (format("downloading ‘%1%’... ") % url); std::cerr.flush(); @@ -163,34 +177,46 @@ struct Curl std::cerr << "\n"; checkInterrupt(); if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; - if (res != CURLE_OK) - throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") + + long httpStatus = -1; + if (res == CURLE_HTTP_RETURNED_ERROR) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + + if (res != CURLE_OK) { + long httpStatus = 0; + Error err = + httpStatus == 404 ? NotFound : + httpStatus == 403 ? Forbidden : Misc; + throw DownloadError(err, format("unable to download ‘%1%’: %2% (%3%)") % url % curl_easy_strerror(res) % res); + } - long httpStatus = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); if (httpStatus == 304) return false; return true; } + + DownloadResult download(string url, const DownloadOptions & options) override + { + DownloadResult res; + if (fetch(url, options)) { + res.cached = false; + res.data = data; + } else + res.cached = true; + res.etag = etag; + return res; + } }; -DownloadResult downloadFile(string url, const DownloadOptions & options) +ref makeDownloader() { - DownloadResult res; - Curl curl; - if (curl.fetch(url, options)) { - res.cached = false; - res.data = curl.data; - } else - res.cached = true; - res.etag = curl.etag; - return res; + return make_ref(); } -Path downloadFileCached(ref store, const string & url, bool unpack) +Path Downloader::downloadCached(ref store, const string & url, bool unpack) { Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; createDirs(cacheDir); @@ -234,7 +260,7 @@ Path downloadFileCached(ref store, const string & url, bool unpack) try { DownloadOptions options; options.expectedETag = expectedETag; - auto res = downloadFile(url, options); + auto res = download(url, options); if (!res.cached) storePath = store->addTextToStore(name, res.data, PathSet(), false); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 7aec8de73..5dd2d2c82 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -10,7 +10,8 @@ struct DownloadOptions { string expectedETag; bool verifyTLS{true}; - bool forceProgress{false}; + enum { yes, no, automatic } showProgress{yes}; + bool head{false}; }; struct DownloadResult @@ -21,11 +22,25 @@ struct DownloadResult class Store; -DownloadResult downloadFile(string url, const DownloadOptions & options); +struct Downloader +{ + virtual DownloadResult download(string url, const DownloadOptions & options) = 0; + + Path downloadCached(ref store, const string & url, bool unpack); + + enum Error { NotFound, Forbidden, Misc }; +}; -Path downloadFileCached(ref store, const string & url, bool unpack); +ref makeDownloader(); -MakeError(DownloadError, Error) +class DownloadError : public Error +{ +public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString & fs) + : Error(fs), error(error) + { } +}; bool isUri(const string & s); -- cgit v1.2.3 From 5a8455c85e35706a34892bea5b2a0c72f25663a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Mar 2016 18:21:48 +0100 Subject: Provide function required by Hydra --- src/libstore/local-binary-cache-store.cc | 14 +++++++++++--- src/libstore/store-api.hh | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a10c9d106..8590aea18 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -65,13 +65,21 @@ std::string LocalBinaryCacheStore::getFile(const std::string & path) return readFile(binaryCacheDir + "/" + path); } +ref openLocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir) +{ + auto store = std::make_shared( + localStore, secretKeyFile, publicKeyFile, binaryCacheDir); + store->init(); + return ref(std::shared_ptr(store)); +} + static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { if (std::string(uri, 0, 7) != "file://") return 0; - auto store = std::make_shared(std::shared_ptr(0), + return openLocalBinaryCacheStore(std::shared_ptr(0), "", "", // FIXME: allow the signing key to be set std::string(uri, 7)); - store->init(); - return store; }); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 97e834ed2..9825d45db 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -453,6 +453,11 @@ ref openStoreAt(const std::string & uri); ref openStore(); +ref openLocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir); + + /* Store implementation registration. */ typedef std::function(const std::string & uri)> OpenStore; -- cgit v1.2.3 From 76f1ba4f3bd8b503a6395649719a444028ff0258 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Mar 2016 18:03:34 +0100 Subject: Add file missing from 201b48de60751979835037a4b4f78128ba3fb7b3 --- src/libstore/http-binary-cache-store.cc | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/libstore/http-binary-cache-store.cc (limited to 'src/libstore') diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000..78f4497e7 --- /dev/null +++ b/src/libstore/http-binary-cache-store.cc @@ -0,0 +1,76 @@ +#include "binary-cache-store.hh" +#include "download.hh" + +namespace nix { + +class HttpBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path cacheUri; + + ref downloader; + +public: + + HttpBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & _cacheUri) + : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile) + , cacheUri(_cacheUri) + , downloader(makeDownloader()) + { + if (cacheUri.back() == '/') + cacheUri.pop_back(); + } + + void init() override + { + // FIXME: do this lazily? + if (!fileExists("nix-cache-info")) + throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + } + +protected: + + bool fileExists(const std::string & path) override + { + try { + DownloadOptions options; + options.showProgress = DownloadOptions::no; + options.head = true; + downloader->download(cacheUri + "/" + path, options); + return true; + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound) + return false; + throw; + } + } + + void upsertFile(const std::string & path, const std::string & data) + { + throw Error("uploading to an HTTP binary cache is not supported"); + } + + std::string getFile(const std::string & path) override + { + DownloadOptions options; + options.showProgress = DownloadOptions::no; + return downloader->download(cacheUri + "/" + path, options).data; + } + +}; + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://") return 0; + auto store = std::make_shared(std::shared_ptr(0), + "", "", // FIXME: allow the signing key to be set + uri); + store->init(); + return store; +}); + +} + -- cgit v1.2.3