From c8a0b9d5cbfe6619f8b38118f5b1d1875d1c5309 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:43:32 +0100 Subject: experimental/optional -> optional --- src/libstore/build.cc | 10 +++++----- src/libstore/parsed-derivations.cc | 4 ++-- src/libstore/parsed-derivations.hh | 8 ++++---- src/libstore/remote-store.hh | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 47ee8b48f..6b88b1307 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2413,7 +2413,7 @@ void DerivationGoal::writeStructuredAttrs() objects consisting entirely of those values. (So nested arrays or objects are not supported.) */ - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional { + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) return shellEscape(value); @@ -3311,8 +3311,8 @@ void DerivationGoal::checkOutputs(const std::map & outputs) struct Checks { bool ignoreSelfRefs = false; - std::experimental::optional maxSize, maxClosureSize; - std::experimental::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; }; /* Compute the closure and closure size of some output. This @@ -3359,7 +3359,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) info.path, closureSize, *checks.maxClosureSize); } - auto checkRefs = [&](const std::experimental::optional & value, bool allowed, bool recursive) + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) { if (!value) return; @@ -3413,7 +3413,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) if (maxClosureSize != output->end()) checks.maxClosureSize = maxClosureSize->get(); - auto get = [&](const std::string & name) -> std::experimental::optional { + auto get = [&](const std::string & name) -> std::optional { auto i = output->find(name); if (i != output->end()) { Strings res; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index dc3286482..17fde00a0 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) } } -std::experimental::optional ParsedDerivation::getStringAttr(const std::string & name) const +std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); @@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const } } -std::experimental::optional ParsedDerivation::getStringsAttr(const std::string & name) const +std::optional ParsedDerivation::getStringsAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0a82c1461..ed07dc652 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -8,22 +8,22 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::experimental::optional structuredAttrs; + std::optional structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::experimental::optional & getStructuredAttrs() const + const std::optional & getStructuredAttrs() const { return structuredAttrs; } - std::experimental::optional getStringAttr(const std::string & name) const; + std::optional getStringAttr(const std::string & name) const; bool getBoolAttr(const std::string & name, bool def = false) const; - std::experimental::optional getStringsAttr(const std::string & name) const; + std::optional getStringsAttr(const std::string & name) const; StringSet getRequiredSystemFeatures() const; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4f554b598..919c6d819 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -149,7 +149,7 @@ public: private: ref openConnection() override; - std::experimental::optional path; + std::optional path; }; -- cgit v1.2.3 From 0cd7f2cd8d99071ebfb06a8f0d6a18efed6cd42e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:44:20 +0100 Subject: pkg-config files: Use c++17 --- src/libstore/nix-store.pc.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faad..6d67b1e03 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 -- cgit v1.2.3 From 529add316c5356a8060c35f987643b7bf5c796dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2019 23:20:50 +0800 Subject: downloadCached: Return ETag This allows fetchFlake() to return the Git revision of a GitHub archive. --- src/libstore/download.cc | 17 +++++++++++++---- src/libstore/download.hh | 12 ++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bb..360d48b09 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -771,7 +771,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); @@ -783,8 +783,11 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa Path expectedStorePath; if (expectedHash) { expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -803,6 +806,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -814,6 +819,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa skip = true; if (effectiveUrl) *effectiveUrl = url_; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -831,6 +837,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa auto res = download(request); if (effectiveUrl) *effectiveUrl = res.effectiveUrl; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; @@ -852,6 +859,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa } catch (DownloadError & e) { if (storePath.empty()) throw; printError(format("warning: %1%; using cached result") % e.msg()); + result.etag = expectedETag; } } @@ -885,7 +893,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa url, expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..8acfe4e1a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -41,6 +41,12 @@ struct DownloadResult uint64_t bodySize = 0; }; +struct CachedDownloadResult +{ + Path path; + std::optional etag; +}; + class Store; struct Downloader @@ -64,8 +70,10 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached( + ref store, const string & uri, bool unpack, string name = "", + const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, + int ttl = settings.tarballTtl); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -- cgit v1.2.3 From b3d33b02e3fc40c7bd8f602334287825e7e6333d Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 10 Apr 2019 12:12:44 +0200 Subject: Added support for private github repositories --- src/libstore/globals.hh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90..80d70fba3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -344,6 +344,9 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + + Setting githubAccessToken{this, "", "github-acces-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; }; -- cgit v1.2.3 From 0cbda84f5b14aba0416cb65f88f8e9d487895207 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:06:27 +0200 Subject: exportGit: Don't clone local repositories This ensures that commands like 'nix flake info /my/nixpkgs' don't copy a gigabyte of crap to ~/.cache/nix. Fixes #60. --- src/libstore/http-binary-cache-store.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..105e1dcdd 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -160,10 +160,11 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared(params, uri); store->init(); return store; -- cgit v1.2.3 From 7dcf5b011a0942ecf953f2b607c4c8d0e9e652c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:09:52 +0200 Subject: Add function for quoting strings --- src/libstore/build.cc | 2 +- src/libstore/store-api.cc | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 53a0c743b..9730c75e2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1805,7 +1805,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), drvPath, settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..8fabeeea4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -726,12 +726,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } -- cgit v1.2.3 From 5c34d665386f4053d666b0899ecca0639e500fbd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 May 2019 15:38:24 +0200 Subject: Make flakes work with 'nix build --store ...' It was getting confused between logical and real store paths. Also, make fetchGit and fetchMercurial update allowedPaths properly. (Maybe the evaluator, rather than the caller of the evaluator, should apply toRealPath(), but that's a bigger change.) --- src/libstore/download.cc | 2 ++ src/libstore/download.hh | 3 +++ 2 files changed, 5 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cb77cdc77..975cfd97d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -804,6 +804,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; + result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); return result; } @@ -912,6 +913,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & url, expectedHash.to_string(), gotHash.to_string()); } + result.storePath = storePath; result.path = store->toRealPath(storePath); return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 8acfe4e1a..aa8c34be2 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -43,6 +43,9 @@ struct DownloadResult struct CachedDownloadResult { + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; Path path; std::optional etag; }; -- cgit v1.2.3 From df3f5a78d5ab0a1f2dc9d288b271b38a9b8b33b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 May 2019 23:36:29 +0200 Subject: Refactor downloadCached() interface --- src/libstore/download.cc | 44 ++++++++++++++++++++++---------------------- src/libstore/download.hh | 20 +++++++++++++++----- 2 files changed, 37 insertions(+), 27 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 975cfd97d..a7c2600f6 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -319,10 +319,10 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); @@ -790,18 +790,20 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; result.storePath = expectedStorePath; @@ -835,10 +837,9 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); @@ -852,18 +853,17 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -883,7 +883,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -906,11 +906,11 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + url, request.expectedHash.to_string(), gotHash.to_string()); } result.storePath = storePath; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index aa8c34be2..b676a1a7b 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -36,11 +36,23 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + struct CachedDownloadResult { // Note: 'storePath' may be different from 'path' when using a @@ -48,6 +60,7 @@ struct CachedDownloadResult Path storePath; Path path; std::optional etag; + std::string effectiveUri; }; class Store; @@ -73,10 +86,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached( - ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, - int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -- cgit v1.2.3 From f0d6d67af93b63c1da1809dc7630026624c19b14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 May 2019 23:43:58 +0200 Subject: Prevent the global registry from being GC'ed Issue #2868. --- src/libstore/download.cc | 3 +++ src/libstore/download.hh | 1 + 2 files changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index a7c2600f6..0d1974d3b 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -913,6 +913,9 @@ CachedDownloadResult Downloader::downloadCached( url, request.expectedHash.to_string(), gotHash.to_string()); } + if (request.gcRoot) + store->addIndirectRoot(fileLink); + result.storePath = storePath; result.path = store->toRealPath(storePath); return result; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index b676a1a7b..404e51195 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -48,6 +48,7 @@ struct CachedDownloadRequest std::string name; Hash expectedHash; unsigned int ttl = settings.tarballTtl; + bool gcRoot = false; CachedDownloadRequest(const std::string & uri) : uri(uri) { } -- cgit v1.2.3 From ae7b56cd9a5ed8810828736fbb930a7c14ea44ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 May 2019 22:35:41 +0200 Subject: Get last commit time of github flakes --- src/libstore/download.cc | 17 ++++++++++++++--- src/libstore/download.hh | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0d1974d3b..0338727c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -808,6 +808,7 @@ CachedDownloadResult Downloader::downloadCached( CachedDownloadResult result; result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); + assert(!request.getLastModified); // FIXME return result; } } @@ -892,16 +893,26 @@ CachedDownloadResult Downloader::downloadCached( store->addTempRoot(unpackedStorePath); if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = ""; + else + result.lastModified = lstat(unpackedLink).st_mtime; } if (unpackedStorePath.empty()) { printInfo(format("unpacking '%1%'...") % url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir}); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + result.lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } - replaceSymlink(unpackedStorePath, unpackedLink); + // Store the last-modified date of the tarball in the symlink + // mtime. This saves us from having to store it somewhere + // else. + replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified); storePath = unpackedStorePath; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 404e51195..43b1c5c09 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -49,6 +49,7 @@ struct CachedDownloadRequest Hash expectedHash; unsigned int ttl = settings.tarballTtl; bool gcRoot = false; + bool getLastModified = false; CachedDownloadRequest(const std::string & uri) : uri(uri) { } @@ -62,6 +63,7 @@ struct CachedDownloadResult Path path; std::optional etag; std::string effectiveUri; + std::optional lastModified; }; class Store; -- cgit v1.2.3 From 63c5c91cc053cbc1fcb8d3fe71c41142c9f51bfa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 May 2019 18:48:28 +0200 Subject: Show hash mismatch warnings in SRI format --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 004be8010..a69592219 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3170,7 +3170,7 @@ void DerivationGoal::registerOutputs() valid. */ delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - dest, h.to_string(), h2.to_string())); + dest, h.to_string(SRI), h2.to_string(SRI))); Path actualDest = worker.store.toRealPath(dest); -- cgit v1.2.3 From 6644b6099be2d3393206bf1c9c091c888c0a0f57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Jun 2019 22:25:48 +0200 Subject: Add flake evaluation cache This exploits the hermetic nature of flake evaluation to speed up repeated evaluations of a flake output attribute. For example (doing 'nix build' on an already present package): $ time nix build nixpkgs:firefox real 0m1.497s user 0m1.160s sys 0m0.139s $ time nix build nixpkgs:firefox real 0m0.052s user 0m0.038s sys 0m0.007s The cache is ~/.cache/nix/eval-cache-v1.sqlite, which has entries like INSERT INTO Attributes VALUES( X'92a907d4efe933af2a46959b082cdff176aa5bfeb47a98fabd234809a67ab195', 'packages.firefox', 1, '/nix/store/pbalzf8x19hckr8cwdv62rd6g0lqgc38-firefox-67.0.drv /nix/store/g6q0gx0v6xvdnizp8lrcw7c4gdkzana0-firefox-67.0 out'); where the hash 92a9... is a fingerprint over the flake store path and the contents of the lockfile. Because flakes are evaluated in pure mode, this uniquely identifies the evaluation result. --- src/libstore/local-store.cc | 9 +-------- src/libstore/nar-info-disk-cache.cc | 7 +------ src/libstore/sqlite.cc | 25 +++++++++++++++++++++++-- src/libstore/sqlite.hh | 6 +++++- 4 files changed, 30 insertions(+), 17 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..f39c73b23 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -294,9 +294,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); + state.db = SQLite(dbPath, create); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -308,11 +306,6 @@ void LocalStore::openDB(State & state, bool create) SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - /* !!! check whether sqlite has been built with foreign key support */ diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 32ad7f2b2..3f6dbbcf5 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -78,12 +78,7 @@ public: state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..eb1daafc5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -25,11 +25,16 @@ namespace nix { throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open SQLite database '%s'") % path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -42,6 +47,12 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { retrySQLite([&]() { @@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 115679b84..78e53fa32 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -52,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null -- cgit v1.2.3 From 8ea842260b4fd93315d35c5ba94b1ff99ab391d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 08:43:45 +0200 Subject: Add '--no-net' convenience flag This flag * Disables substituters. * Sets the tarball-ttl to infinity (ensuring e.g. that the flake registry and any downloaded flakes are considered current). * Disables retrying downloads and sets the connection timeout to the minimum. (So it doesn't completely disable downloads at the moment.) --- src/libstore/download.cc | 18 +----------------- src/libstore/download.hh | 23 ++++++++++++++++++++++- src/libstore/globals.hh | 2 +- src/libstore/http-binary-cache-store.cc | 1 - 4 files changed, 24 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0338727c1..5c1705e2f 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -30,23 +30,7 @@ using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 43b1c5c09..c095ad053 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,34 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 80d70fba3..2aecebe3d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,7 +255,7 @@ public: "Secret keys with which to sign local builds."}; Setting tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 105e1dcdd..11c34fdac 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -84,7 +84,6 @@ protected: try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { -- cgit v1.2.3 From 29ccb2e9697ee2184012dd13854e487928ae4441 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 12:49:18 +0200 Subject: Fix 32-bit overflow with --no-net --no-net causes tarballTtl to be set to the largest 32-bit integer, which causes comparison like 'time + tarballTtl < other_time' to fail on 32-bit systems. So cast them to 64-bit first. https://hydra.nixos.org/build/95076624 --- src/libstore/download.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 5c1705e2f..571c194ec 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -822,7 +822,7 @@ CachedDownloadResult Downloader::downloadCached( auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; result.effectiveUri = request.uri; result.etag = ss[1]; -- cgit v1.2.3 From d132d057a85aa1812c4133feed6c9b34ca70671d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 15:29:05 +0200 Subject: Handle store symlinks in flake directories E.g. 'nix path-info ./result' inside a flake directory now works again. --- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fabeeea4..c5a771030 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -55,7 +55,7 @@ Path Store::followLinksToStore(const Path & _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7a1b31d0f..558ea79af 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -26,6 +26,7 @@ MakeError(InvalidPath, Error) MakeError(Unsupported, Error) MakeError(SubstituteGone, Error) MakeError(SubstituterDisabled, Error) +MakeError(NotInStore, Error) struct BasicDerivation; -- cgit v1.2.3 From 15fa70cd1b853f5e62662b99ccb9ef3da6cfadff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:06:37 +0200 Subject: Downloader: Propagate exceptions from decompressionSink->finish() --- src/libstore/download.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 571c194ec..70cfaf0dd 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -311,8 +311,13 @@ struct CurlDownloader : public Downloader debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; -- cgit v1.2.3 From a67cf5a3585c41dd9f219a2c7aa9cf67fa69520b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:48:52 +0200 Subject: Fix 'error 9 while decompressing xz file' Once we've started writing data to a Sink, we can't restart a download request, because then we end up writing duplicate data to the Sink. Therefore we shouldn't handle retries in Downloader but at a higher level (in particular, in copyStorePath()). Fixes #2952. --- src/libstore/binary-cache-store.cc | 18 ++++--- src/libstore/download.cc | 67 +++++------------------ src/libstore/download.hh | 11 +++- src/libstore/http-binary-cache-store.cc | 57 ++++++++++++++------ src/libstore/store-api.cc | 94 +++++++++++++++++---------------- 5 files changed, 122 insertions(+), 125 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..8b736056e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,6 +10,8 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "retry.hh" +#include "download.hh" #include @@ -79,13 +81,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr BinaryCacheStore::getFile(const std::string & path) { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; + return retry>(downloadSettings.tries, [&]() -> std::shared_ptr { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + }); } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 70cfaf0dd..cf79b2af5 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "retry.hh" #ifdef ENABLE_S3 #include @@ -19,11 +20,9 @@ #include #include -#include #include #include #include -#include #include using namespace std::string_literals; @@ -46,9 +45,6 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; - std::random_device rd; - std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; @@ -61,12 +57,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - struct curl_slist * requestHeaders = 0; std::string encoding; @@ -385,9 +375,7 @@ struct CurlDownloader : public Downloader } } - attempt++; - - auto exc = + fail( code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -398,31 +386,15 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); - - /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); + request.verb(), request.uri, curl_easy_strerror(code), code))); } } }; struct State { - struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { - return i1->embargo > i2->embargo; - } - }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::vector> incoming; }; Sync state_; @@ -435,7 +407,6 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() - : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -529,9 +500,7 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ + /* Add new curl requests from the incoming requests queue. */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -540,22 +509,9 @@ struct CurlDownloader : public Downloader } std::vector> incoming; - auto now = std::chrono::steady_clock::now(); - { auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } + std::swap(state->incoming, incoming); quit = state->quit; } @@ -582,7 +538,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); + state->incoming.clear(); state->quit = true; } } @@ -598,7 +554,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); + state->incoming.push_back(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -681,7 +637,9 @@ std::future Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return enqueueDownload(request).get(); + return retry(request.tries, [&]() { + return enqueueDownload(request).get(); + }); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -868,7 +826,7 @@ CachedDownloadResult Downloader::downloadCached( writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("%s; using cached result", e.msg()); result.etag = expectedETag; } } @@ -931,5 +889,4 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } - } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index c095ad053..7548b83ae 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -99,11 +99,13 @@ struct Downloader std::future enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. */ + /* Synchronously download a file. The request will be retried in + case of transient failures. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + invoked on the thread of the caller. The request will not be + retried in case of transient failures. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs @@ -129,6 +131,11 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } + + bool isTransient() override + { + return error == Downloader::Error::Transient; + } }; bool isUri(const string & s); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 11c34fdac..ff2c10354 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "retry.hh" namespace nix { @@ -113,7 +114,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } @@ -136,21 +136,46 @@ protected: { checkEnabled(); - auto request(makeRequest(path)); - - getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); - maybeDisable(); - callback.rethrow(); - } catch (...) { - callback.rethrow(); - } - }}); + struct State + { + DownloadRequest request; + std::function tryDownload; + unsigned int attempt = 0; + State(DownloadRequest && request) : request(request) {} + }; + + auto state = std::make_shared(makeRequest(path)); + + state->tryDownload = [callback, state, this]() { + getDownloader()->enqueueDownload(state->request, + {[callback, state, this](std::future result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr()); + ++state->attempt; + if (state->attempt < state->request.tries && e.isTransient()) { + auto ms = retrySleepTime(state->attempt); + warn("%s; retrying in %d ms", e.what(), ms); + /* We can't sleep here because that would + block the download thread. So use a + separate thread for sleeping. */ + std::thread([state, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + state->tryDownload(); + }).detach(); + } else { + maybeDisable(); + callback.rethrow(); + } + } catch (...) { + callback.rethrow(); + } + }}); + }; + + state->tryDownload(); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c5a771030..f577799da 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,10 +6,11 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "retry.hh" +#include "download.hh" #include - namespace nix { @@ -572,54 +573,57 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); - - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); - - auto info = srcStore->queryPathInfo(storePath); - - uint64_t total = 0; - - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + retry(downloadSettings.tries, [&]() { - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore->queryPathInfo(storePath); + + uint64_t total = 0; + + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); + }); } -- cgit v1.2.3 From 99e8e58f2de9941353b47ed14fbe4ed76d635519 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 13:48:53 +0200 Subject: Shut up some warnings --- src/libstore/download.hh | 2 ++ src/libstore/fs-accessor.hh | 2 ++ 2 files changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/download.hh b/src/libstore/download.hh index abc4a828c..c5dd893b5 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -94,6 +94,8 @@ class Store; struct Downloader { + virtual ~Downloader() { } + /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f703e1d15..64780a6da 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -19,6 +19,8 @@ public: uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; -- cgit v1.2.3 From 893be6f5e36abb58bbaa9c49055a5218114dd514 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 22 Sep 2019 21:29:33 +0200 Subject: Don't catch exceptions by value --- src/libstore/local-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1062df781..307c00dbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -872,8 +872,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, info->references, narInfo ? narInfo->fileSize : 0, info->narSize}; - } catch (InvalidPath) { - } catch (SubstituterDisabled) { + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) printError(e.what()); -- cgit v1.2.3 From c32bba748902a8603036744d4ebccc134f02958c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Sep 2019 17:28:18 +0200 Subject: Shut up some warnings --- src/libstore/nar-info-disk-cache.hh | 2 ++ src/libstore/sqlite.hh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 88d909732..0646d5a94 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,6 +10,8 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; + virtual ~NarInfoDiskCache() { } + virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 78e53fa32..0f46f6a07 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -5,8 +5,8 @@ #include "types.hh" -class sqlite3; -class sqlite3_stmt; +struct sqlite3; +struct sqlite3_stmt; namespace nix { -- cgit v1.2.3 From 8e478c234100cf03ea1b777d4bd42a9be7be9e8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Oct 2019 17:45:09 +0200 Subject: Add experimental-features setting Experimental features are now opt-in. There are currently two experimental features: "nix-command" (which enables the "nix" command), and "flakes" (which enables support for flakes). This will allow us to merge experimental features more quickly, without committing to supporting them indefinitely. Typical usage: $ nix build --experimental-features 'nix-command flakes' nixpkgs#hello --- src/libstore/globals.cc | 7 +++++++ src/libstore/globals.hh | 5 +++++ 2 files changed, 12 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1c2c08715..249c36673 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -105,6 +105,13 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +void Settings::requireExperimentalFeature(const std::string & name) +{ + auto & f = experimentalFeatures.get(); + if (std::find(f.begin(), f.end(), name) == f.end()) + throw Error("experimental Nix feature '%s' is disabled", name); +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index c0c535a12..0b1a8dac5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -356,6 +356,11 @@ public: Setting githubAccessToken{this, "", "github-acces-token", "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + + Setting experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + void requireExperimentalFeature(const std::string & name); }; -- cgit v1.2.3 From a7aabd7cc785bbf34ad29101672677ced18a7fdd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 16:07:19 +0200 Subject: Add getDefaultProfile() function --- src/libstore/profiles.cc | 18 ++++++++++++++++++ src/libstore/profiles.hh | 4 ++++ 2 files changed, 22 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4c6af567a..29f6f6c17 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile) } +Path getDefaultProfile() +{ + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + return absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + return profileLink; + } +} + + } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 5fa1533de..78645d8b6 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ string optimisticLockProfile(const Path & profile); +/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create + it. */ +Path getDefaultProfile(); + } -- cgit v1.2.3 From b82f75464d1e5ae9a00d8004e5dd7b1ca05059e4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:17:15 +0200 Subject: buildenv: Eliminate global variables, other cleanup --- src/libstore/build.cc | 1 + src/libstore/builtins.hh | 1 - src/libstore/builtins/buildenv.cc | 120 +++++++++++++++++--------------------- src/libstore/builtins/buildenv.hh | 21 +++++++ 4 files changed, 77 insertions(+), 66 deletions(-) create mode 100644 src/libstore/builtins/buildenv.hh (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cdf848c98..1b27d7af0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "builtins/buildenv.hh" #include "download.hh" #include "finally.hh" #include "compression.hh" diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0d2da873e..f9b5f7900 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,6 +6,5 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinBuildenv(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 096593886..c1c85d0bf 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,4 @@ -#include "builtins.hh" +#include "buildenv.hh" #include #include @@ -7,16 +7,14 @@ namespace nix { -typedef std::map Priorities; - -// FIXME: change into local variables. - -static Priorities priorities; - -static unsigned long symlinks; +struct State +{ + std::map priorities; + unsigned long symlinks = 0; +}; /* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { DirEntries srcFiles; @@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); + createLinks(state, srcFile, dstFile, priority); continue; } else if (S_ISLNK(dstSt.st_mode)) { auto target = canonPath(dstFile, true); @@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) throw SysError(format("unlinking '%1%'") % dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); + createLinks(state, target, dstFile, state.priorities[dstFile]); + createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) @@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISLNK(dstSt.st_mode)) { - auto prevPriority = priorities[dstFile]; + auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( "packages '%1%' and '%2%' have the same priority %3%; " @@ -109,41 +107,57 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) } createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; + state.priorities[dstFile] = priority; + state.symlinks++; } } -typedef std::set FileProp; +void buildProfile(const Path & out, Packages && pkgs) +{ + State state; -static FileProp done; -static FileProp postponed = FileProp{}; + std::set done, postponed; -static Path out; + auto addPkg = [&](const Path & pkgDir, int priority) { + if (!done.insert(pkgDir).second) return; + createLinks(state, pkgDir, out, priority); -static void addPkg(const Path & pkgDir, int priority) -{ - if (!done.insert(pkgDir).second) return; - createLinks(pkgDir, out, priority); + try { + for (const auto & p : tokenizeString>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; + } + }; - try { - for (const auto & p : tokenizeString>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) - if (!done.count(p)) - postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; - } -} + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { + return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); + }); + for (const auto & pkg : pkgs) + if (pkg.active) + addPkg(pkg.path, pkg.priority); -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} -}; + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + std::set pkgDirs; + postponed.swap(pkgDirs); + for (const auto & pkgDir : pkgDirs) + addPkg(pkgDir, priorityCounter++); + } -typedef std::vector Packages; + printError("created %d symlinks in user environment", state.symlinks); +} void builtinBuildenv(const BasicDerivation & drv) { @@ -153,7 +167,7 @@ void builtinBuildenv(const BasicDerivation & drv) return i->second; }; - out = getAttr("out"); + Path out = getAttr("out"); createDirs(out); /* Convert the stuff we get from the environment back into a @@ -171,31 +185,7 @@ void builtinBuildenv(const BasicDerivation & drv) } } - /* Symlink to the packages that have been installed explicitly by the - * user. Process in priority order to reduce unnecessary - * symlink/unlink steps. - */ - std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { - return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); - }); - for (const auto & pkg : pkgs) - if (pkg.active) - addPkg(pkg.path, pkg.priority); - - /* Symlink to the packages that have been "propagated" by packages - * installed by the user (i.e., package X declares that it wants Y - * installed as well). We do these later because they have a lower - * priority in case of collisions. - */ - auto priorityCounter = 1000; - while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; - for (const auto & pkgDir : pkgDirs) - addPkg(pkgDir, priorityCounter++); - } - - printError("created %d symlinks in user environment", symlinks); + buildProfile(out, std::move(pkgs)); createSymlink(getAttr("manifest"), out + "/manifest.nix"); } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh new file mode 100644 index 000000000..0a37459b0 --- /dev/null +++ b/src/libstore/builtins/buildenv.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "derivations.hh" +#include "store-api.hh" + +namespace nix { + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector Packages; + +void buildProfile(const Path & out, Packages && pkgs); + +void builtinBuildenv(const BasicDerivation & drv); + +} -- cgit v1.2.3 From 45b740c18b196d0326a94df23d08fa3d68e0863f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 22:11:21 +0200 Subject: Use upstream json_fwd.hpp to speed up compilation --- src/libstore/build.cc | 2 +- src/libstore/parsed-derivations.cc | 6 +++++- src/libstore/parsed-derivations.hh | 10 ++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 21b641f2c..8e795e555 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2521,7 +2521,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; auto json = *structuredAttrs; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 87be8a24e..5553dd863 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,5 +1,7 @@ #include "parsed-derivations.hh" +#include + namespace nix { ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) @@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) auto jsonAttr = drv.env.find("__json"); if (jsonAttr != drv.env.end()) { try { - structuredAttrs = nlohmann::json::parse(jsonAttr->second); + structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); } } } +ParsedDerivation::~ParsedDerivation() { } + std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 9bde4b4dc..6e67e1665 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,6 +1,6 @@ #include "derivations.hh" -#include +#include namespace nix { @@ -8,15 +8,17 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::optional structuredAttrs; + std::unique_ptr structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::optional & getStructuredAttrs() const + ~ParsedDerivation(); + + const nlohmann::json * getStructuredAttrs() const { - return structuredAttrs; + return structuredAttrs.get(); } std::optional getStringAttr(const std::string & name) const; -- cgit v1.2.3 From e30a0155d47a2e8a2eb9d0801b8b1602f71c5fd7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 13:06:32 +0200 Subject: Add "nix profile remove" command --- src/libstore/builtins/buildenv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c1c85d0bf..1b802d908 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -156,7 +156,7 @@ void buildProfile(const Path & out, Packages && pkgs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", state.symlinks); + debug("created %d symlinks in user environment", state.symlinks); } void builtinBuildenv(const BasicDerivation & drv) -- cgit v1.2.3 From 048ef27326c107b8d2a10663cd1391b122ad5fa6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2019 20:34:34 +0100 Subject: Typo --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d9e44e976..ae5a78201 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -354,7 +354,7 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; - Setting githubAccessToken{this, "", "github-acces-token", + Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; Setting experimentalFeatures{this, {}, "experimental-features", -- cgit v1.2.3 From a045f93396a676bf5510700c9ac19f5c16bd69e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Dec 2019 15:07:31 +0100 Subject: Temporarily revert to using 'tar' until we have gzip support --- src/libstore/download.cc | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 2eb39af3a..762b7c303 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -910,6 +910,7 @@ CachedDownloadResult Downloader::downloadCached( printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); +#if 0 unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url))); auto members = readDirectory(tmpDir); if (members.size() != 1) @@ -917,6 +918,12 @@ CachedDownloadResult Downloader::downloadCached( auto topDir = tmpDir + "/" + members.begin()->name; result.lastModified = lstat(topDir).st_mtime; unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); +#else + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", store->toRealPath(store->printStorePath(*storePath)), "-C", tmpDir, "--strip-components", "1"}); + result.lastModified = lstat(tmpDir).st_mtime; + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); +#endif } // Store the last-modified date of the tarball in the symlink // mtime. This saves us from having to store it somewhere -- cgit v1.2.3 From 4da1cd59ba0c371349d3fa86e3b0b6758a0a427f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Dec 2019 17:19:30 +0100 Subject: Fix getting the timestamp of GitHub flakes --- src/libstore/download.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 762b7c303..83737b307 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -912,18 +912,16 @@ CachedDownloadResult Downloader::downloadCached( AutoDelete autoDelete(tmpDir, true); #if 0 unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url))); +#else + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", store->toRealPath(store->printStorePath(*storePath)), "-C", tmpDir}); +#endif auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; result.lastModified = lstat(topDir).st_mtime; unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); -#else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(store->printStorePath(*storePath)), "-C", tmpDir, "--strip-components", "1"}); - result.lastModified = lstat(tmpDir).st_mtime; - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); -#endif } // Store the last-modified date of the tarball in the symlink // mtime. This saves us from having to store it somewhere -- cgit v1.2.3 From 9f4d8c6170517c9452e25dc29c56a6fbb43d40a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jan 2020 16:27:53 +0100 Subject: Pluggable fetchers Flakes are now fetched using an extensible mechanism. Also lots of other flake cleanups. --- src/libstore/fetchers/fetchers.cc | 56 ++++++ src/libstore/fetchers/fetchers.hh | 75 ++++++++ src/libstore/fetchers/git.cc | 382 ++++++++++++++++++++++++++++++++++++++ src/libstore/fetchers/github.cc | 183 ++++++++++++++++++ src/libstore/fetchers/indirect.cc | 114 ++++++++++++ src/libstore/fetchers/parse.cc | 129 +++++++++++++ src/libstore/fetchers/parse.hh | 28 +++ src/libstore/fetchers/regex.hh | 32 ++++ src/libstore/fetchers/registry.cc | 145 +++++++++++++++ src/libstore/fetchers/registry.hh | 47 +++++ src/libstore/globals.hh | 9 + src/libstore/local.mk | 2 +- src/libstore/store-api.cc | 23 +-- 13 files changed, 1203 insertions(+), 22 deletions(-) create mode 100644 src/libstore/fetchers/fetchers.cc create mode 100644 src/libstore/fetchers/fetchers.hh create mode 100644 src/libstore/fetchers/git.cc create mode 100644 src/libstore/fetchers/github.cc create mode 100644 src/libstore/fetchers/indirect.cc create mode 100644 src/libstore/fetchers/parse.cc create mode 100644 src/libstore/fetchers/parse.hh create mode 100644 src/libstore/fetchers/regex.hh create mode 100644 src/libstore/fetchers/registry.cc create mode 100644 src/libstore/fetchers/registry.hh (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc new file mode 100644 index 000000000..7f82d5af0 --- /dev/null +++ b/src/libstore/fetchers/fetchers.cc @@ -0,0 +1,56 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +std::unique_ptr>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::pair> Input::fetchTree(ref store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(store->printStorePath(tree.storePath)); + + if (!tree.narHash) + tree.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.narHash); + + return {std::move(tree), input}; +} + +std::shared_ptr Input::applyOverrides( + std::optional ref, + std::optional rev) const +{ + if (ref) + throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + if (rev) + throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); + return shared_from_this(); +} + +} diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh new file mode 100644 index 000000000..b59b328cc --- /dev/null +++ b/src/libstore/fetchers/fetchers.hh @@ -0,0 +1,75 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + Hash narHash; + std::optional rev; + std::optional revCount; + std::optional lastModified; +}; + +struct Input : std::enable_shared_from_this +{ + std::string type; + std::optional narHash; + + virtual bool operator ==(const Input & other) const { return false; } + + virtual bool isDirect() const { return true; } + + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional getRef() const { return {}; } + + virtual std::optional getRev() const { return {}; } + + virtual std::string to_string() const = 0; + + std::pair> fetchTree(ref store) const; + + virtual std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const; + + virtual std::optional getSourcePath() const { return {}; } + + virtual void clone(const Path & destDir) const + { + throw Error("do not know how to clone input '%s'", to_string()); + } + +private: + + virtual std::pair> fetchTreeInternal(ref store) const = 0; +}; + +struct ParsedURL; + +struct InputScheme +{ + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; +}; + +std::unique_ptr inputFromURL(const ParsedURL & url); + +std::unique_ptr inputFromURL(const std::string & url); + +void registerInputScheme(std::unique_ptr && fetcher); + +} diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc new file mode 100644 index 000000000..bfa862cf5 --- /dev/null +++ b/src/libstore/fetchers/git.cc @@ -0,0 +1,382 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" +#include "regex.hh" + +#include + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) +{ + Path cacheDir = getCacheDir() + "/nix/git-revs-v2"; + std::string linkName = + name == "source" + ? rev.gitRev() + : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false); + return cacheDir + "/" + linkName + ".link"; +} + +static void cacheGitInfo(Store & store, const std::string & name, const Tree & tree) +{ + nlohmann::json json; + json["storePath"] = store.printStorePath(tree.storePath); + json["name"] = name; + json["rev"] = tree.rev->gitRev(); + json["revCount"] = *tree.revCount; + json["lastModified"] = *tree.lastModified; + + auto cacheInfoPath = getCacheInfoPathFor(name, *tree.rev); + createDirs(dirOf(cacheInfoPath)); + writeFile(cacheInfoPath, json.dump()); +} + +static std::optional lookupGitInfo( + ref store, + const std::string & name, + const Hash & rev) +{ + try { + auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev))); + + assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev); + + auto storePath = store->parseStorePath((std::string) json["storePath"]); + + if (store->isValidPath(storePath)) { + Tree tree{ + .actualPath = store->toRealPath(store->printStorePath(storePath)), + .storePath = std::move(storePath), + .rev = rev, + .revCount = json["revCount"], + .lastModified = json["lastModified"], + }; + return tree; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + return {}; +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + GitInput(const ParsedURL & url) : url(url) + { + type = "git"; + } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url.url == other2->url.url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + std::string to_string() const override + { + ParsedURL url2(url); + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url2.to_string(); + } + + void clone(const Path & destDir) const override + { + auto [isLocal, actualUrl] = getActualUrl(); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (ref) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (rev) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + if (!res->ref && res->rev) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); + + return res; + } + + std::optional getSourcePath() const + { + if (url.scheme == "git+file" && !ref && !rev) + return url.path; + return {}; + } + + std::pair getActualUrl() const + { + // Don't clone git+file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "git+file" && !forceHttp; + return {isLocal, isLocal ? url.path : std::string(url.base, 4)}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + if (rev) { + if (auto tree = lookupGitInfo(store, name, *rev)) + return {std::move(*tree), input}; + } + + auto [isLocal, actualUrl] = getActualUrl(); + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + bool haveCommits = !readDirectory(actualUrl + "/.git/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "rev-list", "--count", "HEAD" })) : 0, + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? "HEAD" : "master"; + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + if (auto tree = lookupGitInfo(store, name, *input->rev)) + return {std::move(*tree), input}; + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + unpackTarfile(*source, tmpDir); + + auto storePath = store->addToStore(name, tmpDir); + auto revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + auto tree = Tree { + .actualPath = store->toRealPath(store->printStorePath(storePath)), + .storePath = std::move(storePath), + .rev = input->rev, + .revCount = revCount, + .lastModified = lastModified, + }; + + cacheGitInfo(*store, name, tree); + + return {std::move(tree), input}; + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto input = std::make_unique(url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadURL("Git URL '%s' contains an invalid commit hash", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("Git URL '%s' contains an invalid branch/tag name", url.url); + input->ref = value; + } + } + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc new file mode 100644 index 000000000..c75680649 --- /dev/null +++ b/src/libstore/fetchers/github.cc @@ -0,0 +1,183 @@ +#include "fetchers.hh" +#include "download.hh" +#include "globals.hh" +#include "parse.hh" +#include "regex.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + std::string to_string() const override + { + auto s = fmt("github:%s/%s", owner, repo); + assert(!(ref && rev)); + if (ref) s += "/" + *ref; + if (rev) s += "/" + rev->to_string(Base16, false); + return s; + } + + void clone(const Path & destDir) const override + { + std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + + #if 0 + if (rev) { + if (auto gitInfo = lookupGitInfo(store, "source", *rev)) + return *gitInfo; + } + #endif + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref ? *ref : "master"); + CachedDownloadRequest request(url); + request.ttl = rev ? 1000000000 : settings.tarballTtl; + auto result = getDownloader()->downloadCached(store, request); + auto json = nlohmann::json::parse(readFile(result.path)); + rev = Hash(json["sha"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = 1000000000; + request.getLastModified = true; + auto dresult = getDownloader()->downloadCached(store, request); + + assert(dresult.lastModified); + + Tree result{ + .actualPath = dresult.path, + .storePath = store->parseStorePath(dresult.storePath), + .rev = *rev, + .lastModified = *dresult.lastModified + }; + + #if 0 + // FIXME: this can overwrite a cache file that contains a revCount. + cacheGitInfo("source", gitInfo); + #endif + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + return {std::move(result), input}; + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + input->type = "github"; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadURL("GitHub URL '%s' contains an invalid commit hash", url.url); + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc new file mode 100644 index 000000000..1f9d1e24f --- /dev/null +++ b/src/libstore/fetchers/indirect.cc @@ -0,0 +1,114 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "regex.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInput : Input +{ + std::string id; + std::optional rev; + std::optional ref; + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && id == other2->id + && rev == other2->rev + && ref == other2->ref; + } + + bool isDirect() const override + { + return false; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + bool contains(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && id == other2->id + && (!ref || ref == other2->ref) + && (!rev || rev == other2->rev); + } + + std::string to_string() const override + { + ParsedURL url; + url.scheme = "flake"; + url.path = id; + if (ref) { url.path += '/'; url.path += *ref; }; + if (rev) { url.path += '/'; url.path += rev->gitRev(); }; + return url.to_string(); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + throw Error("indirect input '%s' cannot be fetched directly", to_string()); + } +}; + +struct IndirectInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + input->type = "indirect"; + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + input->rev = Hash(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + input->ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + input->ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + input->rev = Hash(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + // FIXME: forbid query params? + + input->id = path[0]; + if (!std::regex_match(input->id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", input->id); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc new file mode 100644 index 000000000..96a0681e5 --- /dev/null +++ b/src/libstore/fetchers/parse.cc @@ -0,0 +1,129 @@ +#include "parse.hh" +#include "util.hh" +#include "regex.hh" + +namespace nix::fetchers { + +std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex revRegex(revRegexS, std::regex::ECMAScript); + +ParsedURL parseURL(const std::string & url) +{ + static std::regex uriRegex( + "(((" + schemeRegex + "):" + + "(//(" + authorityRegex + "))?" + + "(" + pathRegex + "))" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?" + + ")", + std::regex::ECMAScript); + + std::smatch match; + + if (std::regex_match(url, match, uriRegex)) { + auto & base = match[2]; + std::string scheme = match[3]; + auto authority = match[4].matched + ? std::optional(match[5]) : std::nullopt; + std::string path = match[6]; + auto & query = match[7]; + auto & fragment = match[8]; + + auto isFile = scheme.find("file") != std::string::npos; + + if (authority && *authority != "" && isFile) + throw Error("file:// URL '%s' has unexpected authority '%s'", + url, *authority); + + if (isFile && path.empty()) + path = "/"; + + return ParsedURL{ + .url = url, + .base = base, + .scheme = scheme, + .authority = authority, + .path = path, + .query = decodeQuery(query), + .fragment = percentDecode(std::string(fragment)) + }; + } + + else + throw BadURL("'%s' is not a valid URL", url); +} + +std::string percentDecode(std::string_view in) +{ + std::string decoded; + for (size_t i = 0; i < in.size(); ) { + if (in[i] == '%') { + if (i + 2 >= in.size()) + throw BadURL("invalid URI parameter '%s'", in); + try { + decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw BadURL("invalid URI parameter '%s'", in); + } + } else + decoded += in[i++]; + } + return decoded; +} + +std::map decodeQuery(const std::string & query) +{ + std::map result; + + for (auto s : tokenizeString(query, "&")) { + auto e = s.find('='); + if (e != std::string::npos) + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); + } + + return result; +} + +std::string percentEncode(std::string_view s) +{ + std::string res; + for (auto & c : s) + if ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || strchr("-._~!$&'()*+,;=:@", c)) + res += c; + else + res += fmt("%%%02x", (unsigned int) c); + return res; +} + +std::string encodeQuery(const std::map & ss) +{ + std::string res; + bool first = true; + for (auto & [name, value] : ss) { + if (!first) res += '&'; + first = false; + res += percentEncode(name); + res += '='; + res += percentEncode(value); + } + return res; +} + +std::string ParsedURL::to_string() const +{ + return + scheme + + ":" + + (authority ? "//" + *authority : "") + + path + + (query.empty() ? "" : "?" + encodeQuery(query)) + + (fragment.empty() ? "" : "#" + percentEncode(fragment)); +} + +} diff --git a/src/libstore/fetchers/parse.hh b/src/libstore/fetchers/parse.hh new file mode 100644 index 000000000..22cc57816 --- /dev/null +++ b/src/libstore/fetchers/parse.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "types.hh" + +namespace nix::fetchers { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional authority; + std::string path; + std::map query; + std::string fragment; + + std::string to_string() const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); + +} diff --git a/src/libstore/fetchers/regex.hh b/src/libstore/fetchers/regex.hh new file mode 100644 index 000000000..eb061a048 --- /dev/null +++ b/src/libstore/fetchers/regex.hh @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace nix::fetchers { + +// URI stuff. +const static std::string pctEncoded = "%[0-9a-fA-F][0-9a-fA-F]"; +const static std::string schemeRegex = "[a-z+]+"; +const static std::string authorityRegex = + "(?:(?:[a-z])*@)?" + "[a-zA-Z0-9._~-]*"; +const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; +const static std::string pathRegex = "(?:/?" + segmentRegex + "(?:/" + segmentRegex + ")*|/?)"; +const static std::string pcharRegex = + "(?:[a-zA-Z0-9-._~!$&'()*+,;=:@ ]|" + pctEncoded + ")"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/?])*"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; + +} diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc new file mode 100644 index 000000000..dd74e16c1 --- /dev/null +++ b/src/libstore/fetchers/registry.cc @@ -0,0 +1,145 @@ +#include "registry.hh" +#include "util.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "download.hh" + +#include + +namespace nix::fetchers { + +std::shared_ptr Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared(); + registry->type = type; + + if (!pathExists(path)) + return std::make_shared(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + // FIXME: remove 'uri' soon. + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url)}); + } + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json json; + json["version"] = 1; + for (auto & elem : entries) + json["flakes"][elem.first->to_string()] = { {"url", elem.second->to_string()} }; + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); +} + +void Registry::add( + const std::shared_ptr & from, + const std::shared_ptr & to) +{ + entries.emplace_back(from, to); +} + +void Registry::remove(const std::shared_ptr & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (*i->first == *input) + i = entries.erase(i); + else + ++i; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr getUserRegistry() +{ + return Registry::read(getUserRegistryPath(), Registry::User); +} + +#if 0 +std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) +{ + auto flagRegistry = std::make_shared(); + for (auto const & x : registryOverrides) + flagRegistry->entries.insert_or_assign( + parseFlakeRef2(x.first), + parseFlakeRef2(x.second)); + return flagRegistry; +} +#endif + +static std::shared_ptr getGlobalRegistry(ref store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(path); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref store) +{ + Registries registries; + //registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::shared_ptr lookupInRegistries( + ref store, + std::shared_ptr input) +{ + int n = 0; + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + if (entry.first->contains(*input)) { + input = entry.second->applyOverrides( + !entry.first->getRef() && input->getRef() ? input->getRef() : std::optional(), + !entry.first->getRev() && input->getRev() ? input->getRev() : std::optional()); + goto restart; + } + } + } + + if (!input->isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + + return input; +} + +} diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh new file mode 100644 index 000000000..1757ce323 --- /dev/null +++ b/src/libstore/fetchers/registry.hh @@ -0,0 +1,47 @@ +#pragma once + +#include "types.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + Global = 2, + }; + + RegistryType type; + + std::vector, std::shared_ptr>> entries; + + static std::shared_ptr read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const std::shared_ptr & from, + const std::shared_ptr & to); + + void remove(const std::shared_ptr & input); +}; + +typedef std::vector> Registries; + +std::shared_ptr getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref store); + +std::shared_ptr lookupInRegistries( + ref store, + std::shared_ptr input); + +} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 247fba2f8..d0500be22 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -365,6 +365,15 @@ public: bool isExperimentalFeatureEnabled(const std::string & name); void requireExperimentalFeature(const std::string & name); + + Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ac68c2342..e803ff85a 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,7 +4,7 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/fetchers/*.cc) libstore_LIBS = libutil libnixrust diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c29ca5a12..e37829b17 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,6 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "fetchers/parse.hh" #include @@ -864,27 +865,7 @@ std::pair splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; - } - } + params = fetchers::decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; -- cgit v1.2.3 From b33b94748cc8cc30e8bdf58f34c4934833515440 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jan 2020 23:49:32 +0100 Subject: Convert fetchMercurial to a input type This enables Mercurial flakes. It also fixes a bug in pure mode where you could use a branch/tag name rather than a revision. --- src/libstore/fetchers/mercurial.cc | 274 +++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 src/libstore/fetchers/mercurial.cc (limited to 'src/libstore') diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc new file mode 100644 index 000000000..64d5e84e1 --- /dev/null +++ b/src/libstore/fetchers/mercurial.cc @@ -0,0 +1,274 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" +#include "regex.hh" + +#include + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +struct MercurialInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + MercurialInput(const ParsedURL & url) : url(url) + { + type = "hg"; + } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url.url == other2->url.url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + std::string to_string() const override + { + ParsedURL url2(url); + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url2.to_string(); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::optional getSourcePath() const + { + if (url.scheme == "hg+file" && !ref && !rev) + return url.path; + return {}; + } + + std::pair getActualUrl() const + { + bool isLocal = url.scheme == "hg+file"; + return {isLocal, isLocal ? url.path : std::string(url.base, 3)}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + auto [isLocal, actualUrl] = getActualUrl(); + + // FIXME: return lastModified. + + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + if (!settings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", actualUrl); + + if (settings.warnDirty) + warn("Mercurial tree '%s' is unclean", actualUrl); + + input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input}; + } + } + + if (!input->ref) input->ref = "default"; + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + assert(input->rev || input->ref); + auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + + Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, revOrRef).to_string(Base32, false)); + + /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, + do so now. */ + time_t now = time(0); + struct stat st; + if (stat(stampFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) + { + /* Except that if this is a commit hash that we already have, + we don't have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + } + } + + writeFile(stampFile, ""); + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + input->rev = Hash(tokens[0], htSHA1); + auto revCount = std::stoull(tokens[1]); + input->ref = tokens[2]; + + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + input->rev->gitRev()).to_string(Base32, false); + Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == input->rev->gitRev()); + + auto storePath = store->parseStorePath((std::string) json["storePath"]); + + if (store->isValidPath(storePath)) { + printTalkative("using cached Mercurial store path '%s'", store->printStorePath(storePath)); + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .rev = input->rev, + .revCount = revCount, + }, input}; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + auto storePath = store->addToStore(name, tmpDir); + + nlohmann::json json; + json["storePath"] = store->printStorePath(storePath); + json["uri"] = actualUrl; + json["name"] = name; + json["branch"] = *input->ref; + json["rev"] = input->rev->gitRev(); + json["revCount"] = revCount; + + writeFile(storeLink, json.dump()); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .rev = input->rev, + .revCount = revCount, + }, input}; + } +}; + +struct MercurialInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return nullptr; + + auto input = std::make_unique(url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadURL("Mercurial URL '%s' contains an invalid commit hash", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("Mercurial URL '%s' contains an invalid branch/tag name", url.url); + input->ref = value; + } + } + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} -- cgit v1.2.3 From b5c9dbc84f31a1e9d1e5b6642b1716daa13c18ed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jan 2020 20:00:58 +0100 Subject: Fix --override-flake and add a test --- src/libstore/fetchers/registry.cc | 26 ++++++++++++++------------ src/libstore/fetchers/registry.hh | 8 ++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index dd74e16c1..9c74f118d 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -11,11 +11,10 @@ namespace nix::fetchers { std::shared_ptr Registry::read( const Path & path, RegistryType type) { - auto registry = std::make_shared(); - registry->type = type; + auto registry = std::make_shared(type); if (!pathExists(path)) - return std::make_shared(); + return std::make_shared(type); auto json = nlohmann::json::parse(readFile(path)); @@ -74,17 +73,20 @@ std::shared_ptr getUserRegistry() return Registry::read(getUserRegistryPath(), Registry::User); } -#if 0 -std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) +static std::shared_ptr flagRegistry = + std::make_shared(Registry::Flag); + +std::shared_ptr getFlagRegistry() { - auto flagRegistry = std::make_shared(); - for (auto const & x : registryOverrides) - flagRegistry->entries.insert_or_assign( - parseFlakeRef2(x.first), - parseFlakeRef2(x.second)); return flagRegistry; } -#endif + +void overrideRegistry( + const std::shared_ptr & from, + const std::shared_ptr & to) +{ + flagRegistry->add(from, to); +} static std::shared_ptr getGlobalRegistry(ref store) { @@ -107,7 +109,7 @@ static std::shared_ptr getGlobalRegistry(ref store) Registries getRegistries(ref store) { Registries registries; - //registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getFlagRegistry()); registries.push_back(getUserRegistry()); registries.push_back(getGlobalRegistry(store)); return registries; diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh index 1757ce323..e29f78486 100644 --- a/src/libstore/fetchers/registry.hh +++ b/src/libstore/fetchers/registry.hh @@ -20,6 +20,10 @@ struct Registry std::vector, std::shared_ptr>> entries; + Registry(RegistryType type) + : type(type) + { } + static std::shared_ptr read( const Path & path, RegistryType type); @@ -40,6 +44,10 @@ Path getUserRegistryPath(); Registries getRegistries(ref store); +void overrideRegistry( + const std::shared_ptr & from, + const std::shared_ptr & to); + std::shared_ptr lookupInRegistries( ref store, std::shared_ptr input); -- cgit v1.2.3 From 9a5ca802c711fe0e233d78fd51474e8040338d2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jan 2020 21:26:19 +0100 Subject: clang fixes https://hydra.nixos.org/build/110757171 --- src/libstore/fetchers/fetchers.hh | 4 ++++ src/libstore/fetchers/git.cc | 5 +++-- src/libstore/fetchers/mercurial.cc | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index b59b328cc..1f3be8c2b 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -27,6 +27,8 @@ struct Input : std::enable_shared_from_this std::string type; std::optional narHash; + virtual ~Input() { } + virtual bool operator ==(const Input & other) const { return false; } virtual bool isDirect() const { return true; } @@ -63,6 +65,8 @@ struct ParsedURL; struct InputScheme { + virtual ~InputScheme() { } + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; }; diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index bfa862cf5..16bbcd285 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -142,7 +142,7 @@ struct GitInput : Input return res; } - std::optional getSourcePath() const + std::optional getSourcePath() const override { if (url.scheme == "git+file" && !ref && !rev) return url.path; @@ -172,7 +172,8 @@ struct GitInput : Input return {std::move(*tree), input}; } - auto [isLocal, actualUrl] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 64d5e84e1..304b2d967 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -84,7 +84,8 @@ struct MercurialInput : Input auto input = std::make_shared(*this); - auto [isLocal, actualUrl] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. -- cgit v1.2.3 From cc22cf662b6998abbb5a08afce7678c5b149d204 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jan 2020 22:05:11 +0100 Subject: Respect lock files of inputs + fine-grained lock file control When computing a lock file, we now respect the lock files of flake inputs. This is important for usability / reproducibility. For example, the 'nixops' flake depends on the 'nixops-aws' and 'nixops-hetzner' repositories. So when the 'nixops' flake is used in another flake, we want the versions of 'nixops-aws' and 'nixops-hetzner' locked by the the 'nixops' flake because those presumably have been tested. This can lead to a proliferation of versions of flakes like 'nixpkgs' (since every flake's lock file could depend on a different version of 'nixpkgs'). This is not a major issue when using Nixpkgs overlays or NixOS modules, since then the top-level flake composes those overlays/modules into *its* version of Nixpkgs and all other versions are ignored. Lock file computation has been made a bit more lazy so it won't try to fetch all those versions of 'nixpkgs'. However, in case it's necessary to minimize flake versions, there now are two input attributes that allow this. First, you can copy an input from another flake, as follows: inputs.nixpkgs.follows = "dwarffs/nixpkgs"; This states that the calling flake's 'nixpkgs' input shall be the same as the 'nixpkgs' input of the 'dwarffs' input. Second, you can override inputs of inputs: inputs.nixpkgs.url = github:edolstra/nixpkgs/; inputs.nixops.inputs.nixpkgs.url = github:edolstra/nixpkgs/; or equivalently, using 'follows': inputs.nixpkgs.url = github:edolstra/nixpkgs/; inputs.nixops.inputs.nixpkgs.follows = "nixpkgs"; This states that the 'nixpkgs' input of the 'nixops' input shall be the same as the calling flake's 'nixpkgs' input. Finally, at '-v' Nix now prints the changes to the lock file, e.g. $ nix flake update ~/Misc/eelco-configurations/hagbard inputs of flake 'git+file:///home/eelco/Misc/eelco-configurations?subdir=hagbard' changed: updated 'nixpkgs': 'github:edolstra/nixpkgs/7845bf5f4b3013df1cf036e9c9c3a55a30331db9' -> 'github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd' removed 'nixops' removed 'nixops/nixops-aws' removed 'nixops/nixops-hetzner' removed 'nixops/nixpkgs' --- src/libstore/fetchers/parse.cc | 1 + src/libstore/fetchers/regex.hh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc index 96a0681e5..88c2844d0 100644 --- a/src/libstore/fetchers/parse.cc +++ b/src/libstore/fetchers/parse.cc @@ -6,6 +6,7 @@ namespace nix::fetchers { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { diff --git a/src/libstore/fetchers/regex.hh b/src/libstore/fetchers/regex.hh index eb061a048..ad434860f 100644 --- a/src/libstore/fetchers/regex.hh +++ b/src/libstore/fetchers/regex.hh @@ -27,6 +27,7 @@ extern std::regex revRegex; // A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; } -- cgit v1.2.3 From 5046233b5a6e544eedb4720d81f21d37a7c35d3b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Jan 2020 13:45:49 +0100 Subject: Add Mercurial tests --- src/libstore/fetchers/mercurial.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 304b2d967..e6f252700 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -89,6 +89,8 @@ struct MercurialInput : Input // FIXME: return lastModified. + // FIXME: don't clone local repositories. + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; -- cgit v1.2.3 From 1af7b94c1d4655e8d10529c257ce044294c33137 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jan 2020 13:11:02 +0100 Subject: Add support for tarball flake inputs For example, $ nix flake info https://github.com/edolstra/dwarffs/archive/master.tar.gz Fixes #2929. --- src/libstore/fetchers/fetchers.cc | 4 ++ src/libstore/fetchers/fetchers.hh | 2 +- src/libstore/fetchers/tarball.cc | 91 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/libstore/fetchers/tarball.cc (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 7f82d5af0..efc18dbb8 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -39,6 +39,10 @@ std::pair> Input::fetchTree(ref store) if (input->narHash) assert(input->narHash == tree.narHash); + if (narHash && narHash != input->narHash) + throw Error("NAR hash mismatch in input '%s', expected '%s', got '%s'", + to_string(), narHash->to_string(SRI), input->narHash->to_string(SRI)); + return {std::move(tree), input}; } diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 1f3be8c2b..ccc1683ba 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -25,7 +25,7 @@ struct Tree struct Input : std::enable_shared_from_this { std::string type; - std::optional narHash; + std::optional narHash; // FIXME: implement virtual ~Input() { } diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc new file mode 100644 index 000000000..e82066089 --- /dev/null +++ b/src/libstore/fetchers/tarball.cc @@ -0,0 +1,91 @@ +#include "fetchers.hh" +#include "download.hh" +#include "globals.hh" +#include "parse.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +struct TarballInput : Input +{ + ParsedURL url; + std::optional hash; + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && to_string() == other2->to_string() + && hash == other2->hash; + } + + bool isImmutable() const override + { + return (bool) hash; + } + + std::string to_string() const override + { + auto url2(url); + if (narHash) + url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + return url2.to_string(); + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + CachedDownloadRequest request(url.to_string()); + request.unpack = true; + request.getLastModified = true; + request.name = "source"; + + auto res = getDownloader()->downloadCached(store, request); + + auto input = std::make_shared(*this); + + auto storePath = store->parseStorePath(res.storePath); + + input->narHash = store->queryPathInfo(storePath)->narHash; + + return { + Tree { + .actualPath = res.path, + .storePath = std::move(storePath), + .lastModified = *res.lastModified + }, + input + }; + } +}; + +struct TarballInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + + if (!hasSuffix(url.path, ".zip") + && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tar.gz") + && !hasSuffix(url.path, ".tar.xz") + && !hasSuffix(url.path, ".tar.bz2")) + return nullptr; + + auto input = std::make_unique(); + input->type = "tarball"; + input->url = url; + + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) { + // FIXME: require SRI hash. + input->narHash = Hash(narHash->second); + } + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} -- cgit v1.2.3 From 88b44b1e94735710853bdabb6904073bc77f2ccb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jan 2020 22:53:03 +0100 Subject: Fix flake update check --- src/libstore/fetchers/git.cc | 5 ++++- src/libstore/fetchers/mercurial.cc | 5 ++++- src/libstore/fetchers/parse.cc | 10 ++++++++++ src/libstore/fetchers/parse.hh | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 16bbcd285..1350c5754 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -83,7 +83,7 @@ struct GitInput : Input auto other2 = dynamic_cast(&other); return other2 - && url.url == other2->url.url + && url == other2->url && rev == other2->rev && ref == other2->ref; } @@ -361,6 +361,8 @@ struct GitInputScheme : InputScheme auto input = std::make_unique(url); + input->url.query.clear(); + for (auto &[name, value] : url.query) { if (name == "rev") { if (!std::regex_match(value, revRegex)) @@ -372,6 +374,7 @@ struct GitInputScheme : InputScheme throw BadURL("Git URL '%s' contains an invalid branch/tag name", url.url); input->ref = value; } + else input->url.query.insert_or_assign(name, value); } return input; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index e6f252700..e012f98fc 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -29,7 +29,7 @@ struct MercurialInput : Input auto other2 = dynamic_cast(&other); return other2 - && url.url == other2->url.url + && url == other2->url && rev == other2->rev && ref == other2->ref; } @@ -255,6 +255,8 @@ struct MercurialInputScheme : InputScheme auto input = std::make_unique(url); + input->url.query.clear(); + for (auto &[name, value] : url.query) { if (name == "rev") { if (!std::regex_match(value, revRegex)) @@ -266,6 +268,7 @@ struct MercurialInputScheme : InputScheme throw BadURL("Mercurial URL '%s' contains an invalid branch/tag name", url.url); input->ref = value; } + else input->url.query.insert_or_assign(name, value); } return input; diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc index 88c2844d0..dc1b3efe6 100644 --- a/src/libstore/fetchers/parse.cc +++ b/src/libstore/fetchers/parse.cc @@ -127,4 +127,14 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } +bool ParsedURL::operator ==(const ParsedURL & other) const +{ + return + scheme == other.scheme + && authority == other.authority + && path == other.path + && query == other.query + && fragment == other.fragment; +} + } diff --git a/src/libstore/fetchers/parse.hh b/src/libstore/fetchers/parse.hh index 22cc57816..45d5182b0 100644 --- a/src/libstore/fetchers/parse.hh +++ b/src/libstore/fetchers/parse.hh @@ -15,6 +15,8 @@ struct ParsedURL std::string fragment; std::string to_string() const; + + bool operator ==(const ParsedURL & other) const; }; MakeError(BadURL, Error); -- cgit v1.2.3 From 8414685c0f0c0d744c70d438894e4ecd7a078808 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jan 2020 19:16:40 +0100 Subject: Change lock file format to use an attribute representation of flake refs rather than URLs --- src/libstore/fetchers/fetchers.cc | 50 ++++++++++++++++++++++++++++++++++++++ src/libstore/fetchers/fetchers.hh | 23 +++++++++++++++++- src/libstore/fetchers/git.cc | 27 +++++++++++++++++--- src/libstore/fetchers/github.cc | 27 +++++++++++++++++++- src/libstore/fetchers/indirect.cc | 25 ++++++++++++++++++- src/libstore/fetchers/mercurial.cc | 27 +++++++++++++++++--- src/libstore/fetchers/tarball.cc | 47 ++++++++++++++++++++++++++++++----- 7 files changed, 211 insertions(+), 15 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index efc18dbb8..16f674401 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -2,6 +2,8 @@ #include "parse.hh" #include "store-api.hh" +#include + namespace nix::fetchers { std::unique_ptr>> inputSchemes = nullptr; @@ -26,6 +28,54 @@ std::unique_ptr inputFromURL(const std::string & url) return inputFromURL(parseURL(url)); } +std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs); + if (res) return res; + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else abort(); + } + return json; +} + +Input::Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("type", type()); + return attrs; +} + +std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string", name); +} + +std::string getStrAttr(const Input::Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + std::pair> Input::fetchTree(ref store) const { auto [tree, input] = fetchTreeInternal(store); diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index ccc1683ba..39e004240 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -5,6 +5,9 @@ #include "path.hh" #include +#include + +#include namespace nix { class Store; } @@ -24,9 +27,10 @@ struct Tree struct Input : std::enable_shared_from_this { - std::string type; std::optional narHash; // FIXME: implement + virtual std::string type() const = 0; + virtual ~Input() { } virtual bool operator ==(const Input & other) const { return false; } @@ -43,6 +47,11 @@ struct Input : std::enable_shared_from_this virtual std::string to_string() const = 0; + typedef std::variant Attr; + typedef std::map Attrs; + + Attrs toAttrs() const; + std::pair> fetchTree(ref store) const; virtual std::shared_ptr applyOverrides( @@ -59,6 +68,8 @@ struct Input : std::enable_shared_from_this private: virtual std::pair> fetchTreeInternal(ref store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; }; struct ParsedURL; @@ -68,12 +79,22 @@ struct InputScheme virtual ~InputScheme() { } virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) = 0; }; std::unique_ptr inputFromURL(const ParsedURL & url); std::unique_ptr inputFromURL(const std::string & url); +std::unique_ptr inputFromAttrs(const Input::Attrs & attrs); + void registerInputScheme(std::unique_ptr && fetcher); +nlohmann::json attrsToJson(const Input::Attrs & attrs); + +std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Input::Attrs & attrs, const std::string & name); + } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 1350c5754..2a7ce5432 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -74,9 +74,9 @@ struct GitInput : Input std::optional rev; GitInput(const ParsedURL & url) : url(url) - { - type = "git"; - } + { } + + std::string type() const override { return "git"; } bool operator ==(const Input & other) const override { @@ -105,6 +105,17 @@ struct GitInput : Input return url2.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + void clone(const Path & destDir) const override { auto [isLocal, actualUrl] = getActualUrl(); @@ -379,6 +390,16 @@ struct GitInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index c75680649..13ac4e2f1 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -19,6 +19,8 @@ struct GitHubInput : Input std::optional ref; std::optional rev; + std::string type() const override { return "github"; } + bool operator ==(const Input & other) const override { auto other2 = dynamic_cast(&other); @@ -48,6 +50,18 @@ struct GitHubInput : Input return s; } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + void clone(const Path & destDir) const override { std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); @@ -138,7 +152,6 @@ struct GitHubInputScheme : InputScheme auto path = tokenizeString>(url.path, "/"); auto input = std::make_unique(); - input->type = "github"; if (path.size() == 2) { } else if (path.size() == 3) { @@ -176,6 +189,18 @@ struct GitHubInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc index 1f9d1e24f..d079b3ad3 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libstore/fetchers/indirect.cc @@ -12,6 +12,8 @@ struct IndirectInput : Input std::optional rev; std::optional ref; + std::string type() const override { return "indirect"; } + bool operator ==(const Input & other) const override { auto other2 = dynamic_cast(&other); @@ -51,6 +53,17 @@ struct IndirectInput : Input return url.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("id", id); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + std::shared_ptr applyOverrides( std::optional ref, std::optional rev) const override @@ -79,7 +92,6 @@ struct IndirectInputScheme : InputScheme auto path = tokenizeString>(url.path, "/"); auto input = std::make_unique(); - input->type = "indirect"; if (path.size() == 1) { } else if (path.size() == 2) { @@ -107,6 +119,17 @@ struct IndirectInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + auto input = std::make_unique(); + input->id = getStrAttr(attrs, "id"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index e012f98fc..f0135d512 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -20,9 +20,9 @@ struct MercurialInput : Input std::optional rev; MercurialInput(const ParsedURL & url) : url(url) - { - type = "hg"; - } + { } + + std::string type() const override { return "hg"; } bool operator ==(const Input & other) const override { @@ -51,6 +51,17 @@ struct MercurialInput : Input return url2.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + std::shared_ptr applyOverrides( std::optional ref, std::optional rev) const override @@ -273,6 +284,16 @@ struct MercurialInputScheme : InputScheme return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index e82066089..21c785ada 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -11,6 +11,11 @@ struct TarballInput : Input ParsedURL url; std::optional hash; + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + bool operator ==(const Input & other) const override { auto other2 = dynamic_cast(&other); @@ -22,17 +27,32 @@ struct TarballInput : Input bool isImmutable() const override { - return (bool) hash; + return hash || narHash; } std::string to_string() const override { auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. if (narHash) url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI)); return url2.to_string(); } + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (narHash) + attrs.emplace("narHash", hash->to_string(SRI)); + else if (hash) + attrs.emplace("hash", hash->to_string(SRI)); + return attrs; + } + std::pair> fetchTreeInternal(nix::ref store) const override { CachedDownloadRequest request(url.to_string()); @@ -72,18 +92,33 @@ struct TarballInputScheme : InputScheme && !hasSuffix(url.path, ".tar.bz2")) return nullptr; - auto input = std::make_unique(); - input->type = "tarball"; - input->url = url; + auto input = std::make_unique(url); + + auto hash = url.query.find("hash"); + if (hash != url.query.end()) + // FIXME: require SRI hash. + input->hash = Hash(hash->second); auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) { + if (narHash != url.query.end()) // FIXME: require SRI hash. input->narHash = Hash(narHash->second); - } return input; } + + std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + // FIXME: require SRI hash. + input->hash = Hash(*hash); + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + input->narHash = Hash(*narHash); + return input; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); -- cgit v1.2.3 From 185c3c824015f03027b77b85221df32cdb16e759 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jan 2020 19:35:28 +0100 Subject: Cleanup --- src/libstore/fetchers/git.cc | 37 ++++++++++++++++++++++--------------- src/libstore/fetchers/github.cc | 7 +++++-- src/libstore/fetchers/indirect.cc | 5 +++++ src/libstore/fetchers/mercurial.cc | 37 ++++++++++++++++++++++--------------- src/libstore/fetchers/tarball.cc | 5 +++++ 5 files changed, 59 insertions(+), 32 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 2a7ce5432..5d0448777 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -370,32 +370,39 @@ struct GitInputScheme : InputScheme url.scheme != "git+ssh" && url.scheme != "git+file") return nullptr; - auto input = std::make_unique(url); + auto url2(url); + // FIXME: strip git+ + url2.query.clear(); - input->url.query.clear(); + Input::Attrs attrs; + attrs.emplace("type", "git"); for (auto &[name, value] : url.query) { - if (name == "rev") { - if (!std::regex_match(value, revRegex)) - throw BadURL("Git URL '%s' contains an invalid commit hash", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("Git URL '%s' contains an invalid branch/tag name", url.url); - input->ref = value; - } - else input->url.query.insert_or_assign(name, value); + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); } - return input; + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Git input attribute '%s'", name); + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } if (auto rev = maybeGetStrAttr(attrs, "rev")) input->rev = Hash(*rev, htSHA1); return input; diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index 13ac4e2f1..c8746b723 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -166,8 +166,6 @@ struct GitHubInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev") { - if (!std::regex_match(value, revRegex)) - throw BadURL("GitHub URL '%s' contains an invalid commit hash", url.url); if (input->rev) throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); input->rev = Hash(value, htSHA1); @@ -193,6 +191,11 @@ struct GitHubInputScheme : InputScheme std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "github") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitHub input attribute '%s'", name); + auto input = std::make_unique(); input->owner = getStrAttr(attrs, "owner"); input->repo = getStrAttr(attrs, "repo"); diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc index d079b3ad3..016f5fb39 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libstore/fetchers/indirect.cc @@ -123,6 +123,11 @@ struct IndirectInputScheme : InputScheme std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "id" && name != "ref" && name != "rev") + throw Error("unsupported indirect input attribute '%s'", name); + auto input = std::make_unique(); input->id = getStrAttr(attrs, "id"); input->ref = maybeGetStrAttr(attrs, "ref"); diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index f0135d512..1bdab1dbf 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -264,32 +264,39 @@ struct MercurialInputScheme : InputScheme url.scheme != "hg+ssh" && url.scheme != "hg+file") return nullptr; - auto input = std::make_unique(url); + auto url2(url); + // FIXME: strip hg+ + url2.query.clear(); - input->url.query.clear(); + Input::Attrs attrs; + attrs.emplace("type", "hg"); for (auto &[name, value] : url.query) { - if (name == "rev") { - if (!std::regex_match(value, revRegex)) - throw BadURL("Mercurial URL '%s' contains an invalid commit hash", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("Mercurial URL '%s' contains an invalid branch/tag name", url.url); - input->ref = value; - } - else input->url.query.insert_or_assign(name, value); + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); } - return input; + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Mercurial input attribute '%s'", name); + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + input->ref = *ref; + } if (auto rev = maybeGetStrAttr(attrs, "rev")) input->rev = Hash(*rev, htSHA1); return input; diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 21c785ada..1302299b3 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -110,6 +110,11 @@ struct TarballInputScheme : InputScheme std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "hash" && name != "narHash") + throw Error("unsupported tarball input attribute '%s'", name); + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); if (auto hash = maybeGetStrAttr(attrs, "hash")) // FIXME: require SRI hash. -- cgit v1.2.3 From 8451298b35353abafe385124cb55e8d4911032ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 1 Feb 2020 16:41:54 +0100 Subject: Factor out TreeInfo --- src/libstore/fetchers/fetchers.cc | 6 +++--- src/libstore/fetchers/fetchers.hh | 6 ++---- src/libstore/fetchers/git.cc | 34 ++++++++++++++++++++-------------- src/libstore/fetchers/github.cc | 6 ++++-- src/libstore/fetchers/mercurial.cc | 34 ++++++++++++++++++++++------------ src/libstore/fetchers/tarball.cc | 4 +++- src/libstore/fetchers/tree-info.hh | 13 +++++++++++++ 7 files changed, 67 insertions(+), 36 deletions(-) create mode 100644 src/libstore/fetchers/tree-info.hh (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 16f674401..256ef66f2 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -83,11 +83,11 @@ std::pair> Input::fetchTree(ref store) if (tree.actualPath == "") tree.actualPath = store->toRealPath(store->printStorePath(tree.storePath)); - if (!tree.narHash) - tree.narHash = store->queryPathInfo(tree.storePath)->narHash; + if (!tree.info.narHash) + tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; if (input->narHash) - assert(input->narHash == tree.narHash); + assert(input->narHash == tree.info.narHash); if (narHash && narHash != input->narHash) throw Error("NAR hash mismatch in input '%s', expected '%s', got '%s'", diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 39e004240..7a7ce7d37 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "hash.hh" #include "path.hh" +#include "tree-info.hh" #include #include @@ -19,10 +20,7 @@ struct Tree { Path actualPath; StorePath storePath; - Hash narHash; - std::optional rev; - std::optional revCount; - std::optional lastModified; + TreeInfo info; }; struct Input : std::enable_shared_from_this diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 5d0448777..4ad0f6f34 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -28,11 +28,11 @@ static void cacheGitInfo(Store & store, const std::string & name, const Tree & t nlohmann::json json; json["storePath"] = store.printStorePath(tree.storePath); json["name"] = name; - json["rev"] = tree.rev->gitRev(); - json["revCount"] = *tree.revCount; - json["lastModified"] = *tree.lastModified; + json["rev"] = tree.info.rev->gitRev(); + json["revCount"] = *tree.info.revCount; + json["lastModified"] = *tree.info.lastModified; - auto cacheInfoPath = getCacheInfoPathFor(name, *tree.rev); + auto cacheInfoPath = getCacheInfoPathFor(name, *tree.info.rev); createDirs(dirOf(cacheInfoPath)); writeFile(cacheInfoPath, json.dump()); } @@ -53,9 +53,11 @@ static std::optional lookupGitInfo( Tree tree{ .actualPath = store->toRealPath(store->printStorePath(storePath)), .storePath = std::move(storePath), - .rev = rev, - .revCount = json["revCount"], - .lastModified = json["lastModified"], + .info = TreeInfo { + .rev = rev, + .revCount = json["revCount"], + .lastModified = json["lastModified"], + } }; return tree; } @@ -237,10 +239,12 @@ struct GitInput : Input auto tree = Tree { .actualPath = store->printStorePath(storePath), .storePath = std::move(storePath), - .revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "rev-list", "--count", "HEAD" })) : 0, - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + .info = TreeInfo { + .revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "rev-list", "--count", "HEAD" })) : 0, + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } }; return {std::move(tree), input}; @@ -349,9 +353,11 @@ struct GitInput : Input auto tree = Tree { .actualPath = store->toRealPath(store->printStorePath(storePath)), .storePath = std::move(storePath), - .rev = input->rev, - .revCount = revCount, - .lastModified = lastModified, + .info = TreeInfo { + .rev = input->rev, + .revCount = revCount, + .lastModified = lastModified + } }; cacheGitInfo(*store, name, tree); diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index c8746b723..a4d39d17d 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -113,8 +113,10 @@ struct GitHubInput : Input Tree result{ .actualPath = dresult.path, .storePath = store->parseStorePath(dresult.storePath), - .rev = *rev, - .lastModified = *dresult.lastModified + .info = TreeInfo { + .rev = *rev, + .lastModified = *dresult.lastModified, + }, }; #if 0 diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 1bdab1dbf..b415b7944 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -215,12 +215,17 @@ struct MercurialInput : Input if (store->isValidPath(storePath)) { printTalkative("using cached Mercurial store path '%s'", store->printStorePath(storePath)); - return {Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .rev = input->rev, - .revCount = revCount, - }, input}; + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .rev = input->rev, + .revCount = revCount, + }, + }, + input + }; } } catch (SysError & e) { @@ -246,12 +251,17 @@ struct MercurialInput : Input writeFile(storeLink, json.dump()); - return {Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .rev = input->rev, - .revCount = revCount, - }, input}; + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .rev = input->rev, + .revCount = revCount + } + }, + input + }; } }; diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 1302299b3..fc4d7542b 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -72,7 +72,9 @@ struct TarballInput : Input Tree { .actualPath = res.path, .storePath = std::move(storePath), - .lastModified = *res.lastModified + .info = TreeInfo { + .lastModified = *res.lastModified, + }, }, input }; diff --git a/src/libstore/fetchers/tree-info.hh b/src/libstore/fetchers/tree-info.hh new file mode 100644 index 000000000..30d4f3d6b --- /dev/null +++ b/src/libstore/fetchers/tree-info.hh @@ -0,0 +1,13 @@ +#pragma once + +namespace nix { + +struct TreeInfo +{ + Hash narHash; + std::optional rev; + std::optional revCount; + std::optional lastModified; +}; + +} -- cgit v1.2.3 From b9d64f931893120834fa54ebf084764d2e22ba33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 1 Feb 2020 23:33:44 +0100 Subject: Record TreeInfo in the lock file Necessary for #3253. --- src/libstore/fetchers/tree-info.hh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/tree-info.hh b/src/libstore/fetchers/tree-info.hh index 30d4f3d6b..61500facb 100644 --- a/src/libstore/fetchers/tree-info.hh +++ b/src/libstore/fetchers/tree-info.hh @@ -5,9 +5,18 @@ namespace nix { struct TreeInfo { Hash narHash; - std::optional rev; + std::optional rev; // FIXME: remove std::optional revCount; std::optional lastModified; + + bool operator ==(const TreeInfo & other) const + { + return + narHash == other.narHash + && rev == other.rev + && revCount == other.revCount + && lastModified == other.lastModified; + } }; } -- cgit v1.2.3 From 887730aab39dd63a37d49d72a07c6441ea3b2f92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 1 Feb 2020 23:54:20 +0100 Subject: Remove superfluous TreeInfo::rev field --- src/libstore/fetchers/git.cc | 33 +++++++++++++++++++-------------- src/libstore/fetchers/github.cc | 1 - src/libstore/fetchers/mercurial.cc | 2 -- src/libstore/fetchers/tree-info.hh | 2 -- 4 files changed, 19 insertions(+), 19 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 4ad0f6f34..4f302cd0f 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -23,21 +23,25 @@ static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) return cacheDir + "/" + linkName + ".link"; } -static void cacheGitInfo(Store & store, const std::string & name, const Tree & tree) +static void cacheGitInfo( + Store & store, + const std::string & name, + const Tree & tree, + const Hash & rev) { nlohmann::json json; json["storePath"] = store.printStorePath(tree.storePath); json["name"] = name; - json["rev"] = tree.info.rev->gitRev(); + json["rev"] = rev.gitRev(); json["revCount"] = *tree.info.revCount; json["lastModified"] = *tree.info.lastModified; - auto cacheInfoPath = getCacheInfoPathFor(name, *tree.info.rev); + auto cacheInfoPath = getCacheInfoPathFor(name, rev); createDirs(dirOf(cacheInfoPath)); writeFile(cacheInfoPath, json.dump()); } -static std::optional lookupGitInfo( +static std::optional> lookupGitInfo( ref store, const std::string & name, const Hash & rev) @@ -50,16 +54,14 @@ static std::optional lookupGitInfo( auto storePath = store->parseStorePath((std::string) json["storePath"]); if (store->isValidPath(storePath)) { - Tree tree{ + return {{rev, Tree{ .actualPath = store->toRealPath(store->printStorePath(storePath)), .storePath = std::move(storePath), .info = TreeInfo { - .rev = rev, .revCount = json["revCount"], .lastModified = json["lastModified"], } - }; - return tree; + }}}; } } catch (SysError & e) { @@ -181,8 +183,10 @@ struct GitInput : Input assert(!rev || rev->type == htSHA1); if (rev) { - if (auto tree = lookupGitInfo(store, name, *rev)) - return {std::move(*tree), input}; + if (auto tree = lookupGitInfo(store, name, *rev)) { + input->rev = tree->first; + return {std::move(tree->second), input}; + } } auto [isLocal, actualUrl_] = getActualUrl(); @@ -326,8 +330,10 @@ struct GitInput : Input input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); } - if (auto tree = lookupGitInfo(store, name, *input->rev)) - return {std::move(*tree), input}; + if (auto tree = lookupGitInfo(store, name, *input->rev)) { + assert(*input->rev == tree->first); + return {std::move(tree->second), input}; + } // FIXME: check whether rev is an ancestor of ref. @@ -354,13 +360,12 @@ struct GitInput : Input .actualPath = store->toRealPath(store->printStorePath(storePath)), .storePath = std::move(storePath), .info = TreeInfo { - .rev = input->rev, .revCount = revCount, .lastModified = lastModified } }; - cacheGitInfo(*store, name, tree); + cacheGitInfo(*store, name, tree, *input->rev); return {std::move(tree), input}; } diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index a4d39d17d..0a000e83f 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -114,7 +114,6 @@ struct GitHubInput : Input .actualPath = dresult.path, .storePath = store->parseStorePath(dresult.storePath), .info = TreeInfo { - .rev = *rev, .lastModified = *dresult.lastModified, }, }; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index b415b7944..0eb81d014 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -220,7 +220,6 @@ struct MercurialInput : Input .actualPath = store->printStorePath(storePath), .storePath = std::move(storePath), .info = TreeInfo { - .rev = input->rev, .revCount = revCount, }, }, @@ -256,7 +255,6 @@ struct MercurialInput : Input .actualPath = store->printStorePath(storePath), .storePath = std::move(storePath), .info = TreeInfo { - .rev = input->rev, .revCount = revCount } }, diff --git a/src/libstore/fetchers/tree-info.hh b/src/libstore/fetchers/tree-info.hh index 61500facb..28ead8588 100644 --- a/src/libstore/fetchers/tree-info.hh +++ b/src/libstore/fetchers/tree-info.hh @@ -5,7 +5,6 @@ namespace nix { struct TreeInfo { Hash narHash; - std::optional rev; // FIXME: remove std::optional revCount; std::optional lastModified; @@ -13,7 +12,6 @@ struct TreeInfo { return narHash == other.narHash - && rev == other.rev && revCount == other.revCount && lastModified == other.lastModified; } -- cgit v1.2.3 From 958ec5de568904a07ef050418088d882cbf2ea61 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 2 Feb 2020 11:31:58 +0100 Subject: Cleanup --- src/libstore/fetchers/fetchers.hh | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 7a7ce7d37..0ef79bc45 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -33,8 +33,12 @@ struct Input : std::enable_shared_from_this virtual bool operator ==(const Input & other) const { return false; } + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ virtual bool isDirect() const { return true; } + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ virtual bool isImmutable() const { return (bool) narHash; } virtual bool contains(const Input & other) const { return false; } -- cgit v1.2.3 From 7bcc9f2aaf87a08a7570610f45e99754fd74d3f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 2 Feb 2020 12:28:56 +0100 Subject: Add convenience overload for toRealPath() --- src/libstore/build.cc | 14 +++++++------- src/libstore/derivations.cc | 2 +- src/libstore/download.cc | 6 +++--- src/libstore/fetchers/fetchers.cc | 2 +- src/libstore/fetchers/git.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/store-api.hh | 5 +++++ 7 files changed, 20 insertions(+), 15 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 30077556d..0e3a23a4d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1398,7 +1398,7 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath))); + lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { worker.waitForAWhile(shared_from_this()); @@ -1429,7 +1429,7 @@ void DerivationGoal::tryToBuild() for (auto & i : drv->outputs) { if (worker.store.isValidPath(i.second.path)) continue; debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path))); + deletePath(worker.store.Store::toRealPath(i.second.path)); } /* Don't do a remote build if the derivation has the attribute @@ -1686,7 +1686,7 @@ void DerivationGoal::buildDone() /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second))); + deletePath(worker.store.Store::toRealPath(i.second)); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -2072,7 +2072,7 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot"; + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ @@ -2917,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path) #if __linux__ - Path source = worker.store.toRealPath(worker.store.printStorePath(path)); + Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); debug("bind-mounting %s -> %s", target, source); @@ -3579,7 +3579,7 @@ void DerivationGoal::registerOutputs() if (needsHashRewrite()) { auto r = redirectedOutputs.find(i.second.path); if (r != redirectedOutputs.end()) { - auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second)); + auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair && redirectedBadOutputs.count(i.second.path) && pathExists(redirected)) @@ -3672,7 +3672,7 @@ void DerivationGoal::registerOutputs() BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); - Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); + Path actualDest = worker.store.Store::toRealPath(dest); if (worker.store.isValidPath(dest)) std::rethrow_exception(delayedException); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d9da8769c..a554cb66d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -354,7 +354,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; + readDerivation(store, store.toRealPath(i.first)), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 681b74240..d8c52b151 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -910,7 +910,7 @@ CachedDownloadResult Downloader::downloadCached( printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir); + unpackTarfile(store->toRealPath(*storePath), tmpDir); auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); @@ -928,8 +928,8 @@ CachedDownloadResult Downloader::downloadCached( if (expectedStorePath && *storePath != *expectedStorePath) { unsigned int statusCode = 102; Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first - : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); + ? hashPath(request.expectedHash.type, store->toRealPath(*storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(*storePath)); throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", url, request.expectedHash.to_string(), gotHash.to_string()); } diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 256ef66f2..89cc9e774 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -81,7 +81,7 @@ std::pair> Input::fetchTree(ref store) auto [tree, input] = fetchTreeInternal(store); if (tree.actualPath == "") - tree.actualPath = store->toRealPath(store->printStorePath(tree.storePath)); + tree.actualPath = store->toRealPath(tree.storePath); if (!tree.info.narHash) tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 4f302cd0f..67cb5a9e0 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -55,7 +55,7 @@ static std::optional> lookupGitInfo( if (store->isValidPath(storePath)) { return {{rev, Tree{ - .actualPath = store->toRealPath(store->printStorePath(storePath)), + .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), .info = TreeInfo { .revCount = json["revCount"], @@ -357,7 +357,7 @@ struct GitInput : Input auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); auto tree = Tree { - .actualPath = store->toRealPath(store->printStorePath(storePath)), + .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), .info = TreeInfo { .revCount = revCount, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e214f0659..a337ad0cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1263,7 +1263,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) else hashSink = std::make_unique(info->narHash.type, storePathToHash(printStorePath(info->path))); - dumpPath(toRealPath(printStorePath(i)), *hashSink); + dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 284e201de..902d00216 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -656,6 +656,11 @@ public: return storePath; } + Path toRealPath(const StorePath & storePath) + { + return toRealPath(printStorePath(storePath)); + } + virtual void createUser(const std::string & userName, uid_t userId) { } -- cgit v1.2.3 From fad9faf3545f115e33e522eb11802e2825c8c425 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 2 Feb 2020 12:29:53 +0100 Subject: Add TreeInfo::computeStorePath() --- src/libstore/fetchers/fetchers.cc | 6 ++++++ src/libstore/fetchers/tree-info.hh | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 89cc9e774..373786f38 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -107,4 +107,10 @@ std::shared_ptr Input::applyOverrides( return shared_from_this(); } +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(true, narHash, "source"); +} + } diff --git a/src/libstore/fetchers/tree-info.hh b/src/libstore/fetchers/tree-info.hh index 28ead8588..02e92759b 100644 --- a/src/libstore/fetchers/tree-info.hh +++ b/src/libstore/fetchers/tree-info.hh @@ -1,6 +1,10 @@ #pragma once -namespace nix { +#include "path.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { struct TreeInfo { @@ -15,6 +19,8 @@ struct TreeInfo && revCount == other.revCount && lastModified == other.lastModified; } + + StorePath computeStorePath(Store & store) const; }; } -- cgit v1.2.3 From a9ebc3ea5db2fb0cbf7ab8ea35c9d9d5073abfc8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 2 Feb 2020 13:06:00 +0100 Subject: Remove the git+ and hg+ prefixes from structured input refs --- src/libstore/fetchers/git.cc | 11 ++++++----- src/libstore/fetchers/mercurial.cc | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 67cb5a9e0..0101744bd 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -104,6 +104,7 @@ struct GitInput : Input std::string to_string() const override { ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); if (ref) url2.query.insert_or_assign("ref", *ref); return url2.to_string(); @@ -159,19 +160,19 @@ struct GitInput : Input std::optional getSourcePath() const override { - if (url.scheme == "git+file" && !ref && !rev) + if (url.scheme == "file" && !ref && !rev) return url.path; return {}; } std::pair getActualUrl() const { - // Don't clone git+file:// URIs (but otherwise treat them the + // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing - bool isLocal = url.scheme == "git+file" && !forceHttp; - return {isLocal, isLocal ? url.path : std::string(url.base, 4)}; + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; } std::pair> fetchTreeInternal(nix::ref store) const override @@ -382,7 +383,7 @@ struct GitInputScheme : InputScheme url.scheme != "git+file") return nullptr; auto url2(url); - // FIXME: strip git+ + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); url2.query.clear(); Input::Attrs attrs; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 0eb81d014..9b8c2132c 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -46,6 +46,7 @@ struct MercurialInput : Input std::string to_string() const override { ParsedURL url2(url); + url2.scheme = "hg+" + url2.scheme; if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); if (ref) url2.query.insert_or_assign("ref", *ref); return url2.to_string(); @@ -78,15 +79,15 @@ struct MercurialInput : Input std::optional getSourcePath() const { - if (url.scheme == "hg+file" && !ref && !rev) + if (url.scheme == "file" && !ref && !rev) return url.path; return {}; } std::pair getActualUrl() const { - bool isLocal = url.scheme == "hg+file"; - return {isLocal, isLocal ? url.path : std::string(url.base, 3)}; + bool isLocal = url.scheme == "file"; + return {isLocal, isLocal ? url.path : url.base}; } std::pair> fetchTreeInternal(nix::ref store) const override @@ -273,7 +274,7 @@ struct MercurialInputScheme : InputScheme url.scheme != "hg+file") return nullptr; auto url2(url); - // FIXME: strip hg+ + url2.scheme = std::string(url2.scheme, 3); url2.query.clear(); Input::Attrs attrs; -- cgit v1.2.3 From d5334c466b9ee237a1cce4fd005d790e67d74822 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 2 Feb 2020 16:32:46 +0100 Subject: Automatically do git/hg add on flake.lock --- src/libstore/fetchers/fetchers.hh | 3 +++ src/libstore/fetchers/git.cc | 12 ++++++++++++ src/libstore/fetchers/mercurial.cc | 11 +++++++++++ 3 files changed, 26 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 0ef79bc45..64628d1bb 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -62,6 +62,9 @@ struct Input : std::enable_shared_from_this virtual std::optional getSourcePath() const { return {}; } + // FIXME: should merge with getSourcePath(). + virtual void markChangedFile(std::string_view file) const { assert(false); } + virtual void clone(const Path & destDir) const { throw Error("do not know how to clone input '%s'", to_string()); diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 0101744bd..1c74d92a4 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -165,6 +165,18 @@ struct GitInput : Input return {}; } + void markChangedFile(std::string_view file) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + runProgram("git", true, + { "-C", *sourcePath, "add", + "--force", + "--intent-to-add", + std::string(file) + }); + } + std::pair getActualUrl() const { // Don't clone file:// URIs (but otherwise treat them the diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 9b8c2132c..0825f6b23 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -84,6 +84,17 @@ struct MercurialInput : Input return {}; } + void markChangedFile(std::string_view file) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + // FIXME: shut up if file is already tracked. + runProgram("hg", true, + { "add", + *sourcePath + "/" + std::string(file) + }); + } + std::pair getActualUrl() const { bool isLocal = url.scheme == "file"; -- cgit v1.2.3 From d070e1c5321b43496f1113198e62b2b647433459 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Feb 2020 14:29:34 +0100 Subject: Fix parsing of '#nixosConfigurations."hostname".config...' This is not strictly speaking valid but who cares. --- src/libstore/fetchers/regex.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/regex.hh b/src/libstore/fetchers/regex.hh index ad434860f..504d7bf18 100644 --- a/src/libstore/fetchers/regex.hh +++ b/src/libstore/fetchers/regex.hh @@ -13,7 +13,7 @@ const static std::string authorityRegex = const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string pathRegex = "(?:/?" + segmentRegex + "(?:/" + segmentRegex + ")*|/?)"; const static std::string pcharRegex = - "(?:[a-zA-Z0-9-._~!$&'()*+,;=:@ ]|" + pctEncoded + ")"; + "(?:[a-zA-Z0-9-._~!$&'\"()*+,;=:@ ]|" + pctEncoded + ")"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/?])*"; // A Git ref (i.e. branch or tag name). -- cgit v1.2.3 From a2628b43bbfe4368a3b5963e8b80eb6f463d94c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Feb 2020 15:27:26 +0100 Subject: Fix URL parser Fixes #3062. --- src/libstore/fetchers/parse.cc | 18 ++++++++---------- src/libstore/fetchers/regex.hh | 24 ++++++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc index dc1b3efe6..4f7cb3c6b 100644 --- a/src/libstore/fetchers/parse.cc +++ b/src/libstore/fetchers/parse.cc @@ -11,24 +11,22 @@ std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "(((" + schemeRegex + "):" - + "(//(" + authorityRegex + "))?" - + "(" + pathRegex + "))" + "((" + schemeRegex + "):" + + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?" - + ")", + + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); std::smatch match; if (std::regex_match(url, match, uriRegex)) { - auto & base = match[2]; - std::string scheme = match[3]; + auto & base = match[1]; + std::string scheme = match[2]; auto authority = match[4].matched ? std::optional(match[5]) : std::nullopt; - std::string path = match[6]; - auto & query = match[7]; - auto & fragment = match[8]; + std::string path = match[4].matched ? match[4] : match[5]; + auto & query = match[6]; + auto & fragment = match[7]; auto isFile = scheme.find("file") != std::string::npos; diff --git a/src/libstore/fetchers/regex.hh b/src/libstore/fetchers/regex.hh index 504d7bf18..e0989edfc 100644 --- a/src/libstore/fetchers/regex.hh +++ b/src/libstore/fetchers/regex.hh @@ -5,16 +5,20 @@ namespace nix::fetchers { // URI stuff. -const static std::string pctEncoded = "%[0-9a-fA-F][0-9a-fA-F]"; -const static std::string schemeRegex = "[a-z+]+"; -const static std::string authorityRegex = - "(?:(?:[a-z])*@)?" - "[a-zA-Z0-9._~-]*"; -const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; -const static std::string pathRegex = "(?:/?" + segmentRegex + "(?:/" + segmentRegex + ")*|/?)"; -const static std::string pcharRegex = - "(?:[a-zA-Z0-9-._~!$&'\"()*+,;=:@ ]|" + pctEncoded + ")"; -const static std::string queryRegex = "(?:" + pcharRegex + "|[/?])*"; +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; // A Git ref (i.e. branch or tag name). const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check -- cgit v1.2.3 From 0a4e911cf478d0d38082a8840a5acc8f57d00086 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Feb 2020 18:04:09 +0100 Subject: Install headers in the correct location --- src/libstore/local.mk | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/local.mk b/src/libstore/local.mk index e803ff85a..e8cbe422c 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -60,3 +60,9 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libstore/builtins/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) + +$(foreach i, $(wildcard src/libstore/fetchers/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/fetchers, 0644))) -- cgit v1.2.3 From e2213d77a22a1d3d5d17167eb2760352760405e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Feb 2020 21:55:57 +0100 Subject: Set 'ref' properly for local trees --- src/libstore/fetchers/git.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 1c74d92a4..35a5c18b5 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -23,6 +23,11 @@ static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) return cacheDir + "/" + linkName + ".link"; } +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + static void cacheGitInfo( Store & store, const std::string & name, @@ -268,7 +273,7 @@ struct GitInput : Input } } - if (!input->ref) input->ref = isLocal ? "HEAD" : "master"; + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; Path repoDir; -- cgit v1.2.3 From 9d7fb62db6e8ee6da7f8dbc5f5509271dc12f2ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Feb 2020 14:48:49 +0100 Subject: Add option --commit-lock-file --- src/libstore/fetchers/fetchers.hh | 6 ++++-- src/libstore/fetchers/git.cc | 14 +++++++------- src/libstore/fetchers/mercurial.cc | 11 +++++++---- 3 files changed, 18 insertions(+), 13 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 64628d1bb..9fafbffb6 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -62,8 +62,10 @@ struct Input : std::enable_shared_from_this virtual std::optional getSourcePath() const { return {}; } - // FIXME: should merge with getSourcePath(). - virtual void markChangedFile(std::string_view file) const { assert(false); } + virtual void markChangedFile( + std::string_view file, + std::optional commitMsg) const + { assert(false); } virtual void clone(const Path & destDir) const { diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 35a5c18b5..134d44ecd 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -169,17 +169,17 @@ struct GitInput : Input return url.path; return {}; } - - void markChangedFile(std::string_view file) const override + void markChangedFile(std::string_view file, std::optional commitMsg) const override { auto sourcePath = getSourcePath(); assert(sourcePath); + runProgram("git", true, - { "-C", *sourcePath, "add", - "--force", - "--intent-to-add", - std::string(file) - }); + { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } std::pair getActualUrl() const diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 0825f6b23..6ab0add1d 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -84,15 +84,18 @@ struct MercurialInput : Input return {}; } - void markChangedFile(std::string_view file) const override + void markChangedFile(std::string_view file, std::optional commitMsg) const override { auto sourcePath = getSourcePath(); assert(sourcePath); + // FIXME: shut up if file is already tracked. runProgram("hg", true, - { "add", - *sourcePath + "/" + std::string(file) - }); + { "add", *sourcePath + "/" + std::string(file) }); + + if (commitMsg) + runProgram("hg", true, + { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } std::pair getActualUrl() const -- cgit v1.2.3 From 379852a152cc2299bfd0a02e0229112ad322319c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Feb 2020 14:27:31 +0100 Subject: Registry: Use attr notation instead of URLs --- src/libstore/fetchers/fetchers.cc | 16 +++++++++++++ src/libstore/fetchers/fetchers.hh | 2 ++ src/libstore/fetchers/registry.cc | 50 +++++++++++++++++++++++++++------------ 3 files changed, 53 insertions(+), 15 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 373786f38..90bdc0fc5 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -37,6 +37,22 @@ std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) throw Error("input '%s' is unsupported", attrsToJson(attrs)); } +Input::Attrs jsonToAttrs(const nlohmann::json & json) +{ + fetchers::Input::Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs) { nlohmann::json json; diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 9fafbffb6..4202e8339 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -98,6 +98,8 @@ std::unique_ptr inputFromAttrs(const Input::Attrs & attrs); void registerInputScheme(std::unique_ptr && fetcher); +Input::Attrs jsonToAttrs(const nlohmann::json & json); + nlohmann::json attrsToJson(const Input::Attrs & attrs); std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name); diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index 9c74f118d..1fd42a169 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -19,31 +19,51 @@ std::shared_ptr Registry::read( auto json = nlohmann::json::parse(readFile(path)); auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", path, version); - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - // FIXME: remove 'uri' soon. - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url)}); + // FIXME: remove soon + if (version == 1) { + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url)}); + } } + else if (version == 2) { + for (auto & i : json["flakes"]) + registry->entries.push_back( + { inputFromAttrs(jsonToAttrs(i["from"])) + , inputFromAttrs(jsonToAttrs(i["to"])) + }); + } + + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + + return registry; } void Registry::write(const Path & path) { + nlohmann::json arr; + for (auto & elem : entries) { + nlohmann::json obj; + obj["from"] = attrsToJson(elem.first->toAttrs()); + obj["to"] = attrsToJson(elem.second->toAttrs()); + arr.emplace_back(std::move(obj)); + } + nlohmann::json json; - json["version"] = 1; - for (auto & elem : entries) - json["flakes"][elem.first->to_string()] = { {"url", elem.second->to_string()} }; + json["version"] = 2; + json["flakes"] = std::move(arr); + createDirs(dirOf(path)); - writeFile(path, json.dump(4)); + writeFile(path, json.dump(2)); } void Registry::add( -- cgit v1.2.3 From 442e665d6d3fcbdee7dece2f62a597142f8784b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Feb 2020 23:50:16 +0100 Subject: nix path-info --json: Print hash in SRI format --- src/libstore/store-api.cc | 6 ++++-- src/libstore/store-api.hh | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e37829b17..5ca77b563 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -442,7 +442,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths, void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) + bool includeImpureInfo, bool showClosureSize, + Base hashBase, + AllowInvalidFlag allowInvalid) { auto jsonList = jsonOut.list(); @@ -454,7 +456,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string()) + .attr("narHash", info->narHash.to_string(hashBase)) .attr("narSize", info->narSize); { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0aaec20d1..85851e211 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -541,6 +541,7 @@ public: each path is included. */ void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, + Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, -- cgit v1.2.3 From d4df99a3349cf2228a8ee78dea320afef86eb3ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Feb 2020 23:53:24 +0100 Subject: Parse narHash attribute for all input types --- src/libstore/fetchers/fetchers.cc | 11 ++++++++--- src/libstore/fetchers/tarball.cc | 4 +--- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 90bdc0fc5..0cc6f1c91 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -32,7 +32,12 @@ std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs); - if (res) return res; + if (res) { + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = Hash(*narHash); + return res; + } } throw Error("input '%s' is unsupported", attrsToJson(attrs)); } @@ -106,8 +111,8 @@ std::pair> Input::fetchTree(ref store) assert(input->narHash == tree.info.narHash); if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s', expected '%s', got '%s'", - to_string(), narHash->to_string(SRI), input->narHash->to_string(SRI)); + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); return {std::move(tree), input}; } diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index fc4d7542b..7c0b6690d 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -121,9 +121,7 @@ struct TarballInputScheme : InputScheme if (auto hash = maybeGetStrAttr(attrs, "hash")) // FIXME: require SRI hash. input->hash = Hash(*hash); - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - input->narHash = Hash(*narHash); + return input; } }; -- cgit v1.2.3 From 8a78bcf6a240d8a440cc663d99b76f41a14649dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 15:46:07 +0100 Subject: LocalStore::checkDerivationOutputs(): Improve error message --- src/libstore/local-store.cc | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a337ad0cc..e00556645 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -540,6 +540,18 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName) + { + if (actual != expected) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(actual), printStorePath(expected)); + auto j = drv.env.find(varName); + if (j == drv.env.end() || parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + printStorePath(drvPath), varName, printStorePath(actual)); + }; + + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) @@ -547,24 +559,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat bool recursive; Hash h; out->second.parseHashInfo(recursive, h); - auto outPath = makeFixedOutputPath(recursive, h, drvName); - StringPairs::const_iterator j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath)); + check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); } else { Hash h = hashDerivationModulo(*this, drv, true); - - for (auto & i : drv.outputs) { - auto outPath = makeOutputPath(i.first, h, drvName); - StringPairs::const_iterator j = drv.env.find(i.first); - if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath)); - } + for (auto & i : drv.outputs) + check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); } } -- cgit v1.2.3 From 6529490cc10018d5191e50c482ac1180b96b1a3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 15:53:59 +0100 Subject: nix eval-hydra-jobs: Support job names as aggregate constituents Fixes https://github.com/NixOS/hydra/issues/715. --- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a554cb66d..5dba84aaf 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair) + const Derivation & drv, std::string_view name, RepairFlag repair) { auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) @@ -73,8 +73,8 @@ StorePath writeDerivation(ref store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(*store, false); + auto suffix = std::string(name) + drvExtension; + auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c2df66229..7222d25e5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -79,7 +79,7 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair = NoRepair); + const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Store & store, const Path & drvPath); -- cgit v1.2.3 From b5e3c04c0364abf91550b043df975272bfeecb9e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Feb 2020 12:51:26 +0100 Subject: Fix URL parser Fixes #3361. --- src/libstore/fetchers/parse.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc index 4f7cb3c6b..a5ad14c87 100644 --- a/src/libstore/fetchers/parse.cc +++ b/src/libstore/fetchers/parse.cc @@ -22,8 +22,8 @@ ParsedURL parseURL(const std::string & url) if (std::regex_match(url, match, uriRegex)) { auto & base = match[1]; std::string scheme = match[2]; - auto authority = match[4].matched - ? std::optional(match[5]) : std::nullopt; + auto authority = match[3].matched + ? std::optional(match[3]) : std::nullopt; std::string path = match[4].matched ? match[4] : match[5]; auto & query = match[6]; auto & fragment = match[7]; -- cgit v1.2.3 From c169ea59049f861aaba429f48b828d0820b74d1d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 15 Nov 2019 13:04:42 +0100 Subject: builtins.fetchGit: Fix build when fetching a git worktree Worktrees[1] are a feature of git which allow you to check out a ref in a different directory. While playing around with flakes I realized that git repositories in a worktree checkout break when trying to build a flake: ``` $ git worktree add ../nixpkgs-flakes nixpkgs-flakes $ cd ../nixpkgs-flakes $ nix build .#hello error: opening directory '/home/ma27/Projects/nixpkgs-flakes/.git/refs/heads': Not a directory ``` This issue has been fixed by determining with `git rev-parse --git-common-dir` where the actual `.git` directory is. Please note that this issue only exists on the `flakes` branch, fetching worktree checkouts with Nix master seems to work fine. [1] https://git-scm.com/docs/git-worktree --- src/libstore/fetchers/git.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 134d44ecd..9276b0993 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -217,7 +217,16 @@ struct GitInput : Input /* Check whether this repo has any commits. There are probably better ways to do this. */ - bool haveCommits = !readDirectory(actualUrl + "/.git/refs/heads").empty(); + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); try { if (haveCommits) { -- cgit v1.2.3 From d068f9ffff3d2a98e6dde0834a250e4930d44778 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Feb 2020 22:14:44 +0100 Subject: Restore subdir support in registries Hacky... --- src/libstore/fetchers/registry.cc | 47 ++++++++++++++++++++++++++------------- src/libstore/fetchers/registry.hh | 19 +++++++++++----- 2 files changed, 44 insertions(+), 22 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index 1fd42a169..ebf326bcc 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -29,16 +29,25 @@ std::shared_ptr Registry::read( throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", path, i.key()); registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url)}); + {inputFromURL(i.key()), inputFromURL(url), {}}); } } else if (version == 2) { - for (auto & i : json["flakes"]) + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Input::Attrs extraAttrs; + auto j = toAttrs.find("subdir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } registry->entries.push_back( { inputFromAttrs(jsonToAttrs(i["from"])) - , inputFromAttrs(jsonToAttrs(i["to"])) + , inputFromAttrs(toAttrs) + , extraAttrs }); + } } else @@ -53,8 +62,9 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & elem : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(elem.first->toAttrs()); - obj["to"] = attrsToJson(elem.second->toAttrs()); + obj["from"] = attrsToJson(std::get<0>(elem)->toAttrs()); + obj["to"] = attrsToJson(std::get<1>(elem)->toAttrs()); + obj["to"].update(attrsToJson(std::get<2>(elem))); arr.emplace_back(std::move(obj)); } @@ -68,16 +78,17 @@ void Registry::write(const Path & path) void Registry::add( const std::shared_ptr & from, - const std::shared_ptr & to) + const std::shared_ptr & to, + const Input::Attrs & extraAttrs) { - entries.emplace_back(from, to); + entries.emplace_back(from, to, extraAttrs); } void Registry::remove(const std::shared_ptr & input) { // FIXME: use C++20 std::erase. for (auto i = entries.begin(); i != entries.end(); ) - if (*i->first == *input) + if (*std::get<0>(*i) == *input) i = entries.erase(i); else ++i; @@ -103,9 +114,10 @@ std::shared_ptr getFlagRegistry() void overrideRegistry( const std::shared_ptr & from, - const std::shared_ptr & to) + const std::shared_ptr & to, + const Input::Attrs & extraAttrs) { - flagRegistry->add(from, to); + flagRegistry->add(from, to, extraAttrs); } static std::shared_ptr getGlobalRegistry(ref store) @@ -135,10 +147,11 @@ Registries getRegistries(ref store) return registries; } -std::shared_ptr lookupInRegistries( +std::pair, Input::Attrs> lookupInRegistries( ref store, std::shared_ptr input) { + Input::Attrs extraAttrs; int n = 0; restart: @@ -149,10 +162,12 @@ std::shared_ptr lookupInRegistries( for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { - if (entry.first->contains(*input)) { - input = entry.second->applyOverrides( - !entry.first->getRef() && input->getRef() ? input->getRef() : std::optional(), - !entry.first->getRev() && input->getRev() ? input->getRev() : std::optional()); + auto from = std::get<0>(entry); + if (from->contains(*input)) { + input = std::get<1>(entry)->applyOverrides( + !from->getRef() && input->getRef() ? input->getRef() : std::optional(), + !from->getRev() && input->getRev() ? input->getRev() : std::optional()); + extraAttrs = std::get<2>(entry); goto restart; } } @@ -161,7 +176,7 @@ std::shared_ptr lookupInRegistries( if (!input->isDirect()) throw Error("cannot find flake '%s' in the flake registries", input->to_string()); - return input; + return {input, extraAttrs}; } } diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh index e29f78486..6063f51d6 100644 --- a/src/libstore/fetchers/registry.hh +++ b/src/libstore/fetchers/registry.hh @@ -1,13 +1,12 @@ #pragma once #include "types.hh" +#include "fetchers.hh" namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Registry { enum RegistryType { @@ -18,7 +17,13 @@ struct Registry RegistryType type; - std::vector, std::shared_ptr>> entries; + std::vector< + std::tuple< + std::shared_ptr, // from + std::shared_ptr, // to + Input::Attrs // extra attributes + > + > entries; Registry(RegistryType type) : type(type) @@ -31,7 +36,8 @@ struct Registry void add( const std::shared_ptr & from, - const std::shared_ptr & to); + const std::shared_ptr & to, + const Input::Attrs & extraAttrs); void remove(const std::shared_ptr & input); }; @@ -46,9 +52,10 @@ Registries getRegistries(ref store); void overrideRegistry( const std::shared_ptr & from, - const std::shared_ptr & to); + const std::shared_ptr & to, + const Input::Attrs & extraAttrs); -std::shared_ptr lookupInRegistries( +std::pair, Input::Attrs> lookupInRegistries( ref store, std::shared_ptr input); -- cgit v1.2.3 From 73c98405695837dd93448ed38c0eeb0b56060dfc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Feb 2020 23:44:06 +0100 Subject: Restore subdir -> dir Got this mixed up somewhere. --- src/libstore/fetchers/registry.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index ebf326bcc..fad389d17 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -37,7 +37,7 @@ std::shared_ptr Registry::read( for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); Input::Attrs extraAttrs; - auto j = toAttrs.find("subdir"); + auto j = toAttrs.find("dir"); if (j != toAttrs.end()) { extraAttrs.insert(*j); toAttrs.erase(j); -- cgit v1.2.3 From 4a4521f46253e87a06af9333381c050e983b3c2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Feb 2020 23:47:02 +0100 Subject: Fix nlohmann::json exception --- src/libstore/fetchers/registry.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index fad389d17..721af0c9b 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -64,7 +64,8 @@ void Registry::write(const Path & path) nlohmann::json obj; obj["from"] = attrsToJson(std::get<0>(elem)->toAttrs()); obj["to"] = attrsToJson(std::get<1>(elem)->toAttrs()); - obj["to"].update(attrsToJson(std::get<2>(elem))); + if (!std::get<2>(elem).empty()) + obj["to"].update(attrsToJson(std::get<2>(elem))); arr.emplace_back(std::move(obj)); } -- cgit v1.2.3 From e188fe7c6d4b243ed62ca3d0e47abfd0eec95f79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Mar 2020 16:34:46 +0100 Subject: Move call-flake.nix into libexpr --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e00556645..2025af33e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -343,7 +343,7 @@ void LocalStore::openDB(State & state, bool create) /* Initialise the database schema, if necessary. */ if (create) { - const char * schema = + static const char schema[] = #include "schema.sql.gen.hh" ; db.exec(schema); -- cgit v1.2.3 From fbcb897e21e6c7b866a7aed97129141c0e7caa22 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Mar 2020 13:20:32 +0100 Subject: Add a test for shallow Git clones Also, don't return a revCount anymore for shallow or dirty Git trees, since it's incorrect. Closes #2988. --- src/libstore/fetchers/git.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 9276b0993..46ca187fe 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -34,6 +34,8 @@ static void cacheGitInfo( const Tree & tree, const Hash & rev) { + if (!tree.info.revCount || !tree.info.lastModified) return; + nlohmann::json json; json["storePath"] = store.printStorePath(tree.storePath); json["name"] = name; @@ -169,6 +171,7 @@ struct GitInput : Input return url.path; return {}; } + void markChangedFile(std::string_view file, std::optional commitMsg) const override { auto sourcePath = getSourcePath(); @@ -271,7 +274,6 @@ struct GitInput : Input .actualPath = store->printStorePath(storePath), .storePath = std::move(storePath), .info = TreeInfo { - .revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "rev-list", "--count", "HEAD" })) : 0, // FIXME: maybe we should use the timestamp of the last // modified dirty file? .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, @@ -357,8 +359,11 @@ struct GitInput : Input input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); } + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + if (auto tree = lookupGitInfo(store, name, *input->rev)) { assert(*input->rev == tree->first); + if (isShallow) tree->second.info.revCount.reset(); return {std::move(tree->second), input}; } @@ -380,14 +385,17 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); auto storePath = store->addToStore(name, tmpDir); - auto revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); auto tree = Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), .info = TreeInfo { - .revCount = revCount, + .revCount = + !isShallow + ? std::optional(std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))) + : std::nullopt, .lastModified = lastModified } }; -- cgit v1.2.3 From 2a4e4f6a6e021481f0e92b7d3006345e68e77684 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Mar 2020 20:54:36 +0100 Subject: Unified fetcher caching system --- src/libstore/fetchers/attrs.cc | 71 +++++++++++++++ src/libstore/fetchers/attrs.hh | 26 ++++++ src/libstore/fetchers/cache.cc | 109 ++++++++++++++++++++++ src/libstore/fetchers/cache.hh | 24 +++++ src/libstore/fetchers/fetchers.cc | 50 +---------- src/libstore/fetchers/fetchers.hh | 19 +--- src/libstore/fetchers/git.cc | 4 +- src/libstore/fetchers/github.cc | 89 ++++++++++++++---- src/libstore/fetchers/indirect.cc | 2 +- src/libstore/fetchers/mercurial.cc | 180 +++++++++++++++++++------------------ src/libstore/fetchers/registry.cc | 10 +-- src/libstore/fetchers/registry.hh | 8 +- src/libstore/fetchers/tarball.cc | 2 +- 13 files changed, 411 insertions(+), 183 deletions(-) create mode 100644 src/libstore/fetchers/attrs.cc create mode 100644 src/libstore/fetchers/attrs.hh create mode 100644 src/libstore/fetchers/cache.cc create mode 100644 src/libstore/fetchers/cache.hh (limited to 'src/libstore') diff --git a/src/libstore/fetchers/attrs.cc b/src/libstore/fetchers/attrs.cc new file mode 100644 index 000000000..83b6d9164 --- /dev/null +++ b/src/libstore/fetchers/attrs.cc @@ -0,0 +1,71 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else abort(); + } + return json; +} + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string", name); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +} diff --git a/src/libstore/fetchers/attrs.hh b/src/libstore/fetchers/attrs.hh new file mode 100644 index 000000000..b7f98d71b --- /dev/null +++ b/src/libstore/fetchers/attrs.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "types.hh" + +#include + +#include + +namespace nix::fetchers { + +typedef std::variant Attr; +typedef std::map Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +} diff --git a/src/libstore/fetchers/cache.cc b/src/libstore/fetchers/cache.cc new file mode 100644 index 000000000..4c88b64c5 --- /dev/null +++ b/src/libstore/fetchers/cache.cc @@ -0,0 +1,109 @@ +#include "fetchers/cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional> lookup( + ref store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + if (!immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0))) { + debug("ignoring expired cache entry '%s'", inAttrsJson); + return {}; + } + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return {{jsonToAttrs(nlohmann::json::parse(infoJson)), std::move(storePath)}}; + } +}; + +ref getCache() +{ + static auto cache = std::make_shared(); + return ref(cache); +} + +} diff --git a/src/libstore/fetchers/cache.hh b/src/libstore/fetchers/cache.hh new file mode 100644 index 000000000..ba2d30629 --- /dev/null +++ b/src/libstore/fetchers/cache.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "types.hh" +#include "fetchers/fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional> lookup( + ref store, + const Attrs & inAttrs) = 0; +}; + +ref getCache(); + +} diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 0cc6f1c91..25827ab7c 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -28,7 +28,7 @@ std::unique_ptr inputFromURL(const std::string & url) return inputFromURL(parseURL(url)); } -std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) +std::unique_ptr inputFromAttrs(const Attrs & attrs) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs); @@ -42,36 +42,7 @@ std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) throw Error("input '%s' is unsupported", attrsToJson(attrs)); } -Input::Attrs jsonToAttrs(const nlohmann::json & json) -{ - fetchers::Input::Attrs attrs; - - for (auto & i : json.items()) { - if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get()); - else if (i.value().is_string()) - attrs.emplace(i.key(), i.value().get()); - else - throw Error("unsupported input attribute type in lock file"); - } - - return attrs; -} - -nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs) -{ - nlohmann::json json; - for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { - json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { - json[attr.first] = *v; - } else abort(); - } - return json; -} - -Input::Attrs Input::toAttrs() const +Attrs Input::toAttrs() const { auto attrs = toAttrsInternal(); if (narHash) @@ -80,23 +51,6 @@ Input::Attrs Input::toAttrs() const return attrs; } -std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name) -{ - auto i = attrs.find(name); - if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; - throw Error("input attribute '%s' is not a string", name); -} - -std::string getStrAttr(const Input::Attrs & attrs, const std::string & name) -{ - auto s = maybeGetStrAttr(attrs, name); - if (!s) - throw Error("input attribute '%s' is missing", name); - return *s; -} - std::pair> Input::fetchTree(ref store) const { auto [tree, input] = fetchTreeInternal(store); diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 4202e8339..085a62f47 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -4,11 +4,9 @@ #include "hash.hh" #include "path.hh" #include "tree-info.hh" +#include "attrs.hh" #include -#include - -#include namespace nix { class Store; } @@ -49,9 +47,6 @@ struct Input : std::enable_shared_from_this virtual std::string to_string() const = 0; - typedef std::variant Attr; - typedef std::map Attrs; - Attrs toAttrs() const; std::pair> fetchTree(ref store) const; @@ -87,23 +82,15 @@ struct InputScheme virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) = 0; + virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; }; std::unique_ptr inputFromURL(const ParsedURL & url); std::unique_ptr inputFromURL(const std::string & url); -std::unique_ptr inputFromAttrs(const Input::Attrs & attrs); +std::unique_ptr inputFromAttrs(const Attrs & attrs); void registerInputScheme(std::unique_ptr && fetcher); -Input::Attrs jsonToAttrs(const nlohmann::json & json); - -nlohmann::json attrsToJson(const Input::Attrs & attrs); - -std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name); - -std::string getStrAttr(const Input::Attrs & attrs, const std::string & name); - } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 46ca187fe..179a5fb83 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -420,7 +420,7 @@ struct GitInputScheme : InputScheme if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); url2.query.clear(); - Input::Attrs attrs; + Attrs attrs; attrs.emplace("type", "git"); for (auto &[name, value] : url.query) { @@ -435,7 +435,7 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index 0a000e83f..1772b2828 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -1,8 +1,9 @@ -#include "fetchers.hh" #include "download.hh" +#include "fetchers/cache.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/parse.hh" +#include "fetchers/regex.hh" #include "globals.hh" -#include "parse.hh" -#include "regex.hh" #include "store-api.hh" #include @@ -72,17 +73,36 @@ struct GitHubInput : Input std::pair> fetchTreeInternal(nix::ref store) const override { auto rev = this->rev; + auto ref = this->ref.value_or("master"); - #if 0 - if (rev) { - if (auto gitInfo = lookupGitInfo(store, "source", *rev)) - return *gitInfo; + Attrs mutableAttrs({ + {"type", "github"}, + {"owner", owner}, + {"repo", repo}, + {"ref", ref}, + }); + + if (!rev) { + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = Hash(getStrAttr(res->first, "rev"), htSHA1); + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } } - #endif if (!rev) { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref ? *ref : "master"); + owner, repo, ref); CachedDownloadRequest request(url); request.ttl = rev ? 1000000000 : settings.tarballTtl; auto result = getDownloader()->downloadCached(store, request); @@ -91,6 +111,28 @@ struct GitHubInput : Input debug("HEAD revision for '%s' is %s", url, rev->gitRev()); } + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -118,14 +160,25 @@ struct GitHubInput : Input }, }; - #if 0 - // FIXME: this can overwrite a cache file that contains a revCount. - cacheGitInfo("source", gitInfo); - #endif - - auto input = std::make_shared(*this); - input->ref = {}; - input->rev = *rev; + Attrs infoAttrs({ + {"rev", rev->gitRev()}, + {"lastModified", *result.info.lastModified} + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + result.storePath, + false); + + getCache()->add( + store, + immutableAttrs, + infoAttrs, + result.storePath, + true); return {std::move(result), input}; } @@ -189,7 +242,7 @@ struct GitHubInputScheme : InputScheme return input; } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "github") return {}; diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc index 016f5fb39..963abd85f 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libstore/fetchers/indirect.cc @@ -120,7 +120,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 6ab0add1d..a9c86f1b4 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -1,5 +1,6 @@ -#include "fetchers.hh" -#include "parse.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/cache.hh" +#include "fetchers/parse.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" @@ -7,8 +8,6 @@ #include -#include - using namespace std::string_literals; namespace nix::fetchers { @@ -163,51 +162,80 @@ struct MercurialInput : Input if (!input->ref) input->ref = "default"; - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } assert(input->rev || input->ref); auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, revOrRef).to_string(Base32, false)); + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + if (auto res = getCache()->lookup(store, mutableAttrs)) + return makeResult(res->first, std::move(res->second)); - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(input->rev - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); - - if (pathExists(cacheDir)) { - try { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); } - - writeFile(stampFile, ""); } auto tokens = tokenizeString>( @@ -218,33 +246,8 @@ struct MercurialInput : Input auto revCount = std::stoull(tokens[1]); input->ref = tokens[2]; - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + input->rev->gitRev()).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == input->rev->gitRev()); - - auto storePath = store->parseStorePath((std::string) json["storePath"]); - - if (store->isValidPath(storePath)) { - printTalkative("using cached Mercurial store path '%s'", store->printStorePath(storePath)); - return { - Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = revCount, - }, - }, - input - }; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -255,26 +258,27 @@ struct MercurialInput : Input auto storePath = store->addToStore(name, tmpDir); - nlohmann::json json; - json["storePath"] = store->printStorePath(storePath); - json["uri"] = actualUrl; - json["name"] = name; - json["branch"] = *input->ref; - json["rev"] = input->rev->gitRev(); - json["revCount"] = revCount; - - writeFile(storeLink, json.dump()); - - return { - Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = revCount - } - }, - input - }; + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", revCount}, + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); } }; @@ -291,7 +295,7 @@ struct MercurialInputScheme : InputScheme url2.scheme = std::string(url2.scheme, 3); url2.query.clear(); - Input::Attrs attrs; + Attrs attrs; attrs.emplace("type", "hg"); for (auto &[name, value] : url.query) { @@ -306,7 +310,7 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index 721af0c9b..acbed2109 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -36,7 +36,7 @@ std::shared_ptr Registry::read( else if (version == 2) { for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); - Input::Attrs extraAttrs; + Attrs extraAttrs; auto j = toAttrs.find("dir"); if (j != toAttrs.end()) { extraAttrs.insert(*j); @@ -80,7 +80,7 @@ void Registry::write(const Path & path) void Registry::add( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs) + const Attrs & extraAttrs) { entries.emplace_back(from, to, extraAttrs); } @@ -116,7 +116,7 @@ std::shared_ptr getFlagRegistry() void overrideRegistry( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs) + const Attrs & extraAttrs) { flagRegistry->add(from, to, extraAttrs); } @@ -148,11 +148,11 @@ Registries getRegistries(ref store) return registries; } -std::pair, Input::Attrs> lookupInRegistries( +std::pair, Attrs> lookupInRegistries( ref store, std::shared_ptr input) { - Input::Attrs extraAttrs; + Attrs extraAttrs; int n = 0; restart: diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh index 6063f51d6..d2eb7749b 100644 --- a/src/libstore/fetchers/registry.hh +++ b/src/libstore/fetchers/registry.hh @@ -21,7 +21,7 @@ struct Registry std::tuple< std::shared_ptr, // from std::shared_ptr, // to - Input::Attrs // extra attributes + Attrs // extra attributes > > entries; @@ -37,7 +37,7 @@ struct Registry void add( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs); + const Attrs & extraAttrs); void remove(const std::shared_ptr & input); }; @@ -53,9 +53,9 @@ Registries getRegistries(ref store); void overrideRegistry( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs); + const Attrs & extraAttrs); -std::pair, Input::Attrs> lookupInRegistries( +std::pair, Attrs> lookupInRegistries( ref store, std::shared_ptr input); diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 7c0b6690d..360befd31 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -109,7 +109,7 @@ struct TarballInputScheme : InputScheme return input; } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; -- cgit v1.2.3 From d1165d8791f559352ff6aa7348e1293b2873db1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Mar 2020 21:34:38 +0100 Subject: Require shallow clones to be requested explicitly If you do a fetchTree on a Git repository, whether the result contains a revCount attribute should not depend on whether that repository happens to be a shallow clone or not. That would complicate caching a lot and would be semantically messy. So applying fetchTree/fetchGit to a shallow repository is now an error unless you pass the attribute 'shallow = true'. If 'shallow = true', we don't return revCount, even if the repository is not actually shallow. Note that Nix itself is not doing shallow clones at the moment. But it could do so as an optimisation if the user specifies 'shallow = true'. Issue #2988. --- src/libstore/fetchers/attrs.cc | 25 +++++++++++++++++++++++-- src/libstore/fetchers/attrs.hh | 13 ++++++++++++- src/libstore/fetchers/git.cc | 16 +++++++++++++--- src/libstore/fetchers/mercurial.cc | 2 +- 4 files changed, 49 insertions(+), 7 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/attrs.cc b/src/libstore/fetchers/attrs.cc index 83b6d9164..40c02de42 100644 --- a/src/libstore/fetchers/attrs.cc +++ b/src/libstore/fetchers/attrs.cc @@ -14,6 +14,8 @@ Attrs jsonToAttrs(const nlohmann::json & json) attrs.emplace(i.key(), i.value().get()); else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_boolean()) + attrs.emplace(i.key(), i.value().get()); else throw Error("unsupported input attribute type in lock file"); } @@ -29,6 +31,8 @@ nlohmann::json attrsToJson(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; + } else if (auto v = std::get_if>(&attr.second)) { + json[attr.first] = v->t; } else abort(); } return json; @@ -40,7 +44,7 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin if (i == attrs.end()) return {}; if (auto v = std::get_if(&i->second)) return *v; - throw Error("input attribute '%s' is not a string", name); + throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); } std::string getStrAttr(const Attrs & attrs, const std::string & name) @@ -57,7 +61,7 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & if (i == attrs.end()) return {}; if (auto v = std::get_if(&i->second)) return *v; - throw Error("input attribute '%s' is not a string", name); + throw Error("input attribute '%s' is not an integer", name); } int64_t getIntAttr(const Attrs & attrs, const std::string & name) @@ -68,4 +72,21 @@ int64_t getIntAttr(const Attrs & attrs, const std::string & name) return *s; } +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a Boolean", name); +} + +bool getBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetBoolAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + } diff --git a/src/libstore/fetchers/attrs.hh b/src/libstore/fetchers/attrs.hh index b7f98d71b..2c9e772d2 100644 --- a/src/libstore/fetchers/attrs.hh +++ b/src/libstore/fetchers/attrs.hh @@ -8,7 +8,14 @@ namespace nix::fetchers { -typedef std::variant Attr; +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; +}; + +typedef std::variant> Attr; typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -23,4 +30,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & int64_t getIntAttr(const Attrs & attrs, const std::string & name); +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); + +bool getBoolAttr(const Attrs & attrs, const std::string & name); + } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 179a5fb83..c5d4f019c 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -83,6 +83,7 @@ struct GitInput : Input ParsedURL url; std::optional ref; std::optional rev; + bool shallow = false; GitInput(const ParsedURL & url) : url(url) { } @@ -114,6 +115,7 @@ struct GitInput : Input if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); return url2.to_string(); } @@ -125,6 +127,8 @@ struct GitInput : Input attrs.emplace("ref", *ref); if (rev) attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); return attrs; } @@ -361,9 +365,12 @@ struct GitInput : Input bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + if (auto tree = lookupGitInfo(store, name, *input->rev)) { assert(*input->rev == tree->first); - if (isShallow) tree->second.info.revCount.reset(); + if (shallow) tree->second.info.revCount.reset(); return {std::move(tree->second), input}; } @@ -393,7 +400,7 @@ struct GitInput : Input .storePath = std::move(storePath), .info = TreeInfo { .revCount = - !isShallow + !shallow ? std::optional(std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))) : std::nullopt, .lastModified = lastModified @@ -440,7 +447,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow") throw Error("unsupported Git input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); @@ -451,6 +458,9 @@ struct GitInputScheme : InputScheme } if (auto rev = maybeGetStrAttr(attrs, "rev")) input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + return input; } }; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index a9c86f1b4..3a767ef17 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -260,7 +260,7 @@ struct MercurialInput : Input Attrs infoAttrs({ {"rev", input->rev->gitRev()}, - {"revCount", revCount}, + {"revCount", (int64_t) revCount}, }); if (!this->rev) -- cgit v1.2.3 From 38e360154d8ef6a70e3101a9b0d850e63fbfdfda Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Mar 2020 22:32:26 +0100 Subject: Git: Use unified caching system --- src/libstore/fetchers/git.cc | 166 +++++++++++++++++-------------------- src/libstore/fetchers/mercurial.cc | 11 ++- 2 files changed, 86 insertions(+), 91 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index c5d4f019c..c1044d5a2 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -1,5 +1,6 @@ -#include "fetchers.hh" -#include "parse.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/cache.hh" +#include "fetchers/parse.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" @@ -7,77 +8,15 @@ #include -#include - using namespace std::string_literals; namespace nix::fetchers { -static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) -{ - Path cacheDir = getCacheDir() + "/nix/git-revs-v2"; - std::string linkName = - name == "source" - ? rev.gitRev() - : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false); - return cacheDir + "/" + linkName + ".link"; -} - static std::string readHead(const Path & path) { return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); } -static void cacheGitInfo( - Store & store, - const std::string & name, - const Tree & tree, - const Hash & rev) -{ - if (!tree.info.revCount || !tree.info.lastModified) return; - - nlohmann::json json; - json["storePath"] = store.printStorePath(tree.storePath); - json["name"] = name; - json["rev"] = rev.gitRev(); - json["revCount"] = *tree.info.revCount; - json["lastModified"] = *tree.info.lastModified; - - auto cacheInfoPath = getCacheInfoPathFor(name, rev); - createDirs(dirOf(cacheInfoPath)); - writeFile(cacheInfoPath, json.dump()); -} - -static std::optional> lookupGitInfo( - ref store, - const std::string & name, - const Hash & rev) -{ - try { - auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev))); - - assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev); - - auto storePath = store->parseStorePath((std::string) json["storePath"]); - - if (store->isValidPath(storePath)) { - return {{rev, Tree{ - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = json["revCount"], - .lastModified = json["lastModified"], - } - }}}; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - return {}; -} - struct GitInput : Input { ParsedURL url; @@ -207,11 +146,38 @@ struct GitInput : Input assert(!rev || rev->type == htSHA1); + auto cacheType = shallow ? "git-shallow" : "git"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + if (rev) { - if (auto tree = lookupGitInfo(store, name, *rev)) { - input->rev = tree->first; - return {std::move(tree->second), input}; - } + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); } auto [isLocal, actualUrl_] = getActualUrl(); @@ -290,6 +256,13 @@ struct GitInput : Input if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + Path repoDir; if (isLocal) { @@ -301,6 +274,14 @@ struct GitInput : Input } else { + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; @@ -368,16 +349,15 @@ struct GitInput : Input if (isShallow && !shallow) throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); - if (auto tree = lookupGitInfo(store, name, *input->rev)) { - assert(*input->rev == tree->first); - if (shallow) tree->second.info.revCount.reset(); - return {std::move(tree->second), input}; - } - // FIXME: check whether rev is an ancestor of ref. printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { @@ -395,21 +375,31 @@ struct GitInput : Input auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); - auto tree = Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = - !shallow - ? std::optional(std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))) - : std::nullopt, - .lastModified = lastModified - } - }; - - cacheGitInfo(*store, name, tree, *input->rev); + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); - return {std::move(tree), input}; + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); } }; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 3a767ef17..5cd43a74e 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -174,7 +174,7 @@ struct MercurialInput : Input auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) -> std::pair> { - input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1); + assert(input->rev); assert(!rev || rev == input->rev); return { Tree{ @@ -203,8 +203,13 @@ struct MercurialInput : Input {"ref", *input->ref}, }); - if (auto res = getCache()->lookup(store, mutableAttrs)) - return makeResult(res->first, std::move(res->second)); + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); -- cgit v1.2.3 From 1e7ce1d6da1f571b4dd2ad3d370e557458962a95 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Mar 2020 14:08:25 +0100 Subject: tarball / github fetchers: Use generic caching system --- src/libstore/fetchers/fetchers.hh | 12 ++++ src/libstore/fetchers/git.cc | 2 +- src/libstore/fetchers/github.cc | 72 +++---------------- src/libstore/fetchers/registry.cc | 15 ++-- src/libstore/fetchers/tarball.cc | 142 +++++++++++++++++++++++++++++++------- 5 files changed, 147 insertions(+), 96 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 085a62f47..0a028cf7a 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -93,4 +93,16 @@ std::unique_ptr inputFromAttrs(const Attrs & attrs); void registerInputScheme(std::unique_ptr && fetcher); +StorePath downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index c1044d5a2..ede758544 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -163,7 +163,7 @@ struct GitInput : Input assert(input->rev); assert(!rev || rev == input->rev); return { - Tree{ + Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), .info = TreeInfo { diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index 1772b2828..1041b98a5 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -75,38 +75,13 @@ struct GitHubInput : Input auto rev = this->rev; auto ref = this->ref.value_or("master"); - Attrs mutableAttrs({ - {"type", "github"}, - {"owner", owner}, - {"repo", repo}, - {"ref", ref}, - }); - - if (!rev) { - if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto input = std::make_shared(*this); - input->ref = {}; - input->rev = Hash(getStrAttr(res->first, "rev"), htSHA1); - return { - Tree{ - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, - }, - input - }; - } - } - if (!rev) { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", owner, repo, ref); - CachedDownloadRequest request(url); - request.ttl = rev ? 1000000000 : settings.tarballTtl; - auto result = getDownloader()->downloadCached(store, request); - auto json = nlohmann::json::parse(readFile(result.path)); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false)))); rev = Hash(json["sha"], htSHA1); debug("HEAD revision for '%s' is %s", url, rev->gitRev()); } @@ -143,44 +118,19 @@ struct GitHubInput : Input if (accessToken != "") url += "?access_token=" + accessToken; - CachedDownloadRequest request(url); - request.unpack = true; - request.name = "source"; - request.ttl = 1000000000; - request.getLastModified = true; - auto dresult = getDownloader()->downloadCached(store, request); - - assert(dresult.lastModified); - - Tree result{ - .actualPath = dresult.path, - .storePath = store->parseStorePath(dresult.storePath), - .info = TreeInfo { - .lastModified = *dresult.lastModified, - }, - }; - - Attrs infoAttrs({ - {"rev", rev->gitRev()}, - {"lastModified", *result.info.lastModified} - }); - - if (!this->rev) - getCache()->add( - store, - mutableAttrs, - infoAttrs, - result.storePath, - false); + auto tree = downloadTarball(store, url, "source", true); getCache()->add( store, immutableAttrs, - infoAttrs, - result.storePath, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, true); - return {std::move(result), input}; + return {std::move(tree), input}; } std::shared_ptr applyOverrides( diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index acbed2109..0aac18375 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -1,8 +1,9 @@ -#include "registry.hh" +#include "fetchers/registry.hh" +#include "fetchers/fetchers.hh" #include "util.hh" -#include "fetchers.hh" #include "globals.hh" #include "download.hh" +#include "store-api.hh" #include @@ -126,12 +127,10 @@ static std::shared_ptr getGlobalRegistry(ref store) static auto reg = [&]() { auto path = settings.flakeRegistry; - if (!hasPrefix(path, "/")) { - CachedDownloadRequest request(path); - request.name = "flake-registry.json"; - request.gcRoot = true; - path = getDownloader()->downloadCached(store, request).path; - } + if (!hasPrefix(path, "/")) + // FIXME: register as GC root. + // FIXME: if download fails, use previous version if available. + path = store->toRealPath(downloadFile(store, path, "flake-registry.json", false)); return Registry::read(path, Registry::Global); }(); diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 360befd31..bbc96d70b 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -1,11 +1,114 @@ -#include "fetchers.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/parse.hh" +#include "fetchers/cache.hh" #include "download.hh" #include "globals.hh" -#include "parse.hh" #include "store-api.hh" +#include "archive.hh" +#include "tarfile.hh" namespace nix::fetchers { +StorePath downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + // FIXME: check store + + Attrs inAttrs({ + {"type", "file"}, + {"url", url}, + {"name", name}, + }); + + if (auto res = getCache()->lookup(store, inAttrs)) + return std::move(res->second); + + // FIXME: use ETag. + + DownloadRequest request(url); + auto res = getDownloader()->download(request); + + // FIXME: write to temporary file. + + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + + Attrs infoAttrs({ + {"etag", res.etag}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + info.path.clone(), + immutable); + + return std::move(info.path); +} + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + Attrs inAttrs({ + {"type", "tarball"}, + {"url", url}, + {"name", name}, + }); + + if (auto res = getCache()->lookup(store, inAttrs)) + return Tree { + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }; + + auto tarball = downloadFile(store, url, name, immutable); + + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(tarball), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + auto lastModified = lstat(topDir).st_mtime; + auto unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + + Attrs infoAttrs({ + {"lastModified", lastModified}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + unpackedStorePath, + immutable); + + return Tree { + .actualPath = store->toRealPath(unpackedStorePath), + .storePath = std::move(unpackedStorePath), + .info = TreeInfo { + .lastModified = lastModified, + }, + }; +} + struct TarballInput : Input { ParsedURL url; @@ -55,29 +158,12 @@ struct TarballInput : Input std::pair> fetchTreeInternal(nix::ref store) const override { - CachedDownloadRequest request(url.to_string()); - request.unpack = true; - request.getLastModified = true; - request.name = "source"; - - auto res = getDownloader()->downloadCached(store, request); + auto tree = downloadTarball(store, url.to_string(), "source", false); auto input = std::make_shared(*this); + input->narHash = store->queryPathInfo(tree.storePath)->narHash; - auto storePath = store->parseStorePath(res.storePath); - - input->narHash = store->queryPathInfo(storePath)->narHash; - - return { - Tree { - .actualPath = res.path, - .storePath = std::move(storePath), - .info = TreeInfo { - .lastModified = *res.lastModified, - }, - }, - input - }; + return {std::move(tree), input}; } }; @@ -96,15 +182,19 @@ struct TarballInputScheme : InputScheme auto input = std::make_unique(url); - auto hash = url.query.find("hash"); - if (hash != url.query.end()) + auto hash = input->url.query.find("hash"); + if (hash != input->url.query.end()) { // FIXME: require SRI hash. input->hash = Hash(hash->second); + input->url.query.erase(hash); + } - auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) + auto narHash = input->url.query.find("narHash"); + if (narHash != input->url.query.end()) { // FIXME: require SRI hash. input->narHash = Hash(narHash->second); + input->url.query.erase(narHash); + } return input; } -- cgit v1.2.3 From c5ec95e2c70d15935d02216852bbc22f87f4f5ed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Mar 2020 15:14:23 +0100 Subject: tarball.cc: Use ETags --- src/libstore/fetchers/cache.cc | 24 +++++++--- src/libstore/fetchers/cache.hh | 11 +++++ src/libstore/fetchers/fetchers.hh | 8 +++- src/libstore/fetchers/github.cc | 2 +- src/libstore/fetchers/registry.cc | 2 +- src/libstore/fetchers/tarball.cc | 94 +++++++++++++++++++++++++-------------- 6 files changed, 99 insertions(+), 42 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/cache.cc b/src/libstore/fetchers/cache.cc index 4c88b64c5..14a84744a 100644 --- a/src/libstore/fetchers/cache.cc +++ b/src/libstore/fetchers/cache.cc @@ -65,6 +65,19 @@ struct CacheImpl : Cache std::optional> lookup( ref store, const Attrs & inAttrs) override + { + if (auto res = lookupExpired(store, inAttrs)) { + if (!res->expired) + return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); + debug("ignoring expired cache entry '%s'", + attrsToJson(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired( + ref store, + const Attrs & inAttrs) override { auto state(_state.lock()); @@ -81,11 +94,6 @@ struct CacheImpl : Cache auto immutable = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); - if (!immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0))) { - debug("ignoring expired cache entry '%s'", inAttrsJson); - return {}; - } - store->addTempRoot(storePath); if (!store->isValidPath(storePath)) { // FIXME: we could try to substitute 'storePath'. @@ -96,7 +104,11 @@ struct CacheImpl : Cache debug("using cache entry '%s' -> '%s', '%s'", inAttrsJson, infoJson, store->printStorePath(storePath)); - return {{jsonToAttrs(nlohmann::json::parse(infoJson)), std::move(storePath)}}; + return Result { + .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .storePath = std::move(storePath) + }; } }; diff --git a/src/libstore/fetchers/cache.hh b/src/libstore/fetchers/cache.hh index ba2d30629..a25b05985 100644 --- a/src/libstore/fetchers/cache.hh +++ b/src/libstore/fetchers/cache.hh @@ -17,6 +17,17 @@ struct Cache virtual std::optional> lookup( ref store, const Attrs & inAttrs) = 0; + + struct Result + { + bool expired = false; + Attrs infoAttrs; + StorePath storePath; + }; + + virtual std::optional lookupExpired( + ref store, + const Attrs & inAttrs) = 0; }; ref getCache(); diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 0a028cf7a..9c931dfa1 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -93,7 +93,13 @@ std::unique_ptr inputFromAttrs(const Attrs & attrs); void registerInputScheme(std::unique_ptr && fetcher); -StorePath downloadFile( +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; +}; + +DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index 1041b98a5..0fef456df 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -81,7 +81,7 @@ struct GitHubInput : Input auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false)))); + downloadFile(store, url, "source", false).storePath))); rev = Hash(json["sha"], htSHA1); debug("HEAD revision for '%s' is %s", url, rev->gitRev()); } diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index 0aac18375..638e6e50a 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -130,7 +130,7 @@ static std::shared_ptr getGlobalRegistry(ref store) if (!hasPrefix(path, "/")) // FIXME: register as GC root. // FIXME: if download fails, use previous version if available. - path = store->toRealPath(downloadFile(store, path, "flake-registry.json", false)); + path = store->toRealPath(downloadFile(store, path, "flake-registry.json", false).storePath); return Registry::read(path, Registry::Global); }(); diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index bbc96d70b..53971ec2b 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -9,7 +9,7 @@ namespace nix::fetchers { -StorePath downloadFile( +DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, @@ -23,37 +23,54 @@ StorePath downloadFile( {"name", name}, }); - if (auto res = getCache()->lookup(store, inAttrs)) - return std::move(res->second); + auto cached = getCache()->lookupExpired(store, inAttrs); - // FIXME: use ETag. + if (cached && !cached->expired) + return { + .storePath = std::move(cached->storePath), + .etag = getStrAttr(cached->infoAttrs, "etag") + }; DownloadRequest request(url); + if (cached) + request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); auto res = getDownloader()->download(request); // FIXME: write to temporary file. - StringSink sink; - dumpString(*res.data, sink); - auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - Attrs infoAttrs({ {"etag", res.etag}, }); + std::optional storePath; + + if (res.cached) { + assert(cached); + assert(request.expectedETag == res.etag); + storePath = std::move(cached->storePath); + } else { + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = std::move(info.path); + } + getCache()->add( store, inAttrs, infoAttrs, - info.path.clone(), + *storePath, immutable); - return std::move(info.path); + return { + .storePath = std::move(*storePath), + .etag = res.etag, + }; } Tree downloadTarball( @@ -68,41 +85,52 @@ Tree downloadTarball( {"name", name}, }); - if (auto res = getCache()->lookup(store, inAttrs)) + auto cached = getCache()->lookupExpired(store, inAttrs); + + if (cached && !cached->expired) return Tree { - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), + .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), }, }; - auto tarball = downloadFile(store, url, name, immutable); - - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(tarball), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - auto lastModified = lstat(topDir).st_mtime; - auto unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + auto res = downloadFile(store, url, name, immutable); + + std::optional unpackedStorePath; + time_t lastModified; + + if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { + unpackedStorePath = std::move(cached->storePath); + lastModified = getIntAttr(cached->infoAttrs, "lastModified"); + } else { + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(res.storePath), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + } Attrs infoAttrs({ {"lastModified", lastModified}, + {"etag", res.etag}, }); getCache()->add( store, inAttrs, infoAttrs, - unpackedStorePath, + *unpackedStorePath, immutable); return Tree { - .actualPath = store->toRealPath(unpackedStorePath), - .storePath = std::move(unpackedStorePath), + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), .info = TreeInfo { .lastModified = lastModified, }, -- cgit v1.2.3 From f6ddf48882a068dbf1b1bfff44d487316972d6e9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Mar 2020 17:23:56 +0100 Subject: Get rid of downloadCached() Everything uses the generic caching system now. --- src/libstore/download.cc | 146 -------------------------------------- src/libstore/download.hh | 34 +-------- src/libstore/fetchers/fetchers.hh | 1 + src/libstore/fetchers/tarball.cc | 17 ++++- 4 files changed, 20 insertions(+), 178 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8e0d7d42a..35a35cd78 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -35,10 +35,6 @@ DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); -CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) - : uri(uri), ttl(settings.tarballTtl) -{ } - std::string resolveUri(const std::string & uri) { if (uri.compare(0, 8, "channel:") == 0) @@ -801,148 +797,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached( - ref store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - std::optional expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(*expectedStorePath)) { - CachedDownloadResult result; - result.storePath = store->printStorePath(*expectedStorePath); - result.path = store->toRealPath(result.storePath); - assert(!request.getLastModified); // FIXME - return result; - } - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - - std::optional storePath; - - string expectedETag; - - bool skip = false; - - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = store->parseStorePath(readLink(fileLink)); - // FIXME - store->addTempRoot(*storePath); - if (store->isValidPath(*storePath)) { - auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath.reset(); - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path.clone(); - } - - assert(storePath); - replaceSymlink(store->printStorePath(*storePath), fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (!storePath) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; - } - } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - std::optional unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); - store->addTempRoot(*unpackedStorePath); - if (!store->isValidPath(*unpackedStorePath)) - unpackedStorePath.reset(); - else - result.lastModified = lstat(unpackedLink).st_mtime; - } - if (!unpackedStorePath) { - printInfo("unpacking '%s'...", url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(*storePath), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - result.lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - // Store the last-modified date of the tarball in the symlink - // mtime. This saves us from having to store it somewhere - // else. - replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified); - storePath = std::move(*unpackedStorePath); - } - - if (expectedStorePath && *storePath != *expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(*storePath)).first - : hashFile(request.expectedHash.type, store->toRealPath(*storePath)); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); - } - - if (request.gcRoot) - store->addIndirectRoot(fileLink); - - result.storePath = store->printStorePath(*storePath); - result.path = store->toRealPath(result.storePath); - return result; -} - bool isUri(const string & s) { diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 487036833..28c8a9162 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -65,31 +65,6 @@ struct DownloadResult uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl; - bool gcRoot = false; - bool getLastModified = false; - - CachedDownloadRequest(const std::string & uri); - CachedDownloadRequest() = delete; -}; - -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional etag; - std::string effectiveUri; - std::optional lastModified; -}; - class Store; struct Downloader @@ -111,12 +86,6 @@ struct Downloader invoked on the thread of the caller. */ void download(DownloadRequest && request, Sink & sink); - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; @@ -138,4 +107,7 @@ public: bool isUri(const string & s); +/* Resolve deprecated 'channel:' URLs. */ +std::string resolveUri(const std::string & uri); + } diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 9c931dfa1..deb1c9c59 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -97,6 +97,7 @@ struct DownloadFileResult { StorePath storePath; std::string etag; + std::string effectiveUrl; }; DownloadFileResult downloadFile( diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 53971ec2b..54b59babc 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -28,7 +28,8 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return { .storePath = std::move(cached->storePath), - .etag = getStrAttr(cached->infoAttrs, "etag") + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") }; DownloadRequest request(url); @@ -40,6 +41,7 @@ DownloadFileResult downloadFile( Attrs infoAttrs({ {"etag", res.etag}, + {"url", res.effectiveUri}, }); std::optional storePath; @@ -67,9 +69,22 @@ DownloadFileResult downloadFile( *storePath, immutable); + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + return { .storePath = std::move(*storePath), .etag = res.etag, + .effectiveUrl = res.effectiveUri, }; } -- cgit v1.2.3 From 1c127e6a82dbc128602aa4451dfa8f4c2fe4a751 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Mar 2020 11:42:50 +0100 Subject: downloadFile(): Use expired file if the download fails --- src/libstore/fetchers/tarball.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 54b59babc..62d43ca36 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -25,17 +25,31 @@ DownloadFileResult downloadFile( auto cached = getCache()->lookupExpired(store, inAttrs); - if (cached && !cached->expired) + auto useCached = [&]() -> DownloadFileResult + { return { .storePath = std::move(cached->storePath), .etag = getStrAttr(cached->infoAttrs, "etag"), .effectiveUrl = getStrAttr(cached->infoAttrs, "url") }; + }; + + if (cached && !cached->expired) + return useCached(); DownloadRequest request(url); if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); - auto res = getDownloader()->download(request); + DownloadResult res; + try { + res = getDownloader()->download(request); + } catch (DownloadError & e) { + if (cached) { + warn("%s; using cached version", e.msg()); + return useCached(); + } else + throw; + } // FIXME: write to temporary file. -- cgit v1.2.3 From 8aa354fdfd94db7b63462a4683add04e4eb75c48 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Mar 2020 11:45:34 +0100 Subject: Register flake-registry.json as a GC root again --- src/libstore/fetchers/registry.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index 638e6e50a..69c80a5a9 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -127,10 +127,12 @@ static std::shared_ptr getGlobalRegistry(ref store) static auto reg = [&]() { auto path = settings.flakeRegistry; - if (!hasPrefix(path, "/")) - // FIXME: register as GC root. - // FIXME: if download fails, use previous version if available. - path = store->toRealPath(downloadFile(store, path, "flake-registry.json", false).storePath); + if (!hasPrefix(path, "/")) { + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + if (auto store2 = store.dynamic_pointer_cast()) + store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); + path = store->toRealPath(storePath); + } return Registry::read(path, Registry::Global); }(); -- cgit v1.2.3 From 2287e2f279ac544a2c11921be51f2f556cb78abc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 28 Mar 2020 18:05:50 +0100 Subject: nix flake info: Show flake subdirectory --- src/libstore/fetchers/fetchers.hh | 10 +++++++--- src/libstore/fetchers/git.cc | 4 ++-- src/libstore/fetchers/github.cc | 13 ++++++++----- src/libstore/fetchers/indirect.cc | 4 ++-- src/libstore/fetchers/mercurial.cc | 4 ++-- src/libstore/fetchers/tarball.cc | 4 ++-- 6 files changed, 23 insertions(+), 16 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index deb1c9c59..5e33ec4ca 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -5,6 +5,7 @@ #include "path.hh" #include "tree-info.hh" #include "attrs.hh" +#include "parse.hh" #include @@ -45,7 +46,12 @@ struct Input : std::enable_shared_from_this virtual std::optional getRev() const { return {}; } - virtual std::string to_string() const = 0; + virtual ParsedURL toURL() const = 0; + + std::string to_string() const + { + return toURL().to_string(); + } Attrs toAttrs() const; @@ -74,8 +80,6 @@ private: virtual Attrs toAttrsInternal() const = 0; }; -struct ParsedURL; - struct InputScheme { virtual ~InputScheme() { } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index ede758544..f6b7820b8 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -48,14 +48,14 @@ struct GitInput : Input std::optional getRev() const override { return rev; } - std::string to_string() const override + ParsedURL toURL() const override { ParsedURL url2(url); if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); if (ref) url2.query.insert_or_assign("ref", *ref); if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2.to_string(); + return url2; } Attrs toAttrsInternal() const override diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index 0fef456df..5e34ee051 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -42,13 +42,16 @@ struct GitHubInput : Input std::optional getRev() const override { return rev; } - std::string to_string() const override + ParsedURL toURL() const override { - auto s = fmt("github:%s/%s", owner, repo); + auto path = owner + "/" + repo; assert(!(ref && rev)); - if (ref) s += "/" + *ref; - if (rev) s += "/" + rev->to_string(Base16, false); - return s; + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "github", + .path = path, + }; } Attrs toAttrsInternal() const override diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc index 963abd85f..37e5afbc4 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libstore/fetchers/indirect.cc @@ -43,14 +43,14 @@ struct IndirectInput : Input && (!rev || rev == other2->rev); } - std::string to_string() const override + ParsedURL toURL() const override { ParsedURL url; url.scheme = "flake"; url.path = id; if (ref) { url.path += '/'; url.path += *ref; }; if (rev) { url.path += '/'; url.path += rev->gitRev(); }; - return url.to_string(); + return url; } Attrs toAttrsInternal() const override diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 5cd43a74e..6fb13391b 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -42,13 +42,13 @@ struct MercurialInput : Input std::optional getRev() const override { return rev; } - std::string to_string() const override + ParsedURL toURL() const override { ParsedURL url2(url); url2.scheme = "hg+" + url2.scheme; if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); if (ref) url2.query.insert_or_assign("ref", *ref); - return url2.to_string(); + return url; } Attrs toAttrsInternal() const override diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 62d43ca36..2beb7876d 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -190,7 +190,7 @@ struct TarballInput : Input return hash || narHash; } - std::string to_string() const override + ParsedURL toURL() const override { auto url2(url); // NAR hashes are preferred over file hashes since tar/zip files @@ -199,7 +199,7 @@ struct TarballInput : Input url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); else if (hash) url2.query.insert_or_assign("hash", hash->to_string(SRI)); - return url2.to_string(); + return url2; } Attrs toAttrsInternal() const override -- cgit v1.2.3 From 2287cc64867e76018cb8e88592c618d6de228ed6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 13:31:55 +0200 Subject: Fix segfault --- src/libstore/fetchers/tarball.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 2beb7876d..55244b30e 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -207,7 +207,7 @@ struct TarballInput : Input Attrs attrs; attrs.emplace("url", url.to_string()); if (narHash) - attrs.emplace("narHash", hash->to_string(SRI)); + attrs.emplace("narHash", narHash->to_string(SRI)); else if (hash) attrs.emplace("hash", hash->to_string(SRI)); return attrs; -- cgit v1.2.3 From e0a0ae0467fa8cdcc542f593b9d94283f04508ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:03:28 +0200 Subject: Move fetchers from libstore to libfetchers --- src/libstore/fetchers/attrs.cc | 92 -------- src/libstore/fetchers/attrs.hh | 37 --- src/libstore/fetchers/cache.cc | 121 ---------- src/libstore/fetchers/cache.hh | 35 --- src/libstore/fetchers/fetchers.cc | 91 -------- src/libstore/fetchers/fetchers.hh | 119 ---------- src/libstore/fetchers/git.cc | 460 ------------------------------------- src/libstore/fetchers/github.cc | 218 ------------------ src/libstore/fetchers/indirect.cc | 142 ------------ src/libstore/fetchers/mercurial.cc | 340 --------------------------- src/libstore/fetchers/parse.cc | 138 ----------- src/libstore/fetchers/parse.hh | 30 --- src/libstore/fetchers/regex.hh | 37 --- src/libstore/fetchers/registry.cc | 184 --------------- src/libstore/fetchers/registry.hh | 62 ----- src/libstore/fetchers/tarball.cc | 278 ---------------------- src/libstore/fetchers/tree-info.hh | 26 --- src/libstore/store-api.cc | 4 +- 18 files changed, 2 insertions(+), 2412 deletions(-) delete mode 100644 src/libstore/fetchers/attrs.cc delete mode 100644 src/libstore/fetchers/attrs.hh delete mode 100644 src/libstore/fetchers/cache.cc delete mode 100644 src/libstore/fetchers/cache.hh delete mode 100644 src/libstore/fetchers/fetchers.cc delete mode 100644 src/libstore/fetchers/fetchers.hh delete mode 100644 src/libstore/fetchers/git.cc delete mode 100644 src/libstore/fetchers/github.cc delete mode 100644 src/libstore/fetchers/indirect.cc delete mode 100644 src/libstore/fetchers/mercurial.cc delete mode 100644 src/libstore/fetchers/parse.cc delete mode 100644 src/libstore/fetchers/parse.hh delete mode 100644 src/libstore/fetchers/regex.hh delete mode 100644 src/libstore/fetchers/registry.cc delete mode 100644 src/libstore/fetchers/registry.hh delete mode 100644 src/libstore/fetchers/tarball.cc delete mode 100644 src/libstore/fetchers/tree-info.hh (limited to 'src/libstore') diff --git a/src/libstore/fetchers/attrs.cc b/src/libstore/fetchers/attrs.cc deleted file mode 100644 index 40c02de42..000000000 --- a/src/libstore/fetchers/attrs.cc +++ /dev/null @@ -1,92 +0,0 @@ -#include "attrs.hh" -#include "fetchers.hh" - -#include - -namespace nix::fetchers { - -Attrs jsonToAttrs(const nlohmann::json & json) -{ - Attrs attrs; - - for (auto & i : json.items()) { - if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get()); - else if (i.value().is_string()) - attrs.emplace(i.key(), i.value().get()); - else if (i.value().is_boolean()) - attrs.emplace(i.key(), i.value().get()); - else - throw Error("unsupported input attribute type in lock file"); - } - - return attrs; -} - -nlohmann::json attrsToJson(const Attrs & attrs) -{ - nlohmann::json json; - for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { - json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { - json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { - json[attr.first] = v->t; - } else abort(); - } - return json; -} - -std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) -{ - auto i = attrs.find(name); - if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; - throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); -} - -std::string getStrAttr(const Attrs & attrs, const std::string & name) -{ - auto s = maybeGetStrAttr(attrs, name); - if (!s) - throw Error("input attribute '%s' is missing", name); - return *s; -} - -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) -{ - auto i = attrs.find(name); - if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; - throw Error("input attribute '%s' is not an integer", name); -} - -int64_t getIntAttr(const Attrs & attrs, const std::string & name) -{ - auto s = maybeGetIntAttr(attrs, name); - if (!s) - throw Error("input attribute '%s' is missing", name); - return *s; -} - -std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name) -{ - auto i = attrs.find(name); - if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; - throw Error("input attribute '%s' is not a Boolean", name); -} - -bool getBoolAttr(const Attrs & attrs, const std::string & name) -{ - auto s = maybeGetBoolAttr(attrs, name); - if (!s) - throw Error("input attribute '%s' is missing", name); - return *s; -} - -} diff --git a/src/libstore/fetchers/attrs.hh b/src/libstore/fetchers/attrs.hh deleted file mode 100644 index 2c9e772d2..000000000 --- a/src/libstore/fetchers/attrs.hh +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include "types.hh" - -#include - -#include - -namespace nix::fetchers { - -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ -template -struct Explicit { - T t; -}; - -typedef std::variant> Attr; -typedef std::map Attrs; - -Attrs jsonToAttrs(const nlohmann::json & json); - -nlohmann::json attrsToJson(const Attrs & attrs); - -std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); - -std::string getStrAttr(const Attrs & attrs, const std::string & name); - -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); - -int64_t getIntAttr(const Attrs & attrs, const std::string & name); - -std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); - -bool getBoolAttr(const Attrs & attrs, const std::string & name); - -} diff --git a/src/libstore/fetchers/cache.cc b/src/libstore/fetchers/cache.cc deleted file mode 100644 index 14a84744a..000000000 --- a/src/libstore/fetchers/cache.cc +++ /dev/null @@ -1,121 +0,0 @@ -#include "fetchers/cache.hh" -#include "sqlite.hh" -#include "sync.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -static const char * schema = R"sql( - -create table if not exists Cache ( - input text not null, - info text not null, - path text not null, - immutable integer not null, - timestamp integer not null, - primary key (input) -); -)sql"; - -struct CacheImpl : Cache -{ - struct State - { - SQLite db; - SQLiteStmt add, lookup; - }; - - Sync _state; - - CacheImpl() - { - auto state(_state.lock()); - - auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; - createDirs(dirOf(dbPath)); - - state->db = SQLite(dbPath); - state->db.isCache(); - state->db.exec(schema); - - state->add.create(state->db, - "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); - - state->lookup.create(state->db, - "select info, path, immutable, timestamp from Cache where input = ?"); - } - - void add( - ref store, - const Attrs & inAttrs, - const Attrs & infoAttrs, - const StorePath & storePath, - bool immutable) override - { - _state.lock()->add.use() - (attrsToJson(inAttrs).dump()) - (attrsToJson(infoAttrs).dump()) - (store->printStorePath(storePath)) - (immutable) - (time(0)).exec(); - } - - std::optional> lookup( - ref store, - const Attrs & inAttrs) override - { - if (auto res = lookupExpired(store, inAttrs)) { - if (!res->expired) - return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); - debug("ignoring expired cache entry '%s'", - attrsToJson(inAttrs).dump()); - } - return {}; - } - - std::optional lookupExpired( - ref store, - const Attrs & inAttrs) override - { - auto state(_state.lock()); - - auto inAttrsJson = attrsToJson(inAttrs).dump(); - - auto stmt(state->lookup.use()(inAttrsJson)); - if (!stmt.next()) { - debug("did not find cache entry for '%s'", inAttrsJson); - return {}; - } - - auto infoJson = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); - auto immutable = stmt.getInt(2) != 0; - auto timestamp = stmt.getInt(3); - - store->addTempRoot(storePath); - if (!store->isValidPath(storePath)) { - // FIXME: we could try to substitute 'storePath'. - debug("ignoring disappeared cache entry '%s'", inAttrsJson); - return {}; - } - - debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJson, infoJson, store->printStorePath(storePath)); - - return Result { - .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), - .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), - .storePath = std::move(storePath) - }; - } -}; - -ref getCache() -{ - static auto cache = std::make_shared(); - return ref(cache); -} - -} diff --git a/src/libstore/fetchers/cache.hh b/src/libstore/fetchers/cache.hh deleted file mode 100644 index a25b05985..000000000 --- a/src/libstore/fetchers/cache.hh +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "types.hh" -#include "fetchers/fetchers.hh" - -namespace nix::fetchers { - -struct Cache -{ - virtual void add( - ref store, - const Attrs & inAttrs, - const Attrs & infoAttrs, - const StorePath & storePath, - bool immutable) = 0; - - virtual std::optional> lookup( - ref store, - const Attrs & inAttrs) = 0; - - struct Result - { - bool expired = false; - Attrs infoAttrs; - StorePath storePath; - }; - - virtual std::optional lookupExpired( - ref store, - const Attrs & inAttrs) = 0; -}; - -ref getCache(); - -} diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc deleted file mode 100644 index 25827ab7c..000000000 --- a/src/libstore/fetchers/fetchers.cc +++ /dev/null @@ -1,91 +0,0 @@ -#include "fetchers.hh" -#include "parse.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -std::unique_ptr>> inputSchemes = nullptr; - -void registerInputScheme(std::unique_ptr && inputScheme) -{ - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); -} - -std::unique_ptr inputFromURL(const ParsedURL & url) -{ - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); - if (res) return res; - } - throw Error("input '%s' is unsupported", url.url); -} - -std::unique_ptr inputFromURL(const std::string & url) -{ - return inputFromURL(parseURL(url)); -} - -std::unique_ptr inputFromAttrs(const Attrs & attrs) -{ - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); - return res; - } - } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); -} - -Attrs Input::toAttrs() const -{ - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - attrs.emplace("type", type()); - return attrs; -} - -std::pair> Input::fetchTree(ref store) const -{ - auto [tree, input] = fetchTreeInternal(store); - - if (tree.actualPath == "") - tree.actualPath = store->toRealPath(tree.storePath); - - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; - - if (input->narHash) - assert(input->narHash == tree.info.narHash); - - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); - - return {std::move(tree), input}; -} - -std::shared_ptr Input::applyOverrides( - std::optional ref, - std::optional rev) const -{ - if (ref) - throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); - if (rev) - throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); - return shared_from_this(); -} - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); -} - -} diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh deleted file mode 100644 index 5e33ec4ca..000000000 --- a/src/libstore/fetchers/fetchers.hh +++ /dev/null @@ -1,119 +0,0 @@ -#pragma once - -#include "types.hh" -#include "hash.hh" -#include "path.hh" -#include "tree-info.hh" -#include "attrs.hh" -#include "parse.hh" - -#include - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct Input; - -struct Tree -{ - Path actualPath; - StorePath storePath; - TreeInfo info; -}; - -struct Input : std::enable_shared_from_this -{ - std::optional narHash; // FIXME: implement - - virtual std::string type() const = 0; - - virtual ~Input() { } - - virtual bool operator ==(const Input & other) const { return false; } - - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ - virtual bool isDirect() const { return true; } - - /* Check whether this is an "immutable" input, that is, - one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } - - virtual bool contains(const Input & other) const { return false; } - - virtual std::optional getRef() const { return {}; } - - virtual std::optional getRev() const { return {}; } - - virtual ParsedURL toURL() const = 0; - - std::string to_string() const - { - return toURL().to_string(); - } - - Attrs toAttrs() const; - - std::pair> fetchTree(ref store) const; - - virtual std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const; - - virtual std::optional getSourcePath() const { return {}; } - - virtual void markChangedFile( - std::string_view file, - std::optional commitMsg) const - { assert(false); } - - virtual void clone(const Path & destDir) const - { - throw Error("do not know how to clone input '%s'", to_string()); - } - -private: - - virtual std::pair> fetchTreeInternal(ref store) const = 0; - - virtual Attrs toAttrsInternal() const = 0; -}; - -struct InputScheme -{ - virtual ~InputScheme() { } - - virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; - - virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; -}; - -std::unique_ptr inputFromURL(const ParsedURL & url); - -std::unique_ptr inputFromURL(const std::string & url); - -std::unique_ptr inputFromAttrs(const Attrs & attrs); - -void registerInputScheme(std::unique_ptr && fetcher); - -struct DownloadFileResult -{ - StorePath storePath; - std::string etag; - std::string effectiveUrl; -}; - -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool immutable); - -Tree downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool immutable); - -} diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc deleted file mode 100644 index f6b7820b8..000000000 --- a/src/libstore/fetchers/git.cc +++ /dev/null @@ -1,460 +0,0 @@ -#include "fetchers/fetchers.hh" -#include "fetchers/cache.hh" -#include "fetchers/parse.hh" -#include "globals.hh" -#include "tarfile.hh" -#include "store-api.hh" -#include "regex.hh" - -#include - -using namespace std::string_literals; - -namespace nix::fetchers { - -static std::string readHead(const Path & path) -{ - return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); -} - -struct GitInput : Input -{ - ParsedURL url; - std::optional ref; - std::optional rev; - bool shallow = false; - - GitInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "git"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; - } - - bool isImmutable() const override - { - return (bool) rev; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override - { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - return attrs; - } - - void clone(const Path & destDir) const override - { - auto [isLocal, actualUrl] = getActualUrl(); - - Strings args = {"clone"}; - - args.push_back(actualUrl); - - if (ref) { - args.push_back("--branch"); - args.push_back(*ref); - } - - if (rev) throw Error("cloning a specific revision is not implemented"); - - args.push_back(destDir); - - runProgram("git", true, args); - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - if (!res->ref && res->rev) - throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); - - return res; - } - - std::optional getSourcePath() const override - { - if (url.scheme == "file" && !ref && !rev) - return url.path; - return {}; - } - - void markChangedFile(std::string_view file, std::optional commitMsg) const override - { - auto sourcePath = getSourcePath(); - assert(sourcePath); - - runProgram("git", true, - { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); - - if (commitMsg) - runProgram("git", true, - { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); - } - - std::pair getActualUrl() const - { - // Don't clone file:// URIs (but otherwise treat them the - // same as remote URIs, i.e. don't use the working tree or - // HEAD). - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing - bool isLocal = url.scheme == "file" && !forceHttp; - return {isLocal, isLocal ? url.path : url.base}; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto name = "source"; - - auto input = std::make_shared(*this); - - assert(!rev || rev->type == htSHA1); - - auto cacheType = shallow ? "git-shallow" : "git"; - - auto getImmutableAttrs = [&]() - { - return Attrs({ - {"type", cacheType}, - {"name", name}, - {"rev", input->rev->gitRev()}, - }); - }; - - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> - { - assert(input->rev); - assert(!rev || rev == input->rev); - return { - Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, - }, - input - }; - }; - - if (rev) { - if (auto res = getCache()->lookup(store, getImmutableAttrs())) - return makeResult(res->first, std::move(res->second)); - } - - auto [isLocal, actualUrl_] = getActualUrl(); - auto actualUrl = actualUrl_; // work around clang bug - - // If this is a local directory and no ref or revision is - // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { - bool clean = false; - - /* Check whether this repo has any commits. There are - probably better ways to do this. */ - auto gitDir = actualUrl + "/.git"; - auto commonGitDir = chomp(runProgram( - "git", - true, - { "-C", actualUrl, "rev-parse", "--git-common-dir" } - )); - if (commonGitDir != ".git") - gitDir = commonGitDir; - - bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); - - try { - if (haveCommits) { - runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); - clean = true; - } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - - if (!settings.allowDirty) - throw Error("Git tree '%s' is dirty", actualUrl); - - if (settings.warnDirty) - warn("Git tree '%s' is dirty", actualUrl); - - auto files = tokenizeString>( - runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } - }; - - return {std::move(tree), input}; - } - } - - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; - - Attrs mutableAttrs({ - {"type", cacheType}, - {"name", name}, - {"url", actualUrl}, - {"ref", *input->ref}, - }); - - Path repoDir; - - if (isLocal) { - - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); - - repoDir = actualUrl; - - } else { - - if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; - return makeResult(res->first, std::move(res->second)); - } - } - - Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); - repoDir = cacheDir; - - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", repoDir }); - } - - Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; - - bool doFetch; - time_t now = time(0); - - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (input->rev) { - try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; - } - - if (doFetch) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); - } catch (Error & e) { - if (!pathExists(localRefFile)) throw; - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); - } - - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); - } - - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); - } - - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; - - if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); - - // FIXME: check whether rev is an ancestor of ref. - - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); - - /* Now that we know the ref, check again whether we have it in - the store. */ - if (auto res = getCache()->lookup(store, getImmutableAttrs())) - return makeResult(res->first, std::move(res->second)); - - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - unpackTarfile(*source, tmpDir); - - auto storePath = store->addToStore(name, tmpDir); - - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); - - Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, - {"lastModified", lastModified}, - }); - - if (!shallow) - infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); - - if (!this->rev) - getCache()->add( - store, - mutableAttrs, - infoAttrs, - storePath, - false); - - getCache()->add( - store, - getImmutableAttrs(), - infoAttrs, - storePath, - true); - - return makeResult(infoAttrs, std::move(storePath)); - } -}; - -struct GitInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); - -} diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc deleted file mode 100644 index 5e34ee051..000000000 --- a/src/libstore/fetchers/github.cc +++ /dev/null @@ -1,218 +0,0 @@ -#include "download.hh" -#include "fetchers/cache.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/parse.hh" -#include "fetchers/regex.hh" -#include "globals.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); - -struct GitHubInput : Input -{ - std::string owner; - std::string repo; - std::optional ref; - std::optional rev; - - std::string type() const override { return "github"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; - } - - bool isImmutable() const override - { - return (bool) rev; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override - { - auto path = owner + "/" + repo; - assert(!(ref && rev)); - if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); - return ParsedURL { - .scheme = "github", - .path = path, - }; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - void clone(const Path & destDir) const override - { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); - - if (!rev) { - auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false).storePath))); - rev = Hash(json["sha"], htSHA1); - debug("HEAD revision for '%s' is %s", url, rev->gitRev()); - } - - auto input = std::make_shared(*this); - input->ref = {}; - input->rev = *rev; - - Attrs immutableAttrs({ - {"type", "git-tarball"}, - {"rev", rev->gitRev()}, - }); - - if (auto res = getCache()->lookup(store, immutableAttrs)) { - return { - Tree{ - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, - }, - input - }; - } - - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. - - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); - - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; - - auto tree = downloadTarball(store, url, "source", true); - - getCache()->add( - store, - immutableAttrs, - { - {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} - }, - tree.storePath, - true); - - return {std::move(tree), input}; - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } -}; - -struct GitHubInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "github") return nullptr; - - auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("GitHub URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "github") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported GitHub input attribute '%s'", name); - - auto input = std::make_unique(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); - -} diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc deleted file mode 100644 index 37e5afbc4..000000000 --- a/src/libstore/fetchers/indirect.cc +++ /dev/null @@ -1,142 +0,0 @@ -#include "fetchers.hh" -#include "parse.hh" -#include "regex.hh" - -namespace nix::fetchers { - -std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); - -struct IndirectInput : Input -{ - std::string id; - std::optional rev; - std::optional ref; - - std::string type() const override { return "indirect"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && id == other2->id - && rev == other2->rev - && ref == other2->ref; - } - - bool isDirect() const override - { - return false; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - bool contains(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && id == other2->id - && (!ref || ref == other2->ref) - && (!rev || rev == other2->rev); - } - - ParsedURL toURL() const override - { - ParsedURL url; - url.scheme = "flake"; - url.path = id; - if (ref) { url.path += '/'; url.path += *ref; }; - if (rev) { url.path += '/'; url.path += rev->gitRev(); }; - return url; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("id", id); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - throw Error("indirect input '%s' cannot be fetched directly", to_string()); - } -}; - -struct IndirectInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "flake") return nullptr; - - auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); - - if (path.size() == 1) { - } else if (path.size() == 2) { - if (std::regex_match(path[1], revRegex)) - input->rev = Hash(path[1], htSHA1); - else if (std::regex_match(path[1], refRegex)) - input->ref = path[1]; - else - throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); - } else if (path.size() == 3) { - if (!std::regex_match(path[1], refRegex)) - throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); - input->ref = path[1]; - if (!std::regex_match(path[2], revRegex)) - throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - input->rev = Hash(path[2], htSHA1); - } else - throw BadURL("GitHub URL '%s' is invalid", url.url); - - // FIXME: forbid query params? - - input->id = path[0]; - if (!std::regex_match(input->id, flakeRegex)) - throw BadURL("'%s' is not a valid flake ID", input->id); - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev") - throw Error("unsupported indirect input attribute '%s'", name); - - auto input = std::make_unique(); - input->id = getStrAttr(attrs, "id"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); - -} diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc deleted file mode 100644 index 6fb13391b..000000000 --- a/src/libstore/fetchers/mercurial.cc +++ /dev/null @@ -1,340 +0,0 @@ -#include "fetchers/fetchers.hh" -#include "fetchers/cache.hh" -#include "fetchers/parse.hh" -#include "globals.hh" -#include "tarfile.hh" -#include "store-api.hh" -#include "regex.hh" - -#include - -using namespace std::string_literals; - -namespace nix::fetchers { - -struct MercurialInput : Input -{ - ParsedURL url; - std::optional ref; - std::optional rev; - - MercurialInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "hg"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; - } - - bool isImmutable() const override - { - return (bool) rev; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override - { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - return url; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } - - std::optional getSourcePath() const - { - if (url.scheme == "file" && !ref && !rev) - return url.path; - return {}; - } - - void markChangedFile(std::string_view file, std::optional commitMsg) const override - { - auto sourcePath = getSourcePath(); - assert(sourcePath); - - // FIXME: shut up if file is already tracked. - runProgram("hg", true, - { "add", *sourcePath + "/" + std::string(file) }); - - if (commitMsg) - runProgram("hg", true, - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); - } - - std::pair getActualUrl() const - { - bool isLocal = url.scheme == "file"; - return {isLocal, isLocal ? url.path : url.base}; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto name = "source"; - - auto input = std::make_shared(*this); - - auto [isLocal, actualUrl_] = getActualUrl(); - auto actualUrl = actualUrl_; // work around clang bug - - // FIXME: return lastModified. - - // FIXME: don't clone local repositories. - - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { - - bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked - files. */ - - if (!settings.allowDirty) - throw Error("Mercurial tree '%s' is unclean", actualUrl); - - if (settings.warnDirty) - warn("Mercurial tree '%s' is unclean", actualUrl); - - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); - - auto files = tokenizeString>( - runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - - return {Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - }, input}; - } - } - - if (!input->ref) input->ref = "default"; - - auto getImmutableAttrs = [&]() - { - return Attrs({ - {"type", "hg"}, - {"name", name}, - {"rev", input->rev->gitRev()}, - }); - }; - - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> - { - assert(input->rev); - assert(!rev || rev == input->rev); - return { - Tree{ - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, - }, - input - }; - }; - - if (input->rev) { - if (auto res = getCache()->lookup(store, getImmutableAttrs())) - return makeResult(res->first, std::move(res->second)); - } - - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; - - Attrs mutableAttrs({ - {"type", "hg"}, - {"name", name}, - {"url", actualUrl}, - {"ref", *input->ref}, - }); - - if (auto res = getCache()->lookup(store, mutableAttrs)) { - auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; - return makeResult(res->first, std::move(res->second)); - } - } - - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); - - /* If this is a commit hash that we already have, we don't - have to pull again. */ - if (!(input->rev - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); - - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); - } - } - - auto tokens = tokenizeString>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); - assert(tokens.size() == 3); - - input->rev = Hash(tokens[0], htSHA1); - auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; - - if (auto res = getCache()->lookup(store, getImmutableAttrs())) - return makeResult(res->first, std::move(res->second)); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); - - deletePath(tmpDir + "/.hg_archival.txt"); - - auto storePath = store->addToStore(name, tmpDir); - - Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, - {"revCount", (int64_t) revCount}, - }); - - if (!this->rev) - getCache()->add( - store, - mutableAttrs, - infoAttrs, - storePath, - false); - - getCache()->add( - store, - getImmutableAttrs(), - infoAttrs, - storePath, - true); - - return makeResult(infoAttrs, std::move(storePath)); - } -}; - -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); - -} diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc deleted file mode 100644 index a5ad14c87..000000000 --- a/src/libstore/fetchers/parse.cc +++ /dev/null @@ -1,138 +0,0 @@ -#include "parse.hh" -#include "util.hh" -#include "regex.hh" - -namespace nix::fetchers { - -std::regex refRegex(refRegexS, std::regex::ECMAScript); -std::regex revRegex(revRegexS, std::regex::ECMAScript); -std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); - -ParsedURL parseURL(const std::string & url) -{ - static std::regex uriRegex( - "((" + schemeRegex + "):" - + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); - - std::smatch match; - - if (std::regex_match(url, match, uriRegex)) { - auto & base = match[1]; - std::string scheme = match[2]; - auto authority = match[3].matched - ? std::optional(match[3]) : std::nullopt; - std::string path = match[4].matched ? match[4] : match[5]; - auto & query = match[6]; - auto & fragment = match[7]; - - auto isFile = scheme.find("file") != std::string::npos; - - if (authority && *authority != "" && isFile) - throw Error("file:// URL '%s' has unexpected authority '%s'", - url, *authority); - - if (isFile && path.empty()) - path = "/"; - - return ParsedURL{ - .url = url, - .base = base, - .scheme = scheme, - .authority = authority, - .path = path, - .query = decodeQuery(query), - .fragment = percentDecode(std::string(fragment)) - }; - } - - else - throw BadURL("'%s' is not a valid URL", url); -} - -std::string percentDecode(std::string_view in) -{ - std::string decoded; - for (size_t i = 0; i < in.size(); ) { - if (in[i] == '%') { - if (i + 2 >= in.size()) - throw BadURL("invalid URI parameter '%s'", in); - try { - decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw BadURL("invalid URI parameter '%s'", in); - } - } else - decoded += in[i++]; - } - return decoded; -} - -std::map decodeQuery(const std::string & query) -{ - std::map result; - - for (auto s : tokenizeString(query, "&")) { - auto e = s.find('='); - if (e != std::string::npos) - result.emplace( - s.substr(0, e), - percentDecode(std::string_view(s).substr(e + 1))); - } - - return result; -} - -std::string percentEncode(std::string_view s) -{ - std::string res; - for (auto & c : s) - if ((c >= 'a' && c <= 'z') - || (c >= 'A' && c <= 'Z') - || (c >= '0' && c <= '9') - || strchr("-._~!$&'()*+,;=:@", c)) - res += c; - else - res += fmt("%%%02x", (unsigned int) c); - return res; -} - -std::string encodeQuery(const std::map & ss) -{ - std::string res; - bool first = true; - for (auto & [name, value] : ss) { - if (!first) res += '&'; - first = false; - res += percentEncode(name); - res += '='; - res += percentEncode(value); - } - return res; -} - -std::string ParsedURL::to_string() const -{ - return - scheme - + ":" - + (authority ? "//" + *authority : "") - + path - + (query.empty() ? "" : "?" + encodeQuery(query)) - + (fragment.empty() ? "" : "#" + percentEncode(fragment)); -} - -bool ParsedURL::operator ==(const ParsedURL & other) const -{ - return - scheme == other.scheme - && authority == other.authority - && path == other.path - && query == other.query - && fragment == other.fragment; -} - -} diff --git a/src/libstore/fetchers/parse.hh b/src/libstore/fetchers/parse.hh deleted file mode 100644 index 45d5182b0..000000000 --- a/src/libstore/fetchers/parse.hh +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix::fetchers { - -struct ParsedURL -{ - std::string url; - std::string base; // URL without query/fragment - std::string scheme; - std::optional authority; - std::string path; - std::map query; - std::string fragment; - - std::string to_string() const; - - bool operator ==(const ParsedURL & other) const; -}; - -MakeError(BadURL, Error); - -std::string percentDecode(std::string_view in); - -std::map decodeQuery(const std::string & query); - -ParsedURL parseURL(const std::string & url); - -} diff --git a/src/libstore/fetchers/regex.hh b/src/libstore/fetchers/regex.hh deleted file mode 100644 index e0989edfc..000000000 --- a/src/libstore/fetchers/regex.hh +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -namespace nix::fetchers { - -// URI stuff. -const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+]+)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; -const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; -const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; -const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; -const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; -const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; -const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; -const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; -const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; -const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; -const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; - -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check -extern std::regex refRegex; - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -extern std::regex revRegex; - -// A ref or revision, or a ref followed by a revision. -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - -} diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc deleted file mode 100644 index 69c80a5a9..000000000 --- a/src/libstore/fetchers/registry.cc +++ /dev/null @@ -1,184 +0,0 @@ -#include "fetchers/registry.hh" -#include "fetchers/fetchers.hh" -#include "util.hh" -#include "globals.hh" -#include "download.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -std::shared_ptr Registry::read( - const Path & path, RegistryType type) -{ - auto registry = std::make_shared(type); - - if (!pathExists(path)) - return std::make_shared(type); - - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - - // FIXME: remove soon - if (version == 1) { - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url), {}}); - } - } - - else if (version == 2) { - for (auto & i : json["flakes"]) { - auto toAttrs = jsonToAttrs(i["to"]); - Attrs extraAttrs; - auto j = toAttrs.find("dir"); - if (j != toAttrs.end()) { - extraAttrs.insert(*j); - toAttrs.erase(j); - } - registry->entries.push_back( - { inputFromAttrs(jsonToAttrs(i["from"])) - , inputFromAttrs(toAttrs) - , extraAttrs - }); - } - } - - else - throw Error("flake registry '%s' has unsupported version %d", path, version); - - - return registry; -} - -void Registry::write(const Path & path) -{ - nlohmann::json arr; - for (auto & elem : entries) { - nlohmann::json obj; - obj["from"] = attrsToJson(std::get<0>(elem)->toAttrs()); - obj["to"] = attrsToJson(std::get<1>(elem)->toAttrs()); - if (!std::get<2>(elem).empty()) - obj["to"].update(attrsToJson(std::get<2>(elem))); - arr.emplace_back(std::move(obj)); - } - - nlohmann::json json; - json["version"] = 2; - json["flakes"] = std::move(arr); - - createDirs(dirOf(path)); - writeFile(path, json.dump(2)); -} - -void Registry::add( - const std::shared_ptr & from, - const std::shared_ptr & to, - const Attrs & extraAttrs) -{ - entries.emplace_back(from, to, extraAttrs); -} - -void Registry::remove(const std::shared_ptr & input) -{ - // FIXME: use C++20 std::erase. - for (auto i = entries.begin(); i != entries.end(); ) - if (*std::get<0>(*i) == *input) - i = entries.erase(i); - else - ++i; -} - -Path getUserRegistryPath() -{ - return getHome() + "/.config/nix/registry.json"; -} - -std::shared_ptr getUserRegistry() -{ - return Registry::read(getUserRegistryPath(), Registry::User); -} - -static std::shared_ptr flagRegistry = - std::make_shared(Registry::Flag); - -std::shared_ptr getFlagRegistry() -{ - return flagRegistry; -} - -void overrideRegistry( - const std::shared_ptr & from, - const std::shared_ptr & to, - const Attrs & extraAttrs) -{ - flagRegistry->add(from, to, extraAttrs); -} - -static std::shared_ptr getGlobalRegistry(ref store) -{ - static auto reg = [&]() { - auto path = settings.flakeRegistry; - - if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; - if (auto store2 = store.dynamic_pointer_cast()) - store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); - path = store->toRealPath(storePath); - } - - return Registry::read(path, Registry::Global); - }(); - - return reg; -} - -Registries getRegistries(ref store) -{ - Registries registries; - registries.push_back(getFlagRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getGlobalRegistry(store)); - return registries; -} - -std::pair, Attrs> lookupInRegistries( - ref store, - std::shared_ptr input) -{ - Attrs extraAttrs; - int n = 0; - - restart: - - n++; - if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); - - for (auto & registry : getRegistries(store)) { - // FIXME: O(n) - for (auto & entry : registry->entries) { - auto from = std::get<0>(entry); - if (from->contains(*input)) { - input = std::get<1>(entry)->applyOverrides( - !from->getRef() && input->getRef() ? input->getRef() : std::optional(), - !from->getRev() && input->getRev() ? input->getRev() : std::optional()); - extraAttrs = std::get<2>(entry); - goto restart; - } - } - } - - if (!input->isDirect()) - throw Error("cannot find flake '%s' in the flake registries", input->to_string()); - - return {input, extraAttrs}; -} - -} diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh deleted file mode 100644 index d2eb7749b..000000000 --- a/src/libstore/fetchers/registry.hh +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include "types.hh" -#include "fetchers.hh" - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct Registry -{ - enum RegistryType { - Flag = 0, - User = 1, - Global = 2, - }; - - RegistryType type; - - std::vector< - std::tuple< - std::shared_ptr, // from - std::shared_ptr, // to - Attrs // extra attributes - > - > entries; - - Registry(RegistryType type) - : type(type) - { } - - static std::shared_ptr read( - const Path & path, RegistryType type); - - void write(const Path & path); - - void add( - const std::shared_ptr & from, - const std::shared_ptr & to, - const Attrs & extraAttrs); - - void remove(const std::shared_ptr & input); -}; - -typedef std::vector> Registries; - -std::shared_ptr getUserRegistry(); - -Path getUserRegistryPath(); - -Registries getRegistries(ref store); - -void overrideRegistry( - const std::shared_ptr & from, - const std::shared_ptr & to, - const Attrs & extraAttrs); - -std::pair, Attrs> lookupInRegistries( - ref store, - std::shared_ptr input); - -} diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc deleted file mode 100644 index 55244b30e..000000000 --- a/src/libstore/fetchers/tarball.cc +++ /dev/null @@ -1,278 +0,0 @@ -#include "fetchers/fetchers.hh" -#include "fetchers/parse.hh" -#include "fetchers/cache.hh" -#include "download.hh" -#include "globals.hh" -#include "store-api.hh" -#include "archive.hh" -#include "tarfile.hh" - -namespace nix::fetchers { - -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool immutable) -{ - // FIXME: check store - - Attrs inAttrs({ - {"type", "file"}, - {"url", url}, - {"name", name}, - }); - - auto cached = getCache()->lookupExpired(store, inAttrs); - - auto useCached = [&]() -> DownloadFileResult - { - return { - .storePath = std::move(cached->storePath), - .etag = getStrAttr(cached->infoAttrs, "etag"), - .effectiveUrl = getStrAttr(cached->infoAttrs, "url") - }; - }; - - if (cached && !cached->expired) - return useCached(); - - DownloadRequest request(url); - if (cached) - request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); - DownloadResult res; - try { - res = getDownloader()->download(request); - } catch (DownloadError & e) { - if (cached) { - warn("%s; using cached version", e.msg()); - return useCached(); - } else - throw; - } - - // FIXME: write to temporary file. - - Attrs infoAttrs({ - {"etag", res.etag}, - {"url", res.effectiveUri}, - }); - - std::optional storePath; - - if (res.cached) { - assert(cached); - assert(request.expectedETag == res.etag); - storePath = std::move(cached->storePath); - } else { - StringSink sink; - dumpString(*res.data, sink); - auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = std::move(info.path); - } - - getCache()->add( - store, - inAttrs, - infoAttrs, - *storePath, - immutable); - - if (url != res.effectiveUri) - getCache()->add( - store, - { - {"type", "file"}, - {"url", res.effectiveUri}, - {"name", name}, - }, - infoAttrs, - *storePath, - immutable); - - return { - .storePath = std::move(*storePath), - .etag = res.etag, - .effectiveUrl = res.effectiveUri, - }; -} - -Tree downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool immutable) -{ - Attrs inAttrs({ - {"type", "tarball"}, - {"url", url}, - {"name", name}, - }); - - auto cached = getCache()->lookupExpired(store, inAttrs); - - if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), - }, - }; - - auto res = downloadFile(store, url, name, immutable); - - std::optional unpackedStorePath; - time_t lastModified; - - if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { - unpackedStorePath = std::move(cached->storePath); - lastModified = getIntAttr(cached->infoAttrs, "lastModified"); - } else { - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(res.storePath), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - - Attrs infoAttrs({ - {"lastModified", lastModified}, - {"etag", res.etag}, - }); - - getCache()->add( - store, - inAttrs, - infoAttrs, - *unpackedStorePath, - immutable); - - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, - }, - }; -} - -struct TarballInput : Input -{ - ParsedURL url; - std::optional hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - else if (hash) - attrs.emplace("hash", hash->to_string(SRI)); - return attrs; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - -struct TarballInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; - - if (!hasSuffix(url.path, ".zip") - && !hasSuffix(url.path, ".tar") - && !hasSuffix(url.path, ".tar.gz") - && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash" && name != "narHash") - throw Error("unsupported tarball input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - // FIXME: require SRI hash. - input->hash = Hash(*hash); - - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); - -} diff --git a/src/libstore/fetchers/tree-info.hh b/src/libstore/fetchers/tree-info.hh deleted file mode 100644 index 02e92759b..000000000 --- a/src/libstore/fetchers/tree-info.hh +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "path.hh" - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional revCount; - std::optional lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; -}; - -} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 66a9d9fb5..e5282bb30 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,7 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" -#include "fetchers/parse.hh" +#include "url.hh" #include @@ -867,7 +867,7 @@ std::pair splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - params = fetchers::decodeQuery(uri.substr(q + 1)); + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; -- cgit v1.2.3 From 2c692a3b144523bca68dd6de618124ba6c9bb332 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:29:29 +0200 Subject: Remove global -I flags --- src/libstore/local.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local.mk b/src/libstore/local.mk index e8cbe422c..b6d04834a 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif -libstore_CXXFLAGS = \ +libstore_CXXFLAGS += \ + -I src/libutil -I src/libstore \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ -- cgit v1.2.3 From d15d91cad1676cd08a184339ab0c586f7bcee205 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 16:50:13 +0200 Subject: Makefile cleanup --- src/libstore/local.mk | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b6d04834a..636f74b65 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,7 +4,7 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/fetchers/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) libstore_LIBS = libutil libnixrust @@ -64,6 +64,3 @@ $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) - -$(foreach i, $(wildcard src/libstore/fetchers/*.hh), \ - $(eval $(call install-file-in, $(i), $(includedir)/nix/fetchers, 0644))) -- cgit v1.2.3 From 7a9687ba30d579bc51e0aaf3193e0ab8d86400d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Apr 2020 01:00:56 +0200 Subject: SQLiteStmt: Use std::string_view --- src/libstore/sqlite.cc | 4 ++-- src/libstore/sqlite.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index eb1daafc5..63527a811 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -95,10 +95,10 @@ SQLiteStmt::Use::~Use() sqlite3_reset(stmt); } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throwSQLiteError(stmt.db, "binding argument"); } else bind(); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index fd04c9b07..661a384ef 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -54,7 +54,7 @@ struct SQLiteStmt ~Use(); /* Bind the next parameter. */ - Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null -- cgit v1.2.3 From aaa109565e4fb662e423f23bc48c9ad9831dd281 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Apr 2020 23:04:21 +0200 Subject: Use a more space/time-efficient representation for the eval cache --- src/libstore/local-store.cc | 2 +- src/libstore/sqlite.cc | 5 +++++ src/libstore/sqlite.hh | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ae7513ad8..b6db627b5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -588,7 +588,7 @@ uint64_t LocalStore::addValidPath(State & state, (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); + uint64_t id = state.db.getLastInsertedRowId(); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 63527a811..a1c262f5f 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt) }); } +uint64_t SQLite::getLastInsertedRowId() +{ + return sqlite3_last_insert_rowid(db); +} + void SQLiteStmt::create(sqlite3 * db, const string & sql) { checkInterrupt(); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 661a384ef..50909a35a 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -26,6 +26,8 @@ struct SQLite void isCache(); void exec(const std::string & stmt); + + uint64_t getLastInsertedRowId(); }; /* RAII wrapper to create and destroy SQLite prepared statements. */ -- cgit v1.2.3 From 447ea52b0750785ba8d3b9fc9845d6f8f4c26811 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Jun 2020 14:05:15 +0200 Subject: FileTransfer: Don't store status since curl already does that --- src/libstore/filetransfer.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index e9684b3d4..29edbd0ad 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -56,7 +56,6 @@ struct curlFileTransfer : public FileTransfer Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; unsigned int attempt = 0; @@ -167,7 +166,6 @@ struct curlFileTransfer : public FileTransfer if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; auto ss = tokenizeString>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared(); result.bodySize = 0; acceptRanges = false; @@ -183,7 +181,9 @@ struct curlFileTransfer : public FileTransfer the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - if (result.etag == request.expectedETag && status == "200") { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + if (result.etag == request.expectedETag && httpStatus == 200) { debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } -- cgit v1.2.3 From 29e0748847dfeabbfaabbec8a588088fd1887fb1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Jun 2020 14:20:22 +0200 Subject: Show HTTP status message For example: warning: unable to download 'https://api.github.com/repos/edolstra/dwarffs/commits/master': HTTP error 403 ('rate limit exceeded'); using cached version --- src/libstore/filetransfer.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 29edbd0ad..99316327b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -56,6 +56,7 @@ struct curlFileTransfer : public FileTransfer Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object + std::string statusMsg; unsigned int attempt = 0; @@ -163,11 +164,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); + std::smatch match; + if (std::regex_match(line, match, statusLine)) { result.etag = ""; - auto ss = tokenizeString>(line, " "); result.data = std::make_shared(); result.bodySize = 0; + statusMsg = trim(match[1]); acceptRanges = false; encoding = ""; } else { @@ -404,8 +407,8 @@ struct curlFileTransfer : public FileTransfer ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 ? FileTransferError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) + fmt("unable to %s '%s': HTTP error %d ('%s')", + request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, -- cgit v1.2.3 From fdff09e57c8cae0d4227dfe39804c8279e29452c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Jun 2020 15:18:10 +0200 Subject: Fix coverage build --- src/libstore/filetransfer.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 67af0ab02..cd15cde62 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -22,6 +22,7 @@ #include #include #include +#include using namespace std::string_literals; -- cgit v1.2.3