Merge remote-tracking branch 'origin/master' into flakes

8 files changed, 112 insertions, 83 deletions

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 360d48b09..cb77cdc77 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader
         writeFull(wakeupPipe.writeSide.get(), " ");
     }
 
+#ifdef ENABLE_S3
+    std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
+    {
+        auto [path, params] = splitUriAndParams(uri);
+
+        auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
+            if (slash == std::string::npos)
+                throw nix::Error("bad S3 URI '%s'", path);
+
+        std::string bucketName(path, 5, slash - 5);
+        std::string key(path, slash + 1);
+
+        return {bucketName, key, params};
+    }
+#endif
+
     void enqueueDownload(const DownloadRequest & request,
         Callback<DownloadResult> callback) override
     {
@@ -622,12 +638,15 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
-                auto slash = request.uri.find('/', 5);
-                if (slash == std::string::npos)
-                    throw nix::Error("bad S3 URI '%s'", request.uri);
-                std::string bucketName(request.uri, 5, slash - 5);
-                std::string key(request.uri, slash + 1);
+                auto [bucketName, key, params] = parseS3Uri(request.uri);
+
+                std::string profile = get(params, "profile", "");
+                std::string region = get(params, "region", Aws::Region::US_EAST_1);
+                std::string scheme = get(params, "scheme", "");
+                std::string endpoint = get(params, "endpoint", "");
+
+                S3Helper s3Helper(profile, region, scheme, endpoint);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DownloadResult res;
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index b415d5421..d8a5da0d4 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -129,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
        check if the root is in a directory in or linked from the
        gcroots directory. */
     if (settings.checkRootReachability) {
-        Roots roots = findRoots();
-        if (roots.find(gcRoot) == roots.end())
+        Roots roots = findRoots(false);
+        if (roots[storePath].count(gcRoot) == 0)
             printError(
                 format(
                     "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
@@ -197,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
-{
-    std::set<std::pair<pid_t, Path>> tempRoots;
+static std::string censored = "{censored}";
+
 
+void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
+{
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
@@ -250,14 +251,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
             Path root(contents, pos, end - pos);
             debug("got temporary root '%s'", root);
             assertStorePath(root);
-            tempRoots.emplace(pid, root);
+            tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
 
         fds.push_back(fd); /* keep open */
     }
-
-    return tempRoots;
 }
 
 
@@ -266,7 +265,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
     auto foundRoot = [&](const Path & path, const Path & target) {
         Path storePath = toStorePath(target);
         if (isStorePath(storePath) && isValidPath(storePath))
-            roots[path] = storePath;
+            roots[storePath].emplace(path);
         else
             printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
     };
@@ -306,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         else if (type == DT_REG) {
             Path storePath = storeDir + "/" + baseNameOf(path);
             if (isStorePath(storePath) && isValidPath(storePath))
-                roots[path] = storePath;
+                roots[storePath].emplace(path);
         }
 
     }
@@ -321,10 +320,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 }
 
 
-Roots LocalStore::findRootsNoTemp()
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
 {
-    Roots roots;
-
     /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
@@ -333,32 +330,22 @@ Roots LocalStore::findRootsNoTemp()
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    size_t n = 0;
-    for (auto & root : findRuntimeRoots())
-        roots[fmt("{memory:%d}", n++)] = root;
-
-    return roots;
+    findRuntimeRoots(roots, censor);
 }
 
 
-Roots LocalStore::findRoots()
+Roots LocalStore::findRoots(bool censor)
 {
-    Roots roots = findRootsNoTemp();
+    Roots roots;
+    findRootsNoTemp(roots, censor);
 
     FDs fds;
-    pid_t prev = -1;
-    size_t n = 0;
-    for (auto & root : readTempRoots(fds)) {
-        if (prev != root.first) n = 0;
-        prev = root.first;
-        roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
-    }
+    findTempRoots(fds, roots, censor);
 
     return roots;
 }
 
-
-static void readProcLink(const string & file, StringSet & paths)
+static void readProcLink(const string & file, Roots & roots)
 {
     /* 64 is the starting buffer size gnu readlink uses... */
     auto bufsiz = ssize_t{64};
@@ -377,8 +364,8 @@ try_again:
         goto try_again;
     }
     if (res > 0 && buf[0] == '/')
-        paths.emplace(static_cast<char *>(buf), res);
-    return;
+        roots[std::string(static_cast<char *>(buf), res)]
+            .emplace(file);
 }
 
 static string quoteRegexChars(const string & raw)
@@ -387,20 +374,20 @@ static string quoteRegexChars(const string & raw)
     return std::regex_replace(raw, specialRegex, R"(\$&)");
 }
 
-static void readFileRoots(const char * path, StringSet & paths)
+static void readFileRoots(const char * path, Roots & roots)
 {
     try {
-        paths.emplace(readFile(path));
+        roots[readFile(path)].emplace(path);
     } catch (SysError & e) {
         if (e.errNo != ENOENT && e.errNo != EACCES)
             throw;
     }
 }
 
-PathSet LocalStore::findRuntimeRoots()
+void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
-    PathSet roots;
-    StringSet paths;
+    Roots unchecked;
+
     auto procDir = AutoCloseDir{opendir("/proc")};
     if (procDir) {
         struct dirent * ent;
@@ -410,10 +397,10 @@ PathSet LocalStore::findRuntimeRoots()
         while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
-                readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
-                readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths);
+                readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
+                readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
 
-                auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
+                auto fdStr = fmt("/proc/%s/fd", ent->d_name);
                 auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
@@ -422,9 +409,8 @@ PathSet LocalStore::findRuntimeRoots()
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
-                    if (fd_ent->d_name[0] != '.') {
-                        readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
-                    }
+                    if (fd_ent->d_name[0] != '.')
+                        readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
                 }
                 if (errno) {
                     if (errno == ESRCH)
@@ -434,18 +420,19 @@ PathSet LocalStore::findRuntimeRoots()
                 fdDir.reset();
 
                 try {
-                    auto mapLines =
-                        tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
-                    for (const auto& line : mapLines) {
+                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    for (const auto & line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
-                            paths.emplace(match[1]);
+                            unchecked[match[1]].emplace(mapFile);
                     }
 
-                    auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
+                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
+                    auto envString = readFile(envFile, true);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        paths.emplace(i->str());
+                        unchecked[i->str()].emplace(envFile);
                 } catch (SysError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
@@ -465,7 +452,7 @@ PathSet LocalStore::findRuntimeRoots()
         for (const auto & line : lsofLines) {
             std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
-                paths.emplace(match[1]);
+                unchecked[match[1]].emplace("{lsof}");
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
@@ -473,21 +460,23 @@ PathSet LocalStore::findRuntimeRoots()
 #endif
 
 #if defined(__linux__)
-    readFileRoots("/proc/sys/kernel/modprobe", paths);
-    readFileRoots("/proc/sys/kernel/fbsplash", paths);
-    readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
+    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
+    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
+    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
 #endif
 
-    for (auto & i : paths)
-        if (isInStore(i)) {
-            Path path = toStorePath(i);
-            if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
+    for (auto & [target, links] : unchecked) {
+        if (isInStore(target)) {
+            Path path = toStorePath(target);
+            if (isStorePath(path) && isValidPath(path)) {
                 debug(format("got additional root '%1%'") % path);
-                roots.insert(path);
+                if (censor)
+                    roots[path].insert(censored);
+                else
+                    roots[path].insert(links.begin(), links.end());
             }
         }
-
-    return roots;
+    }
 }
 
 
@@ -754,16 +743,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
     printError(format("finding garbage collector roots..."));
-    Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
+    Roots rootMap;
+    if (!options.ignoreLiveness)
+        findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.second);
+    for (auto & i : rootMap) state.roots.insert(i.first);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
     FDs fds;
-    for (auto & root : readTempRoots(fds))
-        state.tempRoots.insert(root.second);
+    Roots tempRoots;
+    findTempRoots(fds, tempRoots, true);
+    for (auto & root : tempRoots)
+        state.tempRoots.insert(root.first);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index fce963433..6b655647b 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -180,11 +180,11 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
+    void findTempRoots(FDs & fds, Roots & roots, bool censor);
 
 public:
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
@@ -267,9 +267,9 @@ private:
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
-    Roots findRootsNoTemp();
+    void findRootsNoTemp(Roots & roots, bool censor);
 
-    PathSet findRuntimeRoots();
+    void findRuntimeRoots(Roots & roots, bool censor);
 
     void removeUnusedLinks(const GCState & state);
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index def140cfb..15faf78a5 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -596,7 +596,7 @@ void RemoteStore::syncWithGC()
 }
 
 
-Roots RemoteStore::findRoots()
+Roots RemoteStore::findRoots(bool censor)
 {
     auto conn(getConnection());
     conn->to << wopFindRoots;
@@ -606,7 +606,7 @@ Roots RemoteStore::findRoots()
     while (count--) {
         Path link = readString(conn->from);
         Path target = readStorePath(*this, conn->from);
-        result[link] = target;
+        result[target].emplace(link);
     }
     return result;
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 919c6d819..80f18ab71 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -82,7 +82,7 @@ public:
 
     void syncWithGC() override;
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 51de89e0d..cd547a964 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -126,6 +126,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
         res->endpointOverride = endpoint;
     }
     res->requestTimeoutMs = 600 * 1000;
+    res->connectTimeoutMs = 5 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     res->caFile = settings.caFile;
     return res;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 913a11121..c13ff1156 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -842,12 +842,11 @@ namespace nix {
 
 RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
 
-
-ref<Store> openStore(const std::string & uri_,
-    const Store::Params & extraParams)
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
 {
     auto uri(uri_);
-    Store::Params params(extraParams);
+    Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
         for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
@@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_,
         }
         uri = uri_.substr(0, q);
     }
+    return {uri, params};
+}
+
+ref<Store> openStore(const std::string & uri_,
+    const Store::Params & extraParams)
+{
+    auto [uri, uriParams] = splitUriAndParams(uri_);
+    auto params = extraParams;
+    params.insert(uriParams.begin(), uriParams.end());
 
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f504735e0..7a1b31d0f 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -11,6 +11,8 @@
 #include <atomic>
 #include <limits>
 #include <map>
+#include <unordered_map>
+#include <unordered_set>
 #include <memory>
 #include <string>
 
@@ -47,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 const uint32_t exportMagic = 0x4558494e;
 
 
-typedef std::map<Path, Path> Roots;
+typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
 
 
 struct GCOptions
@@ -483,8 +485,10 @@ public:
 
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
-       outside of the Nix store that point to `storePath'.  */
-    virtual Roots findRoots()
+       outside of the Nix store that point to `storePath'. If
+       'censor' is true, privacy-sensitive information about roots
+       found in /proc is censored. */
+    virtual Roots findRoots(bool censor)
     { unsupported("findRoots"); }
 
     /* Perform a garbage collection. */
@@ -798,4 +802,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
    for paths created by makeFixedOutputPath() / addToStore(). */
 std::string makeFixedOutputCA(bool recursive, const Hash & hash);
 
+
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
+
 }