Merge branch 'master' into debug-merge

11 files changed, 113 insertions, 62 deletions

diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index a2d53a7bf..e41037633 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -6,6 +6,8 @@
 
 #include <nlohmann/json_fwd.hpp>
 
+#include <optional>
+
 namespace nix::fetchers {
 
 typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 916e0a8e8..e158d914b 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -200,12 +200,17 @@ void Input::markChangedFile(
     return scheme->markChangedFile(*this, file, commitMsg);
 }
 
+std::string Input::getName() const
+{
+    return maybeGetStrAttr(attrs, "name").value_or("source");
+}
+
 StorePath Input::computeStorePath(Store & store) const
 {
     auto narHash = getNarHash();
     if (!narHash)
         throw Error("cannot compute store path for mutable input '%s'", to_string());
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
 }
 
 std::string Input::getType() const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index a72cfafa4..c43b047a7 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -38,6 +38,9 @@ struct Input
     bool immutable = false;
     bool direct = true;
 
+    /* path of the parent of this input, used for relative path resolution */
+    std::optional<Path> parent;
+
 public:
     static Input fromURL(const std::string & url);
 
@@ -81,6 +84,8 @@ public:
         std::string_view file,
         std::optional<std::string> commitMsg) const;
 
+    std::string getName() const;
+
     StorePath computeStorePath(Store & store) const;
 
     // Convenience functions for common attributes.
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index d8e0dbe0a..544d2ffbf 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -4,6 +4,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "pathlocks.hh"
 
 #include <sys/time.h>
 #include <sys/wait.h>
@@ -12,6 +13,12 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
+// Explicit initial branch of our bare repo to suppress warnings from new version of git.
+// The value itself does not matter, since we always fetch a specific revision or branch.
+// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
+// old version of git, which will ignore unrecognized `-c` options.
+const std::string gitInitialBranch = "__nix_dummy_branch";
+
 static std::string readHead(const Path & path)
 {
     return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
@@ -44,7 +51,7 @@ struct GitInputScheme : InputScheme
         for (auto &[name, value] : url.query) {
             if (name == "rev" || name == "ref")
                 attrs.emplace(name, value);
-            else if (name == "shallow")
+            else if (name == "shallow" || name == "submodules")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -60,7 +67,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -167,10 +174,10 @@ struct GitInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        std::string name = input.getName();
+
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@@ -270,7 +277,7 @@ struct GitInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 // FIXME: maybe we should use the timestamp of the last
                 // modified dirty file?
@@ -317,9 +324,11 @@ struct GitInputScheme : InputScheme
             Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
             repoDir = cacheDir;
 
+            createDirs(dirOf(cacheDir));
+            PathLocks cacheDirLock({cacheDir + ".lock"});
+
             if (!pathExists(cacheDir)) {
-                createDirs(dirOf(cacheDir));
-                runProgram("git", true, { "init", "--bare", repoDir });
+                runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
             }
 
             Path localRefFile =
@@ -386,6 +395,8 @@ struct GitInputScheme : InputScheme
 
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
+
+            // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
 
         bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -406,17 +417,14 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
-        RunOptions checkCommitOpts(
-            "git",
-            { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
-        );
-        checkCommitOpts.searchPath = true;
-        checkCommitOpts.mergeStderrToStdout = true;
-
-        auto result = runProgram(checkCommitOpts);
+        auto result = runProgram(RunOptions {
+            .program = "git",
+            .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
+            .mergeStderrToStdout = true
+        });
         if (WEXITSTATUS(result.first) == 128
-            && result.second.find("bad file") != std::string::npos
-        ) {
+            && result.second.find("bad file") != std::string::npos)
+        {
             throw Error(
                 "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
                     "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
@@ -432,7 +440,7 @@ struct GitInputScheme : InputScheme
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
-            runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+            runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
             // TODO: repoDir might lack the ref (it only checks if rev
             // exists, see FIXME above) so use a big hammer and fetch
             // everything to ensure we get the rev.
@@ -448,9 +456,11 @@ struct GitInputScheme : InputScheme
             // FIXME: should pipe this, or find some better way to extract a
             // revision.
             auto source = sinkToSource([&](Sink & sink) {
-                RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
-                gitOptions.standardOut = &sink;
-                runProgram2(gitOptions);
+                runProgram2({
+                    .program = "git",
+                    .args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
+                    .standardOut = &sink
+                });
             });
 
             unpackTarfile(*source, tmpDir);
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8352ef02d..1c539b80e 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
         input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 
@@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
     void clone(const Input & input, const Path & destDir) override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+        Input::fromURL(fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
-            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
     }
 };
@@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         if ("PAT" == token.substr(0, fldsplit))
             return std::make_pair("Private-token", token.substr(fldsplit+1));
         warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
-        return std::nullopt;
+        return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1));
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
@@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
-        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+        Input::fromURL(fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
-            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
     }
 };
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index cfd705e22..2e8869d83 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
+libfetchers_LDFLAGS += -pthread
+
 libfetchers_LIBS = libutil libstore
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 0eb401e10..d52d4641b 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -11,34 +11,32 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
-namespace {
-
-RunOptions hgOptions(const Strings & args) {
-	RunOptions opts("hg", args);
-	opts.searchPath = true;
-
-	auto env = getEnv();
-	// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
-	env["HGPLAIN"] = "";
-	opts.environment = env;
-
-	return opts;
+static RunOptions hgOptions(const Strings & args)
+{
+    auto env = getEnv();
+    // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+    env["HGPLAIN"] = "";
+
+    return {
+        .program = "hg",
+        .searchPath = true,
+        .args = args,
+        .environment = env
+    };
 }
 
 // runProgram wrapper that uses hgOptions instead of stock RunOptions.
-string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static string runHg(const Strings & args, const std::optional<std::string> & input = {})
 {
-	RunOptions opts = hgOptions(args);
-	opts.input = input;
+    RunOptions opts = hgOptions(args);
+    opts.input = input;
 
-	auto res = runProgram(opts);
+    auto res = runProgram(std::move(opts));
 
-	if (!statusOk(res.first))
-		throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
-
-	return res.second;
-}
+    if (!statusOk(res.first))
+        throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
 
+    return res.second;
 }
 
 struct MercurialInputScheme : InputScheme
@@ -74,7 +72,7 @@ struct MercurialInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "hg") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
                 throw Error("unsupported Mercurial input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -147,10 +145,10 @@ struct MercurialInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        auto name = input.getName();
+
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
@@ -193,7 +191,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {
                     Tree(store->toRealPath(storePath), std::move(storePath)),
@@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme
            have to pull again. */
         if (!(input.getRev()
                 && pathExists(cacheDir)
-                && runProgram(
-                    hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
-                    .killStderr(true)).second == "1"))
+                && runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1"))
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
 
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index d1003de57..fb5702c4c 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -82,18 +82,38 @@ struct PathInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
+        std::string absPath;
         auto path = getStrAttr(input.attrs, "path");
 
-        // FIXME: check whether access to 'path' is allowed.
+        if (path[0] != '/') {
+            if (!input.parent)
+                throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
+
+            auto parent = canonPath(*input.parent);
 
-        auto storePath = store->maybeParseStorePath(path);
+            // the path isn't relative, prefix it
+            absPath = nix::absPath(path, parent);
+
+            // for security, ensure that if the parent is a store path, it's inside it
+            if (store->isInStore(parent)) {
+                auto storePath = store->printStorePath(store->toStorePath(parent).first);
+                if (!isInDir(absPath, storePath))
+                    throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
+            }
+        } else
+            absPath = path;
+
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
+
+        // FIXME: check whether access to 'path' is allowed.
+        auto storePath = store->maybeParseStorePath(absPath);
 
         if (storePath)
             store->addTempRoot(*storePath);
 
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
             // FIXME: try to substitute storePath.
-            storePath = store->addToStore("source", path);
+            storePath = store->addToStore("source", absPath);
 
         return {
             Tree(store->toRealPath(*storePath), std::move(*storePath)),
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 74376adc0..f35359d4b 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
     return userRegistry;
 }
 
+std::shared_ptr<Registry> getCustomRegistry(const Path & p)
+{
+    static auto customRegistry =
+        Registry::read(p, Registry::Custom);
+    return customRegistry;
+}
+
 static std::shared_ptr<Registry> flagRegistry =
     std::make_shared<Registry>(Registry::Flag);
 
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
index 1077af020..260a2c460 100644
--- a/src/libfetchers/registry.hh
+++ b/src/libfetchers/registry.hh
@@ -14,6 +14,7 @@ struct Registry
         User = 1,
         System = 2,
         Global = 3,
+        Custom = 4,
     };
 
     RegistryType type;
@@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
 
 std::shared_ptr<Registry> getUserRegistry();
 
+std::shared_ptr<Registry> getCustomRegistry(const Path & p);
+
 Path getUserRegistryPath();
 
 Registries getRegistries(ref<Store> store);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index b8d7d2c70..031ccc5f7 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -178,7 +178,8 @@ struct TarballInputScheme : InputScheme
             && !hasSuffix(url.path, ".tar")
             && !hasSuffix(url.path, ".tar.gz")
             && !hasSuffix(url.path, ".tar.xz")
-            && !hasSuffix(url.path, ".tar.bz2"))
+            && !hasSuffix(url.path, ".tar.bz2")
+            && !hasSuffix(url.path, ".tar.zst"))
             return {};
 
         Input input;
@@ -195,7 +196,7 @@ struct TarballInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
+            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         Input input;
@@ -225,7 +226,7 @@ struct TarballInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
         return {std::move(tree), input};
     }
 };