aboutsummaryrefslogtreecommitdiff
path: root/src/libfetchers
diff options
context:
space:
mode:
Diffstat (limited to 'src/libfetchers')
-rw-r--r--src/libfetchers/attrs.hh2
-rw-r--r--src/libfetchers/fetchers.cc7
-rw-r--r--src/libfetchers/fetchers.hh5
-rw-r--r--src/libfetchers/git.cc67
-rw-r--r--src/libfetchers/github.cc10
-rw-r--r--src/libfetchers/local.mk2
-rw-r--r--src/libfetchers/mercurial.cc52
-rw-r--r--src/libfetchers/path.cc26
-rw-r--r--src/libfetchers/registry.cc9
-rw-r--r--src/libfetchers/registry.hh3
-rw-r--r--src/libfetchers/tarball.cc7
11 files changed, 124 insertions, 66 deletions
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index a2d53a7bf..e41037633 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -6,6 +6,8 @@
#include <nlohmann/json_fwd.hpp>
+#include <optional>
+
namespace nix::fetchers {
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 916e0a8e8..e158d914b 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -200,12 +200,17 @@ void Input::markChangedFile(
return scheme->markChangedFile(*this, file, commitMsg);
}
+std::string Input::getName() const
+{
+ return maybeGetStrAttr(attrs, "name").value_or("source");
+}
+
StorePath Input::computeStorePath(Store & store) const
{
auto narHash = getNarHash();
if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string());
- return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+ return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
}
std::string Input::getType() const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index a72cfafa4..c43b047a7 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -38,6 +38,9 @@ struct Input
bool immutable = false;
bool direct = true;
+ /* path of the parent of this input, used for relative path resolution */
+ std::optional<Path> parent;
+
public:
static Input fromURL(const std::string & url);
@@ -81,6 +84,8 @@ public:
std::string_view file,
std::optional<std::string> commitMsg) const;
+ std::string getName() const;
+
StorePath computeStorePath(Store & store) const;
// Convenience functions for common attributes.
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 81c647f89..8468d2afc 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -4,13 +4,21 @@
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
+#include "pathlocks.hh"
#include <sys/time.h>
+#include <sys/wait.h>
using namespace std::string_literals;
namespace nix::fetchers {
+// Explicit initial branch of our bare repo to suppress warnings from new version of git.
+// The value itself does not matter, since we always fetch a specific revision or branch.
+// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
+// old version of git, which will ignore unrecognized `-c` options.
+const std::string gitInitialBranch = "__nix_dummy_branch";
+
static std::string readHead(const Path & path)
{
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
@@ -59,7 +67,7 @@ struct GitInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
@@ -153,21 +161,23 @@ struct GitInputScheme : InputScheme
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
- // Don't clone file:// URIs (but otherwise treat them the
- // same as remote URIs, i.e. don't use the working tree or
- // HEAD).
+ // file:// URIs are normally not cloned (but otherwise treated the
+ // same as remote URIs, i.e. we don't use the working tree or
+ // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
+ // repo, treat as a remote URI to force a clone.
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
- bool isLocal = url.scheme == "file" && !forceHttp;
+ bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
+ bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
- auto name = "source";
-
Input input(_input);
+ std::string name = input.getName();
+
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@@ -267,7 +277,7 @@ struct GitInputScheme : InputScheme
return files.count(file);
};
- auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+ auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
@@ -314,11 +324,17 @@ struct GitInputScheme : InputScheme
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
repoDir = cacheDir;
+ Path cacheDirLock = cacheDir + ".lock";
+ createDirs(dirOf(cacheDir));
+ AutoCloseFD lock = openLockFile(cacheDirLock, true);
+ lockFile(lock.get(), ltWrite, true);
+
if (!pathExists(cacheDir)) {
- createDirs(dirOf(cacheDir));
- runProgram("git", true, { "init", "--bare", repoDir });
+ runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
}
+ deleteLockFile(cacheDirLock, lock.get());
+
Path localRefFile =
input.getRef()->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input.getRef()
@@ -363,7 +379,9 @@ struct GitInputScheme : InputScheme
? "refs/*"
: ref->compare(0, 5, "refs/") == 0
? *ref
- : "refs/heads/" + *ref;
+ : ref == "HEAD"
+ ? *ref
+ : "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
@@ -401,17 +419,14 @@ struct GitInputScheme : InputScheme
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
- RunOptions checkCommitOpts(
- "git",
- { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
- );
- checkCommitOpts.searchPath = true;
- checkCommitOpts.mergeStderrToStdout = true;
-
- auto result = runProgram(checkCommitOpts);
+ auto result = runProgram(RunOptions {
+ .program = "git",
+ .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
+ .mergeStderrToStdout = true
+ });
if (WEXITSTATUS(result.first) == 128
- && result.second.find("bad file") != std::string::npos
- ) {
+ && result.second.find("bad file") != std::string::npos)
+ {
throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
"Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
@@ -427,7 +442,7 @@ struct GitInputScheme : InputScheme
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
- runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+ runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
// TODO: repoDir might lack the ref (it only checks if rev
// exists, see FIXME above) so use a big hammer and fetch
// everything to ensure we get the rev.
@@ -443,9 +458,11 @@ struct GitInputScheme : InputScheme
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
- RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
- gitOptions.standardOut = &sink;
- runProgram2(gitOptions);
+ runProgram2({
+ .program = "git",
+ .args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
+ .standardOut = &sink
+ });
});
unpackTarfile(*source, tmpDir);
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8352ef02d..ffc44e9e2 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
- auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+ auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
@@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
void clone(const Input & input, const Path & destDir) override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
- Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ Input::fromURL(fmt("git+https://%s/%s/%s.git",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
- .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};
@@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
- Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ Input::fromURL(fmt("git+https://%s/%s/%s.git",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
- .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index cfd705e22..2e8869d83 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
+libfetchers_LDFLAGS += -pthread
+
libfetchers_LIBS = libutil libstore
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 0eb401e10..d52d4641b 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -11,34 +11,32 @@ using namespace std::string_literals;
namespace nix::fetchers {
-namespace {
-
-RunOptions hgOptions(const Strings & args) {
- RunOptions opts("hg", args);
- opts.searchPath = true;
-
- auto env = getEnv();
- // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
- env["HGPLAIN"] = "";
- opts.environment = env;
-
- return opts;
+static RunOptions hgOptions(const Strings & args)
+{
+ auto env = getEnv();
+ // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+ env["HGPLAIN"] = "";
+
+ return {
+ .program = "hg",
+ .searchPath = true,
+ .args = args,
+ .environment = env
+ };
}
// runProgram wrapper that uses hgOptions instead of stock RunOptions.
-string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static string runHg(const Strings & args, const std::optional<std::string> & input = {})
{
- RunOptions opts = hgOptions(args);
- opts.input = input;
+ RunOptions opts = hgOptions(args);
+ opts.input = input;
- auto res = runProgram(opts);
+ auto res = runProgram(std::move(opts));
- if (!statusOk(res.first))
- throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
-
- return res.second;
-}
+ if (!statusOk(res.first))
+ throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
+ return res.second;
}
struct MercurialInputScheme : InputScheme
@@ -74,7 +72,7 @@ struct MercurialInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
@@ -147,10 +145,10 @@ struct MercurialInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
- auto name = "source";
-
Input input(_input);
+ auto name = input.getName();
+
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
@@ -193,7 +191,7 @@ struct MercurialInputScheme : InputScheme
return files.count(file);
};
- auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+ auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {
Tree(store->toRealPath(storePath), std::move(storePath)),
@@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme
have to pull again. */
if (!(input.getRev()
&& pathExists(cacheDir)
- && runProgram(
- hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
- .killStderr(true)).second == "1"))
+ && runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index d1003de57..fb5702c4c 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -82,18 +82,38 @@ struct PathInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
+ std::string absPath;
auto path = getStrAttr(input.attrs, "path");
- // FIXME: check whether access to 'path' is allowed.
+ if (path[0] != '/') {
+ if (!input.parent)
+ throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
+
+ auto parent = canonPath(*input.parent);
- auto storePath = store->maybeParseStorePath(path);
+ // the path isn't relative, prefix it
+ absPath = nix::absPath(path, parent);
+
+ // for security, ensure that if the parent is a store path, it's inside it
+ if (store->isInStore(parent)) {
+ auto storePath = store->printStorePath(store->toStorePath(parent).first);
+ if (!isInDir(absPath, storePath))
+ throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
+ }
+ } else
+ absPath = path;
+
+ Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
+
+ // FIXME: check whether access to 'path' is allowed.
+ auto storePath = store->maybeParseStorePath(absPath);
if (storePath)
store->addTempRoot(*storePath);
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
// FIXME: try to substitute storePath.
- storePath = store->addToStore("source", path);
+ storePath = store->addToStore("source", absPath);
return {
Tree(store->toRealPath(*storePath), std::move(*storePath)),
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 81b2227de..f35359d4b 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -114,7 +114,7 @@ static std::shared_ptr<Registry> getSystemRegistry()
Path getUserRegistryPath()
{
- return getHome() + "/.config/nix/registry.json";
+ return getConfigDir() + "/nix/registry.json";
}
std::shared_ptr<Registry> getUserRegistry()
@@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
return userRegistry;
}
+std::shared_ptr<Registry> getCustomRegistry(const Path & p)
+{
+ static auto customRegistry =
+ Registry::read(p, Registry::Custom);
+ return customRegistry;
+}
+
static std::shared_ptr<Registry> flagRegistry =
std::make_shared<Registry>(Registry::Flag);
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
index 1077af020..260a2c460 100644
--- a/src/libfetchers/registry.hh
+++ b/src/libfetchers/registry.hh
@@ -14,6 +14,7 @@ struct Registry
User = 1,
System = 2,
Global = 3,
+ Custom = 4,
};
RegistryType type;
@@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
std::shared_ptr<Registry> getUserRegistry();
+std::shared_ptr<Registry> getCustomRegistry(const Path & p);
+
Path getUserRegistryPath();
Registries getRegistries(ref<Store> store);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index b8d7d2c70..031ccc5f7 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -178,7 +178,8 @@ struct TarballInputScheme : InputScheme
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
- && !hasSuffix(url.path, ".tar.bz2"))
+ && !hasSuffix(url.path, ".tar.bz2")
+ && !hasSuffix(url.path, ".tar.zst"))
return {};
Input input;
@@ -195,7 +196,7 @@ struct TarballInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
+ if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
throw Error("unsupported tarball input attribute '%s'", name);
Input input;
@@ -225,7 +226,7 @@ struct TarballInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
- auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+ auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
return {std::move(tree), input};
}
};
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 15:59:06 +0000