aboutsummaryrefslogtreecommitdiff
path: root/src/libfetchers
diff options
context:
space:
mode:
Diffstat (limited to 'src/libfetchers')
-rw-r--r--src/libfetchers/attrs.hh2
-rw-r--r--src/libfetchers/fetchers.cc7
-rw-r--r--src/libfetchers/fetchers.hh13
-rw-r--r--src/libfetchers/git.cc45
-rw-r--r--src/libfetchers/github.cc14
-rw-r--r--src/libfetchers/local.mk2
-rw-r--r--src/libfetchers/mercurial.cc52
-rw-r--r--src/libfetchers/path.cc18
-rw-r--r--src/libfetchers/registry.cc7
-rw-r--r--src/libfetchers/registry.hh3
-rw-r--r--src/libfetchers/tarball.cc24
11 files changed, 106 insertions, 81 deletions
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index a2d53a7bf..e41037633 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -6,6 +6,8 @@
#include <nlohmann/json_fwd.hpp>
+#include <optional>
+
namespace nix::fetchers {
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 916e0a8e8..e158d914b 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -200,12 +200,17 @@ void Input::markChangedFile(
return scheme->markChangedFile(*this, file, commitMsg);
}
+std::string Input::getName() const
+{
+ return maybeGetStrAttr(attrs, "name").value_or("source");
+}
+
StorePath Input::computeStorePath(Store & store) const
{
auto narHash = getNarHash();
if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string());
- return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+ return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
}
std::string Input::getType() const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index c6b219c02..c43b047a7 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -38,6 +38,9 @@ struct Input
bool immutable = false;
bool direct = true;
+ /* path of the parent of this input, used for relative path resolution */
+ std::optional<Path> parent;
+
public:
static Input fromURL(const std::string & url);
@@ -81,6 +84,8 @@ public:
std::string_view file,
std::optional<std::string> commitMsg) const;
+ std::string getName() const;
+
StorePath computeStorePath(Store & store) const;
// Convenience functions for common attributes.
@@ -145,13 +150,7 @@ DownloadFileResult downloadFile(
bool immutable,
const Headers & headers = {});
-struct DownloadTarballMeta
-{
- time_t lastModified;
- std::string effectiveUrl;
-};
-
-std::pair<Tree, DownloadTarballMeta> downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index b9a240b13..5af38dde9 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -4,8 +4,10 @@
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
+#include "pathlocks.hh"
#include <sys/time.h>
+#include <sys/wait.h>
using namespace std::string_literals;
@@ -59,7 +61,7 @@ struct GitInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
@@ -166,12 +168,12 @@ struct GitInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
- auto name = "source";
-
Input input(_input);
+ std::string name = input.getName();
+
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
- bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+ bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(true);
bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
std::string cacheType = "git";
@@ -269,7 +271,7 @@ struct GitInputScheme : InputScheme
return files.count(file);
};
- auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+ auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
@@ -316,11 +318,17 @@ struct GitInputScheme : InputScheme
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
repoDir = cacheDir;
+ Path cacheDirLock = cacheDir + ".lock";
+ createDirs(dirOf(cacheDir));
+ AutoCloseFD lock = openLockFile(cacheDirLock, true);
+ lockFile(lock.get(), ltWrite, true);
+
if (!pathExists(cacheDir)) {
- createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", repoDir });
}
+ deleteLockFile(cacheDirLock, lock.get());
+
Path localRefFile =
input.getRef()->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input.getRef()
@@ -405,17 +413,14 @@ struct GitInputScheme : InputScheme
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
- RunOptions checkCommitOpts(
- "git",
- { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
- );
- checkCommitOpts.searchPath = true;
- checkCommitOpts.mergeStderrToStdout = true;
-
- auto result = runProgram(checkCommitOpts);
+ auto result = runProgram(RunOptions {
+ .program = "git",
+ .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
+ .mergeStderrToStdout = true
+ });
if (WEXITSTATUS(result.first) == 128
- && result.second.find("bad file") != std::string::npos
- ) {
+ && result.second.find("bad file") != std::string::npos)
+ {
throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
"Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
@@ -447,9 +452,11 @@ struct GitInputScheme : InputScheme
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
- RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
- gitOptions.standardOut = &sink;
- runProgram2(gitOptions);
+ runProgram2({
+ .program = "git",
+ .args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
+ .standardOut = &sink
+ });
});
unpackTarfile(*source, tmpDir);
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 3e5ad75a8..ffc44e9e2 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
- auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers);
+ auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
- input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified));
+ input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
- {"lastModified", uint64_t(meta.lastModified)}
+ {"lastModified", uint64_t(lastModified)}
},
tree.storePath,
true);
@@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
void clone(const Input & input, const Path & destDir) override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
- Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ Input::fromURL(fmt("git+https://%s/%s/%s.git",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
- .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};
@@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
- Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ Input::fromURL(fmt("git+https://%s/%s/%s.git",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
- .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index cfd705e22..2e8869d83 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
+libfetchers_LDFLAGS += -pthread
+
libfetchers_LIBS = libutil libstore
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 0eb401e10..d52d4641b 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -11,34 +11,32 @@ using namespace std::string_literals;
namespace nix::fetchers {
-namespace {
-
-RunOptions hgOptions(const Strings & args) {
- RunOptions opts("hg", args);
- opts.searchPath = true;
-
- auto env = getEnv();
- // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
- env["HGPLAIN"] = "";
- opts.environment = env;
-
- return opts;
+static RunOptions hgOptions(const Strings & args)
+{
+ auto env = getEnv();
+ // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+ env["HGPLAIN"] = "";
+
+ return {
+ .program = "hg",
+ .searchPath = true,
+ .args = args,
+ .environment = env
+ };
}
// runProgram wrapper that uses hgOptions instead of stock RunOptions.
-string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static string runHg(const Strings & args, const std::optional<std::string> & input = {})
{
- RunOptions opts = hgOptions(args);
- opts.input = input;
+ RunOptions opts = hgOptions(args);
+ opts.input = input;
- auto res = runProgram(opts);
+ auto res = runProgram(std::move(opts));
- if (!statusOk(res.first))
- throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
-
- return res.second;
-}
+ if (!statusOk(res.first))
+ throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
+ return res.second;
}
struct MercurialInputScheme : InputScheme
@@ -74,7 +72,7 @@ struct MercurialInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
@@ -147,10 +145,10 @@ struct MercurialInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
- auto name = "source";
-
Input input(_input);
+ auto name = input.getName();
+
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
@@ -193,7 +191,7 @@ struct MercurialInputScheme : InputScheme
return files.count(file);
};
- auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+ auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {
Tree(store->toRealPath(storePath), std::move(storePath)),
@@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme
have to pull again. */
if (!(input.getRev()
&& pathExists(cacheDir)
- && runProgram(
- hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
- .killStderr(true)).second == "1"))
+ && runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index d1003de57..b6fcdac9e 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -82,18 +82,30 @@ struct PathInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
+ std::string absPath;
auto path = getStrAttr(input.attrs, "path");
- // FIXME: check whether access to 'path' is allowed.
+ if (path[0] != '/' && input.parent) {
+ auto parent = canonPath(*input.parent);
+
+ // the path isn't relative, prefix it
+ absPath = canonPath(parent + "/" + path);
- auto storePath = store->maybeParseStorePath(path);
+ // for security, ensure that if the parent is a store path, it's inside it
+ if (!parent.rfind(store->storeDir, 0) && absPath.rfind(store->storeDir, 0))
+ throw BadStorePath("relative path '%s' points outside of its parent's store path %s, this is a security violation", path, parent);
+ } else
+ absPath = path;
+
+ // FIXME: check whether access to 'path' is allowed.
+ auto storePath = store->maybeParseStorePath(absPath);
if (storePath)
store->addTempRoot(*storePath);
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
// FIXME: try to substitute storePath.
- storePath = store->addToStore("source", path);
+ storePath = store->addToStore("source", absPath);
return {
Tree(store->toRealPath(*storePath), std::move(*storePath)),
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 74376adc0..f35359d4b 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
return userRegistry;
}
+std::shared_ptr<Registry> getCustomRegistry(const Path & p)
+{
+ static auto customRegistry =
+ Registry::read(p, Registry::Custom);
+ return customRegistry;
+}
+
static std::shared_ptr<Registry> flagRegistry =
std::make_shared<Registry>(Registry::Flag);
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
index 1077af020..260a2c460 100644
--- a/src/libfetchers/registry.hh
+++ b/src/libfetchers/registry.hh
@@ -14,6 +14,7 @@ struct Registry
User = 1,
System = 2,
Global = 3,
+ Custom = 4,
};
RegistryType type;
@@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
std::shared_ptr<Registry> getUserRegistry();
+std::shared_ptr<Registry> getCustomRegistry(const Path & p);
+
Path getUserRegistryPath();
Registries getRegistries(ref<Store> store);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index bd05bb2f1..031ccc5f7 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -109,7 +109,7 @@ DownloadFileResult downloadFile(
};
}
-std::pair<Tree, DownloadTarballMeta> downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
@@ -127,10 +127,7 @@ std::pair<Tree, DownloadTarballMeta> downloadTarball(
if (cached && !cached->expired)
return {
Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
- {
- .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")),
- .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url),
- },
+ getIntAttr(cached->infoAttrs, "lastModified")
};
auto res = downloadFile(store, url, name, immutable, headers);
@@ -155,7 +152,6 @@ std::pair<Tree, DownloadTarballMeta> downloadTarball(
Attrs infoAttrs({
{"lastModified", uint64_t(lastModified)},
- {"effectiveUrl", res.effectiveUrl},
{"etag", res.etag},
});
@@ -168,10 +164,7 @@ std::pair<Tree, DownloadTarballMeta> downloadTarball(
return {
Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
- {
- .lastModified = lastModified,
- .effectiveUrl = res.effectiveUrl,
- },
+ lastModified,
};
}
@@ -185,7 +178,8 @@ struct TarballInputScheme : InputScheme
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
- && !hasSuffix(url.path, ".tar.bz2"))
+ && !hasSuffix(url.path, ".tar.bz2")
+ && !hasSuffix(url.path, ".tar.zst"))
return {};
Input input;
@@ -202,7 +196,7 @@ struct TarballInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
+ if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
throw Error("unsupported tarball input attribute '%s'", name);
Input input;
@@ -230,11 +224,9 @@ struct TarballInputScheme : InputScheme
return true;
}
- std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
- Input input(_input);
- auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false);
- input.attrs.insert_or_assign("url", meta.effectiveUrl);
+ auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
return {std::move(tree), input};
}
};
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 18:50:23 +0000