diff options
34 files changed, 429 insertions, 85 deletions
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index dd481160f..3a2d4de0e 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.7 + uses: zeebe-io/backport-action@v0.0.8 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d01ef4768..aae5b93e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/configure.ac b/configure.ac index 8a01c33ec..789dfdb3c 100644 --- a/configure.ac +++ b/configure.ac @@ -294,6 +294,17 @@ esac AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) +if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then + AC_MSG_CHECKING([whether sandbox-shell has the standalone feature]) + # busybox shell sometimes allows executing other busybox applets, + # even if they are not in the path, breaking our sandbox + if PATH= $sandbox_shell -c "busybox" 2>&1 | grep -qv "not found"; then + AC_MSG_RESULT(enabled) + AC_MSG_ERROR([Please disable busybox FEATURE_SH_STANDALONE]) + else + AC_MSG_RESULT(disabled) + fi +fi # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 90a8f1f94..59ce5cac7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -71,18 +71,6 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 3.0 ``` -To run a functional test: - -```console -make tests/test-name-should-auto-complete.sh.test -``` - -To run the unit-tests for C++ code: - -``` -make check -``` - If you have a flakes-enabled Nix you can replace: ```console @@ -94,3 +82,29 @@ by: ```console $ nix develop ``` + +## Testing + +Nix comes with three different flavors of tests: unit, functional and integration. + +### Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) framework. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +### Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +The whole testsuite can be run with `make install && make installcheck`. +Individual tests can be run with `make tests/{testName}.sh.test`. + +### Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index a808e145a..878916dc9 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -37,3 +37,6 @@ :c Go until end of program, exception, or builtins.break(). :s Go one step ``` + +* `builtins.fetchTree` (and flake inputs) can now be used to fetch plain files + over the `http(s)` and `file` protocols in addition to directory tarballs. diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 24d894898..e3ac42beb 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -9,6 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process +LimitNOFILE=4096 [Install] WantedBy=multi-user.target diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 4bac4b7ba..aee7ff4bf 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -442,9 +442,13 @@ add_nix_vol_fstab_line() { local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" shift - # wrap `ex` to work around a problem with vim plugins breaking exit codes; - # (see https://github.com/NixOS/nix/issues/5468) - # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split + # wrap `ex` to work around a problem with vim plugins breaking exit codes + # (see github.com/NixOS/nix/issues/5468) + # + # the first draft used `--noplugin`, but github.com/NixOS/nix/issues/6462 + # suggests we need the less-semantic `-u NONE` + # + # we'd prefer EDITOR="/usr/bin/ex -u NONE" but vifs doesn't word-split # the EDITOR env. # # TODO: at some point we should switch to `--clean`, but it wasn't added @@ -452,7 +456,7 @@ add_nix_vol_fstab_line() { # minver 10.12.6 seems to have released with vim 7.4 cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF #!/bin/sh -/usr/bin/ex --noplugin "\$@" +/usr/bin/ex -u NONE "\$@" EOF chmod 755 "$SCRATCH/ex_cleanroom_wrapper" @@ -646,8 +650,9 @@ EOF task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2 # technically /etc/synthetic.d/nix is supported in Big Sur+ # but handling both takes even more code... + # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier _sudo "to add Nix to /etc/synthetic.conf" \ - /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF + /usr/bin/ex -u NONE /etc/synthetic.conf <<EOF :a ${NIX_ROOT:1} . @@ -815,7 +820,8 @@ setup_volume_daemon() { local volume_uuid="$2" if ! test_voldaemon; then task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2 - _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF + # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier + _sudo "to install the Nix volume mounter" /usr/bin/ex -u NONE "$NIX_VOLUME_MOUNTD_DEST" <<EOF :a $(generate_mount_daemon "$cmd_type" "$volume_uuid") . diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a94e60aca..635ce19b6 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -621,11 +621,15 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF auto drvPath = attr->forceDerivation(); std::set<std::string> outputsToInstall; + std::optional<NixInt> priority; - if (auto aMeta = attr->maybeGetAttr(state->sMeta)) + if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) for (auto & s : aOutputsToInstall->getListOfStrings()) outputsToInstall.insert(s); + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) { outputsToInstall.clear(); @@ -643,6 +647,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF auto drvInfo = DerivationInfo { .drvPath = std::move(drvPath), .outputsToInstall = std::move(outputsToInstall), + .priority = priority, }; return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 1a5a96153..5d715210e 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -142,6 +142,7 @@ struct InstallableValue : Installable { StorePath drvPath; std::set<std::string> outputsToInstall; + std::optional<NixInt> priority; }; virtual std::vector<DerivationInfo> toDerivations() = 0; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 1be98fc95..d77b25898 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -47,7 +47,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v3"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -175,6 +175,24 @@ struct AttrDb }); } + AttrId setInt( + AttrKey key, + int n) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (symbols[key.second]) + (AttrType::Int) + (n).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + AttrId setListOfStrings( AttrKey key, const std::vector<std::string> & l) @@ -287,6 +305,8 @@ struct AttrDb } case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Int: + return {{rowId, int_t{queryAttribute.getInt(2)}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -426,6 +446,8 @@ Value & AttrCursor::forceValue() cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + else if (v.type() == nInt) + cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}}; else if (v.type() == nAttrs) ; // FIXME: do something? else @@ -621,6 +643,28 @@ bool AttrCursor::getBool() return v.boolean; } +NixInt AttrCursor::getInt() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey()); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto i = std::get_if<int_t>(&cachedValue->second)) { + debug("using cached Integer attribute '%s'", getAttrPathStr()); + return i->x; + } else + throw TypeError("'%s' is not an Integer", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nInt) + throw TypeError("'%s' is not an Integer", getAttrPathStr()); + + return v.integer; +} + std::vector<std::string> AttrCursor::getListOfStrings() { if (root->db) { diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 636e293ad..c93e55b93 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -45,12 +45,14 @@ enum AttrType { Failed = 5, Bool = 6, ListOfStrings = 7, + Int = 8, }; struct placeholder_t {}; struct missing_t {}; struct misc_t {}; struct failed_t {}; +struct int_t { NixInt x; }; typedef uint64_t AttrId; typedef std::pair<AttrId, Symbol> AttrKey; typedef std::pair<std::string, NixStringContext> string_t; @@ -63,6 +65,7 @@ typedef std::variant< misc_t, failed_t, bool, + int_t, std::vector<std::string> > AttrValue; @@ -116,6 +119,8 @@ public: bool getBool(); + NixInt getInt(); + std::vector<std::string> getListOfStrings(); std::vector<Symbol> getAttrs(); diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 92ec27046..3e9d264b4 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply() { - std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"}; + std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"}; for (auto & [name, value] : settings) { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index cbf4f0a6f..35c841897 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -723,6 +723,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V lockFlake(state, flakeRef, LockFlags { .updateLockFile = false, + .writeLockFile = false, .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, .allowMutable = !evalSettings.pureEval, }), diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 96cd84b82..11542b06b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3560,7 +3560,7 @@ static RegisterPrimOp primop_match({ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " ``` - Evaluates to `[ "foo" ]`. + Evaluates to `[ "FOO" ]`. )s", .fun = prim_match, }); diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index cb1c4a977..b95980cab 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -10,6 +10,6 @@ libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libexpr libutil libstore +libexpr-tests_LIBS = libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d23a820a4..a71bff76f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -26,11 +26,6 @@ namespace { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -std::string getGitDir() -{ - return getEnv("GIT_DIR").value_or(".git"); -} - bool isCacheFileWithinTtl(const time_t now, const struct stat & st) { return st.st_mtime + settings.tarballTtl > now; @@ -152,7 +147,7 @@ struct WorkdirInfo WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = getGitDir(); + std::string gitDir(".git"); auto env = getEnv(); // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong @@ -370,7 +365,7 @@ struct GitInputScheme : InputScheme { auto sourcePath = getSourcePath(input); assert(sourcePath); - auto gitDir = getGitDir(); + auto gitDir = ".git"; runProgram("git", true, { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); @@ -396,7 +391,7 @@ struct GitInputScheme : InputScheme std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override { Input input(_input); - auto gitDir = getGitDir(); + auto gitDir = ".git"; std::string name = input.getName(); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index a1084c984..0631fb6e8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -243,7 +243,10 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/commits/%s" + : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -262,7 +265,10 @@ struct GitHubInputScheme : GitArchiveInputScheme // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/tarball/%s" + : "https://%s/api/v3/repos/%s/%s/tarball/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index dde0ad761..6c551bd93 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "tarfile.hh" #include "types.hh" +#include "split.hh" namespace nix::fetchers { @@ -168,24 +169,34 @@ std::pair<Tree, time_t> downloadTarball( }; } -struct TarballInputScheme : InputScheme +// An input scheme corresponding to a curl-downloadable resource. +struct CurlInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) override + virtual const std::string inputType() const = 0; + const std::set<std::string> transportUrlSchemes = {"file", "http", "https"}; + + const bool hasTarballExtension(std::string_view path) const { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; + return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") + || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") + || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2") + || hasSuffix(path, ".tar.zst"); + } - if (!hasSuffix(url.path, ".zip") - && !hasSuffix(url.path, ".tar") - && !hasSuffix(url.path, ".tgz") - && !hasSuffix(url.path, ".tar.gz") - && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2") - && !hasSuffix(url.path, ".tar.zst")) - return {}; + virtual bool isValidURL(const ParsedURL & url) const = 0; + + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (!isValidURL(url)) + return std::nullopt; Input input; - input.attrs.insert_or_assign("type", "tarball"); - input.attrs.insert_or_assign("url", url.to_string()); + + auto urlWithoutApplicationScheme = url; + urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + + input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); @@ -194,14 +205,17 @@ struct TarballInputScheme : InputScheme std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + auto type = maybeGetStrAttr(attrs, "type"); + if (type != inputType()) return {}; + std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") - throw Error("unsupported tarball input attribute '%s'", name); + if (!allowedNames.count(name)) + throw Error("unsupported %s input attribute '%s'", *type, name); Input input; input.attrs = attrs; + //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } @@ -209,14 +223,9 @@ struct TarballInputScheme : InputScheme ParsedURL toURL(const Input & input) override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); - /* - else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) - url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true)); - */ return url; } @@ -225,6 +234,42 @@ struct TarballInputScheme : InputScheme return true; } +}; + +struct FileInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "file"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : !hasTarballExtension(url.path)); + } + + std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override + { + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + return {std::move(file.storePath), input}; + } +}; + +struct TarballInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "tarball"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : hasTarballExtension(url.path)); + } + std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override { auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; @@ -233,5 +278,6 @@ struct TarballInputScheme : InputScheme }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); +static auto rFileInputScheme = OnStartup([] { registerInputScheme(std::make_unique<FileInputScheme>()); }); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 6f6ad57cb..47458a388 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -93,8 +93,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( - "packages '%1%' and '%2%' have the same priority %3%; " + "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "or type 'nix profile install --help' if using 'nix profile' to find out how" "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f6232d255..5b7abeb49 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,6 +1,7 @@ #include "url.hh" #include "url-parts.hh" #include "util.hh" +#include "split.hh" namespace nix { @@ -136,4 +137,21 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +/** + * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' + * into a tuple '(applicationScheme, transportScheme)' + * + * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"} + * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"} + */ +ParsedUrlScheme parseUrlScheme(std::string_view scheme) +{ + auto application = splitPrefixTo(scheme, '+'); + auto transport = scheme; + return ParsedUrlScheme { + .application = application, + .transport = transport, + }; +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 6e77142e3..2a9fb34c1 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -27,4 +27,19 @@ std::map<std::string, std::string> decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); +/* + * Although that’s not really standardized anywhere, an number of tools + * use a scheme of the form 'x+y' in urls, where y is the “transport layer” + * scheme, and x is the “application layer” scheme. + * + * For example git uses `git+https` to designate remotes using a Git + * protocol over http. + */ +struct ParsedUrlScheme { + std::optional<std::string_view> application; + std::string_view transport; +}; + +ParsedUrlScheme parseUrlScheme(std::string_view scheme); + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 519855ea3..426f23905 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -543,8 +543,6 @@ static void main_nix_build(int argc, char * * argv) restoreProcessContext(); - logger->stop(); - execvp(shell->c_str(), argPtrs.data()); throw SysError("executing shell '%s'", *shell); @@ -603,8 +601,6 @@ static void main_nix_build(int argc, char * * argv) outPaths.push_back(outputPath); } - logger->stop(); - for (auto & path : outPaths) std::cout << store->printStorePath(path) << '\n'; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 96f3c3b26..c412bb814 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1489,8 +1489,6 @@ static int main_nix_env(int argc, char * * argv) globals.state->printStats(); - logger->stop(); - return 0; } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 153b84137..9163eefd0 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1095,8 +1095,6 @@ static int main_nix_store(int argc, char * * argv) op(opFlags, opArgs); - logger->stop(); - return 0; } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1190b8348..2a3fc0213 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -18,6 +18,9 @@ struct DevelopSettings : Config Setting<std::string> bashPrompt{this, "", "bash-prompt", "The bash prompt (`PS1`) in `nix develop` shells."}; + Setting<std::string> bashPromptPrefix{this, "", "bash-prompt-prefix", + "Prefix prepended to the `PS1` environment variable in `nix develop` shells."}; + Setting<std::string> bashPromptSuffix{this, "", "bash-prompt-suffix", "Suffix appended to the `PS1` environment variable in `nix develop` shells."}; }; @@ -482,6 +485,9 @@ struct CmdDevelop : Common, MixEnvironment if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt.get())); + if (developSettings.bashPromptPrefix != "") + script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n", + shellEscape(developSettings.bashPromptPrefix.get())); if (developSettings.bashPromptSuffix != "") script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix.get())); @@ -512,9 +518,20 @@ struct CmdDevelop : Common, MixEnvironment Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); - shell = store->printStorePath( - Installable::toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) - + "/bin/bash"; + bool found = false; + + for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { + auto s = store->printStorePath(path) + "/bin/bash"; + if (pathExists(s)) { + shell = s; + found = true; + break; + } + } + + if (!found) + throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); + } catch (Error &) { ignoreException(); } diff --git a/src/nix/develop.md b/src/nix/develop.md index 8bcff66c9..e036ec6b9 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -80,8 +80,8 @@ initialised by `stdenv` and exits. This build environment can be recorded into a profile using `--profile`. The prompt used by the `bash` shell can be customised by setting the -`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in -the flake's `nixConfig` attribute. +`bash-prompt`, `bash-prompt-prefix`, and `bash-prompt-suffix` settings in +`nix.conf` or in the flake's `nixConfig` attribute. # Flake output attributes diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1938ce4e6..8370b8dcf 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -509,7 +509,7 @@ struct CmdFlakeCheck : FlakeCommand std::string_view replacement = name == "defaultPackage" ? "packages.<system>.default" : - name == "defaultApps" ? "apps.<system>.default" : + name == "defaultApp" ? "apps.<system>.default" : name == "defaultTemplate" ? "templates.default" : name == "defaultBundler" ? "bundlers.<system>.default" : name == "overlay" ? "overlays.default" : @@ -1076,9 +1076,13 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else if (attrPath.size() > 0 && attrPathS[0] == "legacyPackages") { if (attrPath.size() == 1) recurse(); - else if (!showLegacy) - logger->warn(fmt("%s: " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); - else { + else if (!showLegacy){ + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); + } + } else { if (visitor.isDerivation()) showDerivation(); else if (attrPath.size() <= 2) diff --git a/src/nix/flake.md b/src/nix/flake.md index c8251eb74..a1ab43281 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -181,9 +181,17 @@ Currently the `type` attribute can be one of the following: * `tarball`: Tarballs. The location of the tarball is specified by the attribute `url`. - In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`, - `.tar.xz`, `.tar.bz2` or `.tar.zst`. + In URL form, the schema must be `tarball+http://`, `tarball+https://` or `tarball+file://`. + If the extension corresponds to a known archive format (`.zip`, `.tar`, + `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` + can be dropped. + +* `file`: Plain files or directory tarballs, either over http(s) or from the local + disk. + + In URL form, the schema must be `file+http://`, `file+https://` or `file+file://`. + If the extension doesn’t correspond to a known archive format (as defined by the + `tarball` fetcher), then the `file+` prefix can be dropped. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: @@ -331,9 +339,10 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-suffix`, - and `flake-registry`) are allowed to be set without confirmation so long as - `accept-flake-config` is not set in the global configuration. + whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, + `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without + confirmation so long as `accept-flake-config` is not set in the global + configuration. ## Flake inputs diff --git a/src/nix/main.cc b/src/nix/main.cc index 6d0f6ce6e..dadb54306 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -261,6 +261,8 @@ void mainWrapped(int argc, char * * argv) } #endif + Finally f([] { logger->stop(); }); + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); @@ -279,8 +281,6 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlInfo; } - Finally f([] { logger->stop(); }); - NixArgs args; if (argc == 2 && std::string(argv[1]) == "__dump-args") { diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 685776bec..3814e7d5a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -37,7 +37,7 @@ struct ProfileElement StorePathSet storePaths; std::optional<ProfileElementSource> source; bool active = true; - // FIXME: priority + int priority = 5; std::string describe() const { @@ -116,6 +116,9 @@ struct ProfileManifest for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; + if(e.contains("priority")) { + element.priority = e["priority"]; + } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { parseFlakeRef(e[sOriginalUrl]), @@ -153,6 +156,7 @@ struct ProfileManifest nlohmann::json obj; obj["storePaths"] = paths; obj["active"] = element.active; + obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); obj["url"] = element.source->resolvedRef.to_string(); @@ -177,7 +181,7 @@ struct ProfileManifest for (auto & element : elements) { for (auto & path : element.storePaths) { if (element.active) - pkgs.emplace_back(store->printStorePath(path), true, 5); + pkgs.emplace_back(store->printStorePath(path), true, element.priority); references.insert(path); } } @@ -259,6 +263,17 @@ builtPathsPerInstallable( struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { + std::optional<int64_t> priority; + + CmdProfileInstall() { + addFlag({ + .longName = "priority", + .description = "The priority of the package to install.", + .labels = {"priority"}, + .handler = {&priority}, + }); + }; + std::string description() override { return "install a package into a profile"; @@ -282,6 +297,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + + if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { // FIXME: make build() return this? auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); @@ -291,8 +308,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, installable2->outputsSpec }; + + if(drv.priority) { + element.priority = *drv.priority; + } } + if(priority) { // if --priority was specified we want to override the priority of the installable + element.priority = *priority; + }; + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); manifest.elements.push_back(std::move(element)); diff --git a/tests/fetchTree-file.sh b/tests/fetchTree-file.sh new file mode 100644 index 000000000..1c0ce39ce --- /dev/null +++ b/tests/fetchTree-file.sh @@ -0,0 +1,105 @@ +source common.sh + +clearStore + +cd "$TEST_ROOT" + +test_fetch_file () { + echo foo > test_input + + input_hash="$(nix hash path test_input)" + + nix eval --impure --file - <<EOF + let + tree = builtins.fetchTree { type = "file"; url = "file://$PWD/test_input"; }; + in + assert (tree.narHash == "$input_hash"); + tree +EOF +} + +# Make sure that `http(s)` and `file` flake inputs are properly extracted when +# they should be, and treated as opaque files when they should be +test_file_flake_input () { + rm -fr "$TEST_ROOT/testFlake"; + mkdir "$TEST_ROOT/testFlake"; + pushd testFlake + + mkdir inputs + echo foo > inputs/test_input_file + tar cfa test_input.tar.gz inputs + cp test_input.tar.gz test_input_no_ext + input_tarball_hash="$(nix hash path test_input.tar.gz)" + input_directory_hash="$(nix hash path inputs)" + + cat <<EOF > flake.nix + { + inputs.no_ext_default_no_unpack = { + url = "file://$PWD/test_input_no_ext"; + flake = false; + }; + inputs.no_ext_explicit_unpack = { + url = "tarball+file://$PWD/test_input_no_ext"; + flake = false; + }; + inputs.tarball_default_unpack = { + url = "file://$PWD/test_input.tar.gz"; + flake = false; + }; + inputs.tarball_explicit_no_unpack = { + url = "file+file://$PWD/test_input.tar.gz"; + flake = false; + }; + outputs = { ... }: {}; + } +EOF + + nix flake update + nix eval --file - <<EOF + with (builtins.fromJSON (builtins.readFile ./flake.lock)); + + # Url inputs whose extension doesn’t match a know archive format should + # not be unpacked by default + assert (nodes.no_ext_default_no_unpack.locked.type == "file"); + assert (nodes.no_ext_default_no_unpack.locked.unpack or false == false); + assert (nodes.no_ext_default_no_unpack.locked.narHash == "$input_tarball_hash"); + + # For backwards compatibility, flake inputs that correspond to the + # old 'tarball' fetcher should still have their type set to 'tarball' + assert (nodes.tarball_default_unpack.locked.type == "tarball"); + # Unless explicitely specified, the 'unpack' parameter shouldn’t appear here + # because that would break older Nix versions + assert (!nodes.tarball_default_unpack.locked ? unpack); + assert (nodes.tarball_default_unpack.locked.narHash == "$input_directory_hash"); + + # Explicitely passing the unpack parameter should enforce the desired behavior + assert (nodes.no_ext_explicit_unpack.locked.narHash == nodes.tarball_default_unpack.locked.narHash); + assert (nodes.tarball_explicit_no_unpack.locked.narHash == nodes.no_ext_default_no_unpack.locked.narHash); + true +EOF + popd + + [[ -z "${NIX_DAEMON_PACKAGE}" ]] && return 0 + + # Ensure that a lockfile generated by the current Nix for tarball inputs + # can still be read by an older Nix + + cat <<EOF > flake.nix + { + inputs.tarball = { + url = "file://$PWD/test_input.tar.gz"; + flake = false; + }; + outputs = { self, tarball }: { + foo = builtins.readFile "${tarball}/test_input_file"; + }; + } + nix flake update + + clearStore + "$NIX_DAEMON_PACKAGE/bin/nix" eval .#foo +EOF +} + +test_fetch_file +test_file_flake_input diff --git a/tests/flakes.sh b/tests/flakes.sh index 24601784f..9a1f0ab6a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -163,6 +163,7 @@ nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Di # But should succeed in impure mode. (! nix build -o $TEST_ROOT/result flake2#bar --impure) nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file +nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure # Building a local flake with an unlocked dependency should fail with --no-update-lock-file. nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' diff --git a/tests/local.mk b/tests/local.mk index e3c4ff4eb..2932d2b13 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -23,6 +23,7 @@ nix_tests = \ fetchGit.sh \ fetchurl.sh \ fetchPath.sh \ + fetchTree-file.sh \ simple.sh \ referrers.sh \ optimise-store.sh \ diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index f8da3d929..7ba3235fa 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -120,3 +120,21 @@ nix profile install "$flake1Dir^man" (! [ -e $TEST_HOME/.nix-profile/bin/hello ]) [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) + +# test priority +nix profile remove 0 + +# Make another flake. +flake2Dir=$TEST_ROOT/flake2 +printf World > $flake1Dir/who +cp -r $flake1Dir $flake2Dir +printf World2 > $flake2Dir/who + +nix profile install $flake1Dir +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 100 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 0 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] +# nix profile install $flake1Dir --priority 100 +# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] |