Merge branch 'path-info' into ca-drv-exotic

16 files changed, 322 insertions, 31 deletions

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 753980fd4..35686a8aa 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -29,7 +29,15 @@ void Args::removeFlag(const std::string & longName)
 
 void Completions::add(std::string completion, std::string description)
 {
-    assert(description.find('\n') == std::string::npos);
+    description = trim(description);
+    // ellipsize overflowing content on the back of the description
+    auto end_index = description.find_first_of(".\n");
+    if (end_index != std::string::npos) {
+        auto needs_ellipsis = end_index != description.size() - 1;
+        description.resize(end_index);
+        if (needs_ellipsis)
+            description.append(" [...]");
+    }
     insert(Completion {
         .completion = completion,
         .description = description
@@ -324,7 +332,7 @@ MultiCommand::MultiCommand(const Commands & commands_)
     expectArgs({
         .label = "subcommand",
         .optional = true,
-        .handler = {[=](std::string s) {
+        .handler = {[=,this](std::string s) {
             assert(!command);
             auto i = commands.find(s);
             if (i == commands.end()) {
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 9bb412b4f..b349f2d80 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -209,7 +209,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
         .description = fmt("Set the `%s` setting.", name),
         .category = category,
         .labels = {"value"},
-        .handler = {[=](std::string s) { overridden = true; set(s); }},
+        .handler = {[this](std::string s) { overridden = true; set(s); }},
     });
 
     if (isAppendable())
@@ -218,7 +218,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
             .description = fmt("Append to the `%s` setting.", name),
             .category = category,
             .labels = {"value"},
-            .handler = {[=](std::string s) { overridden = true; set(s, true); }},
+            .handler = {[this](std::string s) { overridden = true; set(s, true); }},
         });
 }
 
@@ -270,13 +270,13 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
         .longName = name,
         .description = fmt("Enable the `%s` setting.", name),
         .category = category,
-        .handler = {[=]() { override(true); }}
+        .handler = {[this]() { override(true); }}
     });
     args.addFlag({
         .longName = "no-" + name,
         .description = fmt("Disable the `%s` setting.", name),
         .category = category,
-        .handler = {[=]() { override(false); }}
+        .handler = {[this]() { override(false); }}
     });
 }
 
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 79ec0f9cf..7ac43c854 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -250,11 +250,15 @@ public:
     operator const T &() const { return value; }
     operator T &() { return value; }
     const T & get() const { return value; }
-    bool operator ==(const T & v2) const { return value == v2; }
-    bool operator !=(const T & v2) const { return value != v2; }
-    void operator =(const T & v) { assign(v); }
+    template<typename U>
+    bool operator ==(const U & v2) const { return value == v2; }
+    template<typename U>
+    bool operator !=(const U & v2) const { return value != v2; }
+    template<typename U>
+    void operator =(const U & v) { assign(v); }
     virtual void assign(const T & v) { value = v; }
-    void setDefault(const T & v) { if (!overridden) value = v; }
+    template<typename U>
+    void setDefault(const U & v) { if (!overridden) value = v; }
 
     void set(const std::string & str, bool append = false) override;
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 7d236028c..0ebeaba61 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -74,6 +74,8 @@ struct AbstractPos
     virtual void print(std::ostream & out) const = 0;
 
     std::optional<LinesOfCode> getCodeLines() const;
+
+    virtual ~AbstractPos() = default;
 };
 
 std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index e0902971e..58d762ebb 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -16,6 +16,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::ReplFlake, "repl-flake" },
     { Xp::AutoAllocateUids, "auto-allocate-uids" },
     { Xp::Cgroups, "cgroups" },
+    { Xp::DiscardReferences, "discard-references" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index af775feb0..ac372e03e 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -25,6 +25,7 @@ enum struct ExperimentalFeature
     ReplFlake,
     AutoAllocateUids,
     Cgroups,
+    DiscardReferences,
 };
 
 /**
diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc
new file mode 100644
index 000000000..f66accb10
--- /dev/null
+++ b/src/libutil/namespaces.cc
@@ -0,0 +1,97 @@
+#if __linux__
+
+#include "namespaces.hh"
+#include "util.hh"
+#include "finally.hh"
+
+#include <sys/mount.h>
+
+namespace nix {
+
+bool userNamespacesSupported()
+{
+    static auto res = [&]() -> bool
+    {
+        if (!pathExists("/proc/self/ns/user")) {
+            debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y");
+            return false;
+        }
+
+        Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
+        if (!pathExists(maxUserNamespaces) ||
+            trim(readFile(maxUserNamespaces)) == "0")
+        {
+            debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'");
+            return false;
+        }
+
+        Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone";
+        if (pathExists(procSysKernelUnprivilegedUsernsClone)
+            && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0")
+        {
+            debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'");
+            return false;
+        }
+
+        try {
+            Pid pid = startProcess([&]()
+            {
+                _exit(0);
+            }, {
+                .cloneFlags = CLONE_NEWUSER
+            });
+
+            auto r = pid.wait();
+            assert(!r);
+        } catch (SysError & e) {
+            debug("user namespaces do not work on this system: %s", e.msg());
+            return false;
+        }
+
+        return true;
+    }();
+    return res;
+}
+
+bool mountAndPidNamespacesSupported()
+{
+    static auto res = [&]() -> bool
+    {
+        try {
+
+            Pid pid = startProcess([&]()
+            {
+                /* Make sure we don't remount the parent's /proc. */
+                if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
+                    _exit(1);
+
+                /* Test whether we can remount /proc. The kernel disallows
+                   this if /proc is not fully visible, i.e. if there are
+                   filesystems mounted on top of files inside /proc.  See
+                   https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */
+                if (mount("none", "/proc", "proc", 0, 0) == -1)
+                    _exit(2);
+
+                _exit(0);
+            }, {
+                .cloneFlags = CLONE_NEWNS | CLONE_NEWPID | (userNamespacesSupported() ? CLONE_NEWUSER : 0)
+            });
+
+            if (pid.wait()) {
+                debug("PID namespaces do not work on this system: cannot remount /proc");
+                return false;
+            }
+
+        } catch (SysError & e) {
+            debug("mount namespaces do not work on this system: %s", e.msg());
+            return false;
+        }
+
+        return true;
+    }();
+    return res;
+}
+
+}
+
+#endif
diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh
new file mode 100644
index 000000000..e82379b9c
--- /dev/null
+++ b/src/libutil/namespaces.hh
@@ -0,0 +1,13 @@
+#pragma once
+
+namespace nix {
+
+#if __linux__
+
+bool userNamespacesSupported();
+
+bool mountAndPidNamespacesSupported();
+
+#endif
+
+}
diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc
index 412c03030..e4e928b3b 100644
--- a/src/libutil/tests/hash.cc
+++ b/src/libutil/tests/hash.cc
@@ -1,5 +1,12 @@
-#include "hash.hh"
+#include <regex>
+
+#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
+#include <rapidcheck/gtest.h>
+
+#include <hash.hh>
+
+#include "tests/hash.hh"
 
 namespace nix {
 
@@ -73,3 +80,16 @@ namespace nix {
                 "c7d329eeb6dd26545e96e55b874be909");
     }
 }
+
+namespace rc {
+using namespace nix;
+
+Gen<Hash> Arbitrary<Hash>::arbitrary()
+{
+    Hash hash(htSHA1);
+    for (size_t i = 0; i < hash.hashSize; ++i)
+        hash.hash[i] = *gen::arbitrary<uint8_t>();
+    return gen::just(hash);
+}
+
+}
diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh
new file mode 100644
index 000000000..9e9650e6e
--- /dev/null
+++ b/src/libutil/tests/hash.hh
@@ -0,0 +1,15 @@
+#pragma once
+
+#include <rapidcheck/gen/Arbitrary.h>
+
+#include <hash.hh>
+
+namespace rc {
+using namespace nix;
+
+template<>
+struct Arbitrary<Hash> {
+    static Gen<Hash> arbitrary();
+};
+
+}
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index 815e18560..167915439 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -2,14 +2,28 @@ check: libutil-tests_RUN
 
 programs += libutil-tests
 
+libutil-tests-exe_NAME = libnixutil-tests
+
+libutil-tests-exe_DIR := $(d)
+
+libutil-tests-exe_INSTALL_DIR :=
+
+libutil-tests-exe_LIBS = libutil-tests
+
+libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
+
+libraries += libutil-tests
+
+libutil-tests_NAME = libnixutil-tests
+
 libutil-tests_DIR := $(d)
 
 libutil-tests_INSTALL_DIR :=
 
 libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr
+libutil-tests_CXXFLAGS += -I src/libutil
 
 libutil-tests_LIBS = libutil
 
-libutil-tests_LDFLAGS := $(GTEST_LIBS)
+libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc
index c3b233797..a908631e6 100644
--- a/src/libutil/tests/url.cc
+++ b/src/libutil/tests/url.cc
@@ -99,6 +99,27 @@ namespace nix {
         ASSERT_EQ(parsed, expected);
     }
 
+    TEST(parseURL, parsesFilePlusHttpsUrl) {
+        auto s = "file+https://www.example.org/video.mp4";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "file+https://www.example.org/video.mp4",
+            .base = "https://www.example.org/video.mp4",
+            .scheme = "file+https",
+            .authority = "www.example.org",
+            .path = "/video.mp4",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) {
+        auto s = "file://www.example.org/video.mp4";
+        ASSERT_THROW(parseURL(s), Error);
+    }
 
     TEST(parseURL, parseIPv4Address) {
         auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello";
@@ -281,4 +302,37 @@ namespace nix {
         ASSERT_EQ(d, s);
     }
 
+
+    /* ----------------------------------------------------------------------------
+     * percentEncode
+     * --------------------------------------------------------------------------*/
+
+    TEST(percentEncode, encodesUrlEncodedString) {
+        std::string s = percentEncode("==@==");
+        std::string d = "%3D%3D%40%3D%3D";
+        ASSERT_EQ(d, s);
+    }
+
+    TEST(percentEncode, keepArgument) {
+        std::string a = percentEncode("abd / def");
+        std::string b = percentEncode("abd / def", "/");
+        ASSERT_EQ(a, "abd%20%2F%20def");
+        ASSERT_EQ(b, "abd%20/%20def");
+    }
+
+    TEST(percentEncode, inverseOfDecode) {
+        std::string original = "%3D%3D%40%3D%3D";
+        std::string once = percentEncode(original);
+        std::string back = percentDecode(once);
+
+        ASSERT_EQ(back, original);
+    }
+
+    TEST(percentEncode, trailingPercent) {
+        std::string s = percentEncode("==@==%");
+        std::string d = "%3D%3D%40%3D%3D%25";
+
+        ASSERT_EQ(d, s);
+    }
+
 }
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 5b7abeb49..9e44241ac 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url)
         auto & query = match[6];
         auto & fragment = match[7];
 
-        auto isFile = scheme.find("file") != std::string::npos;
+        auto transportIsFile = parseUrlScheme(scheme).transport == "file";
 
-        if (authority && *authority != "" && isFile)
+        if (authority && *authority != "" && transportIsFile)
             throw BadURL("file:// URL '%s' has unexpected authority '%s'",
                 url, *authority);
 
-        if (isFile && path.empty())
+        if (transportIsFile && path.empty())
             path = "/";
 
         return ParsedURL{
@@ -88,17 +88,22 @@ std::map<std::string, std::string> decodeQuery(const std::string & query)
     return result;
 }
 
-std::string percentEncode(std::string_view s)
+const static std::string allowedInQuery = ":@/?";
+const static std::string allowedInPath = ":@/";
+
+std::string percentEncode(std::string_view s, std::string_view keep)
 {
     std::string res;
     for (auto & c : s)
+        // unreserved + keep
         if ((c >= 'a' && c <= 'z')
             || (c >= 'A' && c <= 'Z')
             || (c >= '0' && c <= '9')
-            || strchr("-._~!$&'()*+,;=:@", c))
+            || strchr("-._~", c)
+            || keep.find(c) != std::string::npos)
             res += c;
         else
-            res += fmt("%%%02x", (unsigned int) c);
+            res += fmt("%%%02X", (unsigned int) c);
     return res;
 }
 
@@ -109,9 +114,9 @@ std::string encodeQuery(const std::map<std::string, std::string> & ss)
     for (auto & [name, value] : ss) {
         if (!first) res += '&';
         first = false;
-        res += percentEncode(name);
+        res += percentEncode(name, allowedInQuery);
         res += '=';
-        res += percentEncode(value);
+        res += percentEncode(value, allowedInQuery);
     }
     return res;
 }
@@ -122,7 +127,7 @@ std::string ParsedURL::to_string() const
         scheme
         + ":"
         + (authority ? "//" + *authority : "")
-        + path
+        + percentEncode(path, allowedInPath)
         + (query.empty() ? "" : "?" + encodeQuery(query))
         + (fragment.empty() ? "" : "#" + percentEncode(fragment));
 }
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 2a9fb34c1..ddd673d65 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -22,6 +22,7 @@ struct ParsedURL
 MakeError(BadURL, Error);
 
 std::string percentDecode(std::string_view in);
+std::string percentEncode(std::string_view s, std::string_view keep="");
 
 std::map<std::string, std::string> decodeQuery(const std::string & query);
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 993dc1cb6..885bae69c 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -36,6 +36,7 @@
 #ifdef __linux__
 #include <sys/prctl.h>
 #include <sys/resource.h>
+#include <sys/mman.h>
 
 #include <cmath>
 #endif
@@ -537,6 +538,16 @@ std::string getUserName()
     return name;
 }
 
+Path getHomeOf(uid_t userId)
+{
+    std::vector<char> buf(16384);
+    struct passwd pwbuf;
+    struct passwd * pw;
+    if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0
+        || !pw || !pw->pw_dir || !pw->pw_dir[0])
+        throw Error("cannot determine user's home directory");
+    return pw->pw_dir;
+}
 
 Path getHome()
 {
@@ -558,13 +569,7 @@ Path getHome()
             }
         }
         if (!homeDir) {
-            std::vector<char> buf(16384);
-            struct passwd pwbuf;
-            struct passwd * pw;
-            if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
-                || !pw || !pw->pw_dir || !pw->pw_dir[0])
-                throw Error("cannot determine user's home directory");
-            homeDir = pw->pw_dir;
+            homeDir = getHomeOf(geteuid());
             if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
                 warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
             }
@@ -604,6 +609,19 @@ Path getDataDir()
     return dataDir ? *dataDir : getHome() + "/.local/share";
 }
 
+Path getStateDir()
+{
+    auto stateDir = getEnv("XDG_STATE_HOME");
+    return stateDir ? *stateDir : getHome() + "/.local/state";
+}
+
+Path createNixStateDir()
+{
+    Path dir = getStateDir() + "/nix";
+    createDirs(dir);
+    return dir;
+}
+
 
 std::optional<Path> getSelfExe()
 {
@@ -1047,9 +1065,17 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun)
 }
 
 
+static int childEntry(void * arg)
+{
+    auto main = (std::function<void()> *) arg;
+    (*main)();
+    return 1;
+}
+
+
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
 {
-    auto wrapper = [&]() {
+    std::function<void()> wrapper = [&]() {
         if (!options.allowVfork)
             logger = makeSimpleLogger();
         try {
@@ -1069,7 +1095,27 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
             _exit(1);
     };
 
-    pid_t pid = doFork(options.allowVfork, wrapper);
+    pid_t pid = -1;
+
+    if (options.cloneFlags) {
+        #ifdef __linux__
+        // Not supported, since then we don't know when to free the stack.
+        assert(!(options.cloneFlags & CLONE_VM));
+
+        size_t stackSize = 1 * 1024 * 1024;
+        auto stack = (char *) mmap(0, stackSize,
+            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+        if (stack == MAP_FAILED) throw SysError("allocating stack");
+
+        Finally freeStack([&]() { munmap(stack, stackSize); });
+
+        pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
+        #else
+        throw Error("clone flags are only supported on Linux");
+        #endif
+    } else
+        pid = doFork(options.allowVfork, wrapper);
+
     if (pid == -1) throw SysError("unable to fork");
 
     return pid;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 9b149de80..b5625ecef 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -137,6 +137,9 @@ void deletePath(const Path & path, uint64_t & bytesFreed);
 
 std::string getUserName();
 
+/* Return the given user's home directory from /etc/passwd. */
+Path getHomeOf(uid_t userId);
+
 /* Return $HOME or the user's home directory from /etc/passwd. */
 Path getHome();
 
@@ -155,6 +158,12 @@ Path getDataDir();
 /* Return the path of the current executable. */
 std::optional<Path> getSelfExe();
 
+/* Return $XDG_STATE_HOME or $HOME/.local/state. */
+Path getStateDir();
+
+/* Create the Nix state directory and return the path to it. */
+Path createNixStateDir();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
@@ -298,6 +307,7 @@ struct ProcessOptions
     bool dieWithParent = true;
     bool runExitHandlers = false;
     bool allowVfork = false;
+    int cloneFlags = 0; // use clone() with the specified flags (Linux only)
 };
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());