Merge pull request #7732 from hercules-ci/make-initLibStore-viable-alternative

Make `initLibStore` a viable alternative

8 files changed, 134 insertions, 92 deletions

diff --git a/configure.ac b/configure.ac
index e587bd563..bb3f92e4d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -184,7 +184,7 @@ fi
 
 # Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
 # used by S3BinaryCacheStore.
-PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
+PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
 # Look for libarchive.
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index de91dc28d..10a0c4067 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -27,8 +27,6 @@ static ref<Store> store()
     if (!_store) {
         try {
             initLibStore();
-            loadConfFile();
-            settings.lockCPU = false;
             _store = openStore();
         } catch (Error & e) {
             croak("%s", e.what());
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 37664c065..56f47a4ac 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -10,7 +10,6 @@
 #include <cctype>
 #include <exception>
 #include <iostream>
-#include <mutex>
 
 #include <cstdlib>
 #include <sys/time.h>
@@ -20,16 +19,9 @@
 #ifdef __linux__
 #include <features.h>
 #endif
-#ifdef __GLIBC__
-#include <gnu/lib-names.h>
-#include <nss.h>
-#include <dlfcn.h>
-#endif
 
 #include <openssl/crypto.h>
 
-#include <sodium.h>
-
 
 namespace nix {
 
@@ -115,57 +107,6 @@ std::string getArg(const std::string & opt,
     return *i;
 }
 
-
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
-/* OpenSSL is not thread-safe by default - it will randomly crash
-   unless the user supplies a mutex locking function. So let's do
-   that. */
-static std::vector<std::mutex> opensslLocks;
-
-static void opensslLockCallback(int mode, int type, const char * file, int line)
-{
-    if (mode & CRYPTO_LOCK)
-        opensslLocks[type].lock();
-    else
-        opensslLocks[type].unlock();
-}
-#endif
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-#ifdef __GLIBC__
-        /* On linux, glibc will run every lookup through the nss layer.
-         * That means every lookup goes, by default, through nscd, which acts as a local
-         * cache.
-         * Because we run builds in a sandbox, we also remove access to nscd otherwise
-         * lookups would leak into the sandbox.
-         *
-         * But now we have a new problem, we need to make sure the nss_dns backend that
-         * does the dns lookups when nscd is not available is loaded or available.
-         *
-         * We can't make it available without leaking nix's environment, so instead we'll
-         * load the backend, and configure nss so it does not try to run dns lookups
-         * through nscd.
-         *
-         * This is technically only used for builtins:fetch* functions so we only care
-         * about dns.
-         *
-         * All other platforms are unaffected.
-         */
-        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
-            warn("unable to load nss_dns backend");
-        // FIXME: get hosts entry from nsswitch.conf.
-        __nss_configure_lookup("hosts", "files dns");
-#endif
-    });
-}
-
 static void sigHandler(int signo) { }
 
 
@@ -177,16 +118,7 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
-    /* Initialise OpenSSL locking. */
-    opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
-    CRYPTO_set_locking_callback(opensslLockCallback);
-#endif
-
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
-    loadConfFile();
+    initLibStore();
 
     startSignalHandlerThread();
 
@@ -223,7 +155,10 @@ void initNix()
     if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP");
 #endif
 
-    /* Register a SIGSEGV handler to detect stack overflows. */
+    /* Register a SIGSEGV handler to detect stack overflows.
+       Why not initLibExpr()? initGC() is essentially that, but
+       detectStackOverflow is not an instance of the init function concept, as
+       it may have to be invoked more than once per process. */
     detectStackOverflow();
 
     /* There is no privacy in the Nix system ;-)  At least not for
@@ -236,16 +171,6 @@ void initNix()
     gettimeofday(&tv, 0);
     srandom(tv.tv_usec);
 
-    /* On macOS, don't use the per-session TMPDIR (as set e.g. by
-       sshd). This breaks build users because they don't have access
-       to the TMPDIR, in particular in ‘nix-store --serve’. */
-#if __APPLE__
-    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
-        unsetenv("TMPDIR");
-#endif
-
-    preloadNSS();
-    initLibStore();
 }
 
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 823b4af74..1b38e32fb 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -7,12 +7,20 @@
 
 #include <algorithm>
 #include <map>
+#include <mutex>
 #include <thread>
 #include <dlfcn.h>
 #include <sys/utsname.h>
 
 #include <nlohmann/json.hpp>
 
+#include <sodium/core.h>
+
+#ifdef __GLIBC__
+#include <gnu/lib-names.h>
+#include <nss.h>
+#include <dlfcn.h>
+#endif
 
 namespace nix {
 
@@ -41,7 +49,6 @@ Settings::Settings()
     , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
-    lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
     allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
 
     auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
@@ -281,6 +288,42 @@ void initPlugins()
     settings.pluginFiles.pluginsLoaded = true;
 }
 
+static void preloadNSS()
+{
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    static std::once_flag dns_resolve_flag;
+
+    std::call_once(dns_resolve_flag, []() {
+#ifdef __GLIBC__
+        /* On linux, glibc will run every lookup through the nss layer.
+         * That means every lookup goes, by default, through nscd, which acts as a local
+         * cache.
+         * Because we run builds in a sandbox, we also remove access to nscd otherwise
+         * lookups would leak into the sandbox.
+         *
+         * But now we have a new problem, we need to make sure the nss_dns backend that
+         * does the dns lookups when nscd is not available is loaded or available.
+         *
+         * We can't make it available without leaking nix's environment, so instead we'll
+         * load the backend, and configure nss so it does not try to run dns lookups
+         * through nscd.
+         *
+         * This is technically only used for builtins:fetch* functions so we only care
+         * about dns.
+         *
+         * All other platforms are unaffected.
+         */
+        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
+            warn("unable to load nss_dns backend");
+        // FIXME: get hosts entry from nsswitch.conf.
+        __nss_configure_lookup("hosts", "files dns");
+#endif
+    });
+}
+
 static bool initLibStoreDone = false;
 
 void assertLibStoreInitialized() {
@@ -291,6 +334,24 @@ void assertLibStoreInitialized() {
 }
 
 void initLibStore() {
+
+    initLibUtil();
+
+    if (sodium_init() == -1)
+        throw Error("could not initialise libsodium");
+
+    loadConfFile();
+
+    preloadNSS();
+
+    /* On macOS, don't use the per-session TMPDIR (as set e.g. by
+       sshd). This breaks build users because they don't have access
+       to the TMPDIR, in particular in ‘nix-store --serve’. */
+#if __APPLE__
+    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+        unsetenv("TMPDIR");
+#endif
+
     initLibStoreDone = true;
 }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index f598ed4a8..d6c5d437a 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -458,11 +458,6 @@ public:
         )",
         {"env-keep-derivations"}};
 
-    /**
-     * Whether to lock the Nix client and worker to the same CPU.
-     */
-    bool lockCPU;
-
     Setting<SandboxMode> sandboxMode{
         this,
         #if __linux__
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 0c8f5727d..2c36d9d94 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -1,6 +1,7 @@
 #include <iostream>
 #include <cstring>
 
+#include <openssl/crypto.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
@@ -16,7 +17,6 @@
 
 namespace nix {
 
-
 static size_t regularHashSize(HashType type) {
     switch (type) {
     case htMD5: return md5HashSize;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 843a10eab..21d1c8dcd 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -47,6 +47,9 @@ extern char * * environ __attribute__((weak));
 
 namespace nix {
 
+void initLibUtil() {
+}
+
 std::optional<std::string> getEnv(const std::string & key)
 {
     char * value = getenv(key.c_str());
@@ -1744,14 +1747,40 @@ void triggerInterrupt()
 }
 
 static sigset_t savedSignalMask;
+static bool savedSignalMaskIsSet = false;
 
-void startSignalHandlerThread()
+void setChildSignalMask(sigset_t * sigs)
 {
-    updateWindowSize();
+    assert(sigs); // C style function, but think of sigs as a reference
+
+#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
+    sigemptyset(&savedSignalMask);
+    // There's no "assign" or "copy" function, so we rely on (math) idempotence
+    // of the or operator: a or a = a.
+    sigorset(&savedSignalMask, sigs, sigs);
+#else
+    // Without sigorset, our best bet is to assume that sigset_t is a type that
+    // can be assigned directly, such as is the case for a sigset_t defined as
+    // an integer type.
+    savedSignalMask = *sigs;
+#endif
+
+    savedSignalMaskIsSet = true;
+}
 
+void saveSignalMask() {
     if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
         throw SysError("querying signal mask");
 
+    savedSignalMaskIsSet = true;
+}
+
+void startSignalHandlerThread()
+{
+    updateWindowSize();
+
+    saveSignalMask();
+
     sigset_t set;
     sigemptyset(&set);
     sigaddset(&set, SIGINT);
@@ -1767,6 +1796,20 @@ void startSignalHandlerThread()
 
 static void restoreSignals()
 {
+    // If startSignalHandlerThread wasn't called, that means we're not running
+    // in a proper libmain process, but a process that presumably manages its
+    // own signal handlers. Such a process should call either
+    //  - initNix(), to be a proper libmain process
+    //  - startSignalHandlerThread(), to resemble libmain regarding signal
+    //    handling only
+    //  - saveSignalMask(), for processes that define their own signal handling
+    //    thread
+    // TODO: Warn about this? Have a default signal mask? The latter depends on
+    //       whether we should generally inherit signal masks from the caller.
+    //       I don't know what the larger unix ecosystem expects from us here.
+    if (!savedSignalMaskIsSet)
+        return;
+
     if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
         throw SysError("restoring signals");
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 85ab77b1b..040fed68f 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -32,6 +32,7 @@ namespace nix {
 struct Sink;
 struct Source;
 
+void initLibUtil();
 
 /**
  * The system for which Nix is compiled.
@@ -445,6 +446,8 @@ void setStackSize(size_t stackSize);
 /**
  * Restore the original inherited Unix process context (such as signal
  * masks, stack size).
+
+ * See startSignalHandlerThread(), saveSignalMask().
  */
 void restoreProcessContext(bool restoreMounts = true);
 
@@ -814,9 +817,26 @@ class Callback;
 /**
  * Start a thread that handles various signals. Also block those signals
  * on the current thread (and thus any threads created by it).
+ * Saves the signal mask before changing the mask to block those signals.
+ * See saveSignalMask().
  */
 void startSignalHandlerThread();
 
+/**
+ * Saves the signal mask, which is the signal mask that nix will restore
+ * before creating child processes.
+ * See setChildSignalMask() to set an arbitrary signal mask instead of the
+ * current mask.
+ */
+void saveSignalMask();
+
+/**
+ * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
+ * necessarily match the current thread's mask.
+ * See saveSignalMask() to set the saved mask to the current mask.
+ */
+void setChildSignalMask(sigset_t *sigs);
+
 struct InterruptCallback
 {
     virtual ~InterruptCallback() { };