diff options
| -rw-r--r-- | README.md | 25 | ||||
| -rw-r--r-- | doc/manual/expressions/builder-syntax.xml | 119 | ||||
| -rw-r--r-- | doc/manual/hacking.xml | 45 | ||||
| -rw-r--r-- | mk/precompiled-headers.mk | 4 | ||||
| -rw-r--r-- | src/libstore/build.cc | 2 | ||||
| -rw-r--r-- | src/libstore/daemon.cc | 17 | ||||
| -rw-r--r-- | src/libstore/derivations.cc | 8 | ||||
| -rw-r--r-- | src/libstore/derivations.hh | 4 | ||||
| -rw-r--r-- | src/libstore/filetransfer.cc | 39 | ||||
| -rw-r--r-- | src/libstore/filetransfer.hh | 8 | ||||
| -rw-r--r-- | src/libstore/local-store.cc | 172 | ||||
| -rw-r--r-- | src/libstore/local-store.hh | 2 | ||||
| -rw-r--r-- | src/libstore/references.cc | 15 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 15 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 2 | ||||
| -rw-r--r-- | src/libutil/serialise.cc | 13 | ||||
| -rw-r--r-- | src/libutil/serialise.hh | 15 | ||||
| -rw-r--r-- | src/libutil/util.cc | 2 | ||||
| -rw-r--r-- | src/nix-env/nix-env.cc | 3 | ||||
| -rw-r--r-- | src/nix/registry.cc | 1 | ||||
| -rw-r--r-- | tests/local-store.sh | 20 | ||||
| -rw-r--r-- | tests/local.mk | 2 |
22 files changed, 230 insertions, 303 deletions
@@ -12,7 +12,7 @@ for more details. On Linux and macOS the easiest way to Install Nix is to run the following shell command (as a user other than root): -``` +```console $ curl -L https://nixos.org/nix/install | sh ``` @@ -20,27 +20,8 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -### Building Nix - -You can build Nix using one of the targets provided by [release.nix](./release.nix): - -``` -$ nix-build ./release.nix -A build.aarch64-linux -$ nix-build ./release.nix -A build.x86_64-darwin -$ nix-build ./release.nix -A build.i686-linux -$ nix-build ./release.nix -A build.x86_64-linux -``` - -### Development Environment - -You can use the provided `shell.nix` to get a working development environment: - -``` -$ nix-shell -$ ./bootstrap.sh -$ ./configure -$ make -``` +See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to +build nix from source with nix-build or how to get a development environment. ## Additional Resources diff --git a/doc/manual/expressions/builder-syntax.xml b/doc/manual/expressions/builder-syntax.xml deleted file mode 100644 index e51bade44..000000000 --- a/doc/manual/expressions/builder-syntax.xml +++ /dev/null @@ -1,119 +0,0 @@ -<section xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id='sec-builder-syntax'> - -<title>Builder Syntax</title> - -<example xml:id='ex-hello-builder'><title>Build script for GNU Hello -(<filename>builder.sh</filename>)</title> -<programlisting> -source $stdenv/setup <co xml:id='ex-hello-builder-co-1' /> - -PATH=$perl/bin:$PATH <co xml:id='ex-hello-builder-co-2' /> - -tar xvfz $src <co xml:id='ex-hello-builder-co-3' /> -cd hello-* -./configure --prefix=$out <co xml:id='ex-hello-builder-co-4' /> -make <co xml:id='ex-hello-builder-co-5' /> -make install</programlisting> -</example> - -<para><xref linkend='ex-hello-builder' /> shows the builder referenced -from Hello's Nix expression (stored in -<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>). -The builder can actually be made a lot shorter by using the -<emphasis>generic builder</emphasis> functions provided by -<varname>stdenv</varname>, but here we write out the build steps to -elucidate what a builder does. It performs the following -steps:</para> - -<calloutlist> - - <callout arearefs='ex-hello-builder-co-1'> - - <para>When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the - derivation). For instance, the <envar>PATH</envar> variable is - empty<footnote><para>Actually, it's initialised to - <filename>/path-not-set</filename> to prevent Bash from setting it - to a default value.</para></footnote>. This is done to prevent - undeclared inputs from being used in the build process. If for - example the <envar>PATH</envar> contained - <filename>/usr/bin</filename>, then you might accidentally use - <filename>/usr/bin/gcc</filename>.</para> - - <para>So the first step is to set up the environment. This is - done by calling the <filename>setup</filename> script of the - standard environment. The environment variable - <envar>stdenv</envar> points to the location of the standard - environment being used. (It wasn't specified explicitly as an - attribute in <xref linkend='ex-hello-nix' />, but - <varname>mkDerivation</varname> adds it automatically.)</para> - - </callout> - - <callout arearefs='ex-hello-builder-co-2'> - - <para>Since Hello needs Perl, we have to make sure that Perl is in - the <envar>PATH</envar>. The <envar>perl</envar> environment - variable points to the location of the Perl package (since it - was passed in as an attribute to the derivation), so - <filename><replaceable>$perl</replaceable>/bin</filename> is the - directory containing the Perl interpreter.</para> - - </callout> - - <callout arearefs='ex-hello-builder-co-3'> - - <para>Now we have to unpack the sources. The - <varname>src</varname> attribute was bound to the result of - fetching the Hello source tarball from the network, so the - <envar>src</envar> environment variable points to the location in - the Nix store to which the tarball was downloaded. After - unpacking, we <command>cd</command> to the resulting source - directory.</para> - - <para>The whole build is performed in a temporary directory - created in <varname>/tmp</varname>, by the way. This directory is - removed after the builder finishes, so there is no need to clean - up the sources afterwards. Also, the temporary directory is - always newly created, so you don't have to worry about files from - previous builds interfering with the current build.</para> - - </callout> - - <callout arearefs='ex-hello-builder-co-4'> - - <para>GNU Hello is a typical Autoconf-based package, so we first - have to run its <filename>configure</filename> script. In Nix - every package is stored in a separate location in the Nix store, - for instance - <filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>. - Nix computes this path by cryptographically hashing all attributes - of the derivation. The path is passed to the builder through the - <envar>out</envar> environment variable. So here we give - <filename>configure</filename> the parameter - <literal>--prefix=$out</literal> to cause Hello to be installed in - the expected location.</para> - - </callout> - - <callout arearefs='ex-hello-builder-co-5'> - - <para>Finally we build Hello (<literal>make</literal>) and install - it into the location specified by <envar>out</envar> - (<literal>make install</literal>).</para> - - </callout> - -</calloutlist> - -<para>If you are wondering about the absence of error checking on the -result of various commands called in the builder: this is because the -shell script is evaluated with Bash's <option>-e</option> option, -which causes the script to be aborted if any command fails without an -error check.</para> - -</section>
\ No newline at end of file diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index b671811d3..d25d4b84a 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -4,18 +4,37 @@ <title>Hacking</title> -<para>This section provides some notes on how to hack on Nix. To get +<para>This section provides some notes on how to hack on Nix. To get the latest version of Nix from GitHub: <screen> -$ git clone git://github.com/NixOS/nix.git +$ git clone https://github.com/NixOS/nix.git $ cd nix </screen> </para> -<para>To build it and its dependencies: +<para>To build Nix for the current operating system/architecture use + +<screen> +$ nix-build +</screen> + +or if you have a flakes-enabled nix: + +<screen> +$ nix build +</screen> + +This will build <literal>defaultPackage</literal> attribute defined in the <literal>flake.nix</literal> file. + +To build for other platforms add one of the following suffixes to it: aarch64-linux, +i686-linux, x86_64-darwin, x86_64-linux. + +i.e. + <screen> -$ nix-build release.nix -A build.x86_64-linux +nix-build -A defaultPackage.x86_64-linux </screen> + </para> <para>To build all dependencies and start a shell in which all @@ -27,13 +46,27 @@ $ nix-shell To build Nix itself in this shell: <screen> [nix-shell]$ ./bootstrap.sh -[nix-shell]$ configurePhase -[nix-shell]$ make +[nix-shell]$ ./configure $configureFlags +[nix-shell]$ make -j $NIX_BUILD_CORES </screen> To install it in <literal>$(pwd)/inst</literal> and test it: <screen> [nix-shell]$ make install [nix-shell]$ make installcheck +[nix-shell]$ ./inst/bin/nix --version +nix (Nix) 2.4 +</screen> + +If you have a flakes-enabled nix you can replace: + +<screen> +$ nix-shell +</screen> + +by: + +<screen> +$ nix develop </screen> </para> diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1c0452dc2..500c99e4a 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -21,13 +21,13 @@ clean-files += $(GCH) $(PCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(CXX), g++) + ifeq ($(findstring g++,$(CXX)), g++) GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch GLOBAL_ORDER_AFTER += $(GCH) - else ifeq ($(CXX), clang++) + else ifeq ($(findstring clang++,$(CXX)), clang++) GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3380dbdaf..654ad1933 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2774,7 +2774,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 452b7ff32..df295084a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -350,21 +350,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - std::string s, baseName; + HashType hashAlgo; + std::string baseName; FileIngestionMethod method; { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > (uint8_t) FileIngestionMethod::Recursive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { - s = "sha256"; + hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } + hashAlgo = parseHashType(hashAlgoRaw); } - HashType hashAlgo = parseHashType(s); StringSink saved; TeeSource savedNARSource(from, saved); @@ -382,7 +385,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump(*saved.s, baseName, method, hashAlgo); + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 24e66f1db..593b55077 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -24,14 +24,6 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str }, output); } -const StorePath BasicDerivation::findOutput(const Store & store, const string & id) const -{ - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error("derivation has no output '%s'", id); - return i->second.path(store, name); -} - bool BasicDerivation::isBuiltin() const { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c8f8d10dc..65ba5e1ff 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -70,10 +70,6 @@ struct BasicDerivation BasicDerivation() { } virtual ~BasicDerivation() { }; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - const StorePath findOutput(const Store & store, const std::string & id) const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index beb508e67..4149f8155 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -124,7 +124,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -145,6 +145,7 @@ struct curlFileTransfer : public FileTransfer LambdaSink finalSink; std::shared_ptr<CompressionSink> decompressionSink; + std::optional<StringSink> errorSink; std::exception_ptr writeException; @@ -154,9 +155,19 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + errorSink = StringSink { }; + } + } + if (errorSink) + (*errorSink)((unsigned char *) contents, realSize); (*decompressionSink)((unsigned char *) contents, realSize); return realSize; @@ -412,16 +423,21 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr<std::string> response; + if (errorSink) + response = errorSink->s; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, + response, fmt("unable to %s '%s': HTTP error %d ('%s')", request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, + response, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -679,7 +695,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else @@ -824,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } } +template<typename... Args> +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how + // to print different messages for different verbosity levels. For now + // we add some heuristics for detecting when we want to show the response. + if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0..25ade0add 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,10 +103,12 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr<string> response; // intentionally optional + template<typename... Args> - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d1a3b95c1..1e6b34cd1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1026,82 +1026,26 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(method, h, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} - if (method != FileIngestionMethod::Recursive) - return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - - /* For computing the NAR hash. */ - auto sha256Sink = std::make_unique<HashSink>(htSHA256); - /* For computing the store path. In recursive SHA-256 mode, this - is the same as the NAR hash, so no need to do it again. */ - std::unique_ptr<HashSink> hashSink = - hashAlgo == htSHA256 - ? nullptr - : std::make_unique<HashSink>(hashAlgo); +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + /* For computing the store path. */ + auto hashSink = std::make_unique<HashSink>(hashAlgo); + TeeSource source { source0, *hashSink }; /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1109,55 +1053,49 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, destination store path is already valid, we just delete the temporary path. Otherwise, we move it to the destination store path. */ - bool inMemory = true; - std::string nar; - - auto source = sinkToSource([&](Sink & sink) { - - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*sha256Sink)(buf, len); - if (hashSink) (*hashSink)(buf, len); - - if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); - } else { - nar.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - - dumpPath(srcPath, sink2, filter); - }); + bool inMemory = false; + + std::string dump; + + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 65536; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } std::unique_ptr<AutoDelete> delTempDir; Path tempPath; - try { - /* Wait for the source coroutine to give us some dummy - data. This is so that we don't create the temporary - directory if the NAR fits in memory. */ - readInt(*source); + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; auto tempDir = createTempDir(realStoreDir, "add"); delTempDir = std::make_unique<AutoDelete>(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, *source); + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); - } catch (EndOfFile &) { - if (!inMemory) throw; - /* The NAR fits in memory, so we didn't do restorePath(). */ + dump.clear(); } - auto sha256 = sha256Sink->finish(); - - Hash hash = hashSink ? hashSink->finish().first : sha256.first; + auto [hash, size] = hashSink->finish(); auto dstPath = makeFixedOutputPath(method, hash, name); @@ -1179,22 +1117,34 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, autoGC(); if (inMemory) { + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ - StringSource source(nar); - restorePath(realPath, source); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) throw Error("renaming '%s' to '%s'", tempPath, realPath); } + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath optimisePath(realPath); ValidPathInfo info(dstPath); - info.narHash = sha256.first; - info.narSize = sha256.second; + info.narHash = narHash.first; + info.narSize = narHash.second; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c0e5d0286..355c2814f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -153,7 +153,7 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..968e2e425 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len, struct RefScanSink : Sink { - HashSink hashSink; StringSet hashes; StringSet seen; string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() { } void operator () (const unsigned char * data, size_t len); }; @@ -62,8 +61,6 @@ struct RefScanSink : Sink void RefScanSink::operator () (const unsigned char * data, size_t len) { - hashSink(data, len); - /* It's possible that a reference spans the previous and current fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ @@ -82,7 +79,9 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) PathSet scanForReferences(const string & path, const PathSet & refs, HashResult & hash) { - RefScanSink sink; + RefScanSink refsSink; + HashSink hashSink { htSHA256 }; + TeeSink sink { refsSink, hashSink }; std::map<string, Path> backMap; /* For efficiency (and a higher hit rate), just search for the @@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path, assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); - sink.hashes.insert(s); + refsSink.hashes.insert(s); backMap[s] = i; } @@ -106,13 +105,13 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - for (auto & i : sink.seen) { + for (auto & i : refsSink.seen) { std::map<string, Path>::iterator j; if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } - hash = sink.hashSink.finish(); + hash = hashSink.finish(); return found; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e6e5e281f..0f6f3b98f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -963,12 +963,20 @@ ref<Store> openStore(const std::string & uri_, throw Error("don't know how to open Nix store '%s'", uri); } +static bool isNonUriPath(const std::string & spec) { + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; +} StoreType getStoreType(const std::string & uri, const std::string & stateDir) { if (uri == "daemon") { return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { + } else if (uri == "local" || isNonUriPath(uri)) { return tLocal; } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -992,8 +1000,9 @@ static RegisterStoreImplementation regStore([]( return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); case tLocal: { Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; + if (isNonUriPath(uri)) { + params2["root"] = absPath(uri); + } return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); } default: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a4be0411e..d1cb2035f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -460,7 +460,7 @@ public: std::optional<Hash> expectedCAHash = {}); // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c8b71188f..00c945113 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -322,5 +322,18 @@ void StringSink::operator () (const unsigned char * data, size_t len) s->append((const char *) data, len); } +size_t ChainSource::read(unsigned char * data, size_t len) +{ + if (useSecond) { + return source2.read(data, len); + } else { + try { + return source1.read(data, len); + } catch (EndOfFile &) { + useSecond = true; + return this->read(data, len); + } + } +} } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8386a4991..5d9acf887 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -189,7 +189,7 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, len); + sink(data, n); return n; } }; @@ -256,6 +256,19 @@ struct LambdaSource : Source } }; +/* Chain two sources together so after the first is exhausted, the second is + used */ +struct ChainSource : Source +{ + Source & source1, & source2; + bool useSecond = false; + ChainSource(Source & s1, Source & s2) + : source1(s1), source2(s2) + { } + + size_t read(unsigned char * data, size_t len) override; +}; + /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 93798a765..a0a8ff4d3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1581,7 +1581,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) struct sockaddr_un addr; addr.sun_family = AF_UNIX; - if (path.size() >= sizeof(addr.sun_path)) + if (path.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%1%' is too long", path); strcpy(addr.sun_path, path.c_str()); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1a36dbee3..ddd036070 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -381,7 +381,8 @@ static void queryInstSources(EvalState & state, if (path.isDerivation()) { elem.setDrvPath(state.store->printStorePath(path)); - elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput(*state.store, "out"))); + auto outputs = state.store->queryDerivationOutputMap(path); + elem.setOutPath(state.store->printStorePath(outputs.at("out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 16d7e511f..ebee4545c 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -111,6 +111,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; userRegistry->add(ref.input, resolved, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); } }; diff --git a/tests/local-store.sh b/tests/local-store.sh new file mode 100644 index 000000000..4ec3d64b0 --- /dev/null +++ b/tests/local-store.sh @@ -0,0 +1,20 @@ +source common.sh + +cd $TEST_ROOT + +echo example > example.txt +mkdir -p ./x + +NIX_STORE_DIR=$TEST_ROOT/x + +CORRECT_PATH=$(nix-store --store ./x --add example.txt) + +PATH1=$(nix path-info --store ./x $CORRECT_PATH) +[ $CORRECT_PATH == $PATH1 ] + +PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) +[ $CORRECT_PATH == $PATH2 ] + +# FIXME we could also test the query parameter version: +# PATH3=$(nix path-info --store "local?store=$PWD/x" $CORRECT_PATH) +# [ $CORRECT_PATH == $PATH3 ] diff --git a/tests/local.mk b/tests/local.mk index 81366160b..0f3bfe606 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -6,7 +6,7 @@ nix_tests = \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh \ + local-store.sh remote-store.sh export.sh export-graph.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ |