From 47baa9d43c0339b0a738b9b75c5ddcfb07d7131d Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 1 Jan 2022 19:24:20 +0100 Subject: make Pos smaller reduces peak hep memory use on eval of our test system from 264.4MB to 242.3MB, possibly also a slight performance boost. theoretically memory use could be cut down by another eight bytes per Pos on average by turning it into a tuple containing an index into a global base position table with row and column offsets, but that doesn't seem worth the effort at this point. --- src/libutil/error.hh | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d55e1d701..bb43aa53b 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -53,6 +53,7 @@ typedef enum { lvlVomit } Verbosity; +/* adjust Pos::origin bit width when adding stuff here */ typedef enum { foFile, foStdin, -- cgit v1.2.3 From 8e2eaaaf69d9e216fce3ca6f7913bd0e2048e4b2 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 31 Dec 2021 00:50:23 +0100 Subject: make Finally more local no need for function<> with c++17 deduction. this saves allocations and virtual calls, but has the same semantics otherwise. not going through function has the side effect of giving compilers more insight into the cleanup code, so we need a few local warning disables. --- src/libutil/finally.hh | 7 +++---- src/libutil/util.cc | 15 ++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index 7760cfe9a..dee2e8d2f 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -1,14 +1,13 @@ #pragma once -#include - /* A trivial class to run a function at the end of a scope. */ +template class Finally { private: - std::function fun; + Fn fun; public: - Finally(std::function fun) : fun(fun) { } + Finally(Fn fun) : fun(std::move(fun)) { } ~Finally() { fun(); } }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b833038a9..9f13d5f02 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -682,7 +682,14 @@ std::string drainFD(int fd, bool block, const size_t reserveSize) void drainFD(int fd, Sink & sink, bool block) { - int saved; + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } Finally finally([&]() { if (!block) { @@ -691,12 +698,6 @@ void drainFD(int fd, Sink & sink, bool block) } }); - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - std::vector buf(64 * 1024); while (1) { checkInterrupt(); -- cgit v1.2.3 From 6b1872312f1b505334acb67b8bf7990b0a0fdbd8 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sat, 12 Mar 2022 21:56:08 +0000 Subject: nix store gc: account for auto-optimised store Before the change on a system with `auto-optimise-store = true`: $ nix store gc --verbose --max 1 deleted all the paths instead of one path (we requested 1 byte limit). It happens because every file in `auto-optimise-store = true` has at least 2 links: file itself and a link in /nix/store/.links/ directory. The change conservatively assumes that any file that has one (as before) or two links (assume auto-potimise mode) will free space. Co-authored-by: Sandro --- src/libutil/util.cc | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9f13d5f02..70eaf4f9c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -406,8 +406,29 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) throw SysError("getting status of '%1%'", path); } - if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_size; + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ -- cgit v1.2.3 From 34e20c164cd512c10af99803215e32889a6d965b Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 18 Feb 2022 16:56:35 +0100 Subject: libfetchers/path: set `lastModified` to path's mtime When importing e.g. a local `nixpkgs` in a flake to test a change like { inputs.nixpkgs.url = path:/home/ma27/Projects/nixpkgs; outputs = /* ... */ } then the input is missing a `lastModified`-field that's e.g. used in `nixpkgs.lib.nixosSystem`. Due to the missing `lastMoified`-field, the mtime is set to 19700101: result -> /nix/store/b7dg1lmmsill2rsgyv2w7b6cnmixkvc1-nixos-system-nixos-22.05.19700101.dirty With this change, the `path`-fetcher now sets a `lastModified` attribute to the `mtime` just like it's the case in the `tarball`-fetcher already. When building NixOS systems with `nixpkgs` being a `path`-input and this patch, the output-path now looks like this: result -> /nix/store/ld2qf9c1s98dxmiwcaq5vn9k5ylzrm1s-nixos-system-nixos-22.05.20220217.dirty --- src/libutil/archive.cc | 19 +++++++++++++++---- src/libutil/archive.hh | 4 ++++ 2 files changed, 19 insertions(+), 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index eda004756..30b471af5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -64,11 +64,12 @@ static void dumpContents(const Path & path, off_t size, } -static void dump(const Path & path, Sink & sink, PathFilter & filter) +static time_t dump(const Path & path, Sink & sink, PathFilter & filter) { checkInterrupt(); auto st = lstat(path); + time_t result = st.st_mtime; sink << "("; @@ -103,7 +104,10 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) for (auto & i : unhacked) if (filter(path + "/" + i.first)) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + "/" + i.second, sink, filter); + auto tmp_mtime = dump(path + "/" + i.second, sink, filter); + if (tmp_mtime > result) { + result = tmp_mtime; + } sink << ")"; } } @@ -114,13 +118,20 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else throw Error("file '%1%' has an unsupported type", path); sink << ")"; + + return result; } -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { sink << narVersionMagic1; - dump(path, sink, filter); + return dump(path, sink, filter); +} + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + dumpPathAndGetMtime(path, sink, filter); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fca351605..79ce08df0 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -48,6 +48,10 @@ namespace nix { void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/* Same as `void dumpPath()`, but returns the last modified date of the path */ +time_t dumpPathAndGetMtime(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + void dumpString(std::string_view s, Sink & sink); /* FIXME: fix this API, it sucks. */ -- cgit v1.2.3 From f902f3c2cbfa1a78bec8d135297abe244452e794 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 21:33:20 +0100 Subject: Add experimental feature 'fetch-closure' --- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b49f47e1d..01f318fa3 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -11,6 +11,7 @@ std::map stringifiedXpFeatures = { { Xp::NixCommand, "nix-command" }, { Xp::RecursiveNix, "recursive-nix" }, { Xp::NoUrlLiterals, "no-url-literals" }, + { Xp::FetchClosure, "fetch-closure" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 291a58e32..b5140dcfe 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -19,7 +19,8 @@ enum struct ExperimentalFeature Flakes, NixCommand, RecursiveNix, - NoUrlLiterals + NoUrlLiterals, + FetchClosure, }; /** -- cgit v1.2.3 From c85467a1b65d2a8907fcab3303572f76c071776d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 6 Feb 2022 14:47:07 +0100 Subject: Revert "TarArchive: Small refactoring" This reverts commit 50a35860ee9237d341948437c5f70a7f0987d393. With this change Nix fails to open bzip2 logfiles that were created from builds with no stdout/stderr. --- src/libutil/tarfile.cc | 26 ++++++++++++++------------ src/libutil/tarfile.hh | 3 --- 2 files changed, 14 insertions(+), 15 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 790bc943a..a7db58559 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -39,28 +39,30 @@ void TarArchive::check(int err, const std::string & reason) throw Error(reason, archive_error_string(this->archive)); } -TarArchive::TarArchive(Source & source, bool raw) - : source(&source), buffer(4096) +TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) { - init(); - if (!raw) + this->archive = archive_read_new(); + this->source = &source; + + if (!raw) { + archive_read_support_filter_all(archive); archive_read_support_format_all(archive); - else + } else { + archive_read_support_filter_all(archive); archive_read_support_format_raw(archive); + archive_read_support_format_empty(archive); + } check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); } + TarArchive::TarArchive(const Path & path) { - init(); - archive_read_support_format_all(archive); - check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); -} + this->archive = archive_read_new(); -void TarArchive::init() -{ - archive = archive_read_new(); archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } void TarArchive::close() diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index f107a7e2e..4d9141fd4 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -17,13 +17,10 @@ struct TarArchive { // disable copy constructor TarArchive(const TarArchive &) = delete; - void init(); - void close(); ~TarArchive(); }; - void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); -- cgit v1.2.3