From 8eac7dfad427acff412d2cd1a0de6e6e683aac0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Aug 2021 13:52:19 +0200 Subject: Remove trash directory --- src/libstore/local-store.hh | 1 - 1 file changed, 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a01d48c4b..5215c9f02 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -87,7 +87,6 @@ public: const Path linksDir; const Path reservedPath; const Path schemaPath; - const Path trashDir; const Path tempRootsDir; const Path fnTempRoots; -- cgit v1.2.3 From 9947f1646a26b339fff2e02b77798e9841fac7f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Aug 2021 18:53:14 +0200 Subject: Remove syncWithGC() --- src/libstore/local-store.hh | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5215c9f02..3b3f26162 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -148,8 +148,6 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - private: typedef std::shared_ptr FDPtr; -- cgit v1.2.3 From 8614cf13344eca75074cd4af20fd90238571b0b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Aug 2021 20:03:32 +0200 Subject: Non-blocking garbage collector The garbage collector no longer blocks other processes from adding/building store paths or adding GC roots. To prevent the collector from deleting store paths just added by another process, processes need to connect to the garbage collector via a Unix domain socket to register new temporary roots. --- src/libstore/local-store.hh | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3b3f26162..85c6c4495 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,9 +58,15 @@ private: struct Stmts; std::unique_ptr stmts; + /* The global GC lock */ + AutoCloseFD fdGCLock; + /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; + /* Connection to the garbage collector. */ + AutoCloseFD fdRootsSocket; + /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point lastGCCheck; @@ -150,10 +156,7 @@ public: private: - typedef std::shared_ptr FDPtr; - typedef list FDs; - - void findTempRoots(FDs & fds, Roots & roots, bool censor); + void findTempRoots(Roots & roots, bool censor); public: @@ -235,18 +238,11 @@ private: struct GCState; - void deleteGarbage(GCState & state, const Path & path); - - void tryToDelete(GCState & state, const Path & path); - - bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); - - void deletePathRecursive(GCState & state, const Path & path); - - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); - - AutoCloseFD openGCLock(LockType lockType); + bool tryToDelete( + GCState & state, + StorePathSet & visited, + const Path & path, + bool recursive); void findRoots(const Path & path, unsigned char type, Roots & roots); -- cgit v1.2.3 From ff453b06f94b4305694beac8255d9ff51bed1a63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Aug 2021 22:43:43 +0200 Subject: Fix auto-gc --- src/libstore/local-store.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 85c6c4495..12c057ec3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -158,6 +158,8 @@ private: void findTempRoots(Roots & roots, bool censor); + AutoCloseFD openGCLock(); + public: Roots findRoots(bool censor) override; -- cgit v1.2.3 From 1785ba2980c9dd8274dd4bec5c0e46be02544b7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Oct 2021 14:58:35 +0200 Subject: Simplify --- src/libstore/local-store.hh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 12c057ec3..08bbc1a7d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -243,9 +243,11 @@ private: bool tryToDelete( GCState & state, StorePathSet & visited, - const Path & path, + const StorePath & path, bool recursive); + void deleteFromStore(GCState & state, std::string_view baseName); + void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); -- cgit v1.2.3 From 35c98a59c5f09d8ca376ac809e87d14ff2fcbde1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Oct 2021 16:58:19 +0200 Subject: Fix GC when there are cycles in the referrers graph (where "referrers" includes the reverse of derivation outputs and derivers). Now we do a full traversal to look if we can reach any root. If not, all paths reached can be deleted. --- src/libstore/local-store.hh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 08bbc1a7d..8629e1528 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -240,11 +240,10 @@ private: struct GCState; - bool tryToDelete( + bool canReachRoot( GCState & state, StorePathSet & visited, - const StorePath & path, - bool recursive); + const StorePath & path); void deleteFromStore(GCState & state, std::string_view baseName); -- cgit v1.2.3 From eab934cb2a23595a7ac7c8a72373cd8096b606a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Oct 2021 12:31:21 +0200 Subject: Make the canReachRoots() traversal non-recursive --- src/libstore/local-store.hh | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8629e1528..21a4e60cf 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -240,11 +240,6 @@ private: struct GCState; - bool canReachRoot( - GCState & state, - StorePathSet & visited, - const StorePath & path); - void deleteFromStore(GCState & state, std::string_view baseName); void findRoots(const Path & path, unsigned char type, Roots & roots); -- cgit v1.2.3 From 0317ffdad3b2137d35e501b38155437e2ec5d0cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Oct 2021 13:34:48 +0200 Subject: Move deleteFromStore() --- src/libstore/local-store.hh | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 21a4e60cf..15475f402 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -240,8 +240,6 @@ private: struct GCState; - void deleteFromStore(GCState & state, std::string_view baseName); - void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); -- cgit v1.2.3 From 0154fa30cf2a6da63e69f7488cec9e289f57cc15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Oct 2021 13:52:49 +0200 Subject: Remove GCState --- src/libstore/local-store.hh | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 15475f402..301425eb1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -238,16 +238,12 @@ private: PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); - struct GCState; - void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); void findRuntimeRoots(Roots & roots, bool censor); - void removeUnusedLinks(const GCState & state); - Path createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); -- cgit v1.2.3 From b9234142f5e3c69f3372f15ab2e6df19c7bf6b64 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Sat, 23 Oct 2021 21:30:51 +0300 Subject: addToStore, addToStoreFromDump: add references argument Allow to pass a set of references to be added as info to the added paths. --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a01d48c4b..7a7329dd8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; -- cgit v1.2.3 From 96670ed2163d3d1a296c9b053833362ec8c06985 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 27 Oct 2021 11:36:51 +0200 Subject: Expose an async interface for `queryRealisation` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doesn’t change much so far because everything is still using it synchronously, but should allow the binary cache to fetch stuff in parallel --- src/libstore/local-store.hh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 301425eb1..b172e897a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -207,7 +207,8 @@ public: std::optional queryRealisation_(State & state, const DrvOutput & id); std::optional> queryRealisationCore_(State & state, const DrvOutput & id); - std::optional queryRealisation(const DrvOutput&) override; + void queryRealisationUncached(const DrvOutput&, + Callback> callback) noexcept override; private: -- cgit v1.2.3 From 0b005bc9d67b3f621bc78e5ecb2cd834172d5563 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Tue, 9 Nov 2021 12:24:49 +0300 Subject: addToStore, addToStoreFromDump: refactor: pass refs by const reference Co-Authored-By: Eelco Dolstra --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7a7329dd8..c6441ef01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; -- cgit v1.2.3 From e9a4abdb5d6fe8e128372a77d879b0187b1bacfe Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 11 Jan 2022 11:57:45 +0100 Subject: Make --repair-path also repair corrupt optimised links There already existed a smoke test for the link content length, but it appears that there exists some corruptions pernicious enough to replace the file content with zeros, and keeping the same length. --repair-path now goes as far as checking the content of the link, making it true to its name and actually repairing the path for such coruption cases. --- src/libstore/local-store.hh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 115ea046a..04698ba07 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -172,8 +172,9 @@ public: void optimiseStore() override; - /* Optimise a single store path. */ - void optimisePath(const Path & path); + /* Optimise a single store path. Optionally, test the encountered + symlinks for corruption. */ + void optimisePath(const Path & path, bool repair); bool verifyStore(bool checkContents, RepairFlag repair) override; @@ -253,7 +254,7 @@ private: InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); + void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, bool repair); // Internal versions that are not wrapped in retry_sqlite. bool isValidPath_(State & state, const StorePath & path); -- cgit v1.2.3 From 9f9f39a24b515d5d3ddd2bbd4cdadb0b1924bc3b Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 11 Jan 2022 13:34:57 +0100 Subject: Prefer RepairFlag over bool when applicable --- src/libstore/local-store.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 04698ba07..c4d7b80bd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -174,7 +174,7 @@ public: /* Optimise a single store path. Optionally, test the encountered symlinks for corruption. */ - void optimisePath(const Path & path, bool repair); + void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; @@ -254,7 +254,7 @@ private: InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, bool repair); + void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair); // Internal versions that are not wrapped in retry_sqlite. bool isValidPath_(State & state, const StorePath & path); -- cgit v1.2.3 From 4dda1f92aae05dd9d633152458d65a3815bcd03c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jan 2022 19:45:21 +0100 Subject: Add command 'nix store copy-log' Fixes #5222. --- src/libstore/local-store.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c4d7b80bd..6d867d778 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -280,6 +280,8 @@ private: const std::string_view pathHash ); + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; -- cgit v1.2.3 From 35dbdbedd41dea45bf38ae11d74f72c39eb304c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Jan 2022 21:14:27 +0100 Subject: nix store ping: Report Nix daemon version Fixes #5952. --- src/libstore/local-store.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d867d778..8cf9c68b3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -211,6 +211,8 @@ public: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + std::optional getVersion() override; + private: int getSchema(); -- cgit v1.2.3 From 2d6d9a28ebb17b1ba1fe0dc4d56b6aa311f94d39 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Feb 2022 15:08:06 +0100 Subject: addToStoreFromDump(): Take std::string_view --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8cf9c68b3..46aed9bcb 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -144,7 +144,7 @@ public: void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, -- cgit v1.2.3 From fe9afb65bb35737a144acd612170b2e284298a2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Feb 2022 16:28:23 +0100 Subject: Remove std::set alias --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 46aed9bcb..a9c7475ac 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -292,7 +292,7 @@ private: typedef std::pair Inode; -typedef set InodesSeen; +typedef std::set InodesSeen; /* "Fix", or canonicalise, the meta-data of the files in a store path -- cgit v1.2.3 From df552ff53e68dff8ca360adbdbea214ece1d08ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Feb 2022 16:00:00 +0100 Subject: Remove std::string alias (for real this time) Also use std::string_view in a few more places. --- src/libstore/local-store.hh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/libstore/local-store.hh') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a9c7475ac..1a278c9a8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -147,8 +147,11 @@ public: StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override; + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; void addTempRoot(const StorePath & path) override; @@ -204,7 +207,11 @@ public: derivation 'deriver'. */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; - void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); + void cacheDrvOutputMapping( + State & state, + const uint64_t deriver, + const std::string & outputName, + const StorePath & output); std::optional queryRealisation_(State & state, const DrvOutput & id); std::optional> queryRealisationCore_(State & state, const DrvOutput & id); -- cgit v1.2.3