Merge pull request #6218 from pennae/pos-symbol-tables

reduce the size of Attr from 3 pointers to 2 on 64 bit machines

5 files changed, 124 insertions, 48 deletions

diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh
new file mode 100644
index 000000000..f15af9cd7
--- /dev/null
+++ b/src/libutil/chunked-vector.hh
@@ -0,0 +1,68 @@
+#pragma once
+
+#include <bits/stdint-uintn.h>
+#include <cstdlib>
+#include <vector>
+#include <limits>
+
+namespace nix {
+
+/* Provides an indexable container like vector<> with memory overhead
+   guarantees like list<> by allocating storage in chunks of ChunkSize
+   elements instead of using a contiguous memory allocation like vector<>
+   does. Not using a single vector that is resized reduces memory overhead
+   on large data sets by on average (growth factor)/2, mostly
+   eliminates copies within the vector during resizing, and provides stable
+   references to its elements. */
+template<typename T, size_t ChunkSize>
+class ChunkedVector {
+private:
+    uint32_t size_ = 0;
+    std::vector<std::vector<T>> chunks;
+
+    /* keep this out of the ::add hot path */
+    [[gnu::noinline]]
+    auto & addChunk()
+    {
+        if (size_ >= std::numeric_limits<uint32_t>::max() - ChunkSize)
+            abort();
+        chunks.emplace_back();
+        chunks.back().reserve(ChunkSize);
+        return chunks.back();
+    }
+
+public:
+    ChunkedVector(uint32_t reserve)
+    {
+        chunks.reserve(reserve);
+        addChunk();
+    }
+
+    uint32_t size() const { return size_; }
+
+    std::pair<T &, uint32_t> add(T value)
+    {
+        const auto idx = size_++;
+        auto & chunk = [&] () -> auto & {
+            if (auto & back = chunks.back(); back.size() < ChunkSize)
+                return back;
+            return addChunk();
+        }();
+        auto & result = chunk.emplace_back(std::move(value));
+        return {result, idx};
+    }
+
+    const T & operator[](uint32_t idx) const
+    {
+        return chunks[idx / ChunkSize][idx % ChunkSize];
+    }
+
+    template<typename Fn>
+    void forEach(Fn fn) const
+    {
+        for (const auto & c : chunks)
+            for (const auto & e : c)
+                fn(e);
+    }
+};
+}
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 348018f57..f4706e3ed 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -87,11 +87,7 @@ struct ErrPos {
         origin = pos.origin;
         line = pos.line;
         column = pos.column;
-        // is file symbol null?
-        if (pos.file.set())
-            file = pos.file;
-        else
-            file = "";
+        file = pos.file;
         return *this;
     }
 
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
index 347b81f73..f9578afc7 100644
--- a/src/libutil/ref.hh
+++ b/src/libutil/ref.hh
@@ -99,47 +99,4 @@ make_ref(Args&&... args)
     return ref<T>(p);
 }
 
-
-/* A non-nullable pointer.
-   This is similar to a C++ "& reference", but mutable.
-   This is similar to ref<T> but backed by a regular pointer instead of a smart pointer.
- */
-template<typename T>
-class ptr {
-private:
-    T * p;
-
-public:
-    ptr<T>(const ptr<T> & r)
-        : p(r.p)
-    { }
-
-    explicit ptr<T>(T * p)
-        : p(p)
-    {
-        if (!p)
-            throw std::invalid_argument("null pointer cast to ptr");
-    }
-
-    T* operator ->() const
-    {
-        return &*p;
-    }
-
-    T& operator *() const
-    {
-        return *p;
-    }
-
-    bool operator == (const ptr<T> & other) const
-    {
-        return p == other.p;
-    }
-
-    bool operator != (const ptr<T> & other) const
-    {
-        return p != other.p;
-    }
-};
-
 }
diff --git a/src/libutil/tests/chunked-vector.cc b/src/libutil/tests/chunked-vector.cc
new file mode 100644
index 000000000..868d11f6f
--- /dev/null
+++ b/src/libutil/tests/chunked-vector.cc
@@ -0,0 +1,54 @@
+#include "chunked-vector.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+    TEST(ChunkedVector, InitEmpty) {
+        auto v = ChunkedVector<int, 2>(100);
+        ASSERT_EQ(v.size(), 0);
+    }
+
+    TEST(ChunkedVector, GrowsCorrectly) {
+        auto v = ChunkedVector<int, 2>(100);
+        for (auto i = 1; i < 20; i++) {
+            v.add(i);
+            ASSERT_EQ(v.size(), i);
+        }
+    }
+
+    TEST(ChunkedVector, AddAndGet) {
+        auto v = ChunkedVector<int, 2>(100);
+        for (auto i = 1; i < 20; i++) {
+            auto [i2, idx] = v.add(i);
+            auto & i3 = v[idx];
+            ASSERT_EQ(i, i2);
+            ASSERT_EQ(&i2, &i3);
+        }
+    }
+
+    TEST(ChunkedVector, ForEach) {
+        auto v = ChunkedVector<int, 2>(100);
+        for (auto i = 1; i < 20; i++) {
+            v.add(i);
+        }
+        int count = 0;
+        v.forEach([&count](int elt) {
+            count++;
+        });
+        ASSERT_EQ(count, v.size());
+    }
+
+    TEST(ChunkedVector, OverflowOK) {
+        // Similar to the AddAndGet, but intentionnally use a small
+        // initial ChunkedVector to force it to overflow
+        auto v = ChunkedVector<int, 2>(2);
+        for (auto i = 1; i < 20; i++) {
+            auto [i2, idx] = v.add(i);
+            auto & i3 = v[idx];
+            ASSERT_EQ(i, i2);
+            ASSERT_EQ(&i2, &i3);
+        }
+    }
+
+}
+
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 00ba567c6..6bcbd7e1d 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -5,6 +5,7 @@
 #include <list>
 #include <set>
 #include <string>
+#include <limits>
 #include <map>
 #include <variant>
 #include <vector>