Get rid of the old eval cache

1 files changed, 5 insertions, 497 deletions

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 9e46cc55a..2f2dd65c7 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -11,7 +11,7 @@
 #include "fetchers.hh"
 #include "registry.hh"
 #include "json.hh"
-#include "sqlite.hh"
+#include "flake/eval-cache.hh"
 
 #include <nlohmann/json.hpp>
 #include <queue>
@@ -669,479 +669,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
     }
 };
 
-// FIXME: inefficient representation of attrs
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    primary key (parent, name)
-);
-)sql";
-
-enum AttrType {
-    Placeholder = 0,
-    FullAttrs = 1,
-    String = 2,
-    Missing = 3,
-    Misc = 4,
-    Failed = 5,
-};
-
-struct AttrDb
-{
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    struct placeholder_t {};
-    struct missing_t {};
-    struct misc_t {};
-    struct failed_t {};
-    typedef uint64_t AttrId;
-    typedef std::pair<AttrId, Symbol> AttrKey;
-    typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Fingerprint & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select rowid, type, value from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    AttrId setAttrs(
-        AttrKey key,
-        const std::vector<Symbol> & attrs)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::FullAttrs)
-            (0, false).exec();
-
-        AttrId rowId = state->db.getLastInsertedRowId();
-        assert(rowId);
-
-        for (auto & attr : attrs)
-            state->insertAttribute.use()
-                (rowId)
-                (attr)
-                (AttrType::Placeholder)
-                (0, false).exec();
-
-        return rowId;
-    }
-
-    AttrId setString(
-        AttrKey key,
-        std::string_view s)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::String)
-            (s).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setPlaceholder(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Placeholder)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setMissing(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Missing)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setMisc(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Misc)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setFailed(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Failed)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getAttr(
-        AttrKey key,
-        SymbolTable & symbols)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        if (type == AttrType::Placeholder)
-            return {{rowId, placeholder_t()}};
-        else if (type == AttrType::FullAttrs) {
-            // FIXME: expensive, should separate this out.
-            std::vector<Symbol> attrs;
-            auto queryAttributes(state->queryAttributes.use()(rowId));
-            while (queryAttributes.next())
-                attrs.push_back(symbols.create(queryAttributes.getStr(0)));
-            return {{rowId, attrs}};
-        } else if (type == AttrType::String) {
-            return {{rowId, queryAttribute.getStr(2)}};
-        } else if (type == AttrType::Missing) {
-            return {{rowId, missing_t()}};
-        } else if (type == AttrType::Misc) {
-            return {{rowId, misc_t()}};
-        } else if (type == AttrType::Failed) {
-            return {{rowId, failed_t()}};
-        } else
-            throw Error("unexpected type in evaluation cache");
-    }
-};
-
-struct AttrCursor;
-
-struct AttrRoot : std::enable_shared_from_this<AttrRoot>
-{
-    std::shared_ptr<AttrDb> db;
-    EvalState & state;
-    typedef std::function<Value *()> RootLoader;
-    RootLoader rootLoader;
-    RootValue value;
-
-    AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, RootLoader rootLoader)
-        : db(db)
-        , state(state)
-        , rootLoader(rootLoader)
-    {
-    }
-
-    Value * getRootValue()
-    {
-        if (!value) {
-            debug("getting root value");
-            value = allocRootValue(rootLoader());
-        }
-        return *value;
-    }
-
-    std::shared_ptr<AttrCursor> getRoot()
-    {
-        return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
-    }
-};
-
-struct AttrCursor : std::enable_shared_from_this<AttrCursor>
-{
-    ref<AttrRoot> root;
-    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
-    Parent parent;
-    RootValue _value;
-    std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue;
-
-    AttrCursor(
-        ref<AttrRoot> root,
-        Parent parent,
-        Value * value = nullptr,
-        std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> && cachedValue = {})
-        : root(root), parent(parent), cachedValue(std::move(cachedValue))
-    {
-        if (value)
-            _value = allocRootValue(value);
-    }
-
-    AttrDb::AttrKey getAttrKey()
-    {
-        if (!parent)
-            return {0, root->state.sEpsilon};
-        if (!parent->first->cachedValue) {
-            parent->first->cachedValue = root->db->getAttr(
-                parent->first->getAttrKey(), root->state.symbols);
-            assert(parent->first->cachedValue);
-        }
-        return {parent->first->cachedValue->first, parent->second};
-    }
-
-    Value & getValue()
-    {
-        if (!_value) {
-            if (parent) {
-                auto & vParent = parent->first->getValue();
-                root->state.forceAttrs(vParent);
-                auto attr = vParent.attrs->get(parent->second);
-                if (!attr)
-                    throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
-                _value = allocRootValue(attr->value);
-            } else
-                _value = allocRootValue(root->getRootValue());
-        }
-        return **_value;
-    }
-
-    std::vector<Symbol> getAttrPath() const
-    {
-        if (parent) {
-            auto attrPath = parent->first->getAttrPath();
-            attrPath.push_back(parent->second);
-            return attrPath;
-        } else
-            return {};
-    }
-
-    std::vector<Symbol> getAttrPath(Symbol name) const
-    {
-        auto attrPath = getAttrPath();
-        attrPath.push_back(name);
-        return attrPath;
-    }
-
-    std::string getAttrPathStr() const
-    {
-        return concatStringsSep(".", getAttrPath());
-    }
-
-    std::string getAttrPathStr(Symbol name) const
-    {
-        return concatStringsSep(".", getAttrPath(name));
-    }
-
-    Value & forceValue()
-    {
-        debug("evaluating uncached attribute %s", getAttrPathStr());
-
-        auto & v = getValue();
-
-        try {
-            root->state.forceValue(v);
-        } catch (EvalError &) {
-            debug("setting '%s' to failed", getAttrPathStr());
-            if (root->db)
-                cachedValue = {root->db->setFailed(getAttrKey()), AttrDb::failed_t()};
-            throw;
-        }
-
-        if (root->db && (!cachedValue || std::get_if<AttrDb::placeholder_t>(&cachedValue->second))) {
-            if (v.type == tString)
-                cachedValue = {root->db->setString(getAttrKey(), v.string.s), v.string.s};
-            else if (v.type == tAttrs)
-                ; // FIXME: do something?
-            else
-                cachedValue = {root->db->setMisc(getAttrKey()), AttrDb::misc_t()};
-        }
-
-        return v;
-    }
-
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
-    {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
-
-            if (cachedValue) {
-                if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
-                    for (auto & attr : *attrs)
-                        if (attr == name)
-                            return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
-                    return nullptr;
-                } else if (std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
-                    auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
-                    if (attr) {
-                        if (std::get_if<AttrDb::missing_t>(&attr->second))
-                            return nullptr;
-                        else if (std::get_if<AttrDb::failed_t>(&attr->second))
-                            throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
-                        else
-                            return std::make_shared<AttrCursor>(root,
-                                std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
-                    }
-                    // Incomplete attrset, so need to fall thru and
-                    // evaluate to see whether 'name' exists
-                } else
-                    return nullptr;
-                    //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-            }
-        }
-
-        auto & v = forceValue();
-
-        if (v.type != tAttrs)
-            return nullptr;
-            //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-
-        auto attr = v.attrs->get(name);
-
-        if (!attr) {
-            if (root->db) {
-                assert(cachedValue);
-                root->db->setMissing({cachedValue->first, name});
-            }
-            return nullptr;
-        }
-
-        std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue2;
-        if (root->db) {
-            assert(cachedValue);
-            cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), AttrDb::placeholder_t()};
-        }
-
-        return std::make_shared<AttrCursor>(
-            root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
-    }
-
-    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name)
-    {
-        return maybeGetAttr(root->state.symbols.create(name));
-    }
-
-    std::shared_ptr<AttrCursor> getAttr(Symbol name)
-    {
-        auto p = maybeGetAttr(name);
-        if (!p)
-            throw Error("attribute '%s' does not exist", getAttrPathStr(name));
-        return p;
-    }
-
-    std::shared_ptr<AttrCursor> getAttr(std::string_view name)
-    {
-        return getAttr(root->state.symbols.create(name));
-    }
-
-    std::string getString()
-    {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
-            if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
-                if (auto s = std::get_if<std::string>(&cachedValue->second)) {
-                    debug("using cached string attribute '%s'", getAttrPathStr());
-                    return *s;
-                } else
-                    throw TypeError("'%s' is not a string", getAttrPathStr());
-            }
-        }
-
-        auto & v = forceValue();
-
-        if (v.type != tString)
-            throw TypeError("'%s' is not a string", getAttrPathStr());
-
-        return v.string.s;
-    }
-
-    std::vector<Symbol> getAttrs()
-    {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
-            if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
-                if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
-                    debug("using cached attrset attribute '%s'", getAttrPathStr());
-                    return *attrs;
-                } else
-                    throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-            }
-        }
-
-        auto & v = forceValue();
-
-        if (v.type != tAttrs)
-            throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-
-        std::vector<Symbol> attrs;
-        for (auto & attr : *getValue().attrs)
-            attrs.push_back(attr.name);
-        std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
-            return (const string &) a < (const string &) b;
-        });
-
-        if (root->db)
-            cachedValue = {root->db->setAttrs(getAttrKey(), attrs), attrs};
-
-        return attrs;
-    }
-
-    bool isDerivation()
-    {
-        auto aType = maybeGetAttr("type");
-        return aType && aType->getString() == "derivation";
-    }
-};
-
 struct CmdFlakeShow : FlakeCommand
 {
     bool showLegacy = false;
@@ -1170,9 +697,9 @@ struct CmdFlakeShow : FlakeCommand
         auto state = getEvalState();
         auto flake = lockFlake();
 
-        std::function<void(AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
+        std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
 
-        visit = [&](AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
+        visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@@ -1286,28 +813,9 @@ struct CmdFlakeShow : FlakeCommand
             }
         };
 
-        auto db = useEvalCache ? std::make_shared<AttrDb>(flake.getFingerprint()) : nullptr;
-
-        auto root = std::make_shared<AttrRoot>(db, *state,
-            [&]()
-            {
-                /* For testing whether the evaluation cache is
-                   complete. */
-                if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
-                    throw Error("not everything is cached, but evaluation is not allowed");
-
-                auto vFlake = state->allocValue();
-                flake::callFlake(*state, flake, *vFlake);
-
-                state->forceAttrs(*vFlake);
-
-                auto aOutputs = vFlake->attrs->get(state->symbols.create("outputs"));
-                assert(aOutputs);
-
-                return aOutputs->value;
-            });
+        auto cache = openEvalCache(*state, flake, useEvalCache);
 
-        visit(*root->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
+        visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
     }
 };