* Cache the manifests in /nix/var/nix/manifests in a SQLite database.

  This significantly speeds up the download-using-manifests
  substituter, especially if manifests are very large.  For instance,
  one "nix-build -A geeqie" operation that updated four packages using
  binary patches went from 18.5s to 1.6s.  It also significantly
  reduces memory use.

  The cache is kept in /nix/var/nix/manifests/cache.sqlite.  It's
  updated automatically when manifests are added to or removed from
  /nix/var/nix/manifests.  It might be interesting to have nix-pull
  store manifests directly in the DB, rather than storing them as
  separate flat files, but then we would need a command line interface
  to delete manifests from the DB.

1 files changed, 138 insertions, 4 deletions

diff --git a/scripts/NixManifest.pm.in b/scripts/NixManifest.pm.in
index 21c8b6ba2..998c23ef1 100644
--- a/scripts/NixManifest.pm.in
+++ b/scripts/NixManifest.pm.in
@@ -1,4 +1,7 @@
 use strict;
+use DBI;
+use Cwd;
+use File::stat;
 
 
 sub addPatch {
@@ -34,7 +37,7 @@ sub readManifest {
     my $manifestVersion = 2;
 
     my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType);
-    my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom, $system);
+    my ($narHash, $narSize, $references, $deriver, $copyFrom, $system);
 
     while (<MANIFEST>) {
         chomp;
@@ -59,7 +62,6 @@ sub readManifest {
                 undef $system;
                 $references = "";
                 $deriver = "";
-                $hashAlgo = "md5";
 	    }
 
         } else {
@@ -83,7 +85,7 @@ sub readManifest {
                             { url => $url, hash => $hash, size => $size
                             , narHash => $narHash, narSize => $narSize
                             , references => $references
-                            , deriver => $deriver, hashAlgo => $hashAlgo
+                            , deriver => $deriver
                             , system => $system
                             };
                     }
@@ -95,7 +97,7 @@ sub readManifest {
                         { url => $url, hash => $hash, size => $size
                         , basePath => $basePath, baseHash => $baseHash
                         , narHash => $narHash, narSize => $narSize
-                        , patchType => $patchType, hashAlgo => $hashAlgo
+                        , patchType => $patchType
                         };
                 }
 
@@ -193,4 +195,136 @@ sub writeManifest {
 }
 
 
+sub updateManifestDB {
+    my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests");
+
+    my $dbPath = "$manifestDir/cache.sqlite";
+
+    # Open/create the database.
+    my $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
+        or die "cannot open database `$dbPath'";
+    $dbh->{AutoCommit} = 0;
+    $dbh->{RaiseError} = 1;
+    $dbh->{PrintError} = 0;
+
+    $dbh->do("pragma foreign_keys = on");
+
+    # Initialise the database schema, if necessary.
+    $dbh->do(<<EOF);
+        create table if not exists Manifests (
+            id        integer primary key autoincrement not null,
+            path      text unique not null,
+            timestamp integer not null
+        );
+EOF
+    
+    $dbh->do(<<EOF);
+        create table if not exists NARs (
+            id               integer primary key autoincrement not null,
+            manifest         integer not null,
+            storePath        text not null,
+            url              text not null,
+            hash             text,
+            size             integer,
+            narHash          text,
+            narSize          integer,
+            refs             text,
+            deriver          text,
+            system           text,
+            foreign key (manifest) references Manifests(id) on delete cascade
+        );
+EOF
+
+    $dbh->do("create index if not exists NARs_storePath on NARs(storePath)");
+
+    $dbh->do(<<EOF);
+        create table if not exists Patches (
+            id               integer primary key autoincrement not null,
+            manifest         integer not null,
+            storePath        text not null,
+            basePath         text not null,
+            baseHash         text not null,
+            url              text not null,
+            hash             text,
+            size             integer,
+            narHash          text,
+            narSize          integer,
+            patchType        text not null,
+            foreign key (manifest) references Manifests(id) on delete cascade
+        );
+EOF
+
+    $dbh->do("create index if not exists Patches_storePath on Patches(storePath)");
+
+    # !!! locking?
+
+    # Read each manifest in $manifestDir and add it to the database,
+    # unless we've already done so on a previous run.
+    my %seen;
+    
+    for my $manifest (glob "$manifestDir/*.nixmanifest") {
+        $manifest = Cwd::abs_path($manifest);
+        my $timestamp = lstat($manifest)->mtime;
+        $seen{$manifest} = 1;
+
+        next if scalar @{$dbh->selectcol_arrayref(
+            "select 1 from Manifests where path = ? and timestamp = ?",
+            {}, $manifest, $timestamp)} == 1;
+
+        # !!! Insert directly into the DB.
+        my %narFiles;
+        my %patches;
+        my $version = readManifest($manifest, \%narFiles, \%patches);
+        
+        if ($version < 3) {
+            die "you have an old-style manifest `$manifest'; please delete it";
+        }
+        if ($version >= 10) {
+            die "manifest `$manifest' is too new; please delete it or upgrade Nix";
+        }
+
+        $dbh->do("delete from Manifests where path = ?", {}, $manifest);
+                 
+        $dbh->do("insert into Manifests(path, timestamp) values (?, ?)",
+                 {}, $manifest, $timestamp);
+
+        my $id = $dbh->sqlite_last_insert_rowid();
+
+        foreach my $storePath (keys %narFiles) {
+            my $narFileList = $narFiles{$storePath};
+            foreach my $narFile (@{$narFiles{$storePath}}) {
+                $dbh->do(
+                    "insert into NARs(manifest, storePath, url, hash, size, narHash, " .
+                    "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    {}, $id, $storePath, $narFile->{url}, $narFile->{hash}, $narFile->{size},
+                    $narFile->{narHash}, $narFile->{narSize}, $narFile->{references},
+                    $narFile->{deriver}, $narFile->{system});
+            }
+        }
+
+        foreach my $storePath (keys %patches) {
+            my $patchList = $patches{$storePath};
+            foreach my $patch (@{$patchList}) {
+                $dbh->do(
+                    "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " .
+                    "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    {}, $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url},
+                    $patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize},
+                    $patch->{patchType});
+            }
+        }
+    }
+
+    # Removed cached information for removed manifests from the DB.
+    foreach my $manifest (@{$dbh->selectcol_arrayref("select path from Manifests")}) {
+        next if defined $seen{$manifest};
+        $dbh->do("delete from Manifests where path = ?", {}, $manifest);
+    }
+
+    $dbh->commit;
+
+    return $dbh;
+}
+
+
 return 1;