diff options
| -rw-r--r-- | corepkgs/Makefile.am | 2 | ||||
| -rw-r--r-- | corepkgs/nar/unnar.fix | 8 | ||||
| -rw-r--r-- | corepkgs/nar/unnar.sh | 3 | ||||
| -rw-r--r-- | scripts/Makefile.am | 4 | ||||
| -rw-r--r-- | scripts/nix-pull | 65 | ||||
| -rwxr-xr-x | scripts/nix-pull-prebuilts | 83 | ||||
| -rwxr-xr-x | scripts/nix-push-prebuilts | 44 | ||||
| -rw-r--r-- | scripts/prebuilts.conf | 6 | ||||
| -rw-r--r-- | src/fstate.cc | 6 | ||||
| -rw-r--r-- | src/fstate.hh | 2 | ||||
| -rw-r--r-- | src/nix.cc | 25 | ||||
| -rw-r--r-- | src/store.cc | 45 | ||||
| -rw-r--r-- | src/store.hh | 4 |
13 files changed, 155 insertions, 142 deletions
diff --git a/corepkgs/Makefile.am b/corepkgs/Makefile.am index 9ce9c8c79..9298865bf 100644 --- a/corepkgs/Makefile.am +++ b/corepkgs/Makefile.am @@ -6,3 +6,5 @@ install-data-local: $(INSTALL) -d $(datadir)/fix/nar $(INSTALL_DATA) nar/nar.fix $(datadir)/fix/nar $(INSTALL_DATA) nar/nar.sh $(datadir)/fix/nar + $(INSTALL_DATA) nar/unnar.fix $(datadir)/fix/nar + $(INSTALL_DATA) nar/unnar.sh $(datadir)/fix/nar diff --git a/corepkgs/nar/unnar.fix b/corepkgs/nar/unnar.fix new file mode 100644 index 000000000..db97750aa --- /dev/null +++ b/corepkgs/nar/unnar.fix @@ -0,0 +1,8 @@ +Function(["nar", "name"], + Package( + [ ("name", Var("name")) + , ("build", Relative("nar/unnar.sh")) + , ("nar", Var("nar")) + ] + ) +)
\ No newline at end of file diff --git a/corepkgs/nar/unnar.sh b/corepkgs/nar/unnar.sh new file mode 100644 index 000000000..e6a3f3c1f --- /dev/null +++ b/corepkgs/nar/unnar.sh @@ -0,0 +1,3 @@ +#! /bin/sh + +/tmp/nix/bin/nix --restore "$out" < $nar || exit 1 diff --git a/scripts/Makefile.am b/scripts/Makefile.am index e4602f2a1..2f4dbacc9 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -5,5 +5,5 @@ install-exec-local: $(INSTALL) -d $(sysconfdir)/profile.d $(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh $(INSTALL) -d $(sysconfdir)/nix - # !!! don't overwrite local modifications - $(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf +# !!! don't overwrite local modifications + $(INSTALL_DATA) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf diff --git a/scripts/nix-pull b/scripts/nix-pull index 0b09c8e00..59773a2ba 100644 --- a/scripts/nix-pull +++ b/scripts/nix-pull @@ -1,2 +1,67 @@ #! /usr/bin/perl -w +my $prefix = $ENV{"NIX"} || "/tmp/nix"; # !!! use prefix +my $etcdir = "$prefix/etc/nix"; +my $tmpfile = "$prefix/var/nix/pull.tmp"; + +my $conffile = "$etcdir/prebuilts.conf"; + +open CONFFILE, "<$conffile"; + +while (<CONFFILE>) { + + chomp; + if (/^\s*(\S+)\s*(\#.*)?$/) { + my $url = $1; + + print "obtaining list of Nix archives at $url...\n"; + + system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape + if ($?) { die "`wget' failed"; } + + open INDEX, "<$tmpfile"; + + while (<INDEX>) { + # Get all links to prebuilts, that is, file names of the + # form foo-HASH-HASH.tar.bz2. + next unless (/HREF=\"([^\"]*)\"/); + my $fn = $1; + next if $fn =~ /\.\./; + next if $fn =~ /\//; + next unless $fn =~ /([0-9a-z]{32})-([0-9a-z]{32})\.nar/; + my $hash = $2; + + print "registering $hash -> $url/$fn\n"; + + # Construct a Fix expression that fetches and unpacks a + # Nix archive from the network. + my $fetch = + "App(IncludeFix(\"fetchurl/fetchurl.fix\"), " . + "[(\"url\", \"$url/$fn\"), (\"hash\", \"\")])"; + my $fixexpr = + "App(IncludeFix(\"nar/unnar.fix\"), " . + "[ (\"nar\", $fetch)" . + ", (\"name\", \"fetched-$hash\")" . + "])"; + + my $fixfile = "/tmp/nix-pull-tmp.fix"; + open FIX, ">$fixfile"; + print FIX $fixexpr; + close FIX; + + # Instantiate a Nix expression from the Fix expression. + my $nhash = `fix $fixfile`; + $? and die "instantiating Nix archive expression"; + chomp $nhash; + die unless $nhash =~ /^([0-9a-z]{32})$/; + + system "nix --substitute $hash $nhash"; + if ($?) { die "`nix --substitute' failed"; } + } + + close INDEX; + + unlink $tmpfile; + } + +} diff --git a/scripts/nix-pull-prebuilts b/scripts/nix-pull-prebuilts deleted file mode 100755 index 3d045b463..000000000 --- a/scripts/nix-pull-prebuilts +++ /dev/null @@ -1,83 +0,0 @@ -#! /usr/bin/perl -w - -my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix -my $etcdir = "$prefix/etc/nix"; -my $knowns = "$prefix/var/nix/known-prebuilts"; -my $tmpfile = "$prefix/var/nix/prebuilts.tmp"; - -my $conffile = "$etcdir/prebuilts.conf"; - -umask 0022; - -sub register { - my $fn = shift; - my $url = shift; - return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/; - my $id = $1; - my $pkghash = $2; - my $prebuilthash = $3; - - print "$pkghash => $prebuilthash ($id)\n"; - - system "nix regprebuilt $pkghash $prebuilthash"; - if ($?) { die "`nix regprebuilt' failed"; } - - if ($url =~ /^\//) { - system "nix regfile $url"; - if ($?) { die "`nix regfile' failed"; } - } else { - system "nix regurl $prebuilthash $url"; - if ($?) { die "`nix regurl' failed"; } - } - - print KNOWNS "$pkghash\n"; -} - -open KNOWNS, ">$knowns"; - -open CONFFILE, "<$conffile"; - -while (<CONFFILE>) { - chomp; - if (/^\s*(\S+)\s*(\#.*)?$/) { - my $url = $1; - - print "obtaining prebuilt list from $url...\n"; - - if ($url =~ /^\//) { - - # It's a local path. - - foreach my $fn (glob "$url/*") { - register($fn, $fn); - } - - } else { - - # It's a URL. - - system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape - if ($?) { die "`wget' failed"; } - - open INDEX, "<$tmpfile"; - - while (<INDEX>) { - # Get all links to prebuilts, that is, file names of the - # form foo-HASH-HASH.tar.bz2. - next unless (/HREF=\"([^\"]*)\"/); - my $fn = $1; - next if $fn =~ /\.\./; - next if $fn =~ /\//; - register($fn, "$url/$fn"); - } - - close INDEX; - - unlink $tmpfile; - } - } -} - -close CONFFILE; - -close KNOWNS; diff --git a/scripts/nix-push-prebuilts b/scripts/nix-push-prebuilts deleted file mode 100755 index 2d44e7cda..000000000 --- a/scripts/nix-push-prebuilts +++ /dev/null @@ -1,44 +0,0 @@ -#! /usr/bin/perl -w - -my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix -my $etcdir = "$prefix/etc/nix"; -my $exportdir = "$prefix/var/nix/prebuilts/exports"; -my $knowns = "$prefix/var/nix/known-prebuilts"; - -umask 0022; - -# For performance, put the known hashes in an associative array. -my %knowns = (); -open KNOWNS, "<$knowns"; -while (<KNOWNS>) { - next unless /([0-9a-z]{32})/; - $knowns{$1} = 1; -} -close KNOWNS; - -# For each installed package, check whether a prebuilt is known. - -open PKGS, "nix listinst|"; - -while (<PKGS>) { - chomp; - next unless /([0-9a-z]{32})/; - my $pkghash = $1; - if (!defined $knowns{$1}) { - # No known prebuilt exists for this package; so export it. - print "exporting $pkghash...\n"; - system "nix export '$exportdir' $pkghash"; - if ($?) { die "`nix export' failed"; } - } -} - -close PKGS; - -# Push the prebuilts to the server. !!! FIXME - -system "rsync -av -e ssh '$exportdir'/ eelco\@losser.st-lab.cs.uu.nl:/home/eelco/public_html/nix-prebuilts/"; - -# Rerun `nix-pull-prebuilts' to rescan the prebuilt source locations. - -print "running nix-pull-prebuilts..."; -system "nix-pull-prebuilts"; diff --git a/scripts/prebuilts.conf b/scripts/prebuilts.conf index 9b950cad4..c7bc89c61 100644 --- a/scripts/prebuilts.conf +++ b/scripts/prebuilts.conf @@ -1,4 +1,2 @@ -# A list of URLs or local paths from where we obtain prebuilts. -/nix/var/nix/prebuilts/imports -/nix/var/nix/prebuilts/exports -http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/ +# A list of URLs from where we obtain Nix archives. +http://losser.st-lab.cs.uu.nl/~eelco/nix-dist/ diff --git a/src/fstate.cc b/src/fstate.cc index fdd43d1b1..97532c162 100644 --- a/src/fstate.cc +++ b/src/fstate.cc @@ -147,6 +147,12 @@ Hash hashTerm(ATerm t) } +FState hash2fstate(Hash hash) +{ + return ATmake("Include(<str>)", ((string) hash).c_str()); +} + + ATerm termFromHash(const Hash & hash, string * p) { string path = expandHash(hash); diff --git a/src/fstate.hh b/src/fstate.hh index 159c7ba46..8a873a5ac 100644 --- a/src/fstate.hh +++ b/src/fstate.hh @@ -85,6 +85,8 @@ Error badTerm(const format & f, ATerm t); /* Hash an aterm. */ Hash hashTerm(ATerm t); +FState hash2fstate(Hash hash); + /* Read an aterm from disk, given its hash. */ ATerm termFromHash(const Hash & hash, string * p = 0); diff --git a/src/nix.cc b/src/nix.cc index 4721563fd..53057328d 100644 --- a/src/nix.cc +++ b/src/nix.cc @@ -26,6 +26,8 @@ static ArgType argType = atpUnknown; --add / -A: copy a path to the Nix store --query / -q: query information + --substitute: register a substitute expression + --dump: dump a path as a Nix archive --restore: restore a path from a Nix archive @@ -87,12 +89,6 @@ static Hash argToHash(const string & arg) } -static FState hash2fstate(Hash hash) -{ - return ATmake("Include(<str>)", ((string) hash).c_str()); -} - - /* Realise (or install) paths from the given Nix fstate expressions. */ static void opInstall(Strings opFlags, Strings opArgs) @@ -187,6 +183,21 @@ static void opQuery(Strings opFlags, Strings opArgs) } +static void opSubstitute(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() % 2) throw UsageError("expecting even number of arguments"); + + for (Strings::iterator i = opArgs.begin(); + i != opArgs.end(); ) + { + Hash srcHash = parseHash(*i++); + Hash subHash = parseHash(*i++); + registerSubstitute(srcHash, subHash); + } +} + + /* A sink that writes dump output to stdout. */ struct StdoutSink : DumpSink { @@ -277,6 +288,8 @@ void run(Strings args) op = opAdd; else if (arg == "--query" || arg == "-q") op = opQuery; + else if (arg == "--substitute") + op = opSubstitute; else if (arg == "--dump") op = opDump; else if (arg == "--restore") diff --git a/src/store.cc b/src/store.cc index 5a3a4e067..435ac5cc6 100644 --- a/src/store.cc +++ b/src/store.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include "db.hh" #include "archive.hh" +#include "fstate.hh" struct CopySink : DumpSink @@ -83,6 +84,20 @@ void copyPath(string src, string dst) } +void registerSubstitute(const Hash & srcHash, const Hash & subHash) +{ + Strings subs; + queryListDB(nixDB, dbSubstitutes, srcHash, subs); /* non-existence = ok */ + + for (Strings::iterator it = subs.begin(); it != subs.end(); it++) + if (parseHash(*it) == subHash) return; + + subs.push_back(subHash); + + setListDB(nixDB, dbSubstitutes, srcHash, subs); +} + + Hash registerPath(const string & _path, Hash hash) { string path(canonPath(_path)); @@ -139,8 +154,7 @@ string expandHash(const Hash & hash, const string & target, if (!target.empty() && !isInPrefix(target, prefix)) abort(); - if (!queryListDB(nixDB, dbHash2Paths, hash, paths)) - throw Error(format("no paths known with hash `%1%'") % (string) hash); + queryListDB(nixDB, dbHash2Paths, hash, paths); /* !!! we shouldn't check for staleness by default --- too slow */ @@ -181,8 +195,32 @@ string expandHash(const Hash & hash, const string & target, /* try next one */ } } + + /* Try to realise the substitutes. */ + + Strings subs; + queryListDB(nixDB, dbSubstitutes, hash, subs); /* non-existence = ok */ + + for (Strings::iterator it = subs.begin(); it != subs.end(); it++) { + StringSet dummy; + FState nf = realiseFState(hash2fstate(parseHash(*it)), dummy); + string path = fstatePath(nf); + + if (hashPath(path) != hash) + throw Error(format("bad substitute in `%1%'") % (string) path); + + if (target.empty()) + return path; /* !!! prefix */ + else { + if (path != target) { + copyPath(path, target); + registerPath(target, hash); + } + return target; + } + } - throw Error(format("all paths with hash `%1%' are stale") % (string) hash); + throw Error(format("cannot expand hash `%1%'") % (string) hash); } @@ -193,6 +231,7 @@ void addToStore(string srcPath, string & dstPath, Hash & hash) hash = hashPath(srcPath); try { + /* !!! should not use the substitutes! */ dstPath = expandHash(hash, "", nixStore); return; } catch (...) { diff --git a/src/store.hh b/src/store.hh index 8b02cba99..8b41478a2 100644 --- a/src/store.hh +++ b/src/store.hh @@ -8,8 +8,12 @@ using namespace std; +/* Copy a path recursively. */ void copyPath(string src, string dst); +/* Register a substitute. */ +void registerSubstitute(const Hash & srcHash, const Hash & subHash); + /* Register a path keyed on its hash. */ Hash registerPath(const string & path, Hash hash = Hash()); |