From 7dd91d3779b4f806ac0085e0ccc60416d81c1148 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 25 May 2003 22:42:19 +0000 Subject: * Prebuilt package sharing. We allow transparent binary deployment by sharing package directories (i.e., the result of building a Nix descriptor). `nix-pull-prebuilts' obtains a list of all known prebuilts by consulting the paths and URLs specified in $prefix/etc/nix/prebuilts.conf. The mappings ($pkghash, $prebuilthash) and ($prebuilthash, $location) are registered with Nix so that it can use the prebuilt with hash $prebuilthash when installing a package with hash $pkghash by downloading and unpacking $location. `nix-push-prebuilts' creates prebuilts for all packages for which no prebuilt is known to exist. It can then optionally upload these to the network through rsync. `nix-[pull|push]-prebuilts' just provide a policy. Nix provides the mechanism through the `nix [export|regprebuilt|regurl]' commands. --- scripts/Makefile.am | 6 +++- scripts/nix-generate-regscript | 20 ------------ scripts/nix-pull-prebuilts | 69 ++++++++++++++++++++++++++++++++++++++++++ scripts/nix-push-prebuilts | 40 ++++++++++++++++++++++++ scripts/prebuilts.conf | 4 +++ 5 files changed, 118 insertions(+), 21 deletions(-) delete mode 100755 scripts/nix-generate-regscript create mode 100755 scripts/nix-pull-prebuilts create mode 100755 scripts/nix-push-prebuilts create mode 100644 scripts/prebuilts.conf (limited to 'scripts') diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 4140cdf5b..f1d008f1e 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -1,5 +1,9 @@ -bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage +bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage \ + nix-pull-prebuilts nix-push-prebuilts install-exec-local: $(INSTALL) -d $(sysconfdir)/profile.d $(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh + $(INSTALL) -d $(sysconfdir)/nix + # !!! don't overwrite local modifications + $(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf diff --git a/scripts/nix-generate-regscript b/scripts/nix-generate-regscript deleted file mode 100755 index bf370f8d7..000000000 --- a/scripts/nix-generate-regscript +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/perl -w - -my $dir = shift @ARGV; -$dir || die "missing directory"; -my $url = shift @ARGV; -$url || die "missing base url"; - -chdir $dir || die "cannot chdir to $dir"; - -foreach my $prebuilt (glob("*.tar.bz2")) { - - $prebuilt =~ /-([a-z0-9]+)-([a-z0-9]+).tar.bz2$/ - || die "invalid file name: $prebuilt"; - - my $pkgHash = $1; - my $prebuiltHash = $2; - - print "regprebuilt $pkgHash $prebuiltHash\n"; - print "regurl $prebuiltHash $url/$prebuilt\n"; -} diff --git a/scripts/nix-pull-prebuilts b/scripts/nix-pull-prebuilts new file mode 100755 index 000000000..91bbf8082 --- /dev/null +++ b/scripts/nix-pull-prebuilts @@ -0,0 +1,69 @@ +#! /usr/bin/perl -w + +my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix +my $etcdir = "$prefix/etc/nix"; +my $knowns = "$prefix/var/nix/known-prebuilts"; +my $tmpfile = "$prefix/var/nix/prebuilts.tmp"; + +my $conffile = "$etcdir/prebuilts.conf"; + +sub register { + my $fn = shift; + return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/; + my $id = $1; + my $pkghash = $2; + my $prebuilthash = $3; + print "$pkghash => $prebuilthash ($id)\n"; + system "nix regprebuilt $pkghash $prebuilthash"; + if ($?) { die "`nix regprebuilt' failed"; } + print KNOWNS "$pkghash\n"; +} + +open KNOWNS, ">$knowns"; + +open CONFFILE, "<$conffile"; + +while () { + chomp; + if (/^\s*(\S+)\s*(\#.*)?$/) { + my $url = $1; + + print "obtaining prebuilt list from $url...\n"; + + if ($url =~ /^\//) { + + # It's a local path. + + foreach my $fn (glob "$url/*") { + register $fn; + } + + } else { + + # It's a URL. + + system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape + if ($?) { die "`wget' failed"; } + + open INDEX, "<$tmpfile"; + + while () { + # Get all links to prebuilts, that is, file names of the + # form foo-HASH-HASH.tar.bz2. + next unless (/HREF=\"([^\"]*)\"/); + my $fn = $1; + next if $fn =~ /\.\./; + next if $fn =~ /\//; + register $fn; + } + + close INDEX; + + unlink $tmpfile; + } + } +} + +close CONFFILE; + +close KNOWNS; diff --git a/scripts/nix-push-prebuilts b/scripts/nix-push-prebuilts new file mode 100755 index 000000000..2e3029b16 --- /dev/null +++ b/scripts/nix-push-prebuilts @@ -0,0 +1,40 @@ +#! /usr/bin/perl -w + +my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix +my $etcdir = "$prefix/etc/nix"; +my $exportdir = "$prefix/var/nix/prebuilts/exports"; +my $knowns = "$prefix/var/nix/known-prebuilts"; + +# For performance, put the known hashes in an associative array. +my %knowns = (); +open KNOWNS, "<$knowns"; +while () { + next unless /([0-9a-z]{32})/; + $knowns{$1} = 1; +} +close KNOWNS; + +# For each installed package, check whether a prebuilt is known. + +open PKGS, "nix listinst|"; +open KNOWNS, ">>$knowns"; + +while () { + chomp; + next unless /([0-9a-z]{32})/; + my $pkghash = $1; + if (!defined $knowns{$1}) { + # No known prebuilt exists for this package; so export it. + print "exporting $pkghash...\n"; + system "nix export '$exportdir' $pkghash"; + if ($?) { die "`nix export' failed"; } + print KNOWNS "$pkghash\n"; + } +} + +close KNOWNS; +close PKGS; + +# Push the prebuilts to the server. !!! FIXME + +system "rsync -av -e ssh '$exportdir' losser:/home/eelco/public_html/nix-prebuilts/"; diff --git a/scripts/prebuilts.conf b/scripts/prebuilts.conf new file mode 100644 index 000000000..9b950cad4 --- /dev/null +++ b/scripts/prebuilts.conf @@ -0,0 +1,4 @@ +# A list of URLs or local paths from where we obtain prebuilts. +/nix/var/nix/prebuilts/imports +/nix/var/nix/prebuilts/exports +http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/ -- cgit v1.2.3