diff options
30 files changed, 514 insertions, 86 deletions
diff --git a/.gitignore b/.gitignore index 3f9758b9d..e10c75418 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ perl/Makefile.config /tests/restricted-innocent /tests/shell /tests/shell.drv +/tests/config.nix # /tests/lang/ /tests/lang/*.out @@ -118,3 +119,5 @@ GPATH GRTAGS GSYMS GTAGS + +nix-rust/target @@ -1,6 +1,7 @@ makefiles = \ mk/precompiled-headers.mk \ local.mk \ + nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ src/libmain/local.mk \ diff --git a/configure.ac b/configure.ac index 26f16b846..8e38f2b8e 100644 --- a/configure.ac +++ b/configure.ac @@ -117,26 +117,15 @@ fi ]) NEED_PROG(bash, bash) -NEED_PROG(patch, patch) AC_PATH_PROG(xmllint, xmllint, false) AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) -NEED_PROG(sed, sed) -NEED_PROG(tar, tar) -NEED_PROG(bzip2, bzip2) -NEED_PROG(gzip, gzip) -NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(cat, cat) -NEED_PROG(tr, tr) -AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], - [path of cat, mkdir, etc.]), - coreutils=$withval, coreutils=$(dirname $cat)) -AC_SUBST(coreutils) +AC_SUBST(coreutils, [$(dirname $(type -p cat))]) AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], @@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LIBS="-latomic $LIBS" fi -# Look for OpenSSL, a required dependency. +# Look for OpenSSL, a required dependency. FIXME: this is only (maybe) +# used by S3BinaryCacheStore. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) - # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) - # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) @@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], have_sodium=1], [have_sodium=]) AC_SUBST(HAVE_SODIUM, [$have_sodium]) - # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) - # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 32ce6b399..cb9945944 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -1,29 +1,13 @@ +# FIXME: remove this file? let fromEnv = var: def: let val = builtins.getEnv var; in if val != "" then val else def; in rec { - shell = "@bash@"; - coreutils = "@coreutils@"; - bzip2 = "@bzip2@"; - gzip = "@gzip@"; - xz = "@xz@"; - tar = "@tar@"; - tarFlags = "@tarFlags@"; - tr = "@tr@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; nixLocalstateDir = "@localstatedir@"; nixSysconfDir = "@sysconfdir@"; nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; - - # If Nix is installed in the Nix store, then automatically add it as - # a dependency to the core packages. This ensures that they work - # properly in a chroot. - chrootDeps = - if dirOf nixPrefix == builtins.storeDir then - [ (builtins.storePath nixPrefix) ] - else - [ ]; } diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index d39a20637..10515bc8b 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -1,39 +1,12 @@ -with import <nix/config.nix>; - -let - - builder = builtins.toFile "unpack-channel.sh" - '' - mkdir $out - cd $out - xzpat="\.xz\$" - gzpat="\.gz\$" - if [[ "$src" =~ $xzpat ]]; then - ${xz} -d < $src | ${tar} xf - ${tarFlags} - elif [[ "$src" =~ $gzpat ]]; then - ${gzip} -d < $src | ${tar} xf - ${tarFlags} - else - ${bzip2} -d < $src | ${tar} xf - ${tarFlags} - fi - if [ * != $channelName ]; then - mv * $out/$channelName - fi - ''; - -in - { name, channelName, src }: derivation { - system = builtins.currentSystem; - builder = shell; - args = [ "-e" builder ]; - inherit name channelName src; + builder = "builtin:unpack-channel"; + + system = "builtin"; - PATH = "${nixBinDir}:${coreutils}"; + inherit name channelName src; # No point in doing this remotely. preferLocalBuild = true; - - inherit chrootDeps; } diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock new file mode 100644 index 000000000..0112ed471 --- /dev/null +++ b/nix-rust/Cargo.lock @@ -0,0 +1,84 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "filetime" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix-rust" +version = "0.1.0" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tar" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml new file mode 100644 index 000000000..c4f4ceb09 --- /dev/null +++ b/nix-rust/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "nix-rust" +version = "0.1.0" +authors = ["Eelco Dolstra <edolstra@gmail.com>"] +edition = "2018" + +[lib] +name = "nixrust" +crate-type = ["cdylib"] + +[dependencies] +tar = "0.4" +libc = "0.2" diff --git a/nix-rust/local.mk b/nix-rust/local.mk new file mode 100644 index 000000000..7645d5394 --- /dev/null +++ b/nix-rust/local.mk @@ -0,0 +1,38 @@ +ifeq ($(OPTIMIZE), 1) + RUST_MODE = --release + RUST_DIR = release +else + RUST_MODE = + RUST_DIR = debug +endif + +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) +libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl + +ifeq ($(OS), Darwin) +libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" +else +libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) +libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir) +endif + +$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ + $(libnixrust_BUILD_FLAGS) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.$(SO_EXT) + +$(libnixrust_INSTALL_PATH): $(libnixrust_PATH) + $(target-gen) cp $^ $@ +ifeq ($(OS), Darwin) + install_name_tool -id $@ $@ +endif + +dist-files += $(d)/vendor + +clean: clean-rust + +clean-rust: + $(suppress) rm -rfv nix-rust/target diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs new file mode 100644 index 000000000..519007ea0 --- /dev/null +++ b/nix-rust/src/error.rs @@ -0,0 +1,31 @@ +#[derive(Debug)] +pub enum Error { + IOError(std::io::Error), + Misc(String), + Foreign(CppException), +} + +impl From<std::io::Error> for Error { + fn from(err: std::io::Error) -> Self { + Error::IOError(err) + } +} + +impl From<Error> for CppException { + fn from(err: Error) -> Self { + match err { + Error::Foreign(ex) => ex, + Error::Misc(s) => unsafe { make_error(&s) }, + Error::IOError(err) => unsafe { make_error(&err.to_string()) }, + } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct CppException(*const libc::c_void); // == std::exception_ptr* + +extern "C" { + #[allow(improper_ctypes)] // YOLO + fn make_error(s: &str) -> CppException; +} diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs new file mode 100644 index 000000000..7bce7753c --- /dev/null +++ b/nix-rust/src/foreign.rs @@ -0,0 +1,14 @@ +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs new file mode 100644 index 000000000..1b88ac8af --- /dev/null +++ b/nix-rust/src/lib.rs @@ -0,0 +1,32 @@ +mod error; +mod foreign; +mod tarfile; + +pub use error::Error; + +pub struct CBox<T> { + pub ptr: *mut libc::c_void, + phantom: std::marker::PhantomData<T>, +} + +impl<T> CBox<T> { + fn new(t: T) -> Self { + unsafe { + let size = std::mem::size_of::<T>(); + let ptr = libc::malloc(size); + *(ptr as *mut T) = t; // FIXME: probably UB + Self { + ptr, + phantom: std::marker::PhantomData, + } + } + } +} + +#[no_mangle] +pub extern "C" fn unpack_tarfile( + source: foreign::Source, + dest_dir: &str, +) -> CBox<Result<(), error::CppException>> { + CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into())) +} diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs new file mode 100644 index 000000000..379d9098f --- /dev/null +++ b/nix-rust/src/tarfile.rs @@ -0,0 +1,46 @@ +use crate::{foreign::Source, Error}; +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(source); + + for file in tar.entries()? { + let mut file = file?; + + let dest_file = dest_dir.join(file.path()?); + + fs::create_dir_all(dest_file.parent().unwrap())?; + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file)?; + } + tar::EntryType::Regular => { + let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file)?; + io::copy(&mut file, &mut f)?; + } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; + } + tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} + t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), + } + } + + Ok(()) +} diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index 67a20c3f4..bc1749e60 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; -$bzip2 = "@bzip2@"; -$xz = "@xz@"; -$curl = "@curl@"; - $useBindings = 1; %config = (); diff --git a/release-common.nix b/release-common.nix index 63f39f005..dd5f939d9 100644 --- a/release-common.nix +++ b/release-common.nix @@ -51,6 +51,7 @@ rec { openssl pkgconfig sqlite boehmgc boost nlohmann_json + rustc cargo # Tests git diff --git a/release.nix b/release.nix index 1d7c2e2d6..9cf4c74f2 100644 --- a/release.nix +++ b/release.nix @@ -10,6 +10,50 @@ let jobs = rec { + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock, and include it in the Nix tarball. This allows Nix + # to be built without network access. + vendoredCrates = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); + + files = map (pkg: import <nix/fetchurl.nix> { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor + + cat > $out/vendor/config <<EOF + [source.crates-io] + replace-with = "vendored-sources" + + [source.vendored-sources] + directory = "vendor" + EOF + + ${toString (builtins.map (file: '' + mkdir $out/vendor/tmp + tar xvf ${file} -C $out/vendor/tmp + dir=$(echo $out/vendor/tmp/*) + + # Add just enough metadata to keep Cargo happy. + printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json" + + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- + fi + + mv "$dir" $out/vendor/ + + rm -rf $out/vendor/tmp + '') files)} + ''; + tarball = with pkgs; @@ -38,6 +82,8 @@ let distPhase = '' + cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/ + runHook preDist make dist mkdir -p $out/tarballs @@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; - buildInputs = buildDeps ++ tarballDeps ++ perlDeps; + buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ]; inherit configureFlags; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 7ef3b3823..9d0c64291 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "hash.hh" +#include "tarfile.hh" #include <sys/time.h> @@ -164,14 +165,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, if (e.errNo != ENOENT) throw; } - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + unpackTarfile(*source, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 51a9fa35b..efbb7fc9f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3128,6 +3128,8 @@ void DerivationGoal::runChild() builtinFetchurl(drv2, netrcData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); else throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); _exit(0); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0d2da873e..87d6ce665 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -7,5 +7,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinBuildenv(const BasicDerivation & drv); +void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc new file mode 100644 index 000000000..d18e3ddaf --- /dev/null +++ b/src/libstore/builtins/unpack-channel.cc @@ -0,0 +1,29 @@ +#include "builtins.hh" +#include "tarfile.hh" + +namespace nix { + +void builtinUnpackChannel(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + auto channelName = getAttr("channelName"); + auto src = getAttr("src"); + + createDirs(out); + + unpackTarfile(src, out); + + auto entries = readDirectory(out); + if (entries.size() != 1) + throw Error("channel tarball '%s' contains more than one file", src); + if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) + throw SysError("renaming channel directory"); +} + +} diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e80663dff..61e88c5c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "tarfile.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached( unpackedStorePath = ""; } if (unpackedStorePath.empty()) { - printInfo(format("unpacking '%1%'...") % url); + printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); storePath = unpackedStorePath; diff --git a/src/libutil/local.mk b/src/libutil/local.mk index e41a67d1f..35c1f6c13 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,3 +7,5 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context + +libutil_LIBS = libnixrust diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc new file mode 100644 index 000000000..931d29542 --- /dev/null +++ b/src/libutil/rust-ffi.cc @@ -0,0 +1,12 @@ +#include "logging.hh" +#include "rust-ffi.hh" + +namespace nix { + +extern "C" std::exception_ptr * make_error(rust::StringSlice s) +{ + // FIXME: leak + return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size)))); +} + +} diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh new file mode 100644 index 000000000..663758bfc --- /dev/null +++ b/src/libutil/rust-ffi.hh @@ -0,0 +1,84 @@ +#include "serialise.hh" + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template<typename T> +struct Slice +{ + T * ptr; + size_t size; + + Slice(T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice<char> +{ + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice<uint8_t> data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } +}; + +/* C++ representation of Rust's Result<T, CppException>. */ +template<typename T> +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template<typename T> +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +} diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 128e287f3..5780c93a6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,7 +77,6 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - bool hasData(); protected: diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc new file mode 100644 index 000000000..2cc7793fd --- /dev/null +++ b/src/libutil/tarfile.cc @@ -0,0 +1,36 @@ +#include "rust-ffi.hh" +#include "compression.hh" + +extern "C" { + rust::Result<std::tuple<>> * + unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); +} + +namespace nix { + +void unpackTarfile(Source & source, const Path & destDir) +{ + rust::Source source2(source); + rust::CBox(unpack_tarfile(source2, destDir))->unwrap(); +} + +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional<std::string> baseName) +{ + if (!baseName) baseName = baseNameOf(tarFile); + + auto source = sinkToSource([&](Sink & sink) { + // FIXME: look at first few bytes to determine compression type. + auto decompressor = + // FIXME: add .gz support + hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(tarFile, *decompressor); + decompressor->finish(); + }); + + unpackTarfile(*source, destDir); +} + +} diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh new file mode 100644 index 000000000..ce0911e2a --- /dev/null +++ b/src/libutil/tarfile.hh @@ -0,0 +1,10 @@ +#include "serialise.hh" + +namespace nix { + +void unpackTarfile(Source & source, const Path & destDir); + +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional<std::string> baseName = {}); + +} diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index f54706a8a..78c883833 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -9,6 +9,7 @@ #include "legacy.hh" #include "finally.hh" #include "progress-bar.hh" +#include "tarfile.hh" #include <iostream> @@ -192,8 +193,7 @@ static int _main(int argc, char * * argv) if (hasSuffix(baseNameOf(uri), ".zip")) runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + unpackTarfile(tmpFile, unpacked, baseNameOf(uri)); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ diff --git a/tests/config.nix b/tests/config.nix.in index 6ba91065b..51aed539c 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -1,9 +1,7 @@ -with import <nix/config.nix>; - rec { - inherit shell; + shell = "@bash@"; - path = coreutils; + path = "@coreutils@"; system = builtins.currentSystem; diff --git a/tests/local.mk b/tests/local.mk index 187f96ea2..f1d215eec 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) +installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT) |