From a1ff43045b3f1d075c67c262ff6f623b2b5d569e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 15 Sep 2019 23:09:30 +0200 Subject: Move stuff around --- nix-rust/local.mk | 2 +- nix-rust/src/binary_cache_store.rs | 51 ---------------- nix-rust/src/c.rs | 42 +++++++++++++ nix-rust/src/foreign.rs | 19 ++++++ nix-rust/src/lib.rs | 65 +------------------- nix-rust/src/path_info.rs | 70 --------------------- nix-rust/src/store.rs | 101 ------------------------------- nix-rust/src/store/binary_cache_store.rs | 50 +++++++++++++++ nix-rust/src/store/mod.rs | 7 +++ nix-rust/src/store/path_info.rs | 70 +++++++++++++++++++++ nix-rust/src/store/store.rs | 101 +++++++++++++++++++++++++++++++ nix-rust/src/tarfile.rs | 46 -------------- nix-rust/src/util/mod.rs | 1 + nix-rust/src/util/tarfile.rs | 46 ++++++++++++++ 14 files changed, 339 insertions(+), 332 deletions(-) delete mode 100644 nix-rust/src/binary_cache_store.rs create mode 100644 nix-rust/src/c.rs delete mode 100644 nix-rust/src/path_info.rs delete mode 100644 nix-rust/src/store.rs create mode 100644 nix-rust/src/store/binary_cache_store.rs create mode 100644 nix-rust/src/store/mod.rs create mode 100644 nix-rust/src/store/path_info.rs create mode 100644 nix-rust/src/store/store.rs delete mode 100644 nix-rust/src/tarfile.rs create mode 100644 nix-rust/src/util/mod.rs create mode 100644 nix-rust/src/util/tarfile.rs (limited to 'nix-rust') diff --git a/nix-rust/local.mk b/nix-rust/local.mk index cf1a5c5c5..1157f7894 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -18,7 +18,7 @@ libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir) endif -$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml +$(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ $(libnixrust_BUILD_FLAGS) \ RUSTC_BOOTSTRAP=1 cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ diff --git a/nix-rust/src/binary_cache_store.rs b/nix-rust/src/binary_cache_store.rs deleted file mode 100644 index b7e61eb2d..000000000 --- a/nix-rust/src/binary_cache_store.rs +++ /dev/null @@ -1,51 +0,0 @@ -use crate::store::{Store, StorePath}; -use crate::path_info::PathInfo; -use crate::Error; -use futures::compat::Future01CompatExt; - -pub struct BinaryCacheStore { - base_uri: String, - client: reqwest::r#async::Client, -} - -impl BinaryCacheStore { - pub fn new(base_uri: String) -> Self { - Self { - base_uri, - client: reqwest::r#async::Client::new(), - } - } -} - -impl Store for BinaryCacheStore { - fn query_path_info( - &self, - path: &StorePath, - ) -> std::pin::Pin> + Send>> { - let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash); - let path = path.clone(); - let client = self.client.clone(); - let store_dir = self.store_dir().to_string(); - - Box::pin(async move { - let response = client - .get(&uri) - .send() - .compat() - .await?; - - if response.status() == reqwest::StatusCode::NOT_FOUND || response.status() == reqwest::StatusCode::FORBIDDEN { - return Err(Error::InvalidPath(path)); - } - - let mut response = response.error_for_status()?; - - let body = response - .text() - .compat() - .await?; - - PathInfo::parse_nar_info(&body, &store_dir) - }) - } -} diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs new file mode 100644 index 000000000..1abd34198 --- /dev/null +++ b/nix-rust/src/c.rs @@ -0,0 +1,42 @@ +use super::{foreign::{self, CBox}, error, util, store}; + +#[no_mangle] +pub extern "C" fn unpack_tarfile( + source: foreign::Source, + dest_dir: &str, +) -> CBox> { + CBox::new(util::tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into())) +} + +#[no_mangle] +pub extern "C" fn rust_test() { + use crate::store::Store; + use futures::future::{FutureExt, TryFutureExt}; + use std::path::Path; + + let fut = async move { + let store: Box = Box::new(store::BinaryCacheStore::new( + "https://cache.nixos.org".to_string(), + )); + + let path = store + .parse_store_path(&Path::new( + "/nix/store/7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3", + )) + .unwrap(); + + /* + let info = store.query_path_info(&path).await.unwrap(); + + eprintln!("INFO = {:?}", info); + */ + + let closure = store.compute_path_closure(vec![path].into_iter().collect()).await.unwrap(); + + eprintln!("CLOSURE = {:?}", closure.len()); + + Ok(()) + }; + + tokio::run(fut.boxed().compat()); +} diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs index 7bce7753c..8e04280f3 100644 --- a/nix-rust/src/foreign.rs +++ b/nix-rust/src/foreign.rs @@ -12,3 +12,22 @@ impl std::io::Read for Source { Ok(n) } } + +pub struct CBox { + pub ptr: *mut libc::c_void, + phantom: std::marker::PhantomData, +} + +impl CBox { + pub fn new(t: T) -> Self { + unsafe { + let size = std::mem::size_of::(); + let ptr = libc::malloc(size); + *(ptr as *mut T) = t; // FIXME: probably UB + Self { + ptr, + phantom: std::marker::PhantomData, + } + } + } +} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 7445e005c..2dbe6a217 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -1,70 +1,9 @@ #![feature(await_macro, async_await)] -mod binary_cache_store; +mod c; mod error; mod foreign; mod store; -mod tarfile; -mod path_info; +mod util; pub use error::Error; - -pub struct CBox { - pub ptr: *mut libc::c_void, - phantom: std::marker::PhantomData, -} - -impl CBox { - fn new(t: T) -> Self { - unsafe { - let size = std::mem::size_of::(); - let ptr = libc::malloc(size); - *(ptr as *mut T) = t; // FIXME: probably UB - Self { - ptr, - phantom: std::marker::PhantomData, - } - } - } -} - -#[no_mangle] -pub extern "C" fn unpack_tarfile( - source: foreign::Source, - dest_dir: &str, -) -> CBox> { - CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into())) -} - -#[no_mangle] -pub extern "C" fn rust_test() { - use crate::store::Store; - use futures::future::{FutureExt, TryFutureExt}; - use std::path::Path; - - let fut = async move { - let store: Box = Box::new(binary_cache_store::BinaryCacheStore::new( - "https://cache.nixos.org".to_string(), - )); - - let path = store - .parse_store_path(&Path::new( - "/nix/store/7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3", - )) - .unwrap(); - - /* - let info = store.query_path_info(&path).await.unwrap(); - - eprintln!("INFO = {:?}", info); - */ - - let closure = store.compute_path_closure(vec![path].into_iter().collect()).await.unwrap(); - - eprintln!("CLOSURE = {:?}", closure.len()); - - Ok(()) - }; - - tokio::run(fut.boxed().compat()); -} diff --git a/nix-rust/src/path_info.rs b/nix-rust/src/path_info.rs deleted file mode 100644 index 2759e03d4..000000000 --- a/nix-rust/src/path_info.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::store::StorePath; -use crate::Error; -use std::collections::BTreeSet; - -#[derive(Clone, Debug)] -pub struct PathInfo { - pub path: StorePath, - pub references: BTreeSet, - pub nar_size: u64, - pub deriver: Option, - - // Additional binary cache info. - pub url: Option, - pub compression: Option, - pub file_size: Option, -} - -impl PathInfo { - pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result { - let mut path = None; - let mut references = BTreeSet::new(); - let mut nar_size = None; - let mut deriver = None; - let mut url = None; - let mut compression = None; - let mut file_size = None; - - for line in nar_info.lines() { - let colon = line.find(':').ok_or(Error::BadNarInfo)?; - - let (name, value) = line.split_at(colon); - - if !value.starts_with(": ") { - return Err(Error::BadNarInfo); - } - - let value = &value[2..]; - - if name == "StorePath" { - path = Some(StorePath::new(std::path::Path::new(value), store_dir)?); - } else if name == "NarSize" { - nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?); - } else if name == "References" { - if !value.is_empty() { - for r in value.split(' ') { - references.insert(StorePath::new_short(r)?); - } - } - } else if name == "Deriver" { - deriver = Some(StorePath::new_short(value)?); - } else if name == "URL" { - url = Some(value.into()); - } else if name == "Compression" { - compression = Some(value.into()); - } else if name == "FileSize" { - file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?); - } - } - - Ok(PathInfo { - path: path.ok_or(Error::BadNarInfo)?, - references, - nar_size: nar_size.ok_or(Error::BadNarInfo)?, - deriver, - url: Some(url.ok_or(Error::BadNarInfo)?), - compression, - file_size, - }) - } -} diff --git a/nix-rust/src/store.rs b/nix-rust/src/store.rs deleted file mode 100644 index 256dcf0f3..000000000 --- a/nix-rust/src/store.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::path_info::PathInfo; -use crate::Error; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; - -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub struct StorePath { - pub hash: String, - pub name: String, -} - -pub const STORE_PATH_HASH_CHARS: usize = 32; - -impl StorePath { - pub fn new(path: &Path, store_dir: &str) -> Result { - // FIXME: check store_dir - Self::new_short( - path.file_name() - .ok_or(Error::BadStorePath(path.into()))? - .to_str() - .ok_or(Error::BadStorePath(path.into()))?, - ) - } - - pub fn new_short(base_name: &str) -> Result { - if base_name.len() < STORE_PATH_HASH_CHARS + 2 - || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8 - { - return Err(Error::BadStorePath(base_name.into())); - } - - // FIXME: validate name - - Ok(StorePath { - hash: base_name[0..STORE_PATH_HASH_CHARS].to_string(), - name: base_name[STORE_PATH_HASH_CHARS + 1..].to_string(), - }) - } -} - -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub struct StorePathHash { - bytes: [u8; 20], -} - -/* -impl StorePathHash { - pub fn to_base32(&self) -> String { - "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz".to_string() - } -} -*/ - -pub trait Store: Send + Sync { - fn store_dir(&self) -> &str { - "/nix/store" - } - - fn query_path_info( - &self, - store_path: &StorePath, - ) -> std::pin::Pin> + Send>>; -} - -impl dyn Store { - pub fn parse_store_path(&self, path: &Path) -> Result { - StorePath::new(path, self.store_dir()) - } - - pub async fn compute_path_closure( - &self, - roots: BTreeSet, - ) -> Result, Error> { - let mut done = BTreeSet::new(); - let mut result = BTreeMap::new(); - let mut pending = vec![]; - - for root in roots { - pending.push(self.query_path_info(&root)); - done.insert(root); - } - - while !pending.is_empty() { - let (info, _, remaining) = futures::future::select_all(pending).await; - pending = remaining; - - let info = info?; - - for path in &info.references { - if !done.contains(path) { - pending.push(self.query_path_info(&path)); - done.insert(path.clone()); - } - } - - result.insert(info.path.clone(), info); - } - - Ok(result) - } -} diff --git a/nix-rust/src/store/binary_cache_store.rs b/nix-rust/src/store/binary_cache_store.rs new file mode 100644 index 000000000..f3c7b39bb --- /dev/null +++ b/nix-rust/src/store/binary_cache_store.rs @@ -0,0 +1,50 @@ +use super::{Store, StorePath, PathInfo}; +use crate::Error; +use futures::compat::Future01CompatExt; + +pub struct BinaryCacheStore { + base_uri: String, + client: reqwest::r#async::Client, +} + +impl BinaryCacheStore { + pub fn new(base_uri: String) -> Self { + Self { + base_uri, + client: reqwest::r#async::Client::new(), + } + } +} + +impl Store for BinaryCacheStore { + fn query_path_info( + &self, + path: &StorePath, + ) -> std::pin::Pin> + Send>> { + let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash); + let path = path.clone(); + let client = self.client.clone(); + let store_dir = self.store_dir().to_string(); + + Box::pin(async move { + let response = client + .get(&uri) + .send() + .compat() + .await?; + + if response.status() == reqwest::StatusCode::NOT_FOUND || response.status() == reqwest::StatusCode::FORBIDDEN { + return Err(Error::InvalidPath(path)); + } + + let mut response = response.error_for_status()?; + + let body = response + .text() + .compat() + .await?; + + PathInfo::parse_nar_info(&body, &store_dir) + }) + } +} diff --git a/nix-rust/src/store/mod.rs b/nix-rust/src/store/mod.rs new file mode 100644 index 000000000..a0d1c72ab --- /dev/null +++ b/nix-rust/src/store/mod.rs @@ -0,0 +1,7 @@ +mod binary_cache_store; +mod path_info; +mod store; + +pub use binary_cache_store::BinaryCacheStore; +pub use path_info::PathInfo; +pub use store::{Store, StorePath}; diff --git a/nix-rust/src/store/path_info.rs b/nix-rust/src/store/path_info.rs new file mode 100644 index 000000000..2759e03d4 --- /dev/null +++ b/nix-rust/src/store/path_info.rs @@ -0,0 +1,70 @@ +use crate::store::StorePath; +use crate::Error; +use std::collections::BTreeSet; + +#[derive(Clone, Debug)] +pub struct PathInfo { + pub path: StorePath, + pub references: BTreeSet, + pub nar_size: u64, + pub deriver: Option, + + // Additional binary cache info. + pub url: Option, + pub compression: Option, + pub file_size: Option, +} + +impl PathInfo { + pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result { + let mut path = None; + let mut references = BTreeSet::new(); + let mut nar_size = None; + let mut deriver = None; + let mut url = None; + let mut compression = None; + let mut file_size = None; + + for line in nar_info.lines() { + let colon = line.find(':').ok_or(Error::BadNarInfo)?; + + let (name, value) = line.split_at(colon); + + if !value.starts_with(": ") { + return Err(Error::BadNarInfo); + } + + let value = &value[2..]; + + if name == "StorePath" { + path = Some(StorePath::new(std::path::Path::new(value), store_dir)?); + } else if name == "NarSize" { + nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?); + } else if name == "References" { + if !value.is_empty() { + for r in value.split(' ') { + references.insert(StorePath::new_short(r)?); + } + } + } else if name == "Deriver" { + deriver = Some(StorePath::new_short(value)?); + } else if name == "URL" { + url = Some(value.into()); + } else if name == "Compression" { + compression = Some(value.into()); + } else if name == "FileSize" { + file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?); + } + } + + Ok(PathInfo { + path: path.ok_or(Error::BadNarInfo)?, + references, + nar_size: nar_size.ok_or(Error::BadNarInfo)?, + deriver, + url: Some(url.ok_or(Error::BadNarInfo)?), + compression, + file_size, + }) + } +} diff --git a/nix-rust/src/store/store.rs b/nix-rust/src/store/store.rs new file mode 100644 index 000000000..ceaed6648 --- /dev/null +++ b/nix-rust/src/store/store.rs @@ -0,0 +1,101 @@ +use super::PathInfo; +use crate::Error; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct StorePath { + pub hash: String, + pub name: String, +} + +pub const STORE_PATH_HASH_CHARS: usize = 32; + +impl StorePath { + pub fn new(path: &Path, store_dir: &str) -> Result { + // FIXME: check store_dir + Self::new_short( + path.file_name() + .ok_or(Error::BadStorePath(path.into()))? + .to_str() + .ok_or(Error::BadStorePath(path.into()))?, + ) + } + + pub fn new_short(base_name: &str) -> Result { + if base_name.len() < STORE_PATH_HASH_CHARS + 2 + || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8 + { + return Err(Error::BadStorePath(base_name.into())); + } + + // FIXME: validate name + + Ok(StorePath { + hash: base_name[0..STORE_PATH_HASH_CHARS].to_string(), + name: base_name[STORE_PATH_HASH_CHARS + 1..].to_string(), + }) + } +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct StorePathHash { + bytes: [u8; 20], +} + +/* +impl StorePathHash { + pub fn to_base32(&self) -> String { + "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz".to_string() + } +} +*/ + +pub trait Store: Send + Sync { + fn store_dir(&self) -> &str { + "/nix/store" + } + + fn query_path_info( + &self, + store_path: &StorePath, + ) -> std::pin::Pin> + Send>>; +} + +impl dyn Store { + pub fn parse_store_path(&self, path: &Path) -> Result { + StorePath::new(path, self.store_dir()) + } + + pub async fn compute_path_closure( + &self, + roots: BTreeSet, + ) -> Result, Error> { + let mut done = BTreeSet::new(); + let mut result = BTreeMap::new(); + let mut pending = vec![]; + + for root in roots { + pending.push(self.query_path_info(&root)); + done.insert(root); + } + + while !pending.is_empty() { + let (info, _, remaining) = futures::future::select_all(pending).await; + pending = remaining; + + let info = info?; + + for path in &info.references { + if !done.contains(path) { + pending.push(self.query_path_info(&path)); + done.insert(path.clone()); + } + } + + result.insert(info.path.clone(), info); + } + + Ok(result) + } +} diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs deleted file mode 100644 index 379d9098f..000000000 --- a/nix-rust/src/tarfile.rs +++ /dev/null @@ -1,46 +0,0 @@ -use crate::{foreign::Source, Error}; -use std::fs; -use std::io; -use std::os::unix::fs::OpenOptionsExt; -use std::path::Path; -use tar::Archive; - -pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { - let dest_dir = Path::new(dest_dir); - - let mut tar = Archive::new(source); - - for file in tar.entries()? { - let mut file = file?; - - let dest_file = dest_dir.join(file.path()?); - - fs::create_dir_all(dest_file.parent().unwrap())?; - - match file.header().entry_type() { - tar::EntryType::Directory => { - fs::create_dir(dest_file)?; - } - tar::EntryType::Regular => { - let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { - 0o666 - } else { - 0o777 - }; - let mut f = fs::OpenOptions::new() - .create(true) - .write(true) - .mode(mode) - .open(dest_file)?; - io::copy(&mut file, &mut f)?; - } - tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; - } - tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} - t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), - } - } - - Ok(()) -} diff --git a/nix-rust/src/util/mod.rs b/nix-rust/src/util/mod.rs new file mode 100644 index 000000000..209627893 --- /dev/null +++ b/nix-rust/src/util/mod.rs @@ -0,0 +1 @@ +pub mod tarfile; diff --git a/nix-rust/src/util/tarfile.rs b/nix-rust/src/util/tarfile.rs new file mode 100644 index 000000000..379d9098f --- /dev/null +++ b/nix-rust/src/util/tarfile.rs @@ -0,0 +1,46 @@ +use crate::{foreign::Source, Error}; +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(source); + + for file in tar.entries()? { + let mut file = file?; + + let dest_file = dest_dir.join(file.path()?); + + fs::create_dir_all(dest_file.parent().unwrap())?; + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file)?; + } + tar::EntryType::Regular => { + let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file)?; + io::copy(&mut file, &mut f)?; + } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; + } + tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} + t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), + } + } + + Ok(()) +} -- cgit v1.2.3