diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5d305d99..0163f98c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - target: ["test", "seek", "package"] + target: ["test", "seek", "package", "download"] steps: - uses: actions/checkout@v1 - name: Test with mkl-rust container diff --git a/intel-mkl-tool/Makefile b/intel-mkl-tool/Makefile index 5b3b96b3..5bec8024 100644 --- a/intel-mkl-tool/Makefile +++ b/intel-mkl-tool/Makefile @@ -11,3 +11,6 @@ seek: package: docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- package + +download: + docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- download -o ./test_download diff --git a/intel-mkl-tool/src/cli.rs b/intel-mkl-tool/src/cli.rs index 0a556a2b..7317a27f 100644 --- a/intel-mkl-tool/src/cli.rs +++ b/intel-mkl-tool/src/cli.rs @@ -9,7 +9,11 @@ use structopt::StructOpt; enum Opt { /// Download Intel-MKL library Download { - /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool` + /// Archive name, e.g. "mkl-static-lp64-iomp". Download all archives if None + #[structopt(long = "name")] + name: Option, + /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool/${MKL_VERSION}/` + #[structopt(short = "o", long = "path")] path: Option, }, @@ -22,8 +26,10 @@ enum Opt { /// Package Intel MKL libraries into an archive Package { + #[structopt(long = "name")] name: Option, - out: Option, + #[structopt(short = "o", long = "path")] + path: Option, }, } @@ -34,13 +40,21 @@ fn main() -> Result<()> { let opt = Opt::from_args(); match opt { - Opt::Download { path } => { - let path = if let Some(path) = path { - path + Opt::Download { name, path } => { + let path = path.unwrap_or(xdg_home_path()); + if let Some(name) = name { + let cfg = Config::from_str(&name)?; + cfg.download(&path.join(cfg.name()))?; } else { - xdg_home_path() - }; - download_default(&path)?; + for cfg in Config::possibles() { + info!( + "Download archive {:<22} into {}", + cfg.name(), + path.display() + ); + cfg.download(&path.join(cfg.name()))?; + } + } } Opt::Seek {} => { @@ -56,26 +70,26 @@ fn main() -> Result<()> { } } - Opt::Package { name, out } => { - let out = out.unwrap_or(env::current_dir().unwrap()); + Opt::Package { name, path } => { + let path = path.unwrap_or(env::current_dir().unwrap()); if let Some(name) = name { let cfg = Config::from_str(&name)?; let entry = Entry::from_config(cfg)?; - let out = if let Ok(version) = entry.version() { - out.join(format!("{}.{}", version.0, version.1)) + let path = if let Ok(version) = entry.version() { + path.join(format!("{}.{}", version.0, version.1)) } else { - out + path }; - let package = entry.package(&out)?; + let package = entry.package(&path)?; info!("Pacakge created: {}", package.display()); } else { for entry in Entry::available() { - let out = if let Ok(version) = entry.version() { - out.join(format!("{}.{}", version.0, version.1)) + let path = if let Ok(version) = entry.version() { + path.join(format!("{}.{}", version.0, version.1)) } else { - out.clone() + path.clone() }; - let package = entry.package(&out)?; + let package = entry.package(&path)?; info!("Pacakge created: {}", package.display()); } } diff --git a/intel-mkl-tool/src/config.rs b/intel-mkl-tool/src/config.rs index ea312f00..7884f378 100644 --- a/intel-mkl-tool/src/config.rs +++ b/intel-mkl-tool/src/config.rs @@ -1,5 +1,7 @@ -use anyhow::*; -use derive_more::Display; +use crate::*; +use curl::easy::Easy; +use derive_more::*; +use std::fs; pub const VALID_CONFIGS: &[&str] = &[ "mkl-dynamic-ilp64-iomp", @@ -80,6 +82,13 @@ impl Config { }) } + pub fn possibles() -> Vec { + VALID_CONFIGS + .iter() + .map(|name| Self::from_str(name).unwrap()) + .collect() + } + /// identifier used in pkg-config pub fn name(&self) -> String { format!("mkl-{}-{}-{}", self.link, self.index_size, self.parallel) @@ -130,6 +139,41 @@ impl Config { } } } + + /// Download archive from AWS S3, and expand into `${out_dir}/*.so` + pub fn download>(&self, out_dir: P) -> Result<()> { + let out_dir = out_dir.as_ref(); + if out_dir.exists() { + fs::create_dir_all(&out_dir)?; + } + let data = read_from_url(&format!("{}/{}.tar.zst", s3_addr(), self.name()))?; + let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; + let mut arc = tar::Archive::new(zstd); + arc.unpack(&out_dir)?; + Ok(()) + } +} + +/// Helper for download file from URL +/// +/// - This function expands obtained data into memory space +/// +fn read_from_url(url: &str) -> Result> { + let mut data = Vec::new(); + let mut handle = Easy::new(); + handle.fail_on_error(true)?; + handle.url(url)?; + { + let mut transfer = handle.transfer(); + transfer + .write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + }) + .unwrap(); + transfer.perform().unwrap(); + } + Ok(data) } #[cfg(test)] diff --git a/intel-mkl-tool/src/download.rs b/intel-mkl-tool/src/download.rs deleted file mode 100644 index 468e4c40..00000000 --- a/intel-mkl-tool/src/download.rs +++ /dev/null @@ -1,48 +0,0 @@ -use crate::{mkl, S3_ADDR}; - -use anyhow::*; -use curl::easy::Easy; -use log::*; -use std::path::*; - -fn download_archive_to_buffer(url: &str) -> Result> { - let mut data = Vec::new(); - let mut handle = Easy::new(); - handle.url(url)?; - { - let mut transfer = handle.transfer(); - transfer - .write_function(|new_data| { - data.extend_from_slice(new_data); - Ok(new_data.len()) - }) - .unwrap(); - transfer.perform().unwrap(); - } - Ok(data) -} - -pub fn download(out_dir: &Path, prefix: &str, year: u32, update: u32) -> Result<()> { - let mkl_core = out_dir.join(format!("{}mkl_core.{}", mkl::PREFIX, mkl::EXTENSION_SHARED)); - if mkl_core.exists() { - info!("Archive already exists: {}", out_dir.display()); - return Ok(()); - } - - let filename = format!("{}_{}_{}.tar.zst", prefix, year, update); - info!("Download archive {} into {}", filename, out_dir.display()); - let data = download_archive_to_buffer(&format!("{}/{}", S3_ADDR, filename))?; - let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; - let mut arc = tar::Archive::new(zstd); - arc.unpack(&out_dir)?; - Ok(()) -} - -pub fn download_default(out_dir: &Path) -> Result<()> { - download( - out_dir, - mkl::ARCHIVE, - mkl::VERSION_YEAR, - mkl::VERSION_UPDATE, - ) -} diff --git a/intel-mkl-tool/src/entry.rs b/intel-mkl-tool/src/entry.rs index 374c643f..03e3c6c8 100644 --- a/intel-mkl-tool/src/entry.rs +++ b/intel-mkl-tool/src/entry.rs @@ -1,5 +1,4 @@ use crate::*; -use anyhow::*; use derive_more::Deref; use std::{ collections::{HashMap, HashSet}, diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index da0b21db..161bc0ae 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -90,32 +90,31 @@ //! |mkl_cdft_core | libmkl_cdft_core.so| libmkl_cdft_core.a| //! +use anyhow::*; use log::*; use std::path::*; mod config; -mod download; mod entry; pub use config::*; -pub use download::*; pub use entry::*; const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-intel-mkl"; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] mod mkl { - pub const ARCHIVE: &'static str = "mkl_linux64"; + pub const OS: &str = "linux"; pub const EXTENSION_STATIC: &'static str = "a"; pub const EXTENSION_SHARED: &'static str = "so"; pub const PREFIX: &'static str = "lib"; - pub const VERSION_YEAR: u32 = 2019; - pub const VERSION_UPDATE: u32 = 5; + pub const VERSION_YEAR: u32 = 2020; + pub const VERSION_UPDATE: u32 = 1; } #[cfg(all(target_os = "macos", target_arch = "x86_64"))] mod mkl { - pub const ARCHIVE: &'static str = "mkl_macos64"; + pub const OS: &str = "macos"; pub const EXTENSION_STATIC: &'static str = "a"; pub const EXTENSION_SHARED: &'static str = "dylib"; pub const PREFIX: &'static str = "lib"; @@ -125,7 +124,7 @@ mod mkl { #[cfg(all(target_os = "windows", target_arch = "x86_64"))] mod mkl { - pub const ARCHIVE: &'static str = "mkl_windows64"; + pub const OS: &str = "windows"; pub const EXTENSION_STATIC: &'static str = "lib"; pub const EXTENSION_SHARED: &'static str = "lib"; pub const PREFIX: &'static str = ""; @@ -133,21 +132,26 @@ mod mkl { pub const VERSION_UPDATE: u32 = 5; } -pub fn archive_filename() -> String { +fn s3_addr() -> String { format!( - "{}_{}_{}.tar.zst", - mkl::ARCHIVE, + "{}/{}/{}.{}", + S3_ADDR, + mkl::OS, mkl::VERSION_YEAR, mkl::VERSION_UPDATE ) } pub fn xdg_home_path() -> PathBuf { - dirs::data_local_dir().unwrap().join("intel-mkl-tool") + dirs::data_local_dir().unwrap().join(format!( + "intel-mkl-tool/{}.{}", + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE + )) } pub fn seek_pkg_config() -> Option { - if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-iomp") { + if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-seq") { if lib.libs.len() > 1 { warn!("Found {} MKL libraries. Use first found.", lib.libs.len()) } @@ -156,10 +160,19 @@ pub fn seek_pkg_config() -> Option { None } -pub fn seek_home() -> Option { - let home_lib = xdg_home_path(); - if home_lib.is_dir() { - return Some(home_lib); +pub fn download_default>(out_dir: P) -> Result<()> { + let cfg = Config::from_str("mkl-dynamic-lp64-seq").unwrap(); + cfg.download(out_dir)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn download() -> Result<()> { + download_default("./test_download")?; + Ok(()) } - None }