From c13b178edf697947f277684edb0a0df14ceafa09 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 6 Jun 2020 21:56:20 +0900 Subject: [PATCH 01/10] Fix archive_filename generation --- intel-mkl-tool/src/lib.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 0e57a3b3..7836450d 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -37,13 +37,8 @@ mod mkl { pub const VERSION_UPDATE: u32 = 5; } -pub fn archive_filename() -> String { - format!( - "{}_{}_{}.tar.zst", - mkl::ARCHIVE, - mkl::VERSION_YEAR, - mkl::VERSION_UPDATE - ) +pub fn archive_filename(archive: &str, year: u32, update: u32) -> String { + format!("{}_{}_{}.tar.zst", archive, year, update) } pub fn home_library_path() -> PathBuf { @@ -77,9 +72,10 @@ pub fn download(out_dir: &Path) -> Result<()> { bail!("Not a directory: {}", out_dir.display()); } - let archive = out_dir.join(archive_filename()); + let filename = archive_filename(mkl::ARCHIVE, mkl::VERSION_YEAR, mkl::VERSION_UPDATE); + let archive = out_dir.join(&filename); if !archive.exists() { - let url = format!("{}/{}", S3_ADDR, archive_filename()); + let url = format!("{}/{}", S3_ADDR, filename); info!("Download archive from AWS S3: {}", url); let f = fs::File::create(&archive)?; let mut buf = io::BufWriter::new(f); @@ -139,7 +135,7 @@ pub fn package(mkl_path: &Path) -> Result { } let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; info!("Intel MKL version: {}.{}", year, update); - let out = PathBuf::from(archive_filename()); + let out = PathBuf::from(archive_filename(mkl::ARCHIVE, year, update)); info!("Create archive: {}", out.display()); let shared_libs: Vec<_> = glob( From 75f43eafb6d070fdd3ee47a1d3dfcbcf1fba7180 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sat, 6 Jun 2020 22:48:34 +0900 Subject: [PATCH 02/10] Package static library --- intel-mkl-tool/src/lib.rs | 76 ++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 7836450d..d527ff70 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -12,8 +12,10 @@ const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-inte #[cfg(all(target_os = "linux", target_arch = "x86_64"))] mod mkl { - pub const ARCHIVE: &'static str = "mkl_linux64"; - pub const EXT: &'static str = "so"; + pub const ARCHIVE_SHARED: &'static str = "mkl_linux64_shared"; + pub const ARCHIVE_STATIC: &'static str = "mkl_linux64_static"; + pub const EXTENSION_SHARED: &'static str = "so"; + pub const EXTENSION_STATIC: &'static str = "a"; pub const PREFIX: &'static str = "lib"; pub const VERSION_YEAR: u32 = 2019; pub const VERSION_UPDATE: u32 = 5; @@ -21,8 +23,9 @@ mod mkl { #[cfg(all(target_os = "macos", target_arch = "x86_64"))] mod mkl { - pub const ARCHIVE: &'static str = "mkl_macos64"; - pub const EXT: &'static str = "dylib"; + pub const ARCHIVE_SHARED: &'static str = "mkl_macos64_shared"; + pub const ARCHIVE_STATIC: &'static str = "mkl_macos64_static"; + pub const EXTENSION_SHARED: &'static str = "dylib"; pub const PREFIX: &'static str = "lib"; pub const VERSION_YEAR: u32 = 2019; pub const VERSION_UPDATE: u32 = 3; @@ -30,8 +33,9 @@ mod mkl { #[cfg(all(target_os = "windows", target_arch = "x86_64"))] mod mkl { - pub const ARCHIVE: &'static str = "mkl_windows64"; - pub const EXT: &'static str = "lib"; + pub const ARCHIVE_SHARED: &'static str = "mkl_windows64_shared"; + pub const ARCHIVE_STATIC: &'static str = "mkl_windows64_static"; + pub const EXTENSION_SHARED: &'static str = "lib"; pub const PREFIX: &'static str = ""; pub const VERSION_YEAR: u32 = 2019; pub const VERSION_UPDATE: u32 = 5; @@ -72,7 +76,7 @@ pub fn download(out_dir: &Path) -> Result<()> { bail!("Not a directory: {}", out_dir.display()); } - let filename = archive_filename(mkl::ARCHIVE, mkl::VERSION_YEAR, mkl::VERSION_UPDATE); + let filename = archive_filename(mkl::ARCHIVE_SHARED, mkl::VERSION_YEAR, mkl::VERSION_UPDATE); let archive = out_dir.join(&filename); if !archive.exists() { let url = format!("{}/{}", S3_ADDR, filename); @@ -88,7 +92,7 @@ pub fn download(out_dir: &Path) -> Result<()> { info!("Archive already exists: {}", archive.display()); } - let core = out_dir.join(format!("{}mkl_core.{}", mkl::PREFIX, mkl::EXT)); + let core = out_dir.join(format!("{}mkl_core.{}", mkl::PREFIX, mkl::EXTENSION_SHARED)); if !core.exists() { let f = fs::File::open(&archive)?; let de = zstd::stream::read::Decoder::new(f)?; @@ -129,34 +133,56 @@ fn get_mkl_version(version_header: &Path) -> Result<(u32, u32)> { Ok((year, update)) } -pub fn package(mkl_path: &Path) -> Result { - if !mkl_path.exists() { - bail!("MKL directory not found: {}", mkl_path.display()); +// Create tar.zst archive from path list +fn create_archive(libs: &[PathBuf], out: &Path) -> Result<()> { + if out.exists() { + bail!("Output archive already exits: {}", out.display()); } - let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; - info!("Intel MKL version: {}.{}", year, update); - let out = PathBuf::from(archive_filename(mkl::ARCHIVE, year, update)); info!("Create archive: {}", out.display()); - - let shared_libs: Vec<_> = glob( - mkl_path - .join(format!("lib/intel64/*.{}", mkl::EXT)) - .to_str() - .unwrap(), - )? - .map(|path| path.unwrap()) - .collect(); let f = fs::File::create(&out)?; let buf = io::BufWriter::new(f); let zstd = zstd::stream::write::Encoder::new(buf, 6)?; let mut ar = tar::Builder::new(zstd); ar.mode(tar::HeaderMode::Deterministic); - for lib in &shared_libs { + for lib in libs { info!("Add {}", lib.display()); ar.append_path_with_name(lib, lib.file_name().unwrap())?; } let zstd = ar.into_inner()?; zstd.finish()?; + Ok(()) +} + +pub fn package(mkl_path: &Path) -> Result<()> { + if !mkl_path.exists() { + bail!("MKL directory not found: {}", mkl_path.display()); + } + let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; + info!("Intel MKL version: {}.{}", year, update); + + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_SHARED)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(archive_filename(mkl::ARCHIVE_SHARED, year, update)), + )?; - Ok(out) + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_STATIC)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(archive_filename(mkl::ARCHIVE_STATIC, year, update)), + )?; + + Ok(()) } From 7ff55f56ed5a282f05c9b2d314454b1554601e3a Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sun, 7 Jun 2020 20:48:56 +0900 Subject: [PATCH 03/10] Download new archives --- intel-mkl-src/build.rs | 21 ++++++++++- intel-mkl-tool/src/cli.rs | 18 +++++---- intel-mkl-tool/src/lib.rs | 77 ++++++++++++++++++--------------------- 3 files changed, 66 insertions(+), 50 deletions(-) diff --git a/intel-mkl-src/build.rs b/intel-mkl-src/build.rs index 10e6c469..8dbbd48b 100644 --- a/intel-mkl-src/build.rs +++ b/intel-mkl-src/build.rs @@ -20,6 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +use intel_mkl_tool::*; use std::{env, path::*}; fn main() { @@ -27,12 +28,28 @@ fn main() { path } else { let out_dir = if cfg!(feature = "use-shared") { - intel_mkl_tool::home_library_path() + intel_mkl_tool::xdg_home_path() } else { PathBuf::from(env::var("OUT_DIR").expect("Failed to get OUT_DIR")) }; - intel_mkl_tool::download(&out_dir).expect("Failed to downalod Intel-MKL archive"); + if cfg!(feature = "static") { + download( + &out_dir, + mkl::ARCHIVE_STATIC, + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE, + ) + .expect("Failed to downalod Intel-MKL archive"); + } else { + download( + &out_dir, + mkl::ARCHIVE_SHARED, + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE, + ) + .expect("Failed to downalod Intel-MKL archive"); + } out_dir }; println!("cargo:rustc-link-search={}", out_dir.display()); diff --git a/intel-mkl-tool/src/cli.rs b/intel-mkl-tool/src/cli.rs index 7802a408..ea84b33b 100644 --- a/intel-mkl-tool/src/cli.rs +++ b/intel-mkl-tool/src/cli.rs @@ -1,4 +1,5 @@ use anyhow::*; +use intel_mkl_tool::*; use std::{env, path::PathBuf}; use structopt::StructOpt; @@ -9,6 +10,10 @@ enum Opt { Download { /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool` path: Option, + /// Version of Intel MKL + year: Option, + /// Version of Intel MKL + update: Option, }, /// Seek Intel-MKL library @@ -29,13 +34,12 @@ fn main() -> Result<()> { let opt = Opt::from_args(); match opt { - Opt::Download { path } => { - let path = if let Some(path) = path { - path - } else { - intel_mkl_tool::home_library_path() - }; - intel_mkl_tool::download(&path)?; + Opt::Download { path, year, update } => { + let path = path.unwrap_or(xdg_home_path()); + let year = year.unwrap_or(mkl::VERSION_YEAR); + let update = update.unwrap_or(intel_mkl_tool::mkl::VERSION_UPDATE); + download(&path, mkl::ARCHIVE_SHARED, year, update)?; + download(&path, mkl::ARCHIVE_STATIC, year, update)?; } Opt::Seek {} => { diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index d527ff70..a13bee6a 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -4,25 +4,25 @@ use glob::glob; use log::*; use std::{ fs, - io::{self, BufRead, Write}, + io::{self, BufRead}, path::*, }; const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-intel-mkl"; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] -mod mkl { +pub mod mkl { pub const ARCHIVE_SHARED: &'static str = "mkl_linux64_shared"; pub const ARCHIVE_STATIC: &'static str = "mkl_linux64_static"; pub const EXTENSION_SHARED: &'static str = "so"; pub const EXTENSION_STATIC: &'static str = "a"; pub const PREFIX: &'static str = "lib"; - pub const VERSION_YEAR: u32 = 2019; - pub const VERSION_UPDATE: u32 = 5; + pub const VERSION_YEAR: u32 = 2020; + pub const VERSION_UPDATE: u32 = 1; } #[cfg(all(target_os = "macos", target_arch = "x86_64"))] -mod mkl { +pub mod mkl { pub const ARCHIVE_SHARED: &'static str = "mkl_macos64_shared"; pub const ARCHIVE_STATIC: &'static str = "mkl_macos64_static"; pub const EXTENSION_SHARED: &'static str = "dylib"; @@ -32,9 +32,8 @@ mod mkl { } #[cfg(all(target_os = "windows", target_arch = "x86_64"))] -mod mkl { - pub const ARCHIVE_SHARED: &'static str = "mkl_windows64_shared"; - pub const ARCHIVE_STATIC: &'static str = "mkl_windows64_static"; +pub mod mkl { + pub const ARCHIVE_SHARED: &'static str = "mkl_windows64"; pub const EXTENSION_SHARED: &'static str = "lib"; pub const PREFIX: &'static str = ""; pub const VERSION_YEAR: u32 = 2019; @@ -45,7 +44,7 @@ pub fn archive_filename(archive: &str, year: u32, update: u32) -> String { format!("{}_{}_{}.tar.zst", archive, year, update) } -pub fn home_library_path() -> PathBuf { +pub fn xdg_home_path() -> PathBuf { dirs::data_local_dir().unwrap().join("intel-mkl-tool") } @@ -60,48 +59,44 @@ pub fn seek_pkg_config() -> Option { } pub fn seek_home() -> Option { - let home_lib = home_library_path(); + let home_lib = xdg_home_path(); if home_lib.is_dir() { return Some(home_lib); } None } -pub fn download(out_dir: &Path) -> Result<()> { - if !out_dir.exists() { - info!("Create output directory: {}", out_dir.display()); - fs::create_dir_all(out_dir)?; - } - if !out_dir.is_dir() { - bail!("Not a directory: {}", out_dir.display()); +fn download_archive_to_buffer(url: &str) -> Result> { + let mut data = Vec::new(); + let mut handle = Easy::new(); + handle.url(url)?; + { + let mut transfer = handle.transfer(); + transfer + .write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + }) + .unwrap(); + transfer.perform().unwrap(); } + Ok(data) +} - let filename = archive_filename(mkl::ARCHIVE_SHARED, mkl::VERSION_YEAR, mkl::VERSION_UPDATE); - let archive = out_dir.join(&filename); - if !archive.exists() { - let url = format!("{}/{}", S3_ADDR, filename); - info!("Download archive from AWS S3: {}", url); - let f = fs::File::create(&archive)?; - let mut buf = io::BufWriter::new(f); - let mut easy = Easy::new(); - easy.url(&url)?; - easy.write_function(move |data| Ok(buf.write(data).unwrap()))?; - easy.perform()?; - assert!(archive.exists()); - } else { - info!("Archive already exists: {}", archive.display()); - } +pub fn download(base_dir: &Path, prefix: &str, year: u32, update: u32) -> Result<()> { + let filename = format!("{}_{}_{}.tar.zst", prefix, year, update); + let dest_dir = base_dir.join(&format!("{}_{}_{}", prefix, year, update)); - let core = out_dir.join(format!("{}mkl_core.{}", mkl::PREFIX, mkl::EXTENSION_SHARED)); - if !core.exists() { - let f = fs::File::open(&archive)?; - let de = zstd::stream::read::Decoder::new(f)?; - let mut arc = tar::Archive::new(de); - arc.unpack(&out_dir)?; - assert!(core.exists()); - } else { - info!("Archive has already been extracted"); + if dest_dir.exists() { + bail!("Directory already exists: {}", dest_dir.display()); } + fs::create_dir_all(&dest_dir)?; + + info!("Download archive {} into {}", filename, dest_dir.display()); + let data = download_archive_to_buffer(&format!("{}/{}", S3_ADDR, filename))?; + let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; + let mut arc = tar::Archive::new(zstd); + arc.unpack(&dest_dir)?; Ok(()) } From 4745124ca5722b52d65659733a917d47a32b36a0 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sun, 7 Jun 2020 20:59:13 +0900 Subject: [PATCH 04/10] Split into submodules --- intel-mkl-tool/src/download.rs | 40 ++++++++++ intel-mkl-tool/src/lib.rs | 135 ++------------------------------- intel-mkl-tool/src/package.rs | 102 +++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 128 deletions(-) create mode 100644 intel-mkl-tool/src/download.rs create mode 100644 intel-mkl-tool/src/package.rs diff --git a/intel-mkl-tool/src/download.rs b/intel-mkl-tool/src/download.rs new file mode 100644 index 00000000..ee876395 --- /dev/null +++ b/intel-mkl-tool/src/download.rs @@ -0,0 +1,40 @@ +use crate::S3_ADDR; + +use anyhow::*; +use curl::easy::Easy; +use log::*; +use std::{fs, path::*}; + +fn download_archive_to_buffer(url: &str) -> Result> { + let mut data = Vec::new(); + let mut handle = Easy::new(); + handle.url(url)?; + { + let mut transfer = handle.transfer(); + transfer + .write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + }) + .unwrap(); + transfer.perform().unwrap(); + } + Ok(data) +} + +pub fn download(base_dir: &Path, prefix: &str, year: u32, update: u32) -> Result<()> { + let filename = format!("{}_{}_{}.tar.zst", prefix, year, update); + let dest_dir = base_dir.join(&format!("{}_{}_{}", prefix, year, update)); + + if dest_dir.exists() { + bail!("Directory already exists: {}", dest_dir.display()); + } + fs::create_dir_all(&dest_dir)?; + + info!("Download archive {} into {}", filename, dest_dir.display()); + let data = download_archive_to_buffer(&format!("{}/{}", S3_ADDR, filename))?; + let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; + let mut arc = tar::Archive::new(zstd); + arc.unpack(&dest_dir)?; + Ok(()) +} diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index a13bee6a..19ff4843 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -1,12 +1,11 @@ -use anyhow::*; -use curl::easy::Easy; -use glob::glob; use log::*; -use std::{ - fs, - io::{self, BufRead}, - path::*, -}; +use std::path::*; + +mod download; +mod package; + +pub use download::*; +pub use package::*; const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-intel-mkl"; @@ -40,10 +39,6 @@ pub mod mkl { pub const VERSION_UPDATE: u32 = 5; } -pub fn archive_filename(archive: &str, year: u32, update: u32) -> String { - format!("{}_{}_{}.tar.zst", archive, year, update) -} - pub fn xdg_home_path() -> PathBuf { dirs::data_local_dir().unwrap().join("intel-mkl-tool") } @@ -65,119 +60,3 @@ pub fn seek_home() -> Option { } None } - -fn download_archive_to_buffer(url: &str) -> Result> { - let mut data = Vec::new(); - let mut handle = Easy::new(); - handle.url(url)?; - { - let mut transfer = handle.transfer(); - transfer - .write_function(|new_data| { - data.extend_from_slice(new_data); - Ok(new_data.len()) - }) - .unwrap(); - transfer.perform().unwrap(); - } - Ok(data) -} - -pub fn download(base_dir: &Path, prefix: &str, year: u32, update: u32) -> Result<()> { - let filename = format!("{}_{}_{}.tar.zst", prefix, year, update); - let dest_dir = base_dir.join(&format!("{}_{}_{}", prefix, year, update)); - - if dest_dir.exists() { - bail!("Directory already exists: {}", dest_dir.display()); - } - fs::create_dir_all(&dest_dir)?; - - info!("Download archive {} into {}", filename, dest_dir.display()); - let data = download_archive_to_buffer(&format!("{}/{}", S3_ADDR, filename))?; - let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; - let mut arc = tar::Archive::new(zstd); - arc.unpack(&dest_dir)?; - Ok(()) -} - -// Read mkl_version.h to get MKL version (e.g. 2019.5) -fn get_mkl_version(version_header: &Path) -> Result<(u32, u32)> { - if !version_header.exists() { - bail!("MKL Version file not found: {}", version_header.display()); - } - let f = fs::File::open(version_header)?; - let f = io::BufReader::new(f); - let mut year = 0; - let mut update = 0; - for line in f.lines() { - if let Ok(line) = line { - if !line.starts_with("#define") { - continue; - } - let ss: Vec<&str> = line.split(" ").collect(); - match ss[1] { - "__INTEL_MKL__" => year = ss[2].parse()?, - "__INTEL_MKL_UPDATE__" => update = ss[2].parse()?, - _ => continue, - } - } - } - if year == 0 || update == 0 { - bail!("Cannot determine MKL versions"); - } - Ok((year, update)) -} - -// Create tar.zst archive from path list -fn create_archive(libs: &[PathBuf], out: &Path) -> Result<()> { - if out.exists() { - bail!("Output archive already exits: {}", out.display()); - } - info!("Create archive: {}", out.display()); - let f = fs::File::create(&out)?; - let buf = io::BufWriter::new(f); - let zstd = zstd::stream::write::Encoder::new(buf, 6)?; - let mut ar = tar::Builder::new(zstd); - ar.mode(tar::HeaderMode::Deterministic); - for lib in libs { - info!("Add {}", lib.display()); - ar.append_path_with_name(lib, lib.file_name().unwrap())?; - } - let zstd = ar.into_inner()?; - zstd.finish()?; - Ok(()) -} - -pub fn package(mkl_path: &Path) -> Result<()> { - if !mkl_path.exists() { - bail!("MKL directory not found: {}", mkl_path.display()); - } - let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; - info!("Intel MKL version: {}.{}", year, update); - - create_archive( - &glob( - mkl_path - .join(format!("lib/intel64/*.{}", mkl::EXTENSION_SHARED)) - .to_str() - .unwrap(), - )? - .map(|path| path.unwrap()) - .collect::>(), - &PathBuf::from(archive_filename(mkl::ARCHIVE_SHARED, year, update)), - )?; - - create_archive( - &glob( - mkl_path - .join(format!("lib/intel64/*.{}", mkl::EXTENSION_STATIC)) - .to_str() - .unwrap(), - )? - .map(|path| path.unwrap()) - .collect::>(), - &PathBuf::from(archive_filename(mkl::ARCHIVE_STATIC, year, update)), - )?; - - Ok(()) -} diff --git a/intel-mkl-tool/src/package.rs b/intel-mkl-tool/src/package.rs new file mode 100644 index 00000000..217a73e5 --- /dev/null +++ b/intel-mkl-tool/src/package.rs @@ -0,0 +1,102 @@ +use crate::mkl; + +use anyhow::*; +use glob::glob; +use log::*; +use std::{ + fs, + io::{self, BufRead}, + path::*, +}; + +// Read mkl_version.h to get MKL version (e.g. 2019.5) +fn get_mkl_version(version_header: &Path) -> Result<(u32, u32)> { + if !version_header.exists() { + bail!("MKL Version file not found: {}", version_header.display()); + } + let f = fs::File::open(version_header)?; + let f = io::BufReader::new(f); + let mut year = 0; + let mut update = 0; + for line in f.lines() { + if let Ok(line) = line { + if !line.starts_with("#define") { + continue; + } + let ss: Vec<&str> = line.split(" ").collect(); + match ss[1] { + "__INTEL_MKL__" => year = ss[2].parse()?, + "__INTEL_MKL_UPDATE__" => update = ss[2].parse()?, + _ => continue, + } + } + } + if year == 0 || update == 0 { + bail!("Cannot determine MKL versions"); + } + Ok((year, update)) +} + +// Create tar.zst archive from path list +fn create_archive(libs: &[PathBuf], out: &Path) -> Result<()> { + if out.exists() { + bail!("Output archive already exits: {}", out.display()); + } + info!("Create archive: {}", out.display()); + let f = fs::File::create(&out)?; + let buf = io::BufWriter::new(f); + let zstd = zstd::stream::write::Encoder::new(buf, 6)?; + let mut ar = tar::Builder::new(zstd); + ar.mode(tar::HeaderMode::Deterministic); + for lib in libs { + info!("Add {}", lib.display()); + ar.append_path_with_name(lib, lib.file_name().unwrap())?; + } + let zstd = ar.into_inner()?; + zstd.finish()?; + Ok(()) +} + +pub fn package(mkl_path: &Path) -> Result<()> { + if !mkl_path.exists() { + bail!("MKL directory not found: {}", mkl_path.display()); + } + let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; + info!("Intel MKL version: {}.{}", year, update); + + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_SHARED)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(&format!( + "{}_{}_{}.tar.zst", + mkl::ARCHIVE_SHARED, + year, + update + )), + )?; + + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_STATIC)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(&format!( + "{}_{}_{}.tar.zst", + mkl::ARCHIVE_STATIC, + year, + update + )), + )?; + + Ok(()) +} From 451b8907ba01b2f39445f2d3322801fde2bd6afb Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sun, 7 Jun 2020 21:12:27 +0900 Subject: [PATCH 05/10] Add test and check HTTP status --- intel-mkl-tool/src/download.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/intel-mkl-tool/src/download.rs b/intel-mkl-tool/src/download.rs index ee876395..86ca7fb8 100644 --- a/intel-mkl-tool/src/download.rs +++ b/intel-mkl-tool/src/download.rs @@ -8,6 +8,7 @@ use std::{fs, path::*}; fn download_archive_to_buffer(url: &str) -> Result> { let mut data = Vec::new(); let mut handle = Easy::new(); + handle.fail_on_error(true)?; handle.url(url)?; { let mut transfer = handle.transfer(); @@ -38,3 +39,21 @@ pub fn download(base_dir: &Path, prefix: &str, year: u32, update: u32) -> Result arc.unpack(&dest_dir)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::mkl; + + #[test] + fn download_url() { + let url = format!( + "{}/{}_{}_{}.tar.zst", + S3_ADDR, + mkl::ARCHIVE_STATIC, + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE + ); + let _ar = download_archive_to_buffer(&url).unwrap(); + } +} From 474ee34c84f5fe16652206c234da2c36902906d2 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Sun, 7 Jun 2020 21:45:20 +0900 Subject: [PATCH 06/10] Create static library only on Linux --- intel-mkl-tool/src/lib.rs | 1 - intel-mkl-tool/src/package.rs | 34 ++++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 19ff4843..4692c2cb 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -23,7 +23,6 @@ pub mod mkl { #[cfg(all(target_os = "macos", target_arch = "x86_64"))] pub mod mkl { pub const ARCHIVE_SHARED: &'static str = "mkl_macos64_shared"; - pub const ARCHIVE_STATIC: &'static str = "mkl_macos64_static"; pub const EXTENSION_SHARED: &'static str = "dylib"; pub const PREFIX: &'static str = "lib"; pub const VERSION_YEAR: u32 = 2019; diff --git a/intel-mkl-tool/src/package.rs b/intel-mkl-tool/src/package.rs index 217a73e5..ca8374ab 100644 --- a/intel-mkl-tool/src/package.rs +++ b/intel-mkl-tool/src/package.rs @@ -81,22 +81,24 @@ pub fn package(mkl_path: &Path) -> Result<()> { )), )?; - create_archive( - &glob( - mkl_path - .join(format!("lib/intel64/*.{}", mkl::EXTENSION_STATIC)) - .to_str() - .unwrap(), - )? - .map(|path| path.unwrap()) - .collect::>(), - &PathBuf::from(&format!( - "{}_{}_{}.tar.zst", - mkl::ARCHIVE_STATIC, - year, - update - )), - )?; + if cfg!(target_or = "linux") { + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_STATIC)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(&format!( + "{}_{}_{}.tar.zst", + mkl::ARCHIVE_STATIC, + year, + update + )), + )?; + } Ok(()) } From a91e43d3260b062a4b0f24cb1a241cf7e04422f9 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Thu, 11 Jun 2020 11:57:32 +0900 Subject: [PATCH 07/10] Fix seeker --- intel-mkl-tool/Cargo.toml | 5 --- intel-mkl-tool/src/{cli.rs => bin/main.rs} | 23 ++++++---- intel-mkl-tool/src/lib.rs | 27 +----------- intel-mkl-tool/src/seek.rs | 49 ++++++++++++++++++++++ 4 files changed, 67 insertions(+), 37 deletions(-) rename intel-mkl-tool/src/{cli.rs => bin/main.rs} (69%) create mode 100644 intel-mkl-tool/src/seek.rs diff --git a/intel-mkl-tool/Cargo.toml b/intel-mkl-tool/Cargo.toml index 941bd0c1..e9076746 100644 --- a/intel-mkl-tool/Cargo.toml +++ b/intel-mkl-tool/Cargo.toml @@ -26,8 +26,3 @@ zstd = "0.5.1" # CLI structopt = { version = "0.3.5", optional = true } env_logger = { version = "0.7.1", optional = true } - -[[bin]] -name = "intel-mkl-tool" -path = "src/cli.rs" -required-features = ["cli"] diff --git a/intel-mkl-tool/src/cli.rs b/intel-mkl-tool/src/bin/main.rs similarity index 69% rename from intel-mkl-tool/src/cli.rs rename to intel-mkl-tool/src/bin/main.rs index ea84b33b..b04eb255 100644 --- a/intel-mkl-tool/src/cli.rs +++ b/intel-mkl-tool/src/bin/main.rs @@ -43,15 +43,24 @@ fn main() -> Result<()> { } Opt::Seek {} => { - if let Some(path) = intel_mkl_tool::seek_pkg_config() { - println!("{}", path.display()); - return Ok(()); + println!("pkg-config"); + println!("-----------"); + for (name, _lib) in intel_mkl_tool::seek_pkg_config() { + println!("- {}", name); } - if let Some(path) = intel_mkl_tool::seek_home() { - println!("{}", path.display()); - return Ok(()); + + let title = format!( + "xdg-data-home (base = {})", + intel_mkl_tool::xdg_home_path().display() + ); + println!( + "\n{}\n{}", + title, + std::str::from_utf8(vec!('-' as u8; title.len() + 1).as_slice()).unwrap() + ); + for (name, _path) in intel_mkl_tool::seek_xdg_home() { + println!("- {}", name); } - bail!("Intel-MKL not found."); } Opt::Package { path } => { diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 4692c2cb..bee4399f 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -1,11 +1,10 @@ -use log::*; -use std::path::*; - mod download; mod package; +mod seek; pub use download::*; pub use package::*; +pub use seek::*; const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-intel-mkl"; @@ -37,25 +36,3 @@ pub mod mkl { pub const VERSION_YEAR: u32 = 2019; pub const VERSION_UPDATE: u32 = 5; } - -pub fn xdg_home_path() -> PathBuf { - dirs::data_local_dir().unwrap().join("intel-mkl-tool") -} - -pub fn seek_pkg_config() -> Option { - if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-iomp") { - if lib.libs.len() > 1 { - warn!("Found {} MKL libraries. Use first found.", lib.libs.len()) - } - return Some(PathBuf::from(lib.libs[0].clone())); - } - None -} - -pub fn seek_home() -> Option { - let home_lib = xdg_home_path(); - if home_lib.is_dir() { - return Some(home_lib); - } - None -} diff --git a/intel-mkl-tool/src/seek.rs b/intel-mkl-tool/src/seek.rs new file mode 100644 index 00000000..ff2dc09e --- /dev/null +++ b/intel-mkl-tool/src/seek.rs @@ -0,0 +1,49 @@ +use std::path::*; + +pub fn xdg_home_path() -> PathBuf { + dirs::data_local_dir().unwrap().join("intel-mkl-tool") +} + +/// Following pkg-config settings are included in Intel MKL distribution +const PKG_CONFIG_TARGETS: [&str; 8] = [ + "mkl-dynamic-ilp64-iomp", + "mkl-dynamic-ilp64-seq", + "mkl-dynamic-lp64-iomp", + "mkl-dynamic-lp64-seq", + "mkl-static-ilp64-iomp", + "mkl-static-ilp64-seq", + "mkl-static-lp64-iomp", + "mkl-static-lp64-seq", +]; + +pub fn seek_pkg_config() -> Vec<(String, pkg_config::Library)> { + PKG_CONFIG_TARGETS + .iter() + .filter_map(|target| { + let lib = pkg_config::Config::new() + .cargo_metadata(false) + .probe(target) + .ok()?; + Some((target.to_string(), lib)) + }) + .collect() +} + +pub fn seek_xdg_home() -> Vec<(String, PathBuf)> { + let base = xdg_home_path(); + if !base.exists() { + return Vec::new(); + } + base.read_dir() + .unwrap() + .flat_map(|entry| { + let path = entry.unwrap().path(); + if path.is_dir() { + let name = path.file_name()?.to_str()?.into(); + Some((name, path)) + } else { + None + } + }) + .collect() +} From 1b77faa08a9f57fffb990c4494655e3e7fec02cd Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Thu, 11 Jun 2020 21:18:20 +0900 Subject: [PATCH 08/10] Add configure sub-module --- intel-mkl-tool/Cargo.toml | 1 + intel-mkl-tool/src/config.rs | 55 ++++++++++++++++++++++++++++++++++++ intel-mkl-tool/src/lib.rs | 2 ++ 3 files changed, 58 insertions(+) create mode 100644 intel-mkl-tool/src/config.rs diff --git a/intel-mkl-tool/Cargo.toml b/intel-mkl-tool/Cargo.toml index e9076746..fcdda5b5 100644 --- a/intel-mkl-tool/Cargo.toml +++ b/intel-mkl-tool/Cargo.toml @@ -26,3 +26,4 @@ zstd = "0.5.1" # CLI structopt = { version = "0.3.5", optional = true } env_logger = { version = "0.7.1", optional = true } +derive_more = "0.99.7" diff --git a/intel-mkl-tool/src/config.rs b/intel-mkl-tool/src/config.rs new file mode 100644 index 00000000..f1084282 --- /dev/null +++ b/intel-mkl-tool/src/config.rs @@ -0,0 +1,55 @@ +use derive_more::Display; + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum Link { + #[display(fmt = "static")] + Static, + #[display(fmt = "dynamic")] + Shared, +} + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum IndexSize { + #[display(fmt = "lp64")] + LP64, + #[display(fmt = "ilp64")] + ILP64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum Parallel { + #[display(fmt = "iomp")] + OpenMP, + #[display(fmt = "seq")] + Sequential, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Config { + link: Link, + index_size: IndexSize, + parallel: Parallel, +} + +impl Config { + fn as_pkg_config_name(&self) -> String { + format!("mkl-{}-{}-{}", self.link, self.index_size, self.parallel) + } + + /// Check if pkg-config has a corresponding setting + pub fn is_pkg_config_managed(&self) -> bool { + pkg_config::Config::new() + .cargo_metadata(false) + .probe(&self.as_pkg_config_name()) + .is_ok() + } + + /// Check if archive is cached in $XDG_DATA_HOME + pub fn is_cached(&self) -> bool { + todo!() + } + + pub fn print_cargo_metadata(&self) { + // + } +} diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index bee4399f..93182905 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -1,7 +1,9 @@ +mod config; mod download; mod package; mod seek; +pub use config::*; pub use download::*; pub use package::*; pub use seek::*; From 81f5c8d13c6f9d1332cdd51e2924db64687050e9 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Fri, 12 Jun 2020 15:35:44 +0900 Subject: [PATCH 09/10] WIP: impl Config --- intel-mkl-tool/Cargo.toml | 2 +- intel-mkl-tool/src/config.rs | 72 +++++++++++++++++++++++++++++++--- intel-mkl-tool/src/download.rs | 1 + 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/intel-mkl-tool/Cargo.toml b/intel-mkl-tool/Cargo.toml index fcdda5b5..bf988d80 100644 --- a/intel-mkl-tool/Cargo.toml +++ b/intel-mkl-tool/Cargo.toml @@ -16,6 +16,7 @@ cli = ["structopt", "env_logger"] [dependencies] anyhow = "1.0.31" curl = "0.4.25" +derive_more = "0.99.7" dirs = "2.0.2" glob = "0.3.0" log = "0.4.8" @@ -26,4 +27,3 @@ zstd = "0.5.1" # CLI structopt = { version = "0.3.5", optional = true } env_logger = { version = "0.7.1", optional = true } -derive_more = "0.99.7" diff --git a/intel-mkl-tool/src/config.rs b/intel-mkl-tool/src/config.rs index f1084282..470f25fd 100644 --- a/intel-mkl-tool/src/config.rs +++ b/intel-mkl-tool/src/config.rs @@ -1,4 +1,6 @@ +use anyhow::*; use derive_more::Display; +use std::path::*; #[derive(Debug, Clone, Copy, PartialEq, Display)] pub enum Link { @@ -32,24 +34,82 @@ pub struct Config { } impl Config { - fn as_pkg_config_name(&self) -> String { + pub fn from_name(_name: &str) -> Result { + todo!() + } + + /// identifier used in pkg-config + pub fn name(&self) -> String { format!("mkl-{}-{}-{}", self.link, self.index_size, self.parallel) } + fn base_dir(&self) -> PathBuf { + todo!() + } + + /// Static and shared library lists to be linked + pub fn libs( + &self, + ) -> ( + Vec, /* static */ + Vec, /* shared */ + ) { + // FIXME this implementation is for Linux, fix for Windows and macOS + let mut static_libs = Vec::new(); + let mut shared_libs = vec!["pthread".into(), "m".into(), "dl".into()]; + + let mut add = |name: &str| match self.link { + Link::Static => { + let base_dir: PathBuf = self.base_dir(); + let path = base_dir.join(format!("lib{}.a", name)); + assert!(path.exists()); + static_libs.push(path); + } + Link::Shared => { + shared_libs.push(name.to_string()); + } + }; + + add("mkl_core"); + match self.index_size { + IndexSize::LP64 => { + add("mkl_intel_lp64"); + } + IndexSize::ILP64 => { + add("mkl_intel_ilp64"); + } + }; + match self.parallel { + Parallel::OpenMP => { + add("iomp5"); + add("mkl_intel_thread"); + } + Parallel::Sequential => { + add("mkl_sequential"); + } + }; + (static_libs, shared_libs) + } + /// Check if pkg-config has a corresponding setting - pub fn is_pkg_config_managed(&self) -> bool { + pub fn pkg_config(&self) -> Option { pkg_config::Config::new() .cargo_metadata(false) - .probe(&self.as_pkg_config_name()) - .is_ok() + .probe(&self.name()) + .ok() } /// Check if archive is cached in $XDG_DATA_HOME - pub fn is_cached(&self) -> bool { + pub fn exists(&self) -> bool { + todo!() + } + + /// Download MKL archive and cache into $XDG_DATA_HOME + pub fn download(&self) -> PathBuf { todo!() } pub fn print_cargo_metadata(&self) { - // + todo!() } } diff --git a/intel-mkl-tool/src/download.rs b/intel-mkl-tool/src/download.rs index 86ca7fb8..00972d74 100644 --- a/intel-mkl-tool/src/download.rs +++ b/intel-mkl-tool/src/download.rs @@ -45,6 +45,7 @@ mod tests { use super::*; use crate::mkl; + #[ignore] #[test] fn download_url() { let url = format!( From 6e98f786fcf3cc0c2f4887648e12837a573f7e80 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Fri, 12 Jun 2020 16:33:55 +0900 Subject: [PATCH 10/10] Config::from_str --- intel-mkl-tool/src/config.rs | 84 +++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/intel-mkl-tool/src/config.rs b/intel-mkl-tool/src/config.rs index 470f25fd..80dde458 100644 --- a/intel-mkl-tool/src/config.rs +++ b/intel-mkl-tool/src/config.rs @@ -34,8 +34,39 @@ pub struct Config { } impl Config { - pub fn from_name(_name: &str) -> Result { - todo!() + pub fn from_str(name: &str) -> Result { + let parts: Vec<_> = name.split("-").collect(); + if parts.len() != 4 { + bail!("Invalid name: {}", name); + } + + if parts[0] != "mkl" { + bail!("Name must start with 'mkl': {}", name); + } + + let link = match parts[1] { + "static" => Link::Static, + "dynamic" => Link::Shared, + another => bail!("Invalid link spec: {}", another), + }; + + let index_size = match parts[2] { + "lp64" => IndexSize::LP64, + "ilp64" => IndexSize::ILP64, + another => bail!("Invalid index spec: {}", another), + }; + + let parallel = match parts[3] { + "iomp" => Parallel::OpenMP, + "seq" => Parallel::Sequential, + another => bail!("Invalid parallel spec: {}", another), + }; + + Ok(Config { + link, + index_size, + parallel, + }) } /// identifier used in pkg-config @@ -113,3 +144,52 @@ impl Config { todo!() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_to_config() -> Result<()> { + let cfg = Config::from_str("mkl-static-lp64-iomp")?; + assert_eq!( + cfg, + Config { + link: Link::Static, + index_size: IndexSize::LP64, + parallel: Parallel::OpenMP + } + ); + Ok(()) + } + + #[test] + fn name_to_config_to_name() -> Result<()> { + let valid_names = [ + "mkl-dynamic-ilp64-iomp", + "mkl-dynamic-ilp64-seq", + "mkl-dynamic-lp64-iomp", + "mkl-dynamic-lp64-seq", + "mkl-static-ilp64-iomp", + "mkl-static-ilp64-seq", + "mkl-static-lp64-iomp", + "mkl-static-lp64-seq", + ]; + for name in &valid_names { + let cfg = Config::from_str(name)?; + assert_eq!(&cfg.name(), name); + } + Ok(()) + } + + #[test] + fn invalid_names() -> Result<()> { + assert!(Config::from_str("").is_err()); + assert!(Config::from_str("static-lp64-iomp").is_err()); + assert!(Config::from_str("mkll-static-lp64-iomp").is_err()); + assert!(Config::from_str("mkl-sttic-lp64-iomp").is_err()); + assert!(Config::from_str("mkl-static-l64-iomp").is_err()); + assert!(Config::from_str("mkl-static-lp64-omp").is_err()); + Ok(()) + } +}