diff --git a/intel-mkl-src/build.rs b/intel-mkl-src/build.rs index 10e6c469..8dbbd48b 100644 --- a/intel-mkl-src/build.rs +++ b/intel-mkl-src/build.rs @@ -20,6 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +use intel_mkl_tool::*; use std::{env, path::*}; fn main() { @@ -27,12 +28,28 @@ fn main() { path } else { let out_dir = if cfg!(feature = "use-shared") { - intel_mkl_tool::home_library_path() + intel_mkl_tool::xdg_home_path() } else { PathBuf::from(env::var("OUT_DIR").expect("Failed to get OUT_DIR")) }; - intel_mkl_tool::download(&out_dir).expect("Failed to downalod Intel-MKL archive"); + if cfg!(feature = "static") { + download( + &out_dir, + mkl::ARCHIVE_STATIC, + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE, + ) + .expect("Failed to downalod Intel-MKL archive"); + } else { + download( + &out_dir, + mkl::ARCHIVE_SHARED, + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE, + ) + .expect("Failed to downalod Intel-MKL archive"); + } out_dir }; println!("cargo:rustc-link-search={}", out_dir.display()); diff --git a/intel-mkl-tool/Cargo.toml b/intel-mkl-tool/Cargo.toml index 941bd0c1..bf988d80 100644 --- a/intel-mkl-tool/Cargo.toml +++ b/intel-mkl-tool/Cargo.toml @@ -16,6 +16,7 @@ cli = ["structopt", "env_logger"] [dependencies] anyhow = "1.0.31" curl = "0.4.25" +derive_more = "0.99.7" dirs = "2.0.2" glob = "0.3.0" log = "0.4.8" @@ -26,8 +27,3 @@ zstd = "0.5.1" # CLI structopt = { version = "0.3.5", optional = true } env_logger = { version = "0.7.1", optional = true } - -[[bin]] -name = "intel-mkl-tool" -path = "src/cli.rs" -required-features = ["cli"] diff --git a/intel-mkl-tool/src/bin/main.rs b/intel-mkl-tool/src/bin/main.rs new file mode 100644 index 00000000..b04eb255 --- /dev/null +++ b/intel-mkl-tool/src/bin/main.rs @@ -0,0 +1,71 @@ +use anyhow::*; +use intel_mkl_tool::*; +use std::{env, path::PathBuf}; +use structopt::StructOpt; + +/// CLI tool for intel-mkl crate +#[derive(Debug, StructOpt)] +enum Opt { + /// Download Intel-MKL library + Download { + /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool` + path: Option, + /// Version of Intel MKL + year: Option, + /// Version of Intel MKL + update: Option, + }, + + /// Seek Intel-MKL library + /// + /// 1. pkg-config + /// 2. `$XDG_DATA_HOME/intel-mkl-tool` + /// will be sought. + Seek {}, + + /// Package Intel MKL libraries into an archive + Package { path: PathBuf }, +} + +fn main() -> Result<()> { + env::set_var("RUST_LOG", "info"); + env_logger::init(); + + let opt = Opt::from_args(); + + match opt { + Opt::Download { path, year, update } => { + let path = path.unwrap_or(xdg_home_path()); + let year = year.unwrap_or(mkl::VERSION_YEAR); + let update = update.unwrap_or(intel_mkl_tool::mkl::VERSION_UPDATE); + download(&path, mkl::ARCHIVE_SHARED, year, update)?; + download(&path, mkl::ARCHIVE_STATIC, year, update)?; + } + + Opt::Seek {} => { + println!("pkg-config"); + println!("-----------"); + for (name, _lib) in intel_mkl_tool::seek_pkg_config() { + println!("- {}", name); + } + + let title = format!( + "xdg-data-home (base = {})", + intel_mkl_tool::xdg_home_path().display() + ); + println!( + "\n{}\n{}", + title, + std::str::from_utf8(vec!('-' as u8; title.len() + 1).as_slice()).unwrap() + ); + for (name, _path) in intel_mkl_tool::seek_xdg_home() { + println!("- {}", name); + } + } + + Opt::Package { path } => { + let _out = intel_mkl_tool::package(&path)?; + } + } + Ok(()) +} diff --git a/intel-mkl-tool/src/cli.rs b/intel-mkl-tool/src/cli.rs deleted file mode 100644 index 7802a408..00000000 --- a/intel-mkl-tool/src/cli.rs +++ /dev/null @@ -1,58 +0,0 @@ -use anyhow::*; -use std::{env, path::PathBuf}; -use structopt::StructOpt; - -/// CLI tool for intel-mkl crate -#[derive(Debug, StructOpt)] -enum Opt { - /// Download Intel-MKL library - Download { - /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool` - path: Option, - }, - - /// Seek Intel-MKL library - /// - /// 1. pkg-config - /// 2. `$XDG_DATA_HOME/intel-mkl-tool` - /// will be sought. - Seek {}, - - /// Package Intel MKL libraries into an archive - Package { path: PathBuf }, -} - -fn main() -> Result<()> { - env::set_var("RUST_LOG", "info"); - env_logger::init(); - - let opt = Opt::from_args(); - - match opt { - Opt::Download { path } => { - let path = if let Some(path) = path { - path - } else { - intel_mkl_tool::home_library_path() - }; - intel_mkl_tool::download(&path)?; - } - - Opt::Seek {} => { - if let Some(path) = intel_mkl_tool::seek_pkg_config() { - println!("{}", path.display()); - return Ok(()); - } - if let Some(path) = intel_mkl_tool::seek_home() { - println!("{}", path.display()); - return Ok(()); - } - bail!("Intel-MKL not found."); - } - - Opt::Package { path } => { - let _out = intel_mkl_tool::package(&path)?; - } - } - Ok(()) -} diff --git a/intel-mkl-tool/src/config.rs b/intel-mkl-tool/src/config.rs new file mode 100644 index 00000000..80dde458 --- /dev/null +++ b/intel-mkl-tool/src/config.rs @@ -0,0 +1,195 @@ +use anyhow::*; +use derive_more::Display; +use std::path::*; + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum Link { + #[display(fmt = "static")] + Static, + #[display(fmt = "dynamic")] + Shared, +} + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum IndexSize { + #[display(fmt = "lp64")] + LP64, + #[display(fmt = "ilp64")] + ILP64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum Parallel { + #[display(fmt = "iomp")] + OpenMP, + #[display(fmt = "seq")] + Sequential, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Config { + link: Link, + index_size: IndexSize, + parallel: Parallel, +} + +impl Config { + pub fn from_str(name: &str) -> Result { + let parts: Vec<_> = name.split("-").collect(); + if parts.len() != 4 { + bail!("Invalid name: {}", name); + } + + if parts[0] != "mkl" { + bail!("Name must start with 'mkl': {}", name); + } + + let link = match parts[1] { + "static" => Link::Static, + "dynamic" => Link::Shared, + another => bail!("Invalid link spec: {}", another), + }; + + let index_size = match parts[2] { + "lp64" => IndexSize::LP64, + "ilp64" => IndexSize::ILP64, + another => bail!("Invalid index spec: {}", another), + }; + + let parallel = match parts[3] { + "iomp" => Parallel::OpenMP, + "seq" => Parallel::Sequential, + another => bail!("Invalid parallel spec: {}", another), + }; + + Ok(Config { + link, + index_size, + parallel, + }) + } + + /// identifier used in pkg-config + pub fn name(&self) -> String { + format!("mkl-{}-{}-{}", self.link, self.index_size, self.parallel) + } + + fn base_dir(&self) -> PathBuf { + todo!() + } + + /// Static and shared library lists to be linked + pub fn libs( + &self, + ) -> ( + Vec, /* static */ + Vec, /* shared */ + ) { + // FIXME this implementation is for Linux, fix for Windows and macOS + let mut static_libs = Vec::new(); + let mut shared_libs = vec!["pthread".into(), "m".into(), "dl".into()]; + + let mut add = |name: &str| match self.link { + Link::Static => { + let base_dir: PathBuf = self.base_dir(); + let path = base_dir.join(format!("lib{}.a", name)); + assert!(path.exists()); + static_libs.push(path); + } + Link::Shared => { + shared_libs.push(name.to_string()); + } + }; + + add("mkl_core"); + match self.index_size { + IndexSize::LP64 => { + add("mkl_intel_lp64"); + } + IndexSize::ILP64 => { + add("mkl_intel_ilp64"); + } + }; + match self.parallel { + Parallel::OpenMP => { + add("iomp5"); + add("mkl_intel_thread"); + } + Parallel::Sequential => { + add("mkl_sequential"); + } + }; + (static_libs, shared_libs) + } + + /// Check if pkg-config has a corresponding setting + pub fn pkg_config(&self) -> Option { + pkg_config::Config::new() + .cargo_metadata(false) + .probe(&self.name()) + .ok() + } + + /// Check if archive is cached in $XDG_DATA_HOME + pub fn exists(&self) -> bool { + todo!() + } + + /// Download MKL archive and cache into $XDG_DATA_HOME + pub fn download(&self) -> PathBuf { + todo!() + } + + pub fn print_cargo_metadata(&self) { + todo!() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_to_config() -> Result<()> { + let cfg = Config::from_str("mkl-static-lp64-iomp")?; + assert_eq!( + cfg, + Config { + link: Link::Static, + index_size: IndexSize::LP64, + parallel: Parallel::OpenMP + } + ); + Ok(()) + } + + #[test] + fn name_to_config_to_name() -> Result<()> { + let valid_names = [ + "mkl-dynamic-ilp64-iomp", + "mkl-dynamic-ilp64-seq", + "mkl-dynamic-lp64-iomp", + "mkl-dynamic-lp64-seq", + "mkl-static-ilp64-iomp", + "mkl-static-ilp64-seq", + "mkl-static-lp64-iomp", + "mkl-static-lp64-seq", + ]; + for name in &valid_names { + let cfg = Config::from_str(name)?; + assert_eq!(&cfg.name(), name); + } + Ok(()) + } + + #[test] + fn invalid_names() -> Result<()> { + assert!(Config::from_str("").is_err()); + assert!(Config::from_str("static-lp64-iomp").is_err()); + assert!(Config::from_str("mkll-static-lp64-iomp").is_err()); + assert!(Config::from_str("mkl-sttic-lp64-iomp").is_err()); + assert!(Config::from_str("mkl-static-l64-iomp").is_err()); + assert!(Config::from_str("mkl-static-lp64-omp").is_err()); + Ok(()) + } +} diff --git a/intel-mkl-tool/src/download.rs b/intel-mkl-tool/src/download.rs new file mode 100644 index 00000000..00972d74 --- /dev/null +++ b/intel-mkl-tool/src/download.rs @@ -0,0 +1,60 @@ +use crate::S3_ADDR; + +use anyhow::*; +use curl::easy::Easy; +use log::*; +use std::{fs, path::*}; + +fn download_archive_to_buffer(url: &str) -> Result> { + let mut data = Vec::new(); + let mut handle = Easy::new(); + handle.fail_on_error(true)?; + handle.url(url)?; + { + let mut transfer = handle.transfer(); + transfer + .write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + }) + .unwrap(); + transfer.perform().unwrap(); + } + Ok(data) +} + +pub fn download(base_dir: &Path, prefix: &str, year: u32, update: u32) -> Result<()> { + let filename = format!("{}_{}_{}.tar.zst", prefix, year, update); + let dest_dir = base_dir.join(&format!("{}_{}_{}", prefix, year, update)); + + if dest_dir.exists() { + bail!("Directory already exists: {}", dest_dir.display()); + } + fs::create_dir_all(&dest_dir)?; + + info!("Download archive {} into {}", filename, dest_dir.display()); + let data = download_archive_to_buffer(&format!("{}/{}", S3_ADDR, filename))?; + let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; + let mut arc = tar::Archive::new(zstd); + arc.unpack(&dest_dir)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mkl; + + #[ignore] + #[test] + fn download_url() { + let url = format!( + "{}/{}_{}_{}.tar.zst", + S3_ADDR, + mkl::ARCHIVE_STATIC, + mkl::VERSION_YEAR, + mkl::VERSION_UPDATE + ); + let _ar = download_archive_to_buffer(&url).unwrap(); + } +} diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 0e57a3b3..93182905 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -1,166 +1,40 @@ -use anyhow::*; -use curl::easy::Easy; -use glob::glob; -use log::*; -use std::{ - fs, - io::{self, BufRead, Write}, - path::*, -}; +mod config; +mod download; +mod package; +mod seek; + +pub use config::*; +pub use download::*; +pub use package::*; +pub use seek::*; const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-intel-mkl"; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] -mod mkl { - pub const ARCHIVE: &'static str = "mkl_linux64"; - pub const EXT: &'static str = "so"; +pub mod mkl { + pub const ARCHIVE_SHARED: &'static str = "mkl_linux64_shared"; + pub const ARCHIVE_STATIC: &'static str = "mkl_linux64_static"; + pub const EXTENSION_SHARED: &'static str = "so"; + pub const EXTENSION_STATIC: &'static str = "a"; pub const PREFIX: &'static str = "lib"; - pub const VERSION_YEAR: u32 = 2019; - pub const VERSION_UPDATE: u32 = 5; + pub const VERSION_YEAR: u32 = 2020; + pub const VERSION_UPDATE: u32 = 1; } #[cfg(all(target_os = "macos", target_arch = "x86_64"))] -mod mkl { - pub const ARCHIVE: &'static str = "mkl_macos64"; - pub const EXT: &'static str = "dylib"; +pub mod mkl { + pub const ARCHIVE_SHARED: &'static str = "mkl_macos64_shared"; + pub const EXTENSION_SHARED: &'static str = "dylib"; pub const PREFIX: &'static str = "lib"; pub const VERSION_YEAR: u32 = 2019; pub const VERSION_UPDATE: u32 = 3; } #[cfg(all(target_os = "windows", target_arch = "x86_64"))] -mod mkl { - pub const ARCHIVE: &'static str = "mkl_windows64"; - pub const EXT: &'static str = "lib"; +pub mod mkl { + pub const ARCHIVE_SHARED: &'static str = "mkl_windows64"; + pub const EXTENSION_SHARED: &'static str = "lib"; pub const PREFIX: &'static str = ""; pub const VERSION_YEAR: u32 = 2019; pub const VERSION_UPDATE: u32 = 5; } - -pub fn archive_filename() -> String { - format!( - "{}_{}_{}.tar.zst", - mkl::ARCHIVE, - mkl::VERSION_YEAR, - mkl::VERSION_UPDATE - ) -} - -pub fn home_library_path() -> PathBuf { - dirs::data_local_dir().unwrap().join("intel-mkl-tool") -} - -pub fn seek_pkg_config() -> Option { - if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-iomp") { - if lib.libs.len() > 1 { - warn!("Found {} MKL libraries. Use first found.", lib.libs.len()) - } - return Some(PathBuf::from(lib.libs[0].clone())); - } - None -} - -pub fn seek_home() -> Option { - let home_lib = home_library_path(); - if home_lib.is_dir() { - return Some(home_lib); - } - None -} - -pub fn download(out_dir: &Path) -> Result<()> { - if !out_dir.exists() { - info!("Create output directory: {}", out_dir.display()); - fs::create_dir_all(out_dir)?; - } - if !out_dir.is_dir() { - bail!("Not a directory: {}", out_dir.display()); - } - - let archive = out_dir.join(archive_filename()); - if !archive.exists() { - let url = format!("{}/{}", S3_ADDR, archive_filename()); - info!("Download archive from AWS S3: {}", url); - let f = fs::File::create(&archive)?; - let mut buf = io::BufWriter::new(f); - let mut easy = Easy::new(); - easy.url(&url)?; - easy.write_function(move |data| Ok(buf.write(data).unwrap()))?; - easy.perform()?; - assert!(archive.exists()); - } else { - info!("Archive already exists: {}", archive.display()); - } - - let core = out_dir.join(format!("{}mkl_core.{}", mkl::PREFIX, mkl::EXT)); - if !core.exists() { - let f = fs::File::open(&archive)?; - let de = zstd::stream::read::Decoder::new(f)?; - let mut arc = tar::Archive::new(de); - arc.unpack(&out_dir)?; - assert!(core.exists()); - } else { - info!("Archive has already been extracted"); - } - Ok(()) -} - -// Read mkl_version.h to get MKL version (e.g. 2019.5) -fn get_mkl_version(version_header: &Path) -> Result<(u32, u32)> { - if !version_header.exists() { - bail!("MKL Version file not found: {}", version_header.display()); - } - let f = fs::File::open(version_header)?; - let f = io::BufReader::new(f); - let mut year = 0; - let mut update = 0; - for line in f.lines() { - if let Ok(line) = line { - if !line.starts_with("#define") { - continue; - } - let ss: Vec<&str> = line.split(" ").collect(); - match ss[1] { - "__INTEL_MKL__" => year = ss[2].parse()?, - "__INTEL_MKL_UPDATE__" => update = ss[2].parse()?, - _ => continue, - } - } - } - if year == 0 || update == 0 { - bail!("Cannot determine MKL versions"); - } - Ok((year, update)) -} - -pub fn package(mkl_path: &Path) -> Result { - if !mkl_path.exists() { - bail!("MKL directory not found: {}", mkl_path.display()); - } - let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; - info!("Intel MKL version: {}.{}", year, update); - let out = PathBuf::from(archive_filename()); - info!("Create archive: {}", out.display()); - - let shared_libs: Vec<_> = glob( - mkl_path - .join(format!("lib/intel64/*.{}", mkl::EXT)) - .to_str() - .unwrap(), - )? - .map(|path| path.unwrap()) - .collect(); - let f = fs::File::create(&out)?; - let buf = io::BufWriter::new(f); - let zstd = zstd::stream::write::Encoder::new(buf, 6)?; - let mut ar = tar::Builder::new(zstd); - ar.mode(tar::HeaderMode::Deterministic); - for lib in &shared_libs { - info!("Add {}", lib.display()); - ar.append_path_with_name(lib, lib.file_name().unwrap())?; - } - let zstd = ar.into_inner()?; - zstd.finish()?; - - Ok(out) -} diff --git a/intel-mkl-tool/src/package.rs b/intel-mkl-tool/src/package.rs new file mode 100644 index 00000000..ca8374ab --- /dev/null +++ b/intel-mkl-tool/src/package.rs @@ -0,0 +1,104 @@ +use crate::mkl; + +use anyhow::*; +use glob::glob; +use log::*; +use std::{ + fs, + io::{self, BufRead}, + path::*, +}; + +// Read mkl_version.h to get MKL version (e.g. 2019.5) +fn get_mkl_version(version_header: &Path) -> Result<(u32, u32)> { + if !version_header.exists() { + bail!("MKL Version file not found: {}", version_header.display()); + } + let f = fs::File::open(version_header)?; + let f = io::BufReader::new(f); + let mut year = 0; + let mut update = 0; + for line in f.lines() { + if let Ok(line) = line { + if !line.starts_with("#define") { + continue; + } + let ss: Vec<&str> = line.split(" ").collect(); + match ss[1] { + "__INTEL_MKL__" => year = ss[2].parse()?, + "__INTEL_MKL_UPDATE__" => update = ss[2].parse()?, + _ => continue, + } + } + } + if year == 0 || update == 0 { + bail!("Cannot determine MKL versions"); + } + Ok((year, update)) +} + +// Create tar.zst archive from path list +fn create_archive(libs: &[PathBuf], out: &Path) -> Result<()> { + if out.exists() { + bail!("Output archive already exits: {}", out.display()); + } + info!("Create archive: {}", out.display()); + let f = fs::File::create(&out)?; + let buf = io::BufWriter::new(f); + let zstd = zstd::stream::write::Encoder::new(buf, 6)?; + let mut ar = tar::Builder::new(zstd); + ar.mode(tar::HeaderMode::Deterministic); + for lib in libs { + info!("Add {}", lib.display()); + ar.append_path_with_name(lib, lib.file_name().unwrap())?; + } + let zstd = ar.into_inner()?; + zstd.finish()?; + Ok(()) +} + +pub fn package(mkl_path: &Path) -> Result<()> { + if !mkl_path.exists() { + bail!("MKL directory not found: {}", mkl_path.display()); + } + let (year, update) = get_mkl_version(&mkl_path.join("include/mkl_version.h"))?; + info!("Intel MKL version: {}.{}", year, update); + + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_SHARED)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(&format!( + "{}_{}_{}.tar.zst", + mkl::ARCHIVE_SHARED, + year, + update + )), + )?; + + if cfg!(target_or = "linux") { + create_archive( + &glob( + mkl_path + .join(format!("lib/intel64/*.{}", mkl::EXTENSION_STATIC)) + .to_str() + .unwrap(), + )? + .map(|path| path.unwrap()) + .collect::>(), + &PathBuf::from(&format!( + "{}_{}_{}.tar.zst", + mkl::ARCHIVE_STATIC, + year, + update + )), + )?; + } + + Ok(()) +} diff --git a/intel-mkl-tool/src/seek.rs b/intel-mkl-tool/src/seek.rs new file mode 100644 index 00000000..ff2dc09e --- /dev/null +++ b/intel-mkl-tool/src/seek.rs @@ -0,0 +1,49 @@ +use std::path::*; + +pub fn xdg_home_path() -> PathBuf { + dirs::data_local_dir().unwrap().join("intel-mkl-tool") +} + +/// Following pkg-config settings are included in Intel MKL distribution +const PKG_CONFIG_TARGETS: [&str; 8] = [ + "mkl-dynamic-ilp64-iomp", + "mkl-dynamic-ilp64-seq", + "mkl-dynamic-lp64-iomp", + "mkl-dynamic-lp64-seq", + "mkl-static-ilp64-iomp", + "mkl-static-ilp64-seq", + "mkl-static-lp64-iomp", + "mkl-static-lp64-seq", +]; + +pub fn seek_pkg_config() -> Vec<(String, pkg_config::Library)> { + PKG_CONFIG_TARGETS + .iter() + .filter_map(|target| { + let lib = pkg_config::Config::new() + .cargo_metadata(false) + .probe(target) + .ok()?; + Some((target.to_string(), lib)) + }) + .collect() +} + +pub fn seek_xdg_home() -> Vec<(String, PathBuf)> { + let base = xdg_home_path(); + if !base.exists() { + return Vec::new(); + } + base.read_dir() + .unwrap() + .flat_map(|entry| { + let path = entry.unwrap().path(); + if path.is_dir() { + let name = path.file_name()?.to_str()?.into(); + Some((name, path)) + } else { + None + } + }) + .collect() +}