extensive wip

check passes

Running in parallel successfully

progress bar

Update progress bar

Speed up extensive tests by chunking

Clean up pb

Rename the env var to enable extensive tests

Author: tgross35

ci/run-docker.sh

@@ -29,6 +29,7 @@ run() {
         --rm \
         --user "$(id -u):$(id -g)" \
         -e RUSTFLAGS \
+        -e CI \
         -e CARGO_HOME=/cargo \
         -e CARGO_TARGET_DIR=/target \
         -e "EMULATED=$emulated" \

crates/libm-macros/src/enums.rs

@@ -31,43 +31,74 @@ pub fn function_enum(
 
     let enum_name = &item.ident;
     let mut as_str_arms = Vec::new();
+    let mut from_str_arms = Vec::new();
     let mut base_arms = Vec::new();
+    let mut math_op_arms = Vec::new();
 
     for func in ALL_FUNCTIONS_FLAT.iter() {
         let fn_name = func.name;
+        let name_ident = Ident::new(fn_name, Span::call_site());
         let ident = Ident::new(&fn_name.to_upper_camel_case(), Span::call_site());
         let bname_ident = Ident::new(&base_name(fn_name).to_upper_camel_case(), Span::call_site());
+        let fty_ident =
+            Ident::new(&func.base_fty.to_string().to_upper_camel_case(), Span::call_site());
 
         // Match arm for `fn as_str(self)` matcher
         as_str_arms.push(quote! { Self::#ident => #fn_name });
+        from_str_arms.push(quote! { #fn_name => Self::#ident });
 
         // Match arm for `fn base_name(self)` matcher
         base_arms.push(quote! { Self::#ident => #base_enum::#bname_ident });
+        math_op_arms.push(quote! {
+            Self::#ident =>
+                crate::op::MathOpInfo::new::<crate::op::#name_ident::Routine>(
+                    crate::FloatTy::#fty_ident
+                )
+        });
 
         let variant =
             Variant { attrs: Vec::new(), ident, fields: Fields::Unit, discriminant: None };
 
         item.variants.push(variant);
     }
 
+    let variants = item.variants.iter();
+
     let res = quote! {
         // Instantiate the enum
         #item
 
         impl #enum_name {
+            pub const ALL: &[Self] = &[
+                #( Self::#variants, )*
+            ];
+
             /// The stringified version of this function name.
             pub const fn as_str(self) -> &'static str {
                 match self {
                     #( #as_str_arms , )*
                 }
             }
 
+            pub fn from_str(s: &str) -> Self {
+                match s {
+                    #( #from_str_arms , )*
+                    _ => panic!("Unrecognized operation `{s}`"),
+                }
+            }
+
             /// The base name enum for this function.
             pub const fn base_name(self) -> #base_enum {
                 match self {
                     #( #base_arms, )*
                 }
             }
+
+            pub fn math_op(self) -> crate::op::MathOpInfo {
+                match self {
+                    #( #math_op_arms , )*
+                }
+            }
         }
     };
 

crates/libm-macros/src/lib.rs

@@ -1,6 +1,7 @@
 mod enums;
 mod parse;
 
+use std::fmt;
 use std::sync::LazyLock;
 
 use parse::{Invocation, StructuredInput};
@@ -206,6 +207,26 @@ enum Ty {
     MutCInt,
 }
 
+impl fmt::Display for Ty {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let s = match self {
+            Ty::F16 => "f16",
+            Ty::F32 => "f32",
+            Ty::F64 => "f64",
+            Ty::F128 => "f128",
+            Ty::I32 => "i32",
+            Ty::CInt => "core::ffi::c_int",
+            Ty::MutF16 => "&mut f16",
+            Ty::MutF32 => "&mut f32",
+            Ty::MutF64 => "&mut f64",
+            Ty::MutF128 => "&mut f128",
+            Ty::MutI32 => "&mut i32",
+            Ty::MutCInt => "&mut core::ffi::c_int",
+        };
+        f.write_str(s)
+    }
+}
+
 impl ToTokens for Ty {
     fn to_tokens(&self, tokens: &mut pm2::TokenStream) {
         let ts = match self {

crates/libm-macros/tests/enum.rs

@@ -17,3 +17,34 @@ fn basename() {
     assert_eq!(Function::Sin.base_name(), BaseName::Sin);
     assert_eq!(Function::Sinf.base_name(), BaseName::Sin);
 }
+
+mod op {
+    macro_rules! simple_mod {
+        (
+            fn_name: $fn_name:ident,
+        ) => {
+            pub mod $fn_name {
+                pub struct Routine {}
+            }
+        };
+    }
+
+    // Test with no extra, no skip, and no attributes
+    libm_macros::for_each_function! {
+        callback: simple_mod,
+    }
+
+    pub struct MathOpInfo {}
+    impl MathOpInfo {
+        pub fn new<T>(_fty: crate::FloatTy) -> Self {
+            unimplemented!()
+        }
+    }
+}
+
+pub enum FloatTy {
+    F16,
+    F32,
+    F64,
+    F128,
+}

crates/libm-test/Cargo.toml

@@ -24,12 +24,14 @@ short-benchmarks = []
 [dependencies]
 anyhow = "1.0.90"
 az = { version = "1.2.1", optional = true }
+indicatif = { version = "0.17.9", default-features = false }
 libm = { path = "../..", features = ["unstable-test-support"] }
 libm-macros = { path = "../libm-macros" }
 musl-math-sys = { path = "../musl-math-sys", optional = true }
 paste = "1.0.15"
 rand = "0.8.5"
 rand_chacha = "0.3.1"
+rayon = "1.10.0"
 rug = { version = "1.26.1", optional = true, default-features = false, features = ["float", "std"] }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
@@ -41,7 +43,15 @@ rand = { version = "0.8.5", optional = true }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", default-features = false, features = ["cargo_bench_support"] }
+# FIXME: use the crates.io version once it supports runtime skipping of tests
+libtest-mimic = { git = "https://github.com/tgross35/libtest-mimic.git", rev = "4c8413b493e1b499bb941d2ced1f4c3d8462f53c" }
 
 [[bench]]
 name = "random"
 harness = false
+
+[[test]]
+# No harness so that we can skip tests at runtime based on env. Prefixed with
+# `z` so these tests get run last.
+name = "z_extensive"
+harness = false

crates/libm-test/examples/plot_domains.rs

@@ -9,6 +9,7 @@ use std::path::Path;
 use std::process::Command;
 use std::{env, fs};
 
+use libm_test::Identifier;
 use libm_test::domain::Domain;
 use libm_test::gen::domain_logspace;
 
@@ -24,35 +25,35 @@ fn main() {
     let mut j_args = Vec::new();
 
     // Plot a few domains with some functions that use them.
-    plot_one(out_dir, "sqrt", Domain::SQRT, &mut j_args);
-    plot_one(out_dir, "cos", Domain::TRIG, &mut j_args);
-    plot_one(out_dir, "cbrt", Domain::UNBOUNDED, &mut j_args);
+    plot_one(out_dir, Identifier::Sqrt, Domain::SQRT, &mut j_args);
+    plot_one(out_dir, Identifier::Cos, Domain::TRIG, &mut j_args);
+    plot_one(out_dir, Identifier::Cbrt, Domain::UNBOUNDED, &mut j_args);
 
     println!("launching script");
     let mut cmd = Command::new("julia");
-    if !cfg!(debug_assertions) {
+    if cfg!(optimizations_enabled) {
         cmd.arg("-O3");
     }
     cmd.arg(jl_script).args(j_args).status().unwrap();
 }
 
 /// Plot a single domain.
-fn plot_one(out_dir: &Path, name: &str, domain: Domain<f32>, j_args: &mut Vec<String>) {
-    let base_name = out_dir.join(format!("domain-inputs-{name}"));
+fn plot_one(out_dir: &Path, id: Identifier, domain: Domain<f32>, j_args: &mut Vec<String>) {
+    let base_name = out_dir.join(format!("domain-inputs-{id}"));
     let text_file = base_name.with_extension("txt");
 
     {
         // Scope for file and writer
         let f = fs::File::create(&text_file).unwrap();
         let mut w = BufWriter::new(f);
 
-        for input in domain_logspace::get_test_cases_inner::<f32>(domain) {
+        for input in domain_logspace::get_test_cases_inner::<f32>(domain, id) {
             writeln!(w, "{:e}", input.0).unwrap();
         }
         w.flush().unwrap();
     }
 
     // The julia script expects `name1 path1 name2 path2...` args
-    j_args.push(name.to_owned());
+    j_args.push(id.as_str().to_owned());
     j_args.push(base_name.to_str().unwrap().to_owned());
 }

crates/libm-test/src/gen.rs

@@ -3,6 +3,7 @@
 use crate::GenerateInput;
 pub mod domain_logspace;
 pub mod edge_cases;
+pub mod extensive;
 pub mod random;
 
 /// Helper type to turn any reusable input into a generator.

crates/libm-test/src/gen/domain_logspace.rs

@@ -1,31 +1,10 @@
 //! A generator that makes use of domain bounds.
 
-use std::ops::Bound;
-
 use libm::support::{Float, MinInt};
 
 use crate::domain::{Domain, HasDomain};
-use crate::{MathOp, logspace};
-
-/// Number of tests to run.
-// FIXME(ntests): replace this with a more logical algorithm
-const NTESTS: usize = {
-    if cfg!(optimizations_enabled) {
-        if crate::emulated()
-            || !cfg!(target_pointer_width = "64")
-            || cfg!(all(target_arch = "x86_64", target_vendor = "apple"))
-        {
-            // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run
-            // in QEMU.
-            100_000
-        } else {
-            5_000_000
-        }
-    } else {
-        // Without optimizations just run a quick check
-        800
-    }
-};
+use crate::run_cfg::{TestAction, TestTy};
+use crate::{MathOp, logspace, op};
 
 /// Create a range of logarithmically spaced inputs within a function's domain.
 ///
@@ -34,26 +13,26 @@ const NTESTS: usize = {
 pub fn get_test_cases<Op>() -> impl Iterator<Item = (Op::FTy,)>
 where
     Op: MathOp + HasDomain<Op::FTy>,
-    <Op::FTy as Float>::Int: TryFrom<usize>,
+    <Op::FTy as Float>::Int: TryFrom<u64>,
 {
-    get_test_cases_inner::<Op::FTy>(Op::D)
+    get_test_cases_inner::<Op::FTy>(Op::D, Op::IDENTIFIER)
 }
 
-pub fn get_test_cases_inner<F>(domain: Domain<F>) -> impl Iterator<Item = (F,)>
+pub fn get_test_cases_inner<F>(domain: Domain<F>, id: op::Identifier) -> impl Iterator<Item = (F,)>
 where
-    F: Float<Int: TryFrom<usize>>,
+    F: Float<Int: TryFrom<u64>>,
 {
-    // We generate logspaced inputs within a specific range, excluding values that are out of
-    // range in order to make iterations useful (random tests still cover the full range).
-    let range_start = match domain.start {
-        Bound::Included(v) | Bound::Excluded(v) => v,
-        Bound::Unbounded => F::NEG_INFINITY,
-    };
-    let range_end = match domain.end {
-        Bound::Included(v) | Bound::Excluded(v) => v,
-        Bound::Unbounded => F::INFINITY,
+    let action = crate::run_cfg::get_iterations(id, TestTy::Logspace, 0);
+    let ntests = match action {
+        TestAction::Iterations(n) => n,
+        TestAction::Run => unimplemented!(),
+        TestAction::Skip => unimplemented!(),
     };
 
-    let steps = F::Int::try_from(NTESTS).unwrap_or(F::Int::MAX);
-    logspace(range_start, range_end, steps).map(|v| (v,))
+    // We generate logspaced inputs within a specific range, excluding values that are out of
+    // range in order to make iterations useful (random tests still cover the full range).
+    let start = domain.range_start();
+    let end = domain.range_end();
+    let steps = F::Int::try_from(ntests).unwrap_or(F::Int::MAX);
+    logspace(start, end, steps).map(|v| (v,))
 }