feat(harness): json output support

crates/libtest2-harness/Cargo.toml

@@ -24,12 +24,15 @@ pre-release-replacements = [
 
 [features]
 default = []
+json = ["dep:serde", "dep:serde_json"]
 
 [dependencies]
 anstream = "0.3.1"
 anstyle = "1.0.0"
 lexarg = { version = "0.1.0", path = "../lexarg" }
 lexarg-error = { version = "0.1.0", path = "../lexarg-error" }
 libtest-lexarg = { version = "0.1.0", path = "../libtest-lexarg" }
+serde = { version = "1.0.160", features = ["derive"], optional = true }
+serde_json = { version = "1.0.96", optional = true }
 
 [dev-dependencies]

crates/libtest2-harness/src/case.rs

@@ -50,84 +50,62 @@ pub enum Source {
 pub type RunResult = Result<(), RunError>;
 
 #[derive(Debug)]
-pub struct RunError(pub(crate) RunErrorInner);
+pub struct RunError {
+    status: notify::RunStatus,
+    cause: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
+}
 
 impl RunError {
-    pub fn cause(cause: impl Into<Fail>) -> Self {
-        Self(RunErrorInner::Failed(cause.into()))
+    pub fn with_cause(cause: impl std::error::Error + Send + Sync + 'static) -> Self {
+        Self {
+            status: notify::RunStatus::Failed,
+            cause: Some(Box::new(cause)),
+        }
     }
 
-    pub fn msg(cause: impl std::fmt::Display) -> Self {
-        Self::cause(FailMessage(cause.to_string()))
+    pub fn fail(cause: impl std::fmt::Display) -> Self {
+        Self::with_cause(Message(cause.to_string()))
     }
 
     pub(crate) fn ignore() -> Self {
-        Self(RunErrorInner::Ignored(Ignore { reason: None }))
+        Self {
+            status: notify::RunStatus::Ignored,
+            cause: None,
+        }
     }
 
     pub(crate) fn ignore_for(reason: String) -> Self {
-        Self(RunErrorInner::Ignored(Ignore {
-            reason: Some(reason),
-        }))
+        Self {
+            status: notify::RunStatus::Ignored,
+            cause: Some(Box::new(Message(reason))),
+        }
     }
-}
 
-impl<E> From<E> for RunError
-where
-    E: std::error::Error + Send + Sync + 'static,
-{
-    fn from(error: E) -> Self {
-        Self::cause(error)
+    pub(crate) fn status(&self) -> notify::RunStatus {
+        self.status
     }
-}
-
-#[derive(Debug)]
-pub(crate) enum RunErrorInner {
-    Failed(Fail),
-    Ignored(Ignore),
-}
 
-#[derive(Debug)]
-pub struct Fail {
-    inner: Box<dyn std::error::Error + Send + Sync + 'static>,
+    pub(crate) fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync)> {
+        self.cause.as_ref().map(|b| b.as_ref())
+    }
 }
 
-impl<E> From<E> for Fail
+impl<E> From<E> for RunError
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    #[cold]
     fn from(error: E) -> Self {
-        Fail {
-            inner: Box::new(error),
-        }
-    }
-}
-
-impl std::fmt::Display for Fail {
-    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.inner.fmt(formatter)
+        Self::with_cause(error)
     }
 }
 
 #[derive(Debug)]
-pub struct FailMessage(String);
+struct Message(String);
 
-impl std::fmt::Display for FailMessage {
+impl std::fmt::Display for Message {
     fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         self.0.fmt(formatter)
     }
 }
 
-impl std::error::Error for FailMessage {}
-
-#[derive(Debug)]
-pub struct Ignore {
-    reason: Option<String>,
-}
-
-impl Ignore {
-    pub fn reason(&self) -> Option<&str> {
-        self.reason.as_deref()
-    }
-}
+impl std::error::Error for Message {}

crates/libtest2-harness/src/harness.rs

@@ -1,3 +1,5 @@
+use libtest_lexarg::OutputFormat;
+
 use crate::*;
 
 pub struct Harness {
@@ -30,13 +32,10 @@ impl Harness {
 
     pub fn main(mut self) -> ! {
         let mut parser = cli::Parser::new(&self.raw);
-        let opts = match parse(&mut parser) {
-            Ok(opts) => opts,
-            Err(err) => {
-                eprintln!("{}", err);
-                std::process::exit(1);
-            }
-        };
+        let opts = parse(&mut parser).unwrap_or_else(|err| {
+            eprintln!("{}", err);
+            std::process::exit(1)
+        });
 
         match opts.color {
             libtest_lexarg::ColorConfig::AutoColor => anstream::ColorChoice::Auto,
@@ -45,45 +44,27 @@ impl Harness {
         }
         .write_global();
 
-        let total = self.cases.len();
-        let matches_filter = |case: &dyn Case, filter: &str| {
-            let test_name = case.name();
+        let mut notifier = notifier(&opts).unwrap_or_else(|err| {
+            eprintln!("{}", err);
+            std::process::exit(1)
+        });
+        discover(&opts, &mut self.cases, notifier.as_mut()).unwrap_or_else(|err| {
+            eprintln!("{}", err);
+            std::process::exit(1)
+        });
 
-            match opts.filter_exact {
-                true => test_name == filter,
-                false => test_name.contains(filter),
+        if !opts.list {
+            match run(&opts, &self.cases, notifier.as_mut()) {
+                Ok(true) => {}
+                Ok(false) => std::process::exit(ERROR_EXIT_CODE),
+                Err(e) => {
+                    eprintln!("error: io error when listing tests: {e:?}");
+                    std::process::exit(ERROR_EXIT_CODE)
+                }
             }
-        };
-        // Remove tests that don't match the test filter
-        if !opts.filters.is_empty() {
-            self.cases.retain(|case| {
-                opts.filters
-                    .iter()
-                    .any(|filter| matches_filter(case.as_ref(), filter))
-            });
-        }
-        // Skip tests that match any of the skip filters
-        if !opts.skip.is_empty() {
-            self.cases
-                .retain(|case| !opts.skip.iter().any(|sf| matches_filter(case.as_ref(), sf)));
         }
-        let num_filtered_out = total - self.cases.len();
 
-        self.cases
-            .sort_unstable_by_key(|case| case.name().to_owned());
-        let seed = crate::shuffle::get_shuffle_seed(&opts);
-        if let Some(seed) = seed {
-            crate::shuffle::shuffle_tests(seed, &mut self.cases);
-        }
-
-        match run(&opts, &self.cases, num_filtered_out) {
-            Ok(true) => std::process::exit(0),
-            Ok(false) => std::process::exit(ERROR_EXIT_CODE),
-            Err(e) => {
-                eprintln!("error: io error when listing tests: {e:?}");
-                std::process::exit(ERROR_EXIT_CODE)
-            }
-        }
+        std::process::exit(0)
     }
 }
 
@@ -138,12 +119,81 @@ fn parse(parser: &mut cli::Parser) -> cli::Result<libtest_lexarg::TestOpts> {
     test_opts.finish()
 }
 
+fn notifier(opts: &libtest_lexarg::TestOpts) -> std::io::Result<Box<dyn notify::Notifier>> {
+    let stdout = anstream::stdout();
+    let notifier: Box<dyn notify::Notifier> = match opts.format {
+        #[cfg(feature = "json")]
+        OutputFormat::Json => Box::new(notify::JsonNotifier::new(stdout)),
+        #[cfg(not(feature = "json"))]
+        OutputFormat::Json => {
+            return Err(std::io::Error::new(std::io::ErrorKind::Other, ""));
+        }
+        _ if opts.list => Box::new(notify::TerseListNotifier::new(stdout)),
+        OutputFormat::Pretty => Box::new(notify::PrettyRunNotifier::new(stdout)),
+        OutputFormat::Terse => Box::new(notify::TerseRunNotifier::new(stdout)),
+        OutputFormat::Junit => todo!(),
+    };
+    Ok(notifier)
+}
+
+fn discover(
+    opts: &libtest_lexarg::TestOpts,
+    cases: &mut Vec<Box<dyn Case>>,
+    notifier: &mut dyn notify::Notifier,
+) -> std::io::Result<()> {
+    notifier.notify(notify::Event::DiscoverStart)?;
+    let timer = std::time::Instant::now();
+
+    // Do this first so it applies to both discover and running
+    cases.sort_unstable_by_key(|case| case.name().to_owned());
+    let seed = shuffle::get_shuffle_seed(&opts);
+    if let Some(seed) = seed {
+        shuffle::shuffle_tests(seed, cases);
+    }
+
+    let matches_filter = |case: &dyn Case, filter: &str| {
+        let test_name = case.name();
+
+        match opts.filter_exact {
+            true => test_name == filter,
+            false => test_name.contains(filter),
+        }
+    };
+    let mut retain_cases = Vec::with_capacity(cases.len());
+    for case in cases.iter() {
+        let filtered_in = opts.filters.is_empty()
+            || opts
+                .filters
+                .iter()
+                .any(|filter| matches_filter(case.as_ref(), filter));
+        let filtered_out =
+            !opts.skip.is_empty() && opts.skip.iter().any(|sf| matches_filter(case.as_ref(), sf));
+        let retain_case = filtered_in && !filtered_out;
+        retain_cases.push(retain_case);
+        notifier.notify(notify::Event::DiscoverCase {
+            name: case.name().to_owned(),
+            mode: notify::CaseMode::Test,
+            run: retain_case,
+        })?;
+    }
+    let mut retain_cases = retain_cases.into_iter();
+    cases.retain(|_| retain_cases.next().unwrap());
+
+    notifier.notify(notify::Event::DiscoverComplete {
+        elapsed_s: notify::Elapsed(timer.elapsed()),
+        seed,
+    })?;
+
+    Ok(())
+}
+
 fn run(
     opts: &libtest_lexarg::TestOpts,
     cases: &[Box<dyn Case>],
-    num_filtered_out: usize,
+    notifier: &mut dyn notify::Notifier,
 ) -> std::io::Result<bool> {
-    let mut outcomes = Outcomes::new(&opts, cases, num_filtered_out)?;
+    notifier.notify(notify::Event::SuiteStart)?;
+    let timer = std::time::Instant::now();
 
     if opts.force_run_in_process {
         todo!("`--force-run-in-process` is not yet supported");
@@ -166,28 +216,43 @@ fn run(
     if opts.options.panic_abort {
         todo!("panic-abort is not yet supported");
     }
+    if opts.logfile.is_some() {
+        todo!("`--logfile` is not yet supported");
+    }
 
-    if opts.list {
-        outcomes.list(cases)?;
-        Ok(true)
-    } else {
-        outcomes.start_suite()?;
+    let mut state = State::new();
+    let run_ignored = match opts.run_ignored {
+        libtest_lexarg::RunIgnored::Yes | libtest_lexarg::RunIgnored::Only => true,
+        libtest_lexarg::RunIgnored::No => false,
+    };
+    state.run_ignored(run_ignored);
 
-        let mut state = State::new();
-        let run_ignored = match opts.run_ignored {
-            libtest_lexarg::RunIgnored::Yes | libtest_lexarg::RunIgnored::Only => true,
-            libtest_lexarg::RunIgnored::No => false,
-        };
-        state.run_ignored(run_ignored);
+    let mut success = true;
+    for case in cases {
+        notifier.notify(notify::Event::CaseStart {
+            name: case.name().to_owned(),
+        })?;
+        let timer = std::time::Instant::now();
 
-        for case in cases {
-            outcomes.start_case(case.as_ref())?;
-            let outcome = case.run(&state);
-            outcomes.finish_case(case.as_ref(), outcome)?;
-        }
+        let outcome = case.run(&state);
 
-        outcomes.finish_suite()?;
+        let err = outcome.as_ref().err();
+        let status = err.map(|e| e.status());
+        let message = err.and_then(|e| e.cause().map(|c| c.to_string()));
+        notifier.notify(notify::Event::CaseComplete {
+            name: case.name().to_owned(),
+            mode: notify::CaseMode::Test,
+            status,
+            message,
+            elapsed_s: Some(notify::Elapsed(timer.elapsed())),
+        })?;
 
-        Ok(!outcomes.has_failed())
+        success &= status != Some(notify::RunStatus::Failed);
     }
+
+    notifier.notify(notify::Event::SuiteComplete {
+        elapsed_s: notify::Elapsed(timer.elapsed()),
+    })?;
+
+    Ok(success)
 }

crates/libtest2-harness/src/lib.rs

@@ -22,13 +22,12 @@
 
 mod case;
 mod harness;
-mod outcomes;
+mod notify;
 mod shuffle;
 mod state;
 
 pub mod cli;
 
 pub use case::*;
 pub use harness::*;
-pub(crate) use outcomes::*;
 pub use state::*;