Revise revisions

README.md

@@ -32,7 +32,9 @@ their command specifies, or the test will fail without even being run.
 * `//@error-pattern: XXX` make sure the stderr output contains `XXX`
 * `//@revisions: XXX YYY` runs the test once for each space separated name in the list
     * emits one stderr file per revision
-    * `//~` comments can be restricted to specific revisions by adding the revision name before the `~` in square brackets: `//[XXX]~`
+    * `//~` comments can be restricted to specific revisions by adding the revision name after the `~` in square brackets: `//~[XXX]`
+    * `//@` comments can be restricted to specific revisions by adding the revision name after the `@` in square brackets: `//@[XXX]`
+        * Note that you cannot add revisions to the `revisions` command.
 * `//@compile-flags: XXX` appends `XXX` to the command line arguments passed to the rustc driver
     * you can specify this multiple times, and all the flags will accumulate
 * `//@rustc-env: XXX=YYY` sets the env var `XXX` to `YYY` for the rustc driver execution.

src/lib.rs

@@ -15,7 +15,7 @@ use regex::bytes::Regex;
 use rustc_stderr::{Level, Message};
 use std::collections::VecDeque;
 use std::ffi::OsString;
-use std::fmt::{Display, Write as _};
+use std::fmt::Display;
 use std::io::Write as _;
 use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
@@ -488,28 +488,22 @@ fn parse_and_test_file(path: PathBuf, config: &Config) -> Vec<TestRun> {
             }]
         }
     };
-    // Ignore file if only/ignore rules do (not) apply
-    if !test_file_conditions(&comments, config) {
-        return vec![TestRun {
-            result: TestResult::Ignored,
-            path,
-            revision: "".into(),
-        }];
-    }
     // Run the test for all revisions
     comments
         .revisions
         .clone()
         .unwrap_or_else(|| vec![String::new()])
         .into_iter()
         .map(|revision| {
-            let (command, errors, stderr) = run_test(&path, config, &revision, &comments);
-
-            // Using a single `eprintln!` to prevent messages from threads from getting intermingled.
-            let mut msg = format!("{}", path.display());
-            if !revision.is_empty() {
-                write!(msg, " (revision `{revision}`) ").unwrap();
+            // Ignore file if only/ignore rules do (not) apply
+            if !test_file_conditions(&comments, config, &revision) {
+                return TestRun {
+                    result: TestResult::Ignored,
+                    path: path.clone(),
+                    revision,
+                };
             }
+            let (command, errors, stderr) = run_test(&path, config, &revision, &comments);
             let result = if errors.is_empty() {
                 TestResult::Ok
             } else {
@@ -569,11 +563,19 @@ fn build_command(path: &Path, config: &Config, revision: &str, comments: &Commen
     if !revision.is_empty() {
         cmd.arg(format!("--cfg={revision}"));
     }
-    for arg in &comments.compile_flags {
+    for arg in comments
+        .for_revision(revision)
+        .flat_map(|r| r.compile_flags.iter())
+    {
         cmd.arg(arg);
     }
     cmd.args(config.trailing_args.iter());
-    cmd.envs(comments.env_vars.iter().map(|(k, v)| (k, v)));
+    cmd.envs(
+        comments
+            .for_revision(revision)
+            .flat_map(|r| r.env_vars.iter())
+            .map(|(k, v)| (k, v)),
+    );
 
     cmd
 }
@@ -627,6 +629,7 @@ fn check_test_result(
         &config.stderr_filters,
         config,
         comments,
+        revision,
     );
     check_output(
         stdout,
@@ -636,6 +639,7 @@ fn check_test_result(
         &config.stdout_filters,
         config,
         comments,
+        revision,
     );
     // Check error annotations in the source against output
     check_annotations(
@@ -659,7 +663,17 @@ fn check_annotations(
     revision: &str,
     comments: &Comments,
 ) {
-    if let Some((ref error_pattern, definition_line)) = comments.error_pattern {
+    let error_pattern = comments.find_one_for_revision(
+        revision,
+        |r| r.error_pattern.as_ref(),
+        |(_, line)| {
+            errors.push(Error::InvalidComment {
+                msg: "same revision defines pattern twice".into(),
+                line: *line,
+            })
+        },
+    );
+    if let Some((error_pattern, definition_line)) = error_pattern {
         // first check the diagnostics messages outside of our file. We check this first, so that
         // you can mix in-file annotations with //@error-pattern annotations, even if there is overlap
         // in the messages.
@@ -671,7 +685,7 @@ fn check_annotations(
         } else {
             errors.push(Error::PatternNotFound {
                 pattern: error_pattern.clone(),
-                definition_line,
+                definition_line: *definition_line,
             });
         }
     }
@@ -680,20 +694,17 @@ fn check_annotations(
     // We will ensure that *all* diagnostics of level at least `lowest_annotation_level`
     // are matched.
     let mut lowest_annotation_level = Level::Error;
+    let mut seen_error_match = false;
     for &ErrorMatch {
         ref pattern,
-        revision: ref rev,
         definition_line,
         line,
         level,
-    } in &comments.error_matches
+    } in comments
+        .for_revision(revision)
+        .flat_map(|r| r.error_matches.iter())
     {
-        if let Some(rev) = rev {
-            if rev != revision {
-                continue;
-            }
-        }
-
+        seen_error_match = true;
         // If we found a diagnostic with a level annotation, make sure that all
         // diagnostics of that level have annotations, even if we don't end up finding a matching diagnostic
         // for this pattern.
@@ -715,18 +726,21 @@ fn check_annotations(
         });
     }
 
-    let filter = |msgs: Vec<Message>| -> Vec<_> {
-        msgs.into_iter()
-            .filter(|msg| {
-                msg.level
-                    >= comments
-                        .require_annotations_for_level
-                        .unwrap_or(lowest_annotation_level)
+    let filter = |mut msgs: Vec<Message>, errors: &mut Vec<_>| -> Vec<_> {
+        let error = |_| {
+            errors.push(Error::InvalidComment {
+                msg: "`require_annotations_for_level` specified twice for same revision".into(),
+                line: 0,
             })
-            .collect()
+        };
+        let required_annotation_level = comments
+            .find_one_for_revision(revision, |r| r.require_annotations_for_level, error)
+            .unwrap_or(lowest_annotation_level);
+        msgs.retain(|msg| msg.level >= required_annotation_level);
+        msgs
     };
 
-    let messages_from_unknown_file_or_line = filter(messages_from_unknown_file_or_line);
+    let messages_from_unknown_file_or_line = filter(messages_from_unknown_file_or_line, errors);
     if !messages_from_unknown_file_or_line.is_empty() {
         errors.push(Error::ErrorsWithoutPattern {
             path: None,
@@ -735,7 +749,7 @@ fn check_annotations(
     }
 
     for (line, msgs) in messages.into_iter().enumerate() {
-        let msgs = filter(msgs);
+        let msgs = filter(msgs, errors);
         if !msgs.is_empty() {
             errors.push(Error::ErrorsWithoutPattern {
                 path: Some((path.to_path_buf(), line)),
@@ -744,10 +758,7 @@ fn check_annotations(
         }
     }
 
-    match (
-        config.mode,
-        comments.error_pattern.is_some() || !comments.error_matches.is_empty(),
-    ) {
+    match (config.mode, error_pattern.is_some() || seen_error_match) {
         (Mode::Pass, true) | (Mode::Panic, true) => errors.push(Error::PatternFoundInPassTest),
         (
             Mode::Fail {
@@ -767,10 +778,11 @@ fn check_output(
     filters: &Filter,
     config: &Config,
     comments: &Comments,
+    revision: &str,
 ) {
     let target = config.target.as_ref().unwrap();
-    let output = normalize(path, output, filters, comments);
-    let path = output_path(path, comments, kind, target);
+    let output = normalize(path, output, filters, comments, revision);
+    let path = output_path(path, comments, kind, target, revision);
     match config.output_conflict_handling {
         OutputConflictHandling::Bless => {
             if output.is_empty() {
@@ -793,8 +805,17 @@ fn check_output(
     }
 }
 
-fn output_path(path: &Path, comments: &Comments, kind: String, target: &str) -> PathBuf {
-    if comments.stderr_per_bitwidth {
+fn output_path(
+    path: &Path,
+    comments: &Comments,
+    kind: String,
+    target: &str,
+    revision: &str,
+) -> PathBuf {
+    if comments
+        .for_revision(revision)
+        .any(|r| r.stderr_per_bitwidth)
+    {
         return path.with_extension(format!("{}bit.{kind}", get_pointer_width(target)));
     }
     path.with_extension(kind)
@@ -810,11 +831,18 @@ fn test_condition(condition: &Condition, config: &Config) -> bool {
 }
 
 /// Returns whether according to the in-file conditions, this file should be run.
-fn test_file_conditions(comments: &Comments, config: &Config) -> bool {
-    if comments.ignore.iter().any(|c| test_condition(c, config)) {
+fn test_file_conditions(comments: &Comments, config: &Config, revision: &str) -> bool {
+    if comments
+        .for_revision(revision)
+        .flat_map(|r| r.ignore.iter())
+        .any(|c| test_condition(c, config))
+    {
         return false;
     }
-    comments.only.iter().all(|c| test_condition(c, config))
+    comments
+        .for_revision(revision)
+        .flat_map(|r| r.only.iter())
+        .all(|c| test_condition(c, config))
 }
 
 // Taken 1:1 from compiletest-rs
@@ -830,7 +858,13 @@ fn get_pointer_width(triple: &str) -> u8 {
     }
 }
 
-fn normalize(path: &Path, text: &[u8], filters: &Filter, comments: &Comments) -> Vec<u8> {
+fn normalize(
+    path: &Path,
+    text: &[u8],
+    filters: &Filter,
+    comments: &Comments,
+    revision: &str,
+) -> Vec<u8> {
     // Useless paths
     let mut text = text.replace(&path.parent().unwrap().display().to_string(), "$DIR");
     if let Some(lib_path) = option_env!("RUSTC_LIB_PATH") {
@@ -841,7 +875,10 @@ fn normalize(path: &Path, text: &[u8], filters: &Filter, comments: &Comments) ->
         text = regex.replace_all(&text, *replacement).into_owned();
     }
 
-    for (from, to) in &comments.normalize_stderr {
+    for (from, to) in comments
+        .for_revision(revision)
+        .flat_map(|r| r.normalize_stderr.iter())
+    {
         text = from.replace_all(&text, to).into_owned();
     }
     text

src/parser.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{collections::HashMap, path::Path};
 
 use bstr::{ByteSlice, Utf8Error};
 use regex::bytes::Regex;
@@ -17,6 +17,44 @@ mod tests;
 pub(crate) struct Comments {
     /// List of revision names to execute. Can only be speicified once
     pub revisions: Option<Vec<String>>,
+    /// Comments that are only available under specific revisions.
+    /// The defaults are in key `vec![]`
+    pub revisioned: HashMap<Vec<String>, Revisioned>,
+}
+
+impl Comments {
+    /// Check that a comment isn't specified twice across multiple differently revisioned statements.
+    /// e.g. `//@[foo, bar] error-pattern: bop` and `//@[foo, baz] error-pattern boop` would end up
+    /// specifying two error patterns that are available in revision `foo`.
+    pub fn find_one_for_revision<'a, T: 'a>(
+        &'a self,
+        revision: &'a str,
+        f: impl Fn(&'a Revisioned) -> Option<T>,
+        error: impl FnOnce(T),
+    ) -> Option<T> {
+        let mut rev = self.for_revision(revision).filter_map(f);
+        let result = rev.next();
+        if let Some(next) = rev.next() {
+            error(next);
+        }
+        result
+    }
+
+    /// Returns an iterator over all revisioned comments that match the revision.
+    pub fn for_revision<'a>(&'a self, revision: &'a str) -> impl Iterator<Item = &'a Revisioned> {
+        self.revisioned.iter().filter_map(move |(k, v)| {
+            if k.is_empty() || k.iter().any(|rev| rev == revision) {
+                Some(v)
+            } else {
+                None
+            }
+        })
+    }
+}
+
+#[derive(Default, Debug)]
+/// Comments that can be filtered for specific revisions.
+pub(crate) struct Revisioned {
     /// Don't run this test if any of these filters apply
     pub ignore: Vec<Condition>,
     /// Only run this test if all of these filters apply
@@ -37,25 +75,25 @@ pub(crate) struct Comments {
     pub require_annotations_for_level: Option<Level>,
 }
 
-#[derive(Default, Debug)]
-struct CommentParser {
+#[derive(Debug)]
+struct CommentParser<T> {
     /// The comments being built.
-    comments: Comments,
+    comments: T,
     /// Any errors that ocurred during comment parsing.
     errors: Vec<Error>,
     /// The line currently being parsed.
     line: usize,
 }
 
-impl std::ops::Deref for CommentParser {
-    type Target = Comments;
+impl<T> std::ops::Deref for CommentParser<T> {
+    type Target = T;
 
     fn deref(&self) -> &Self::Target {
         &self.comments
     }
 }
 
-impl std::ops::DerefMut for CommentParser {
+impl<T> std::ops::DerefMut for CommentParser<T> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.comments
     }
@@ -81,7 +119,6 @@ pub(crate) enum Pattern {
 #[derive(Debug)]
 pub(crate) struct ErrorMatch {
     pub pattern: Pattern,
-    pub revision: Option<String>,
     pub level: Level,
     /// The line where the message was defined, for reporting issues with it (e.g. in case it wasn't found).
     pub definition_line: usize,
@@ -119,7 +156,11 @@ impl Comments {
     pub(crate) fn parse(
         content: &(impl AsRef<[u8]> + ?Sized),
     ) -> std::result::Result<Self, Vec<Error>> {
-        let mut parser = CommentParser::default();
+        let mut parser = CommentParser {
+            comments: Comments::default(),
+            errors: vec![],
+            line: 0,
+        };
 
         let mut fallthrough_to = None; // The line that a `|` will refer to.
         for (l, line) in content.as_ref().lines().enumerate() {
@@ -141,7 +182,7 @@ impl Comments {
     }
 }
 
-impl CommentParser {
+impl CommentParser<Comments> {
     fn parse_checked_line(
         &mut self,
         fallthrough_to: &mut Option<usize>,
@@ -150,15 +191,18 @@ impl CommentParser {
         if let Some((_, command)) = line.split_once_str("//@") {
             self.parse_command(command.trim().to_str()?)
         } else if let Some((_, pattern)) = line.split_once_str("//~") {
-            self.parse_pattern(pattern.to_str()?, fallthrough_to)
-        } else if let Some((_, pattern)) = line.split_once_str("//[") {
-            self.parse_revisioned_pattern(pattern.to_str()?, fallthrough_to)
+            let (revisions, pattern) = self.parse_revisions(pattern.to_str()?);
+            self.revisioned(revisions, |this| {
+                this.parse_pattern(pattern, fallthrough_to)
+            })
         } else {
             *fallthrough_to = None;
         }
         Ok(())
     }
+}
 
+impl<CommentsType> CommentParser<CommentsType> {
     fn error(&mut self, s: impl Into<String>) {
         self.errors.push(Error::InvalidComment {
             msg: s.into(),
@@ -176,8 +220,12 @@ impl CommentParser {
         self.check(opt.is_some(), s);
         opt
     }
+}
 
+impl CommentParser<Comments> {
     fn parse_command(&mut self, command: &str) {
+        let (revisions, command) = self.parse_revisions(command);
+
         // Commands are letters or dashes, grab everything until the first character that is neither of those.
         let (command, args) = match command
             .chars()
@@ -199,11 +247,36 @@ impl CommentParser {
             }
         };
 
+        if command == "revisions" {
+            self.check(
+                revisions.is_empty(),
+                "cannot declare revisions under a revision",
+            );
+            self.check(self.revisions.is_none(), "cannot specify `revisions` twice");
+            self.revisions = Some(args.split_whitespace().map(|s| s.to_string()).collect());
+            return;
+        }
+        self.revisioned(revisions, |this| this.parse_command(command, args));
+    }
+
+    fn revisioned(
+        &mut self,
+        revisions: Vec<String>,
+        f: impl FnOnce(&mut CommentParser<&mut Revisioned>),
+    ) {
+        let mut this = CommentParser {
+            errors: std::mem::take(&mut self.errors),
+            line: self.line,
+            comments: self.revisioned.entry(revisions).or_default(),
+        };
+        f(&mut this);
+        self.errors = this.errors;
+    }
+}
+
+impl CommentParser<&mut Revisioned> {
+    fn parse_command(&mut self, command: &str, args: &str) {
         match command {
-            "revisions" => {
-                self.check(self.revisions.is_none(), "cannot specify `revisions` twice");
-                self.revisions = Some(args.split_whitespace().map(|s| s.to_string()).collect());
-            }
             "compile-flags" => {
                 self.compile_flags
                     .extend(args.split_whitespace().map(|s| s.to_string()));
@@ -287,7 +360,9 @@ impl CommentParser {
             }
         }
     }
+}
 
+impl<CommentsType> CommentParser<CommentsType> {
     fn parse_regex(&mut self, regex: &str) -> Option<Regex> {
         match Regex::new(regex) {
             Ok(regex) => Some(regex),
@@ -331,32 +406,35 @@ impl CommentParser {
         }
     }
 
-    fn parse_pattern(&mut self, pattern: &str, fallthrough_to: &mut Option<usize>) {
-        self.parse_pattern_inner(pattern, fallthrough_to, None)
-    }
-
-    fn parse_revisioned_pattern(&mut self, pattern: &str, fallthrough_to: &mut Option<usize>) {
-        let (revision, pattern) = match pattern.split_once(']') {
-            Some(it) => it,
-            None => {
-                self.error("`//[` without corresponding `]`");
-                return;
+    // parse something like \[[a-z]+(,[a-z]+)*\]
+    fn parse_revisions<'a>(&mut self, pattern: &'a str) -> (Vec<String>, &'a str) {
+        match pattern.chars().next() {
+            Some('[') => {
+                // revisions
+                let s = &pattern[1..];
+                let end = s.char_indices().find_map(|(i, c)| match c {
+                    ']' => Some(i),
+                    _ => None,
+                });
+                let Some(end) = end else {
+                    self.error("`[` without corresponding `]`");
+                    return (vec![], pattern);
+                };
+                let (revision, pattern) = s.split_at(end);
+                (
+                    revision.split(',').map(|s| s.trim().to_string()).collect(),
+                    // 1.. because `split_at` includes the separator
+                    pattern[1..].trim_start(),
+                )
             }
-        };
-        if let Some(pattern) = pattern.strip_prefix('~') {
-            self.parse_pattern_inner(pattern, fallthrough_to, Some(revision.to_owned()))
-        } else {
-            self.error("revisioned pattern must have `~` following the `]`");
+            _ => (vec![], pattern),
         }
     }
+}
 
-    // parse something like (?P<offset>\||[\^]+)? *(?P<level>ERROR|HELP|WARN|NOTE): (?P<text>.*)
-    fn parse_pattern_inner(
-        &mut self,
-        pattern: &str,
-        fallthrough_to: &mut Option<usize>,
-        revision: Option<String>,
-    ) {
+impl CommentParser<&mut Revisioned> {
+    // parse something like (\[[a-z]+(,[a-z]+)*\])?(?P<offset>\||[\^]+)? *(?P<level>ERROR|HELP|WARN|NOTE): (?P<text>.*)
+    fn parse_pattern(&mut self, pattern: &str, fallthrough_to: &mut Option<usize>) {
         let (match_line, pattern) = match pattern.chars().next() {
             Some('|') => (
                 match fallthrough_to {
@@ -418,7 +496,6 @@ impl CommentParser {
         let definition_line = self.line;
         self.error_matches.push(ErrorMatch {
             pattern,
-            revision,
             level,
             definition_line,
             line: match_line,
@@ -435,7 +512,7 @@ impl Pattern {
     }
 }
 
-impl CommentParser {
+impl<CommentsType> CommentParser<CommentsType> {
     fn parse_error_pattern(&mut self, pattern: &str) -> Pattern {
         if let Some(regex) = pattern.strip_prefix('/') {
             match regex.strip_suffix('/') {

src/parser/tests.rs

@@ -16,9 +16,10 @@ fn main() {
     ";
     let comments = Comments::parse(s).unwrap();
     println!("parsed comments: {:#?}", comments);
-    assert_eq!(comments.error_matches[0].definition_line, 5);
-    assert_eq!(comments.error_matches[0].revision, None);
-    match &comments.error_matches[0].pattern {
+    assert_eq!(comments.revisioned.len(), 1);
+    let revisioned = &comments.revisioned[&vec![]];
+    assert_eq!(revisioned.error_matches[0].definition_line, 5);
+    match &revisioned.error_matches[0].pattern {
         Pattern::SubString(s) => {
             assert_eq!(
                 s,
@@ -56,7 +57,9 @@ use std::mem;
     ";
     let comments = Comments::parse(s).unwrap();
     println!("parsed comments: {:#?}", comments);
-    let pat = comments.error_pattern.unwrap();
+    assert_eq!(comments.revisioned.len(), 1);
+    let revisioned = &comments.revisioned[&vec![]];
+    let pat = revisioned.error_pattern.as_ref().unwrap();
     assert_eq!(format!("{:?}", pat.0), r#"SubString("foomp")"#);
     assert_eq!(pat.1, 2);
 }
@@ -70,7 +73,9 @@ use std::mem;
     ";
     let comments = Comments::parse(s).unwrap();
     println!("parsed comments: {:#?}", comments);
-    let pat = comments.error_pattern.unwrap();
+    assert_eq!(comments.revisioned.len(), 1);
+    let revisioned = &comments.revisioned[&vec![]];
+    let pat = revisioned.error_pattern.as_ref().unwrap();
     assert_eq!(format!("{:?}", pat.0), r#"Regex(foomp)"#);
     assert_eq!(pat.1, 2);
 }
@@ -122,8 +127,10 @@ fn parse_x86_64() {
     let s = r"//@ only-target-x86_64-unknown-linux";
     let comments = Comments::parse(s).unwrap();
     println!("parsed comments: {:#?}", comments);
-    assert_eq!(comments.only.len(), 1);
-    match &comments.only[0] {
+    assert_eq!(comments.revisioned.len(), 1);
+    let revisioned = &comments.revisioned[&vec![]];
+    assert_eq!(revisioned.only.len(), 1);
+    match &revisioned.only[0] {
         Condition::Target(t) => assert_eq!(t, "x86_64-unknown-linux"),
         _ => unreachable!(),
     }

tests/integration.rs

@@ -6,12 +6,7 @@ use ui_test::*;
 
 fn main() -> Result<()> {
     run("integrations", Mode::Pass)?;
-    run(
-        "integrations",
-        Mode::Fail {
-            require_patterns: false,
-        },
-    )?;
+    run("integrations", Mode::Panic)?;
 
     Ok(())
 }
@@ -28,6 +23,7 @@ fn run(name: &str, mode: Mode) -> Result<()> {
             "never".into(),
             "--jobs".into(),
             "1".into(),
+            "--no-fail-fast".into(),
             "--target-dir".into(),
             path.parent().unwrap().join("target").into(),
             "--manifest-path".into(),
@@ -80,8 +76,11 @@ fn run(name: &str, mode: Mode) -> Result<()> {
             && path.parent().unwrap().parent().unwrap() == root_dir
             && match mode {
                 Mode::Pass => !fail,
-                Mode::Panic => unreachable!(),
-                Mode::Fail { .. } => fail,
+                // This is weird, but `cargo test` returns 101 instead of 1 when
+                // multiple [[test]]s exist. If there's only one test, it returns
+                // 1 on failure.
+                Mode::Panic => fail,
+                Mode::Fail { .. } => unreachable!(),
             }
     })
 }

tests/integrations/basic-fail/Cargo.stderr

@@ -114,3 +114,63 @@ error: test failed, to rerun pass `--test ui_tests`
 
 Caused by:
   process didn't exit successfully: `$DIR/target/debug/ui_tests-HASH --test-threads 1` (exit status: 1)
+     Running tests/ui_tests_bless.rs 
+   Compiler flags: ["--error-format=json", "--out-dir", "$TMP, "--edition=2021"]
+   Building test dependencies...
+tests/actual_tests_bless/revised_revision.rs ... FAILED
+tests/actual_tests_bless/revisions.rs (foo) ... ok
+tests/actual_tests_bless/revisions.rs (bar) ... ok
+tests/actual_tests_bless/revisions_bad.rs (foo) ... ok
+tests/actual_tests_bless/revisions_bad.rs (bar) ... FAILED
+tests/actual_tests_bless/revisions_filter.rs (foo) ... ignored (in-test comment)
+tests/actual_tests_bless/revisions_filter.rs (bar) ... ignored (in-test comment)
+tests/actual_tests_bless/revisions_filter2.rs (foo) ... ignored (in-test comment)
+tests/actual_tests_bless/revisions_filter2.rs (bar) ... ok
+tests/actual_tests_bless/revisions_multiple_per_annotation.rs (foo) ... ok
+tests/actual_tests_bless/revisions_multiple_per_annotation.rs (bar) ... ok
+tests/actual_tests_bless/revisions_same_everywhere.rs (foo) ... ok
+tests/actual_tests_bless/revisions_same_everywhere.rs (bar) ... ok
+
+tests/actual_tests_bless/revised_revision.rs FAILED:
+command: "parse comments"
+
+Could not parse comment in tests/actual_tests_bless/revised_revision.rs:2 because cannot declare revisions under a revision
+
+full stderr:
+
+
+
+tests/actual_tests_bless/revisions_bad.rs (revision `bar`) FAILED:
+command: "rustc" "--error-format=json" "--out-dir" "$TMP "--edition=2021" "--extern" "basic_fail=$DIR/$DIR/../../../target/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/$DIR/../../../target/debug" "-L" "$DIR/$DIR/../../../target/debug" "tests/actual_tests_bless/revisions_bad.rs" "--cfg=bar"
+
+substring ``main` function not found in crate `revisions_bad`` not found in stderr output
+expected because of pattern here: tests/actual_tests_bless/revisions_bad.rs:4
+
+There were 1 unmatched diagnostics at tests/actual_tests_bless/revisions_bad.rs:10
+    Error: `main` function not found in crate `revisions_bad`
+
+full stderr:
+error[E0601]: `main` function not found in crate `revisions_bad`
+  --> tests/actual_tests_bless/revisions_bad.rs:10:2
+   |
+10 | }
+   |  ^ consider adding a `main` function to `tests/actual_tests_bless/revisions_bad.rs`
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0601`.
+
+
+FAILURES:
+    tests/actual_tests_bless/revised_revision.rs
+    tests/actual_tests_bless/revisions_bad.rs
+
+test result: FAIL. 2 tests failed, 8 tests passed, 3 ignored, 0 filtered out
+error: test failed, to rerun pass `--test ui_tests_bless`
+
+Caused by:
+  process didn't exit successfully: `$DIR/target/debug/ui_tests_bless-HASH --test-threads 1` (exit status: 1)
+   Doc-tests basic_fail
+error: 2 targets failed:
+    `--test ui_tests`
+    `--test ui_tests_bless`

tests/integrations/basic-fail/Cargo.stdout

@@ -3,3 +3,8 @@ running 0 tests
 
 test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished 
 
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished 
+

tests/integrations/basic-fail/Cargo.toml

@@ -12,3 +12,7 @@ tempfile = "3.3.0"
 [[test]]
 name = "ui_tests"
 harness = false
+
+[[test]]
+name = "ui_tests_bless"
+harness = false

tests/integrations/basic-fail/tests/actual_tests_bless/revised_revision.rs

@@ -0,0 +1,13 @@
+use basic_fail::add;
+//@[foo] revisions: foo bar
+
+#[cfg(foo)]
+fn main() {
+    add("42", 3); //~[foo] ERROR: mismatched types
+}
+
+#[cfg(bar)]
+fn main() {
+    add("42", 3);
+    //~[bar]^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/actual_tests_bless/revisions.bar.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+  --> $DIR/revisions.rs:11:9
+   |
+11 |     add("42", 3);
+   |     --- ^^^^ expected `usize`, found `&str`
+   |     |
+   |     arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+   |
+1  | pub fn add(left: usize, right: usize) -> usize {
+   |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions.foo.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/revisions.rs:6:9
+  |
+6 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions.rs

@@ -0,0 +1,13 @@
+use basic_fail::add;
+//@ revisions: foo bar
+
+#[cfg(foo)]
+fn main() {
+    add("42", 3); //~[foo] ERROR: mismatched types
+}
+
+#[cfg(bar)]
+fn main() {
+    add("42", 3);
+    //~[bar]^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_bad.bar.stderr

@@ -0,0 +1,9 @@
+error[E0601]: `main` function not found in crate `revisions_bad`
+  --> $DIR/revisions_bad.rs:10:2
+   |
+10 | }
+   |  ^ consider adding a `main` function to `$DIR/revisions_bad.rs`
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0601`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_bad.foo.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/revisions_bad.rs:8:9
+  |
+8 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_bad.rs

@@ -0,0 +1,10 @@
+#[cfg(foo)]
+use basic_fail::add;
+//@ revisions: foo bar
+//@[bar] error-pattern: `main` function not found in crate `revisions_bad`
+
+#[cfg(foo)]
+fn main() {
+    add("42", 3);
+    //~[foo]^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_filter.rs

@@ -0,0 +1,14 @@
+use basic_fail::add;
+//@ignore-on-host
+//@ revisions: foo bar
+
+#[cfg(foo)]
+fn main() {
+    add("42", 3);
+}
+
+#[cfg(bar)]
+fn main() {
+    add("42", 3);
+    //~[bar]^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_filter2.bar.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+  --> $DIR/revisions_filter2.rs:12:9
+   |
+12 |     add("42", 3);
+   |     --- ^^^^ expected `usize`, found `&str`
+   |     |
+   |     arguments to this function are incorrect
+   |
+note: function defined here
+  --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+   |
+1  | pub fn add(left: usize, right: usize) -> usize {
+   |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_filter2.rs

@@ -0,0 +1,14 @@
+use basic_fail::add;
+//@[foo] ignore-on-host
+//@ revisions: foo bar
+
+#[cfg(foo)]
+fn main() {
+    add("42", 3);
+}
+
+#[cfg(bar)]
+fn main() {
+    add("42", 3);
+    //~[bar]^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_multiple_per_annotation.bar.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/revisions_multiple_per_annotation.rs:5:9
+  |
+5 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_multiple_per_annotation.foo.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/revisions_multiple_per_annotation.rs:5:9
+  |
+5 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_multiple_per_annotation.rs

@@ -0,0 +1,7 @@
+use basic_fail::add;
+//@ revisions: foo bar
+
+fn main() {
+    add("42", 3);
+    //~[bar, foo]^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_same_everywhere.bar.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/revisions_same_everywhere.rs:5:9
+  |
+5 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_same_everywhere.foo.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/revisions_same_everywhere.rs:5:9
+  |
+5 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/actual_tests_bless/revisions_same_everywhere.rs

@@ -0,0 +1,7 @@
+use basic_fail::add;
+//@ revisions: foo bar
+
+fn main() {
+    add("42", 3);
+    //~^ ERROR: mismatched types
+}

tests/integrations/basic-fail/tests/ui_tests_bless.rs

@@ -0,0 +1,44 @@
+use std::num::NonZeroUsize;
+use ui_test::*;
+
+fn main() -> ui_test::color_eyre::Result<()> {
+    let path = "../../../target";
+    let mut config = Config {
+        quiet: false,
+        root_dir: "tests/actual_tests_bless".into(),
+        dependencies_crate_manifest_path: Some("Cargo.toml".into()),
+        output_conflict_handling: if std::env::var_os("BLESS").is_some() {
+            OutputConflictHandling::Bless
+        } else {
+            OutputConflictHandling::Error
+        },
+        // Make sure our tests are ordered for reliable output.
+        num_test_threads: NonZeroUsize::new(1).unwrap(),
+        ..Config::default()
+    };
+    config
+        .dependency_builder
+        .envs
+        .push(("CARGO_TARGET_DIR".into(), path.into()));
+
+    // hide binaries generated for successfully passing tests
+    let tmp_dir = tempfile::tempdir()?;
+    config.args.push("--out-dir".into());
+    config.args.push(tmp_dir.path().as_os_str().to_owned());
+
+    config.args.push("--edition=2021".into());
+    config.stderr_filter("in ([0-9]m )?[0-9\\.]+s", "");
+    config.stdout_filter("in ([0-9]m )?[0-9\\.]+s", "");
+    config.stderr_filter(r"[^ ]*/\.?cargo/registry/.*/", "$$CARGO_REGISTRY");
+    config.stderr_filter(
+        &std::path::Path::new(path)
+            .canonicalize()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .display()
+            .to_string(),
+        "$$DIR",
+    );
+    ui_test::run_tests(config)
+}