Auto merge of #10426 - oli-obk:ui_test, r=Manishearth

Port clippy away from compiletest to ui_test

Reasons to do this:

* runs completely on stable Rust
* is easier to extend with new features
* has its own dogfood test suite, so changes can be tested in [the `ui_test` repo](https://github.com/oli-obk/ui_test)
* supports dependencies from crates.io without having to manually fiddle with command line flags
* supports `ui-cargo`, `ui`, `ui-toml` out of the box, no need to find and run the tests ourselves

One thing that is a big difference to `compiletest` is that if a test emits *any* error, you need to mark all of them with `//~ ERROR:` annotations. Since many clippy tests did not have annotations, I changed many lints to be `warn` in their test so that only the `stderr` output is tested.

TODO:

* [ ] check that this still works as a subtree in the rustc repo

changelog: none
<!-- changelog_checked -->

Note: at present the latest changes needed for clippy are only available as a git dependency, but I expect to publish a new crates.io version soon

Author: bors

.cargo/config.toml

@@ -1,5 +1,7 @@
 [alias]
 uitest = "test --test compile-test"
+uibless = "test --test compile-test -- -- --bless"
+bless = "test -- -- --bless"
 dev = "run --package clippy_dev --bin clippy_dev --manifest-path clippy_dev/Cargo.toml --"
 lintcheck = "run --package lintcheck --bin lintcheck --manifest-path lintcheck/Cargo.toml  -- "
 collect-metadata = "test --test dogfood --features internal -- run_metadata_collection_lint --ignored"

Cargo.toml

@@ -27,27 +27,14 @@ tempfile = { version = "3.2", optional = true }
 termize = "0.1"
 
 [dev-dependencies]
-compiletest_rs = { version = "0.10", features = ["tmp"] }
+ui_test = "0.11.5"
 tester = "0.9"
 regex = "1.5"
 toml = "0.7.3"
 walkdir = "2.3"
 # This is used by the `collect-metadata` alias.
 filetime = "0.2"
-
-# UI test dependencies
-clap = { version = "4.1.4", features = ["derive"] }
-clippy_utils = { path = "clippy_utils" }
-derive-new = "0.5"
-if_chain = "1.0"
 itertools = "0.10.1"
-quote = "1.0"
-serde = { version = "1.0.125", features = ["derive"] }
-syn = { version = "2.0", features = ["full"] }
-futures = "0.3"
-parking_lot = "0.12"
-tokio = { version = "1", features = ["io-util"] }
-rustc-semver = "1.1"
 
 [build-dependencies]
 rustc_tools_util = "0.3.0"
@@ -60,3 +47,7 @@ internal = ["clippy_lints/internal", "tempfile"]
 [package.metadata.rust-analyzer]
 # This package uses #[feature(rustc_private)]
 rustc_private = true
+
+[[test]]
+name = "compile-test"
+harness = false

book/src/development/adding_lints.md

@@ -122,20 +122,17 @@ fn main() {
 }
 ```
 
-Now we can run the test with `TESTNAME=foo_functions cargo uitest`, currently
+Now we can run the test with `TESTNAME=foo_functions cargo uibless`, currently
 this test is meaningless though.
 
 While we are working on implementing our lint, we can keep running the UI test.
-That allows us to check if the output is turning into what we want.
+That allows us to check if the output is turning into what we want by checking the
+`.stderr` file that gets updated on every test run.
 
-Once we are satisfied with the output, we need to run `cargo dev bless` to
-update the `.stderr` file for our lint. Please note that, we should run
-`TESTNAME=foo_functions cargo uitest` every time before running `cargo dev
-bless`. Running `TESTNAME=foo_functions cargo uitest` should pass then. When we
+Running `TESTNAME=foo_functions cargo uitest` should pass on its own. When we
 commit our lint, we need to commit the generated `.stderr` files, too. In
-general, you should only commit files changed by `cargo dev bless` for the
-specific lint you are creating/editing. Note that if the generated files are
-empty, they should be removed.
+general, you should only commit files changed by `cargo bless` for the
+specific lint you are creating/editing.
 
 > _Note:_ you can run multiple test files by specifying a comma separated list:
 > `TESTNAME=foo_functions,test2,test3`.
@@ -169,7 +166,7 @@ additionally run [rustfix] for that test. Rustfix will apply the suggestions
 from the lint to the code of the test file and compare that to the contents of a
 `.fixed` file.
 
-Use `cargo dev bless` to automatically generate the `.fixed` file after running
+Use `cargo bless` to automatically generate the `.fixed` file while running
 the tests.
 
 [rustfix]: https://github.com/rust-lang/rustfix

book/src/development/basics.md

@@ -66,7 +66,7 @@ If the output of a [UI test] differs from the expected output, you can update
 the reference file with:
 
 ```bash
-cargo dev bless
+cargo bless
 ```
 
 For example, this is necessary if you fix a typo in an error message of a lint,

clippy_dev/src/bless.rs

@@ -14,7 +14,6 @@ use std::io;
 use std::path::PathBuf;
 use std::process::{self, ExitStatus};
 
-pub mod bless;
 pub mod dogfood;
 pub mod fmt;
 pub mod lint;

clippy_dev/src/lib.rs

@@ -3,16 +3,16 @@
 #![warn(rust_2018_idioms, unused_lifetimes)]
 
 use clap::{Arg, ArgAction, ArgMatches, Command};
-use clippy_dev::{bless, dogfood, fmt, lint, new_lint, serve, setup, update_lints};
+use clippy_dev::{dogfood, fmt, lint, new_lint, serve, setup, update_lints};
 use indoc::indoc;
 use std::convert::Infallible;
 
 fn main() {
     let matches = get_clap_config();
 
     match matches.subcommand() {
-        Some(("bless", matches)) => {
-            bless::bless(matches.get_flag("ignore-timestamp"));
+        Some(("bless", _)) => {
+            eprintln!("use `cargo bless` to automatically replace `.stderr` and `.fixed` files as tests are being run");
         },
         Some(("dogfood", matches)) => {
             dogfood::dogfood(

clippy_dev/src/main.rs

@@ -96,7 +96,7 @@ fn create_test(lint: &LintData<'_>) -> io::Result<()> {
 
         path.push("src");
         fs::create_dir(&path)?;
-        let header = format!("// compile-flags: --crate-name={lint_name}");
+        let header = format!("//@compile-flags: --crate-name={lint_name}");
         write_file(path.join("main.rs"), get_test_file_contents(lint_name, Some(&header)))?;
 
         Ok(())

clippy_dev/src/new_lint.rs

@@ -0,0 +1,25 @@
+[package]
+name = "clippy_test_deps"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = { version = "4.1.4", features = ["derive"] }
+clippy_utils = { path = "../clippy_utils" }
+derive-new = "0.5"
+if_chain = "1.0"
+itertools = "0.10.1"
+quote = "1.0"
+serde = { version = "1.0.125", features = ["derive"] }
+syn = { version = "2.0", features = ["full"] }
+futures = "0.3"
+parking_lot = "0.12"
+tokio = { version = "1", features = ["io-util"] }
+rustc-semver = "1.1"
+regex = "1.5"
+clippy_lints = { path = "../clippy_lints" }
+
+[features]
+internal = ["clippy_lints/internal"]

clippy_test_deps/Cargo.toml

@@ -0,0 +1,14 @@
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}

clippy_test_deps/src/lib.rs

@@ -2397,7 +2397,7 @@ fn with_test_item_names(tcx: TyCtxt<'_>, module: LocalDefId, f: impl Fn(&[Symbol
 
 /// Checks if the function containing the given `HirId` is a `#[test]` function
 ///
-/// Note: Add `// compile-flags: --test` to UI tests with a `#[test]` function
+/// Note: Add `//@compile-flags: --test` to UI tests with a `#[test]` function
 pub fn is_in_test_function(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
     with_test_item_names(tcx, tcx.parent_module(id), |names| {
         tcx.hir()
@@ -2419,7 +2419,7 @@ pub fn is_in_test_function(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
 
 /// Checks if the item containing the given `HirId` has `#[cfg(test)]` attribute applied
 ///
-/// Note: Add `// compile-flags: --test` to UI tests with a `#[cfg(test)]` function
+/// Note: Add `//@compile-flags: --test` to UI tests with a `#[cfg(test)]` function
 pub fn is_in_cfg_test(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
     fn is_cfg_test(attr: &Attribute) -> bool {
         if attr.has_name(sym::cfg)
@@ -2441,7 +2441,7 @@ pub fn is_in_cfg_test(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
 /// Checks whether item either has `test` attribute applied, or
 /// is a module with `test` in its name.
 ///
-/// Note: Add `// compile-flags: --test` to UI tests with a `#[test]` function
+/// Note: Add `//@compile-flags: --test` to UI tests with a `#[test]` function
 pub fn is_test_module_or_function(tcx: TyCtxt<'_>, item: &Item<'_>) -> bool {
     is_in_test_function(tcx, item.hir_id())
         || matches!(item.kind, ItemKind::Mod(..))