Auto merge of #53410 - djrenren:custom-test-frameworks, r=<try>

Introduce Custom Test Frameworks

Introduces `#[test_case]` and `#[test_runner]` and re-implements `#[test]` and `#[bench]` in terms of them.

Details found here: https://blog.jrenner.net/rust/testing/2018/08/06/custom-test-framework-prop.html

Author: bors

src/Cargo.lock

@@ -2770,6 +2770,7 @@ name = "syntax_ext"
 version = "0.0.0"
 dependencies = [
  "fmt_macros 0.0.0",
+ "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "proc_macro 0.0.0",
  "rustc_data_structures 0.0.0",
  "rustc_errors 0.0.0",

src/librustc_lint/builtin.rs

@@ -1842,43 +1842,56 @@ impl EarlyLintPass for EllipsisInclusiveRangePatterns {
 }
 
 declare_lint! {
-    UNNAMEABLE_TEST_FUNCTIONS,
+    UNNAMEABLE_TEST_ITEMS,
     Warn,
-    "detects an function that cannot be named being marked as #[test]"
+    "detects an item that cannot be named being marked as #[test_case]",
+    report_in_external_macro: true
 }
 
-pub struct UnnameableTestFunctions;
+pub struct UnnameableTestItems {
+    boundary: ast::NodeId, // NodeId of the item under which things are not nameable
+    items_nameable: bool,
+}
+
+impl UnnameableTestItems {
+    pub fn new() -> Self {
+        Self {
+            boundary: ast::DUMMY_NODE_ID,
+            items_nameable: true
+        }
+    }
+}
 
-impl LintPass for UnnameableTestFunctions {
+impl LintPass for UnnameableTestItems {
     fn get_lints(&self) -> LintArray {
-        lint_array!(UNNAMEABLE_TEST_FUNCTIONS)
+        lint_array!(UNNAMEABLE_TEST_ITEMS)
     }
 }
 
-impl<'a, 'tcx> LateLintPass<'a, 'tcx> for UnnameableTestFunctions {
+impl<'a, 'tcx> LateLintPass<'a, 'tcx> for UnnameableTestItems {
     fn check_item(&mut self, cx: &LateContext, it: &hir::Item) {
-        match it.node {
-            hir::ItemKind::Fn(..) => {
-                for attr in &it.attrs {
-                    if attr.name() == "test" {
-                        let parent = cx.tcx.hir.get_parent(it.id);
-                        match cx.tcx.hir.find(parent) {
-                            Some(hir_map::NodeItem(hir::Item {node: hir::ItemKind::Mod(_), ..})) |
-                            None => {}
-                            _ => {
-                                cx.struct_span_lint(
-                                    UNNAMEABLE_TEST_FUNCTIONS,
-                                    attr.span,
-                                    "cannot test inner function",
-                                ).emit();
-                            }
-                        }
-                        break;
-                    }
-                }
+        if self.items_nameable {
+            if let hir::ItemKind::Mod(..) = it.node {}
+            else {
+                self.items_nameable = false;
+                self.boundary = it.id;
             }
-            _ => return,
-        };
+            return;
+        }
+
+        if let Some(attr) = attr::find_by_name(&it.attrs, "test_case") {
+            cx.struct_span_lint(
+                UNNAMEABLE_TEST_ITEMS,
+                attr.span,
+                "cannot test inner items",
+            ).emit();
+        }
+    }
+
+    fn check_item_post(&mut self, _cx: &LateContext, it: &hir::Item) {
+        if !self.items_nameable && self.boundary == it.id {
+            self.items_nameable = true;
+        }
     }
 }
 

src/librustc_lint/lib.rs

@@ -148,7 +148,7 @@ pub fn register_builtins(store: &mut lint::LintStore, sess: Option<&Session>) {
         MutableTransmutes: MutableTransmutes,
         UnionsWithDropFields: UnionsWithDropFields,
         UnreachablePub: UnreachablePub,
-        UnnameableTestFunctions: UnnameableTestFunctions,
+        UnnameableTestItems: UnnameableTestItems::new(),
         TypeAliasBounds: TypeAliasBounds,
         UnusedBrokenConst: UnusedBrokenConst,
         TrivialConstraints: TrivialConstraints,

src/librustc_resolve/macros.rs

@@ -475,6 +475,10 @@ impl<'a, 'cl> Resolver<'a, 'cl> {
             return def;
         }
 
+        if kind == MacroKind::Attr && *&path[0].as_str() == "test" {
+            return Ok(self.macro_prelude.get(&path[0].name).unwrap().def())
+        }
+
         let legacy_resolution = self.resolve_legacy_scope(&invocation.legacy_scope, path[0], false);
         let result = if let Some((legacy_binding, _)) = legacy_resolution {
             Ok(legacy_binding.def())

src/libsyntax/ast.rs

@@ -1582,7 +1582,7 @@ impl TyKind {
         if let TyKind::ImplicitSelf = *self { true } else { false }
     }
 
-    crate fn is_unit(&self) -> bool {
+    pub fn is_unit(&self) -> bool {
         if let TyKind::Tup(ref tys) = *self { tys.is_empty() } else { false }
     }
 }

src/libsyntax/config.rs

@@ -119,7 +119,7 @@ impl<'a> StripUnconfigured<'a> {
     pub fn in_cfg(&mut self, attrs: &[ast::Attribute]) -> bool {
         attrs.iter().all(|attr| {
             // When not compiling with --test we should not compile the #[test] functions
-            if !self.should_test && is_test_or_bench(attr) {
+            if !self.should_test && is_test(attr) {
                 return false;
             }
 
@@ -249,7 +249,7 @@ impl<'a> StripUnconfigured<'a> {
         //
         // NB: This is intentionally not part of the fold_expr() function
         //     in order for fold_opt_expr() to be able to avoid this check
-        if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(a) || is_test_or_bench(a)) {
+        if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(a) || is_test(a)) {
             let msg = "removing an expression is not supported in this position";
             self.sess.span_diagnostic.span_err(attr.span, msg);
         }
@@ -353,6 +353,6 @@ fn is_cfg(attr: &ast::Attribute) -> bool {
     attr.check_name("cfg")
 }
 
-pub fn is_test_or_bench(attr: &ast::Attribute) -> bool {
-    attr.check_name("test") || attr.check_name("bench")
+pub fn is_test(att: &ast::Attribute) -> bool {
+    att.check_name("test_case")
 }

src/libsyntax/ext/expand.rs

@@ -12,10 +12,9 @@ use ast::{self, Block, Ident, NodeId, PatKind, Path};
 use ast::{MacStmtStyle, StmtKind, ItemKind};
 use attr::{self, HasAttrs};
 use source_map::{ExpnInfo, MacroBang, MacroAttribute, dummy_spanned, respan};
-use config::{is_test_or_bench, StripUnconfigured};
+use config::StripUnconfigured;
 use errors::{Applicability, FatalError};
 use ext::base::*;
-use ext::build::AstBuilder;
 use ext::derive::{add_derived_markers, collect_derives};
 use ext::hygiene::{self, Mark, SyntaxContext};
 use ext::placeholders::{placeholder, PlaceholderExpander};
@@ -1370,51 +1369,25 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
                 self.cx.current_expansion.directory_ownership = orig_directory_ownership;
                 result
             }
-            // Ensure that test functions are accessible from the test harness.
+
+            // Ensure that test items can be exported by the harness generator.
             // #[test] fn foo() {}
             // becomes:
             // #[test] pub fn foo_gensym(){}
-            // #[allow(unused)]
-            // use foo_gensym as foo;
-            ast::ItemKind::Fn(..) if self.cx.ecfg.should_test => {
-                if self.tests_nameable && item.attrs.iter().any(|attr| is_test_or_bench(attr)) {
-                    let orig_ident = item.ident;
-                    let orig_vis   = item.vis.clone();
-
+              ast::ItemKind::Const(..)
+            | ast::ItemKind::Static(..)
+            | ast::ItemKind::Fn(..) if self.cx.ecfg.should_test => {
+                if self.tests_nameable && attr::contains_name(&item.attrs, "test_case") {
                     // Publicize the item under gensymed name to avoid pollution
+                    // This means #[test_case] items can't be referenced by user code
                     item = item.map(|mut item| {
                         item.vis = respan(item.vis.span, ast::VisibilityKind::Public);
                         item.ident = item.ident.gensym();
                         item
                     });
-
-                    // Use the gensymed name under the item's original visibility
-                    let mut use_item = self.cx.item_use_simple_(
-                        item.ident.span,
-                        orig_vis,
-                        Some(orig_ident),
-                        self.cx.path(item.ident.span,
-                            vec![keywords::SelfValue.ident(), item.ident]));
-
-                    // #[allow(unused)] because the test function probably isn't being referenced
-                    use_item = use_item.map(|mut ui| {
-                        ui.attrs.push(
-                            self.cx.attribute(DUMMY_SP, attr::mk_list_item(DUMMY_SP,
-                                Ident::from_str("allow"), vec![
-                                    attr::mk_nested_word_item(Ident::from_str("unused"))
-                                ]
-                            ))
-                        );
-
-                        ui
-                    });
-
-                    OneVector::many(
-                        self.fold_unnameable(item).into_iter()
-                            .chain(self.fold_unnameable(use_item)))
-                } else {
-                    self.fold_unnameable(item)
                 }
+
+                self.fold_unnameable(item)
             }
             _ => self.fold_unnameable(item),
         }

src/libsyntax/feature_gate.rs

@@ -503,6 +503,10 @@ declare_features! (
 
     // unsized rvalues at arguments and parameters
     (active, unsized_locals, "1.30.0", Some(48055), None),
+
+    // #![test_runner]
+    // #[test_case]
+    (active, custom_test_frameworks, "1.30.0", Some(50297), None),
 );
 
 declare_features! (
@@ -757,6 +761,10 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
     ("no_link", Normal, Ungated),
     ("derive", Normal, Ungated),
     ("should_panic", Normal, Ungated),
+    ("test_case", Normal, Gated(Stability::Unstable,
+                                "custom_test_frameworks",
+                                "Custom test frameworks are experimental",
+                                cfg_fn!(custom_test_frameworks))),
     ("ignore", Normal, Ungated),
     ("no_implicit_prelude", Normal, Ungated),
     ("reexport_test_harness_main", Normal, Ungated),
@@ -1123,6 +1131,10 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
     ("no_builtins", CrateLevel, Ungated),
     ("recursion_limit", CrateLevel, Ungated),
     ("type_length_limit", CrateLevel, Ungated),
+    ("test_runner", CrateLevel, Gated(Stability::Unstable,
+                    "custom_test_frameworks",
+                    "Custom Test Frameworks is an unstable feature",
+                    cfg_fn!(custom_test_frameworks))),
 ];
 
 // cfg(...)'s that are feature gated

src/libsyntax/test.rs

@@ -15,4 +15,5 @@ rustc_errors = { path = "../librustc_errors" }
 syntax = { path = "../libsyntax" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc_data_structures = { path = "../librustc_data_structures" }
-rustc_target = { path = "../librustc_target" }
+rustc_target = { path = "../librustc_target" }
+log = "0.4"

src/libsyntax_ext/Cargo.toml

@@ -18,7 +18,7 @@
 #![feature(decl_macro)]
 #![cfg_attr(not(stage0), feature(nll))]
 #![feature(str_escape)]
-
+#![feature(quote)]
 #![feature(rustc_diagnostic_macros)]
 
 extern crate fmt_macros;
@@ -29,6 +29,7 @@ extern crate proc_macro;
 extern crate rustc_data_structures;
 extern crate rustc_errors as errors;
 extern crate rustc_target;
+#[macro_use] extern crate log;
 
 mod diagnostics;
 
@@ -48,6 +49,7 @@ mod format_foreign;
 mod global_asm;
 mod log_syntax;
 mod trace_macros;
+mod test;
 
 pub mod proc_macro_registrar;
 
@@ -56,7 +58,7 @@ pub mod proc_macro_impl;
 
 use rustc_data_structures::sync::Lrc;
 use syntax::ast;
-use syntax::ext::base::{MacroExpanderFn, NormalTT, NamedSyntaxExtension};
+use syntax::ext::base::{MacroExpanderFn, NormalTT, NamedSyntaxExtension, MultiModifier};
 use syntax::ext::hygiene;
 use syntax::symbol::Symbol;
 
@@ -127,6 +129,9 @@ pub fn register_builtins(resolver: &mut dyn syntax::ext::base::Resolver,
         assert: assert::expand_assert,
     }
 
+    register(Symbol::intern("test"), MultiModifier(Box::new(test::expand_test)));
+    register(Symbol::intern("bench"), MultiModifier(Box::new(test::expand_bench)));
+
     // format_args uses `unstable` things internally.
     register(Symbol::intern("format_args"),
              NormalTT {

src/libsyntax_ext/lib.rs

@@ -0,0 +1,321 @@
+// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+/// The expansion from a test function to the appropriate test struct for libtest
+/// Ideally, this code would be in libtest but for efficiency and error messages it lives here.
+
+use syntax::ext::base::*;
+use syntax::ext::build::AstBuilder;
+use syntax::ext::hygiene::{self, Mark, SyntaxContext};
+use syntax::attr;
+use syntax::ast;
+use syntax::print::pprust;
+use syntax::symbol::Symbol;
+use syntax_pos::{DUMMY_SP, Span};
+use syntax::source_map::{ExpnInfo, MacroAttribute};
+use std::iter;
+
+pub fn expand_test(
+    cx: &mut ExtCtxt,
+    attr_sp: Span,
+    _meta_item: &ast::MetaItem,
+    item: Annotatable,
+) -> Vec<Annotatable> {
+    expand_test_or_bench(cx, attr_sp, item, false)
+}
+
+pub fn expand_bench(
+    cx: &mut ExtCtxt,
+    attr_sp: Span,
+    _meta_item: &ast::MetaItem,
+    item: Annotatable,
+) -> Vec<Annotatable> {
+    expand_test_or_bench(cx, attr_sp, item, true)
+}
+
+pub fn expand_test_or_bench(
+    cx: &mut ExtCtxt,
+    attr_sp: Span,
+    item: Annotatable,
+    is_bench: bool
+) -> Vec<Annotatable> {
+    // If we're not in test configuration, remove the annotated item
+    if !cx.ecfg.should_test { return vec![]; }
+
+    let item =
+        if let Annotatable::Item(i) = item { i }
+        else {
+            cx.parse_sess.span_diagnostic.span_fatal(item.span(),
+                "#[test] attribute is only allowed on fn items").raise();
+        };
+
+    // has_*_signature will report any errors in the type so compilation
+    // will fail. We shouldn't try to expand in this case because the errors
+    // would be spurious.
+    if (!is_bench && !has_test_signature(cx, &item)) ||
+        (is_bench && !has_bench_signature(cx, &item)) {
+        return vec![Annotatable::Item(item)];
+    }
+
+    let (sp, attr_sp) = {
+        let mark = Mark::fresh(Mark::root());
+        mark.set_expn_info(ExpnInfo {
+            call_site: DUMMY_SP,
+            def_site: None,
+            format: MacroAttribute(Symbol::intern("test")),
+            allow_internal_unstable: true,
+            allow_internal_unsafe: false,
+            local_inner_macros: false,
+            edition: hygiene::default_edition(),
+        });
+        (item.span.with_ctxt(SyntaxContext::empty().apply_mark(mark)),
+         attr_sp.with_ctxt(SyntaxContext::empty().apply_mark(mark)))
+    };
+
+    // Gensym "test" so we can extern crate without conflicting with any local names
+    let test_id = cx.ident_of("test").gensym();
+
+    // creates test::$name
+    let test_path = |name| {
+        cx.path(sp, vec![test_id, cx.ident_of(name)])
+    };
+
+    // creates test::$name
+    let should_panic_path = |name| {
+        cx.path(sp, vec![test_id, cx.ident_of("ShouldPanic"), cx.ident_of(name)])
+    };
+
+    // creates $name: $expr
+    let field = |name, expr| cx.field_imm(sp, cx.ident_of(name), expr);
+
+    let test_fn = if is_bench {
+        // A simple ident for a lambda
+        let b = cx.ident_of("b");
+
+        cx.expr_call(sp, cx.expr_path(test_path("StaticBenchFn")), vec![
+            // |b| self::test::assert_test_result(
+            cx.lambda1(sp,
+                cx.expr_call(sp, cx.expr_path(test_path("assert_test_result")), vec![
+                    // super::$test_fn(b)
+                    cx.expr_call(sp,
+                        cx.expr_path(cx.path(sp, vec![item.ident])),
+                        vec![cx.expr_ident(sp, b)])
+                ]),
+                b
+            )
+            // )
+        ])
+    } else {
+        cx.expr_call(sp, cx.expr_path(test_path("StaticTestFn")), vec![
+            // || {
+            cx.lambda0(sp,
+                // test::assert_test_result(
+                cx.expr_call(sp, cx.expr_path(test_path("assert_test_result")), vec![
+                    // $test_fn()
+                    cx.expr_call(sp, cx.expr_path(cx.path(sp, vec![item.ident])), vec![])
+                // )
+                ])
+            // }
+            )
+        // )
+        ])
+    };
+
+    let mut test_const = cx.item(sp, item.ident.gensym(),
+        // #[test_case]
+        vec![cx.attribute(attr_sp, cx.meta_word(attr_sp, Symbol::intern("test_case")))],
+        // const $ident: test::TestDescAndFn =
+        ast::ItemKind::Const(cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))),
+            // test::TestDescAndFn {
+            cx.expr_struct(sp, test_path("TestDescAndFn"), vec![
+                // desc: test::TestDesc {
+                field("desc", cx.expr_struct(sp, test_path("TestDesc"), vec![
+                    // name: "path::to::test"
+                    field("name", cx.expr_call(sp, cx.expr_path(test_path("StaticTestName")),
+                        vec![
+                            cx.expr_str(sp, Symbol::intern(&item_path(
+                                &cx.current_expansion.module.mod_path,
+                                &item.ident
+                            )))
+                        ])),
+                    // ignore: true | false
+                    field("ignore", cx.expr_bool(sp, should_ignore(&item))),
+                    // allow_fail: true | false
+                    field("allow_fail", cx.expr_bool(sp, should_fail(&item))),
+                    // should_panic: ...
+                    field("should_panic", match should_panic(cx, &item) {
+                        // test::ShouldPanic::No
+                        ShouldPanic::No => cx.expr_path(should_panic_path("No")),
+                        // test::ShouldPanic::Yes
+                        ShouldPanic::Yes(None) => cx.expr_path(should_panic_path("Yes")),
+                        // test::ShouldPanic::YesWithMessage("...")
+                        ShouldPanic::Yes(Some(sym)) => cx.expr_call(sp,
+                            cx.expr_path(should_panic_path("YesWithMessage")),
+                            vec![cx.expr_str(sp, sym)]),
+                    }),
+                // },
+                ])),
+                // testfn: test::StaticTestFn(...) | test::StaticBenchFn(...)
+                field("testfn", test_fn)
+            // }
+            ])
+        // }
+        ));
+    test_const = test_const.map(|mut tc| { tc.vis.node = ast::VisibilityKind::Public; tc});
+
+    // extern crate test as test_gensym
+    let test_extern = cx.item(sp,
+        test_id,
+        vec![],
+        ast::ItemKind::ExternCrate(Some(Symbol::intern("test")))
+    );
+
+    debug!("Synthetic test item:\n{}\n", pprust::item_to_string(&test_const));
+
+    vec![
+        // Access to libtest under a gensymed name
+        Annotatable::Item(test_extern),
+        // The generated test case
+        Annotatable::Item(test_const),
+        // The original item
+        Annotatable::Item(item)
+    ]
+}
+
+fn item_path(mod_path: &[ast::Ident], item_ident: &ast::Ident) -> String {
+    mod_path.iter().chain(iter::once(item_ident))
+        .map(|x| x.to_string()).collect::<Vec<String>>().join("::")
+}
+
+enum ShouldPanic {
+    No,
+    Yes(Option<Symbol>),
+}
+
+fn should_ignore(i: &ast::Item) -> bool {
+    attr::contains_name(&i.attrs, "ignore")
+}
+
+fn should_fail(i: &ast::Item) -> bool {
+    attr::contains_name(&i.attrs, "allow_fail")
+}
+
+fn should_panic(cx: &ExtCtxt, i: &ast::Item) -> ShouldPanic {
+    match attr::find_by_name(&i.attrs, "should_panic") {
+        Some(attr) => {
+            let ref sd = cx.parse_sess.span_diagnostic;
+            if attr.is_value_str() {
+                sd.struct_span_warn(
+                    attr.span(),
+                    "attribute must be of the form: \
+                     `#[should_panic]` or \
+                     `#[should_panic(expected = \"error message\")]`"
+                ).note("Errors in this attribute were erroneously allowed \
+                        and will become a hard error in a future release.")
+                .emit();
+                return ShouldPanic::Yes(None);
+            }
+            match attr.meta_item_list() {
+                // Handle #[should_panic]
+                None => ShouldPanic::Yes(None),
+                // Handle #[should_panic(expected = "foo")]
+                Some(list) => {
+                    let msg = list.iter()
+                        .find(|mi| mi.check_name("expected"))
+                        .and_then(|mi| mi.meta_item())
+                        .and_then(|mi| mi.value_str());
+                    if list.len() != 1 || msg.is_none() {
+                        sd.struct_span_warn(
+                            attr.span(),
+                            "argument must be of the form: \
+                             `expected = \"error message\"`"
+                        ).note("Errors in this attribute were erroneously \
+                                allowed and will become a hard error in a \
+                                future release.").emit();
+                        ShouldPanic::Yes(None)
+                    } else {
+                        ShouldPanic::Yes(msg)
+                    }
+                },
+            }
+        }
+        None => ShouldPanic::No,
+    }
+}
+
+fn has_test_signature(cx: &ExtCtxt, i: &ast::Item) -> bool {
+    let has_should_panic_attr = attr::contains_name(&i.attrs, "should_panic");
+    let ref sd = cx.parse_sess.span_diagnostic;
+    if let ast::ItemKind::Fn(ref decl, ref header, ref generics, _) = i.node {
+        if header.unsafety == ast::Unsafety::Unsafe {
+            sd.span_err(
+                i.span,
+                "unsafe functions cannot be used for tests"
+            );
+            return false
+        }
+        if header.asyncness.is_async() {
+            sd.span_err(
+                i.span,
+                "async functions cannot be used for tests"
+            );
+            return false
+        }
+
+
+        // If the termination trait is active, the compiler will check that the output
+        // type implements the `Termination` trait as `libtest` enforces that.
+        let has_output = match decl.output {
+            ast::FunctionRetTy::Default(..) => false,
+            ast::FunctionRetTy::Ty(ref t) if t.node.is_unit() => false,
+            _ => true
+        };
+
+        if !decl.inputs.is_empty() {
+            sd.span_err(i.span, "functions used as tests can not have any arguments");
+            return false;
+        }
+
+        match (has_output, has_should_panic_attr) {
+            (true, true) => {
+                sd.span_err(i.span, "functions using `#[should_panic]` must return `()`");
+                false
+            },
+            (true, false) => if !generics.params.is_empty() {
+                sd.span_err(i.span,
+                                "functions used as tests must have signature fn() -> ()");
+                false
+            } else {
+                true
+            },
+            (false, _) => true
+        }
+    } else {
+        sd.span_err(i.span, "only functions may be used as tests");
+        false
+    }
+}
+
+fn has_bench_signature(cx: &ExtCtxt, i: &ast::Item) -> bool {
+    let has_sig = if let ast::ItemKind::Fn(ref decl, _, _, _) = i.node {
+        // NB: inadequate check, but we're running
+        // well before resolve, can't get too deep.
+        decl.inputs.len() == 1
+    } else {
+        false
+    };
+
+    if !has_sig {
+        cx.parse_sess.span_diagnostic.span_err(i.span, "functions used as benches must have \
+            signature `fn(&mut Bencher) -> impl Termination`");
+    }
+
+    has_sig
+}

src/libsyntax_ext/test.rs

@@ -40,6 +40,7 @@
 #![feature(panic_unwind)]
 #![feature(staged_api)]
 #![feature(termination_trait_lib)]
+#![feature(test)]
 
 extern crate getopts;
 #[cfg(any(unix, target_os = "cloudabi"))]
@@ -301,7 +302,7 @@ pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) {
 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
 // semantics into parallel test runners, which in turn requires a Vec<>
 // rather than a &[].
-pub fn test_main_static(tests: &[TestDescAndFn]) {
+pub fn test_main_static(tests: &[&mut TestDescAndFn]) {
     let args = env::args().collect::<Vec<_>>();
     let owned_tests = tests
         .iter()

src/libtest/lib.rs

@@ -907,7 +907,8 @@ mod tests {
 
 #[cfg(test)]
 mod bench {
-    use Bencher;
+    extern crate test;
+    use self::test::Bencher;
     use stats::Stats;
 
     #[bench]

src/libtest/stats.rs

@@ -15,12 +15,11 @@
 #![feature(rustc_attrs)]
 #![crate_type = "rlib"]
 
-#![rustc_partition_codegened(module="issue_49595-__test", cfg="cfail2")]
+#![rustc_partition_codegened(module="issue_49595-tests", cfg="cfail2")]
 #![rustc_partition_codegened(module="issue_49595-lit_test", cfg="cfail3")]
 
 mod tests {
-    #[cfg_attr(not(cfail1), ignore)]
-    #[test]
+    #[cfg_attr(not(cfail1), test)]
     fn test() {
     }
 }

src/test/incremental/issue-49595/issue_49595.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 // check-stdout
-// error-pattern:thread 'test_foo' panicked at
+// error-pattern:thread 'test_panic::test_foo' panicked at
 // compile-flags: --test
 // ignore-emscripten
 

src/test/run-fail/test-panic.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 // check-stdout
-// error-pattern:thread 'test_foo' panicked at
+// error-pattern:thread 'test_should_fail_bad_message::test_foo' panicked at
 // compile-flags: --test
 // ignore-emscripten
 

src/test/run-fail/test-should-fail-bad-message.rs

@@ -1,10 +1,10 @@
 { "type": "suite", "event": "started", "test_count": "4" }
-{ "type": "test", "event": "started", "name": "a" }
-{ "type": "test", "name": "a", "event": "ok" }
-{ "type": "test", "event": "started", "name": "b" }
-{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:5\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" }
-{ "type": "test", "event": "started", "name": "c" }
-{ "type": "test", "name": "c", "event": "ok" }
-{ "type": "test", "event": "started", "name": "d" }
-{ "type": "test", "name": "d", "event": "ignored" }
+{ "type": "test", "event": "started", "name": "f::a" }
+{ "type": "test", "name": "f::a", "event": "ok" }
+{ "type": "test", "event": "started", "name": "f::b" }
+{ "type": "test", "name": "f::b", "event": "failed", "stdout": "thread 'f::b' panicked at 'assertion failed: false', f.rs:18:5\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" }
+{ "type": "test", "event": "started", "name": "f::c" }
+{ "type": "test", "name": "f::c", "event": "ok" }
+{ "type": "test", "event": "started", "name": "f::d" }
+{ "type": "test", "name": "f::d", "event": "ignored" }
 { "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": "0" }

src/test/run-make-fulldeps/libtest-json/output.json

@@ -9,6 +9,7 @@
 // except according to those terms.
 
 #![feature(stmt_expr_attributes)]
+#![feature(custom_test_frameworks)]
 
 fn main() {
     let _ = #[cfg(unset)] ();
@@ -17,6 +18,6 @@ fn main() {
     //~^ ERROR removing an expression is not supported in this position
     let _ = [1, 2, 3][#[cfg(unset)] 1];
     //~^ ERROR removing an expression is not supported in this position
-    let _ = #[test] ();
+    let _ = #[test_case] ();
     //~^ ERROR removing an expression is not supported in this position
 }

src/test/ui/cfg-non-opt-expr.rs

@@ -1,26 +1,26 @@
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:14:13
+  --> $DIR/cfg-non-opt-expr.rs:15:13
    |
 LL |     let _ = #[cfg(unset)] ();
    |             ^^^^^^^^^^^^^
 
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:16:21
+  --> $DIR/cfg-non-opt-expr.rs:17:21
    |
 LL |     let _ = 1 + 2 + #[cfg(unset)] 3;
    |                     ^^^^^^^^^^^^^
 
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:18:23
+  --> $DIR/cfg-non-opt-expr.rs:19:23
    |
 LL |     let _ = [1, 2, 3][#[cfg(unset)] 1];
    |                       ^^^^^^^^^^^^^
 
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:20:13
+  --> $DIR/cfg-non-opt-expr.rs:21:13
    |
-LL |     let _ = #[test] ();
-   |             ^^^^^^^
+LL |     let _ = #[test_case] ();
+   |             ^^^^^^^^^^^^
 
 error: aborting due to 4 previous errors
 

src/test/ui/cfg-non-opt-expr.stderr

@@ -0,0 +1,32 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: --test
+// run-pass
+
+#![feature(custom_test_frameworks)]
+#![test_runner(crate::foo_runner)]
+
+#[cfg(test)]
+fn foo_runner(ts: &[&Fn(usize)->()]) {
+    for (i, t) in ts.iter().enumerate() {
+        t(i);
+    }
+}
+
+#[test_case]
+fn test1(i: usize) {
+    println!("Hi #{}", i);
+}
+
+#[test_case]
+fn test2(i: usize) {
+    println!("Hey #{}", i);
+}

src/test/ui/custom-test-frameworks-simple.rs

@@ -0,0 +1,13 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![test_runner(main)] //~ ERROR Custom Test Frameworks is an unstable feature
+
+fn main() {}

src/test/ui/feature-gate-custom_test_frameworks.rs

@@ -0,0 +1,11 @@
+error[E0658]: Custom Test Frameworks is an unstable feature (see issue #50297)
+  --> $DIR/feature-gate-custom_test_frameworks.rs:11:1
+   |
+LL | #![test_runner(main)] //~ ERROR Custom Test Frameworks is an unstable feature
+   | ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: add #![feature(custom_test_frameworks)] to the crate attributes to enable
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0658`.

src/test/ui/feature-gate-custom_test_frameworks.stderr

@@ -2,7 +2,7 @@ error[E0432]: unresolved import `__test`
   --> $DIR/inaccessible-test-modules.rs:15:5
    |
 LL | use __test as x; //~ ERROR unresolved import `__test`
-   |     ^^^^^^^^^^^ no `__test` in the root. Did you mean to use `__test`?
+   |     ^^^^^^^^^^^ no `__test` in the root. Did you mean to use `test`?
 
 error[E0432]: unresolved import `__test_reexports`
   --> $DIR/inaccessible-test-modules.rs:16:5

src/test/ui/inaccessible-test-modules.stderr

@@ -10,5 +10,5 @@
 
 fn main() {
     concat!(test!());
-    //~^ ERROR cannot find macro `test!` in this scope
+    //~^ error: `test` can only be used in attributes
 }

src/test/ui/issues/issue-11692-2.rs

@@ -1,4 +1,4 @@
-error: cannot find macro `test!` in this scope
+error: `test` can only be used in attributes
   --> $DIR/issue-11692-2.rs:12:13
    |
 LL |     concat!(test!());

src/test/ui/issues/issue-11692-2.stderr

@@ -5,7 +5,7 @@ LL | fn bar(x: isize) { }
    | ^^^^^^^^^^^^^^^^^^^^ expected isize, found mutable reference
    |
    = note: expected type `isize`
-              found type `&mut __test::test::Bencher`
+              found type `&mut test::Bencher`
 
 error: aborting due to previous error
 

src/test/ui/issues/issue-12997-2.stderr

@@ -8,19 +8,19 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-// compile-flags: --test -D unnameable_test_functions
+// compile-flags: --test -D unnameable_test_items
 
 #[test]
 fn foo() {
-    #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+    #[test] //~ ERROR cannot test inner items [unnameable_test_items]
     fn bar() {}
     bar();
 }
 
 mod x {
     #[test]
     fn foo() {
-        #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+        #[test] //~ ERROR cannot test inner items [unnameable_test_items]
         fn bar() {}
         bar();
     }

src/test/ui/lint/test-inner-fn.rs

@@ -1,15 +1,15 @@
-error: cannot test inner function
+error: cannot test inner items
   --> $DIR/test-inner-fn.rs:15:5
    |
-LL |     #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+LL |     #[test] //~ ERROR cannot test inner items [unnameable_test_items]
    |     ^^^^^^^
    |
-   = note: requested on the command line with `-D unnameable-test-functions`
+   = note: requested on the command line with `-D unnameable-test-items`
 
-error: cannot test inner function
+error: cannot test inner items
   --> $DIR/test-inner-fn.rs:23:9
    |
-LL |         #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+LL |         #[test] //~ ERROR cannot test inner items [unnameable_test_items]
    |         ^^^^^^^
 
 error: aborting due to 2 previous errors

src/test/ui/lint/test-inner-fn.stderr

@@ -7,7 +7,7 @@ LL | | }
    | |_^ `main` can only return types that implement `std::process::Termination`
    |
    = help: the trait `std::process::Termination` is not implemented for `std::result::Result<f32, std::num::ParseIntError>`
-   = note: required by `__test::test::assert_test_result`
+   = note: required by `test::assert_test_result`
 
 error: aborting due to previous error