[e2e] Fix incorrect test results when one test passes then another fails (flutter#2866)

* [e2e] Fix incorrect test results when one test passes then another fails

For example, the following test will result in an error reported for the first test case.

```
void main() {
  testWidgets('a test that passes', (tester) async {
    expect(true, true);
  });

  testWidgets('a test that fails', (tester) async {
    expect(true, false);
  });
}
```

We need to reset `reportTestException` back to the previous value after completion of `runTest`, or repeated failures will cause the exception handler for a previous test to be invoked, as they "stack". Instead of reseting it, however, do this once in the constructor because the test description is already provided by the function signature.

* Add a mechanism for testing test results

Author: jiahaog

packages/e2e/CHANGELOG.md

@@ -1,6 +1,10 @@
+## 0.6.2+1
+
+* Fix incorrect test results when one test passes then another fails
+
 ## 0.6.2
 
-* Fix `setSurfaceSize` for e2e tests.
+* Fix `setSurfaceSize` for e2e tests
 
 ## 0.6.1
 

packages/e2e/lib/e2e.dart

@@ -32,13 +32,26 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
         }
         await _channel.invokeMethod<void>(
           'allTestsFinished',
-          <String, dynamic>{'results': _results},
+          <String, dynamic>{'results': results},
         );
       } on MissingPluginException {
         print('Warning: E2E test plugin was not detected.');
       }
       if (!_allTestsPassed.isCompleted) _allTestsPassed.complete(true);
     });
+
+    // TODO(jackson): Report the results individually instead of all at once
+    // See https://github.com/flutter/flutter/issues/38985
+    final TestExceptionReporter oldTestExceptionReporter = reportTestException;
+    reportTestException =
+        (FlutterErrorDetails details, String testDescription) {
+      results[testDescription] = 'failed';
+      _failureMethodsDetails.add(Failure(testDescription, details.toString()));
+      if (!_allTestsPassed.isCompleted) {
+        _allTestsPassed.complete(false);
+      }
+      oldTestExceptionReporter(details, testDescription);
+    };
   }
 
   // TODO(dnfield): Remove the ignore once we bump the minimum Flutter version
@@ -100,7 +113,12 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
 
   static const MethodChannel _channel = MethodChannel('plugins.flutter.io/e2e');
 
-  static Map<String, String> _results = <String, String>{};
+  /// Test results that will be populated after the tests have completed.
+  ///
+  /// Keys are the test descriptions, and values are either `success` or
+  /// `failed`.
+  @visibleForTesting
+  Map<String, String> results = <String, String>{};
 
   /// The extra data for the reported result.
   ///
@@ -158,24 +176,12 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
     String description = '',
     Duration timeout,
   }) async {
-    // TODO(jackson): Report the results individually instead of all at once
-    // See https://github.com/flutter/flutter/issues/38985
-    final TestExceptionReporter oldTestExceptionReporter = reportTestException;
-    reportTestException =
-        (FlutterErrorDetails details, String testDescription) {
-      _results[description] = 'failed';
-      _failureMethodsDetails.add(Failure(testDescription, details.toString()));
-      if (!_allTestsPassed.isCompleted) {
-        _allTestsPassed.complete(false);
-      }
-      oldTestExceptionReporter(details, testDescription);
-    };
     await super.runTest(
       testBody,
       invariantTester,
       description: description,
       timeout: timeout,
     );
-    _results[description] ??= 'success';
+    results[description] ??= 'success';
   }
 }

packages/e2e/pubspec.yaml

@@ -1,6 +1,6 @@
 name: e2e
 description: Runs tests that use the flutter_test API as integration tests.
-version: 0.6.2
+version: 0.6.2+1
 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e
 
 environment:

packages/e2e/test/binding_fail_test.dart

@@ -0,0 +1,81 @@
+import 'dart:async';
+import 'dart:io';
+import 'dart:convert';
+
+import 'package:flutter_test/flutter_test.dart';
+
+// Assumes that the flutter command is in `$PATH`.
+const String _flutterBin = 'flutter';
+const String _e2eResultsPrefix = 'E2EWidgetsFlutterBinding test results:';
+
+void main() async {
+  group('E2E binding result', () {
+    test('when multiple tests pass', () async {
+      final Map<String, dynamic> results =
+          await _runTest('test/data/pass_test_script.dart');
+
+      expect(
+          results,
+          equals({
+            'passing test 1': 'success',
+            'passing test 2': 'success',
+          }));
+    });
+
+    test('when multiple tests fail', () async {
+      final Map<String, dynamic> results =
+          await _runTest('test/data/fail_test_script.dart');
+
+      expect(
+          results,
+          equals({
+            'failing test 1': 'failed',
+            'failing test 2': 'failed',
+          }));
+    });
+
+    test('when one test passes, then another fails', () async {
+      final Map<String, dynamic> results =
+          await _runTest('test/data/pass_then_fail_test_script.dart');
+
+      expect(
+          results,
+          equals({
+            'passing test': 'success',
+            'failing test': 'failed',
+          }));
+    });
+  });
+}
+
+/// Runs a test script and returns the [E2EWidgetsFlutterBinding.result].
+///
+/// [scriptPath] is relative to the package root.
+Future<Map<String, dynamic>> _runTest(String scriptPath) async {
+  final Process process =
+      await Process.start(_flutterBin, ['test', '--machine', scriptPath]);
+
+  // In the test [tearDownAll] block, the test results are encoded into JSON and
+  // are printed with the [_e2eResultsPrefix] prefix.
+  //
+  // See the following for the test event spec which we parse the printed lines
+  // out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md
+  final String testResults = (await process.stdout
+          .transform(utf8.decoder)
+          .expand((String text) => text.split('\n'))
+          .map((String line) {
+            try {
+              return jsonDecode(line);
+            } on FormatException {
+              // Only interested in test events which are JSON.
+            }
+          })
+          .where((dynamic testEvent) =>
+              testEvent != null && testEvent['type'] == 'print')
+          .map((dynamic printEvent) => printEvent['message'] as String)
+          .firstWhere(
+              (String message) => message.startsWith(_e2eResultsPrefix)))
+      .replaceAll(_e2eResultsPrefix, '');
+
+  return jsonDecode(testResults);
+}

packages/e2e/test/data/README.md

@@ -0,0 +1,4 @@
+Files in this directory are not invoked directly by the test command.
+
+They are used as inputs for the other test files outside of this directory, so
+that failures can be tested.

packages/e2e/test/data/fail_test_script.dart

@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized();
+
+  testWidgets('failing test 1', (WidgetTester tester) async {
+    expect(true, false);
+  });
+
+  testWidgets('failing test 2', (WidgetTester tester) async {
+    expect(true, false);
+  });
+
+  tearDownAll(() {
+    print(
+        'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+  });
+}

packages/e2e/test/data/pass_test_script.dart

@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized();
+
+  testWidgets('passing test 1', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  testWidgets('passing test 2', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  tearDownAll(() {
+    print(
+        'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+  });
+}

packages/e2e/test/data/pass_then_fail_test_script.dart

@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized();
+
+  testWidgets('passing test', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  testWidgets('failing test', (WidgetTester tester) async {
+    expect(true, false);
+  });
+
+  tearDownAll(() {
+    print(
+        'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+  });
+}