From 44a5471d6739e48dfbb4b93aabf3e05044430e74 Mon Sep 17 00:00:00 2001 From: Jia Hao Goh Date: Thu, 23 Jul 2020 13:07:22 +0800 Subject: [PATCH 1/2] [e2e] Fix incorrect test results when one test passes then another fails For example, the following test will result in an error reported for the first test case. ``` void main() { testWidgets('a test that passes', (tester) async { expect(true, true); }); testWidgets('a test that fails', (tester) async { expect(true, false); }); } ``` We need to reset `reportTestException` back to the previous value after completion of `runTest`, or repeated failures will cause the exception handler for a previous test to be invoked, as they "stack". Instead of reseting it, however, do this once in the constructor because the test description is already provided by the function signature. --- packages/e2e/CHANGELOG.md | 4 ++++ packages/e2e/lib/e2e.dart | 25 +++++++++++++------------ packages/e2e/pubspec.yaml | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md index 255d7ce08c2d..8adbbb67b6a6 100644 --- a/packages/e2e/CHANGELOG.md +++ b/packages/e2e/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.6.1+1 + +* Fix incorrect test results when one test passes then another fails + ## 0.6.1 * Added `data` in the reported json. diff --git a/packages/e2e/lib/e2e.dart b/packages/e2e/lib/e2e.dart index a9bf83041e40..b3b2ba8df4fc 100644 --- a/packages/e2e/lib/e2e.dart +++ b/packages/e2e/lib/e2e.dart @@ -39,6 +39,19 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { } if (!_allTestsPassed.isCompleted) _allTestsPassed.complete(true); }); + + // TODO(jackson): Report the results individually instead of all at once + // See https://github.com/flutter/flutter/issues/38985 + final TestExceptionReporter oldTestExceptionReporter = reportTestException; + reportTestException = + (FlutterErrorDetails details, String testDescription) { + _results[testDescription] = 'failed'; + _failureMethodsDetails.add(Failure(testDescription, details.toString())); + if (!_allTestsPassed.isCompleted) { + _allTestsPassed.complete(false); + } + oldTestExceptionReporter(details, testDescription); + }; } // TODO(dnfield): Remove the ignore once we bump the minimum Flutter version @@ -134,18 +147,6 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { String description = '', Duration timeout, }) async { - // TODO(jackson): Report the results individually instead of all at once - // See https://github.com/flutter/flutter/issues/38985 - final TestExceptionReporter oldTestExceptionReporter = reportTestException; - reportTestException = - (FlutterErrorDetails details, String testDescription) { - _results[description] = 'failed'; - _failureMethodsDetails.add(Failure(testDescription, details.toString())); - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(false); - } - oldTestExceptionReporter(details, testDescription); - }; await super.runTest( testBody, invariantTester, diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index 70e57d0cb4f7..d52662fdbd6c 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -1,6 +1,6 @@ name: e2e description: Runs tests that use the flutter_test API as integration tests. -version: 0.6.1 +version: 0.6.1+1 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e environment: From 865e40c12c06cd7e1b9224be0d7ca94b3192f368 Mon Sep 17 00:00:00 2001 From: Jia Hao Goh Date: Fri, 24 Jul 2020 22:07:44 +0800 Subject: [PATCH 2/2] Add a mechanism for testing test results --- packages/e2e/lib/e2e.dart | 13 ++- packages/e2e/test/binding_fail_test.dart | 81 +++++++++++++++++++ packages/e2e/test/data/README.md | 4 + packages/e2e/test/data/fail_test_script.dart | 22 +++++ packages/e2e/test/data/pass_test_script.dart | 22 +++++ .../test/data/pass_then_fail_test_script.dart | 22 +++++ 6 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 packages/e2e/test/binding_fail_test.dart create mode 100644 packages/e2e/test/data/README.md create mode 100644 packages/e2e/test/data/fail_test_script.dart create mode 100644 packages/e2e/test/data/pass_test_script.dart create mode 100644 packages/e2e/test/data/pass_then_fail_test_script.dart diff --git a/packages/e2e/lib/e2e.dart b/packages/e2e/lib/e2e.dart index b3b2ba8df4fc..ee1f2e518298 100644 --- a/packages/e2e/lib/e2e.dart +++ b/packages/e2e/lib/e2e.dart @@ -32,7 +32,7 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { } await _channel.invokeMethod( 'allTestsFinished', - {'results': _results}, + {'results': results}, ); } on MissingPluginException { print('Warning: E2E test plugin was not detected.'); @@ -45,7 +45,7 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { final TestExceptionReporter oldTestExceptionReporter = reportTestException; reportTestException = (FlutterErrorDetails details, String testDescription) { - _results[testDescription] = 'failed'; + results[testDescription] = 'failed'; _failureMethodsDetails.add(Failure(testDescription, details.toString())); if (!_allTestsPassed.isCompleted) { _allTestsPassed.complete(false); @@ -89,7 +89,12 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { static const MethodChannel _channel = MethodChannel('plugins.flutter.io/e2e'); - static Map _results = {}; + /// Test results that will be populated after the tests have completed. + /// + /// Keys are the test descriptions, and values are either `success` or + /// `failed`. + @visibleForTesting + Map results = {}; /// The extra data for the reported result. /// @@ -153,6 +158,6 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { description: description, timeout: timeout, ); - _results[description] ??= 'success'; + results[description] ??= 'success'; } } diff --git a/packages/e2e/test/binding_fail_test.dart b/packages/e2e/test/binding_fail_test.dart new file mode 100644 index 000000000000..0b00e1177e55 --- /dev/null +++ b/packages/e2e/test/binding_fail_test.dart @@ -0,0 +1,81 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; + +// Assumes that the flutter command is in `$PATH`. +const String _flutterBin = 'flutter'; +const String _e2eResultsPrefix = 'E2EWidgetsFlutterBinding test results:'; + +void main() async { + group('E2E binding result', () { + test('when multiple tests pass', () async { + final Map results = + await _runTest('test/data/pass_test_script.dart'); + + expect( + results, + equals({ + 'passing test 1': 'success', + 'passing test 2': 'success', + })); + }); + + test('when multiple tests fail', () async { + final Map results = + await _runTest('test/data/fail_test_script.dart'); + + expect( + results, + equals({ + 'failing test 1': 'failed', + 'failing test 2': 'failed', + })); + }); + + test('when one test passes, then another fails', () async { + final Map results = + await _runTest('test/data/pass_then_fail_test_script.dart'); + + expect( + results, + equals({ + 'passing test': 'success', + 'failing test': 'failed', + })); + }); + }); +} + +/// Runs a test script and returns the [E2EWidgetsFlutterBinding.result]. +/// +/// [scriptPath] is relative to the package root. +Future> _runTest(String scriptPath) async { + final Process process = + await Process.start(_flutterBin, ['test', '--machine', scriptPath]); + + // In the test [tearDownAll] block, the test results are encoded into JSON and + // are printed with the [_e2eResultsPrefix] prefix. + // + // See the following for the test event spec which we parse the printed lines + // out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md + final String testResults = (await process.stdout + .transform(utf8.decoder) + .expand((String text) => text.split('\n')) + .map((String line) { + try { + return jsonDecode(line); + } on FormatException { + // Only interested in test events which are JSON. + } + }) + .where((dynamic testEvent) => + testEvent != null && testEvent['type'] == 'print') + .map((dynamic printEvent) => printEvent['message'] as String) + .firstWhere( + (String message) => message.startsWith(_e2eResultsPrefix))) + .replaceAll(_e2eResultsPrefix, ''); + + return jsonDecode(testResults); +} diff --git a/packages/e2e/test/data/README.md b/packages/e2e/test/data/README.md new file mode 100644 index 000000000000..e52aca112ce4 --- /dev/null +++ b/packages/e2e/test/data/README.md @@ -0,0 +1,4 @@ +Files in this directory are not invoked directly by the test command. + +They are used as inputs for the other test files outside of this directory, so +that failures can be tested. \ No newline at end of file diff --git a/packages/e2e/test/data/fail_test_script.dart b/packages/e2e/test/data/fail_test_script.dart new file mode 100644 index 000000000000..cbca5900fe29 --- /dev/null +++ b/packages/e2e/test/data/fail_test_script.dart @@ -0,0 +1,22 @@ +import 'dart:convert'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() async { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('failing test 1', (WidgetTester tester) async { + expect(true, false); + }); + + testWidgets('failing test 2', (WidgetTester tester) async { + expect(true, false); + }); + + tearDownAll(() { + print( + 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + }); +} diff --git a/packages/e2e/test/data/pass_test_script.dart b/packages/e2e/test/data/pass_test_script.dart new file mode 100644 index 000000000000..194f71cdfe9b --- /dev/null +++ b/packages/e2e/test/data/pass_test_script.dart @@ -0,0 +1,22 @@ +import 'dart:convert'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() async { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('passing test 1', (WidgetTester tester) async { + expect(true, true); + }); + + testWidgets('passing test 2', (WidgetTester tester) async { + expect(true, true); + }); + + tearDownAll(() { + print( + 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + }); +} diff --git a/packages/e2e/test/data/pass_then_fail_test_script.dart b/packages/e2e/test/data/pass_then_fail_test_script.dart new file mode 100644 index 000000000000..ffb7cac5cd5d --- /dev/null +++ b/packages/e2e/test/data/pass_then_fail_test_script.dart @@ -0,0 +1,22 @@ +import 'dart:convert'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() async { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('passing test', (WidgetTester tester) async { + expect(true, true); + }); + + testWidgets('failing test', (WidgetTester tester) async { + expect(true, false); + }); + + tearDownAll(() { + print( + 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + }); +}