diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md index 2df897d9ab5f..bc844ce72086 100644 --- a/packages/e2e/CHANGELOG.md +++ b/packages/e2e/CHANGELOG.md @@ -1,6 +1,10 @@ +## 0.6.2+1 + +* Fix incorrect test results when one test passes then another fails + ## 0.6.2 -* Fix `setSurfaceSize` for e2e tests. +* Fix `setSurfaceSize` for e2e tests ## 0.6.1 diff --git a/packages/e2e/lib/e2e.dart b/packages/e2e/lib/e2e.dart index d773b9b7a43b..93ce1f2c6cac 100644 --- a/packages/e2e/lib/e2e.dart +++ b/packages/e2e/lib/e2e.dart @@ -32,13 +32,26 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { } await _channel.invokeMethod( 'allTestsFinished', - {'results': _results}, + {'results': results}, ); } on MissingPluginException { print('Warning: E2E test plugin was not detected.'); } if (!_allTestsPassed.isCompleted) _allTestsPassed.complete(true); }); + + // TODO(jackson): Report the results individually instead of all at once + // See https://github.com/flutter/flutter/issues/38985 + final TestExceptionReporter oldTestExceptionReporter = reportTestException; + reportTestException = + (FlutterErrorDetails details, String testDescription) { + results[testDescription] = 'failed'; + _failureMethodsDetails.add(Failure(testDescription, details.toString())); + if (!_allTestsPassed.isCompleted) { + _allTestsPassed.complete(false); + } + oldTestExceptionReporter(details, testDescription); + }; } // TODO(dnfield): Remove the ignore once we bump the minimum Flutter version @@ -100,7 +113,12 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { static const MethodChannel _channel = MethodChannel('plugins.flutter.io/e2e'); - static Map _results = {}; + /// Test results that will be populated after the tests have completed. + /// + /// Keys are the test descriptions, and values are either `success` or + /// `failed`. + @visibleForTesting + Map results = {}; /// The extra data for the reported result. /// @@ -158,24 +176,12 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding { String description = '', Duration timeout, }) async { - // TODO(jackson): Report the results individually instead of all at once - // See https://github.com/flutter/flutter/issues/38985 - final TestExceptionReporter oldTestExceptionReporter = reportTestException; - reportTestException = - (FlutterErrorDetails details, String testDescription) { - _results[description] = 'failed'; - _failureMethodsDetails.add(Failure(testDescription, details.toString())); - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(false); - } - oldTestExceptionReporter(details, testDescription); - }; await super.runTest( testBody, invariantTester, description: description, timeout: timeout, ); - _results[description] ??= 'success'; + results[description] ??= 'success'; } } diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index 956a23f2b930..e3f39c05334b 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -1,6 +1,6 @@ name: e2e description: Runs tests that use the flutter_test API as integration tests. -version: 0.6.2 +version: 0.6.2+1 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e environment: diff --git a/packages/e2e/test/binding_fail_test.dart b/packages/e2e/test/binding_fail_test.dart new file mode 100644 index 000000000000..0b00e1177e55 --- /dev/null +++ b/packages/e2e/test/binding_fail_test.dart @@ -0,0 +1,81 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; + +// Assumes that the flutter command is in `$PATH`. +const String _flutterBin = 'flutter'; +const String _e2eResultsPrefix = 'E2EWidgetsFlutterBinding test results:'; + +void main() async { + group('E2E binding result', () { + test('when multiple tests pass', () async { + final Map results = + await _runTest('test/data/pass_test_script.dart'); + + expect( + results, + equals({ + 'passing test 1': 'success', + 'passing test 2': 'success', + })); + }); + + test('when multiple tests fail', () async { + final Map results = + await _runTest('test/data/fail_test_script.dart'); + + expect( + results, + equals({ + 'failing test 1': 'failed', + 'failing test 2': 'failed', + })); + }); + + test('when one test passes, then another fails', () async { + final Map results = + await _runTest('test/data/pass_then_fail_test_script.dart'); + + expect( + results, + equals({ + 'passing test': 'success', + 'failing test': 'failed', + })); + }); + }); +} + +/// Runs a test script and returns the [E2EWidgetsFlutterBinding.result]. +/// +/// [scriptPath] is relative to the package root. +Future> _runTest(String scriptPath) async { + final Process process = + await Process.start(_flutterBin, ['test', '--machine', scriptPath]); + + // In the test [tearDownAll] block, the test results are encoded into JSON and + // are printed with the [_e2eResultsPrefix] prefix. + // + // See the following for the test event spec which we parse the printed lines + // out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md + final String testResults = (await process.stdout + .transform(utf8.decoder) + .expand((String text) => text.split('\n')) + .map((String line) { + try { + return jsonDecode(line); + } on FormatException { + // Only interested in test events which are JSON. + } + }) + .where((dynamic testEvent) => + testEvent != null && testEvent['type'] == 'print') + .map((dynamic printEvent) => printEvent['message'] as String) + .firstWhere( + (String message) => message.startsWith(_e2eResultsPrefix))) + .replaceAll(_e2eResultsPrefix, ''); + + return jsonDecode(testResults); +} diff --git a/packages/e2e/test/data/README.md b/packages/e2e/test/data/README.md new file mode 100644 index 000000000000..e52aca112ce4 --- /dev/null +++ b/packages/e2e/test/data/README.md @@ -0,0 +1,4 @@ +Files in this directory are not invoked directly by the test command. + +They are used as inputs for the other test files outside of this directory, so +that failures can be tested. \ No newline at end of file diff --git a/packages/e2e/test/data/fail_test_script.dart b/packages/e2e/test/data/fail_test_script.dart new file mode 100644 index 000000000000..cbca5900fe29 --- /dev/null +++ b/packages/e2e/test/data/fail_test_script.dart @@ -0,0 +1,22 @@ +import 'dart:convert'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() async { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('failing test 1', (WidgetTester tester) async { + expect(true, false); + }); + + testWidgets('failing test 2', (WidgetTester tester) async { + expect(true, false); + }); + + tearDownAll(() { + print( + 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + }); +} diff --git a/packages/e2e/test/data/pass_test_script.dart b/packages/e2e/test/data/pass_test_script.dart new file mode 100644 index 000000000000..194f71cdfe9b --- /dev/null +++ b/packages/e2e/test/data/pass_test_script.dart @@ -0,0 +1,22 @@ +import 'dart:convert'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() async { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('passing test 1', (WidgetTester tester) async { + expect(true, true); + }); + + testWidgets('passing test 2', (WidgetTester tester) async { + expect(true, true); + }); + + tearDownAll(() { + print( + 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + }); +} diff --git a/packages/e2e/test/data/pass_then_fail_test_script.dart b/packages/e2e/test/data/pass_then_fail_test_script.dart new file mode 100644 index 000000000000..ffb7cac5cd5d --- /dev/null +++ b/packages/e2e/test/data/pass_then_fail_test_script.dart @@ -0,0 +1,22 @@ +import 'dart:convert'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() async { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('passing test', (WidgetTester tester) async { + expect(true, true); + }); + + testWidgets('failing test', (WidgetTester tester) async { + expect(true, false); + }); + + tearDownAll(() { + print( + 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + }); +}