diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index f493135bc..3f2c7b297 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -1,8 +1,10 @@ -## 1.11.2-dev +## 1.12.0-dev * Bump minimum SDK to `2.4.0` for safer usage of for-loop elements. * Deprecate `PhantomJS` and provide warning when used. Support for `PhantomJS` will be removed in version `2.0.0`. +* Support coverage collection for the Chrome platform. See `README.md` for usage + details. ## 1.11.1 diff --git a/pkgs/test/README.md b/pkgs/test/README.md index a661323d8..6e174095c 100644 --- a/pkgs/test/README.md +++ b/pkgs/test/README.md @@ -186,7 +186,7 @@ specifying it at all, meaning the test order will remain as-is. ### Collecting Code Coverage To collect code coverage, you can run tests with the `--coverage ` -argument. The directory specified can be an absolute or relative path. +argument. The directory specified can be an absolute or relative path. If a directory does not exist at the path specified, a directory will be created. If a directory does exist, files may be overwritten with the latest coverage data, if they conflict. @@ -196,7 +196,8 @@ and the resulting coverage files will be outputted in the directory specified. The files can then be formatted using the `package:coverage` `format_coverage` executable. -Coverage gathering is currently only implemented for tests run in the Dart VM. +Coverage gathering is currently only implemented for tests run on the Dart VM or +Chrome. ### Restricting Tests to Certain Platforms @@ -765,7 +766,7 @@ A configuration file can do much more than just set global defaults. See Tests can be debugged interactively using platforms' built-in development tools. Tests running on browsers can use those browsers' development consoles to inspect -the document, set breakpoints, and step through code. Those running on the Dart +the document, set breakpoints, and step through code. Those running on the Dart VM use [the Dart Observatory][observatory]'s . [observatory]: https://dart-lang.github.io/observatory/ diff --git a/pkgs/test/lib/src/runner/browser/browser_manager.dart b/pkgs/test/lib/src/runner/browser/browser_manager.dart index ce8b74d71..2339908ad 100644 --- a/pkgs/test/lib/src/runner/browser/browser_manager.dart +++ b/pkgs/test/lib/src/runner/browser/browser_manager.dart @@ -8,17 +8,15 @@ import 'dart:convert'; import 'package:async/async.dart'; import 'package:pool/pool.dart'; import 'package:stream_channel/stream_channel.dart'; -import 'package:web_socket_channel/web_socket_channel.dart'; - import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:test_api/src/util/stack_trace_mapper.dart'; // ignore: implementation_imports -import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports -import 'package:test_core/src/runner/environment.dart'; // ignore: implementation_imports -import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports - import 'package:test_core/src/runner/application_exception.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/environment.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports +import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports import 'package:test_core/src/util/io.dart'; // ignore: implementation_imports +import 'package:web_socket_channel/web_socket_channel.dart'; import '../executable_settings.dart'; import 'browser.dart'; @@ -238,8 +236,17 @@ class BrowserManager { }); try { - controller = deserializeSuite(path, currentPlatform(_runtime), - suiteConfig, await _environment, suiteChannel, message); + controller = deserializeSuite( + path, + currentPlatform(_runtime), + suiteConfig, + await _environment, + suiteChannel, + message, gatherCoverage: () async { + var browser = _browser; + if (browser is Chrome) return browser.gatherCoverage(); + return {}; + }); controller.channel('test.browser.mapper').sink.add(mapper?.serialize()); diff --git a/pkgs/test/lib/src/runner/browser/chrome.dart b/pkgs/test/lib/src/runner/browser/chrome.dart index de76d3545..624e2b5d7 100644 --- a/pkgs/test/lib/src/runner/browser/chrome.dart +++ b/pkgs/test/lib/src/runner/browser/chrome.dart @@ -3,17 +3,21 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'dart:io'; +import 'package:coverage/coverage.dart'; +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as p; import 'package:pedantic/pedantic.dart'; import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:test_core/src/util/io.dart'; // ignore: implementation_imports +import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart'; import '../executable_settings.dart'; import 'browser.dart'; import 'default_settings.dart'; -// TODO(nweiz): move this into its own package? /// A class for running an instance of Chrome. /// /// Most of the communication with the browser is expected to happen via HTTP, @@ -28,11 +32,16 @@ class Chrome extends Browser { @override final Future remoteDebuggerUrl; + final Future _tabConnection; + final Map _idToUrl; + /// Starts a new instance of Chrome open to the given [url], which may be a /// [Uri] or a [String]. factory Chrome(Uri url, {ExecutableSettings settings, bool debug = false}) { settings ??= defaultSettings[Runtime.chrome]; var remoteDebuggerCompleter = Completer.sync(); + var connectionCompleter = Completer(); + var idToUrl = {}; return Chrome._(() async { var tryPort = ([int port]) async { var dir = createTempDir(); @@ -73,6 +82,8 @@ class Chrome extends Browser { if (port != null) { remoteDebuggerCompleter.complete( getRemoteDebuggerUrl(Uri.parse('http://localhost:$port'))); + + connectionCompleter.complete(_connect(process, port, idToUrl)); } else { remoteDebuggerCompleter.complete(null); } @@ -85,9 +96,83 @@ class Chrome extends Browser { if (!debug) return tryPort(); return getUnusedPort(tryPort); - }, remoteDebuggerCompleter.future); + }, remoteDebuggerCompleter.future, connectionCompleter.future, idToUrl); } - Chrome._(Future Function() startBrowser, this.remoteDebuggerUrl) + /// Returns a Dart based hit-map containing coverage report, suitable for use + /// with `package:coverage`. + Future> gatherCoverage() async { + var tabConnection = await _tabConnection; + var response = await tabConnection.debugger.connection + .sendCommand('Profiler.takePreciseCoverage', {}); + var result = response.result['result']; + var coverage = await parseChromeCoverage( + (result as List).cast(), + _sourceProvider, + _sourceMapProvider, + _sourceUriProvider, + ); + return coverage; + } + + Chrome._(Future Function() startBrowser, this.remoteDebuggerUrl, + this._tabConnection, this._idToUrl) : super(startBrowser); + + Future _sourceUriProvider(String sourceUrl, String scriptId) async { + var script = _idToUrl[scriptId]; + if (script == null) return null; + var uri = Uri.parse(script); + var path = p.join( + p.joinAll(uri.pathSegments.sublist(1, uri.pathSegments.length - 1)), + sourceUrl); + return path.contains('/packages/') + ? Uri(scheme: 'package', path: path.split('/packages/').last) + : Uri.file(p.absolute(path)); + } + + Future _sourceMapProvider(String scriptId) async { + var script = _idToUrl[scriptId]; + if (script == null) return null; + var mapResponse = await http.get('$script.map'); + if (mapResponse.statusCode != HttpStatus.ok) return null; + return mapResponse.body; + } + + Future _sourceProvider(String scriptId) async { + var script = _idToUrl[scriptId]; + if (script == null) return null; + var scriptResponse = await http.get(script); + if (scriptResponse.statusCode != HttpStatus.ok) return null; + return scriptResponse.body; + } +} + +Future _connect( + Process process, int port, Map idToUrl) async { + // Wait for Chrome to be in a ready state. + await process.stderr + .transform(utf8.decoder) + .transform(LineSplitter()) + .firstWhere((line) => line.startsWith('DevTools listening')); + + var chromeConnection = ChromeConnection('localhost', port); + var tab = (await chromeConnection.getTabs()).first; + var tabConnection = await tab.connect(); + + // Enable debugging. + await tabConnection.debugger.enable(); + + // Coverage reports are in terms of scriptIds so keep note of URLs. + tabConnection.debugger.onScriptParsed.listen((data) { + var script = data.script; + if (script.url.isNotEmpty) idToUrl[script.scriptId] = script.url; + }); + + // Enable coverage collection. + await tabConnection.debugger.connection.sendCommand('Profiler.enable', {}); + await tabConnection.debugger.connection.sendCommand( + 'Profiler.startPreciseCoverage', {'detailed': true, 'callCount': false}); + + return tabConnection; } diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml index feb3f4b2c..789b4a4c4 100644 --- a/pkgs/test/pubspec.yaml +++ b/pkgs/test/pubspec.yaml @@ -1,5 +1,5 @@ name: test -version: 1.11.2-dev +version: 1.12.0-dev description: A full featured library for writing and running Dart tests. homepage: https://github.com/dart-lang/test/blob/master/pkgs/test @@ -10,6 +10,7 @@ dependencies: analyzer: ">=0.36.0 <0.40.0" async: ^2.0.0 boolean_selector: ">=1.0.0 <3.0.0" + coverage: ^0.13.4 http_multi_server: ^2.0.0 io: ^0.3.0 js: ^0.6.0 @@ -28,6 +29,7 @@ dependencies: stream_channel: ">=1.7.0 <3.0.0" typed_data: ^1.0.0 web_socket_channel: ^1.0.0 + webkit_inspection_protocol: ^0.5.0 yaml: ^2.0.0 # Use an exact version until the test_api and test_core package are stable. test_api: 0.2.14 diff --git a/pkgs/test/test/runner/coverage_test.dart b/pkgs/test/test/runner/coverage_test.dart index edd1c0380..36177e4fe 100644 --- a/pkgs/test/test/runner/coverage_test.dart +++ b/pkgs/test/test/runner/coverage_test.dart @@ -11,12 +11,26 @@ import 'package:test_descriptor/test_descriptor.dart' as d; import 'package:path/path.dart' as p; import 'package:test/test.dart'; +import 'package:test_process/test_process.dart'; import '../io.dart'; void main() { group('with the --coverage flag,', () { - test('gathers coverage for VM tests', () async { + Directory coverageDirectory; + + Future _validateCoverage( + TestProcess test, String coveragePath) async { + expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); + await test.shouldExit(0); + + final coverageFile = File(p.join(coverageDirectory.path, coveragePath)); + final coverage = await coverageFile.readAsString(); + final jsonCoverage = json.decode(coverage); + expect(jsonCoverage['coverage'], isNotEmpty); + } + + setUp(() async { await d.file('test.dart', ''' import 'package:test/test.dart'; @@ -27,24 +41,24 @@ void main() { } ''').create(); - final coverageDirectory = - Directory(p.join(Directory.current.path, 'test_coverage')); - expect(await coverageDirectory.exists(), isFalse, - reason: - 'Coverage directory exists, cannot safely run coverage tests. Delete the ${coverageDirectory.path} directory to fix.'); + coverageDirectory = + await Directory.systemTemp.createTemp('test_coverage'); + }); + + tearDown(() async { + await coverageDirectory.delete(recursive: true); + }); + test('gathers coverage for VM tests', () async { var test = await runTest(['--coverage', coverageDirectory.path, 'test.dart']); - expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); - await test.shouldExit(0); - - final coverageFile = - File(p.join(coverageDirectory.path, 'test.dart.vm.json')); - final coverage = await coverageFile.readAsString(); - final jsonCoverage = json.decode(coverage); - expect(jsonCoverage['coverage'], isNotEmpty); + await _validateCoverage(test, 'test.dart.vm.json'); + }); - await coverageDirectory.delete(recursive: true); + test('gathers coverage for Chrome tests', () async { + var test = await runTest( + ['--coverage', coverageDirectory.path, 'test.dart', '-p', 'chrome']); + await _validateCoverage(test, 'test.dart.chrome.json'); }); }); }