Clean up testing/benchmark (#26313)

Author: zanderso

DEPS

@@ -398,9 +398,21 @@ deps = {
   'src/third_party/pkg/archive':
   Var('github_git') + '/brendan-duncan/archive.git' + '@' + '3.1.2',
 
+  'src/third_party/pkg/equatable':
+  Var('github_git') + '/felangel/equatable.git' + '@' + '0ba67c72db8bed75877fc1caafa74112ee0bd921', # 2.0.2
+
   'src/third_party/pkg/file':
   Var('github_git') + '/google/file.dart.git' + '@' + '427bb20ccc852425d67f2880da2a9b4707c266b4', # 6.1.0
 
+  'src/third_party/pkg/flutter_packages':
+  Var('github_git') + '/flutter/packages.git' + '@' + '5617d089f26dd52da3bf05c9fa4620ef11a7419b', # various
+
+  'src/third_party/pkg/gcloud':
+  Var('github_git') + '/dart-lang/gcloud.git' + '@' + '92a33a9d95ea94a4354b052a28b98088d660e0e7', # 0.8.0-dev
+
+  'src/third_party/pkg/googleapis':
+  Var('github_git') + '/google/googleapis.dart.git' + '@' + '07f01b7aa6985e4cafd0fd4b98724841bc9e85a1', # various
+
   'src/third_party/pkg/platform':
   Var('github_git') + '/google/platform.dart.git' + '@' + 'f63fd0bc3021354a0687dc935962c9acc003f47e', # 3.0.1
 

ci/analyze.sh

@@ -82,6 +82,12 @@ analyze \
   --options "$FLUTTER_DIR/analysis_options.yaml" \
   "$FLUTTER_DIR/testing/litetest"
 
+echo "Analyzing testing/benchmark"
+analyze \
+  --packages="$FLUTTER_DIR/testing/benchmark/.dart_tool/package_config.json" \
+  --options "$FLUTTER_DIR/analysis_options.yaml" \
+  "$FLUTTER_DIR/testing/benchmark"
+
 echo "Analyzing testing/smoke_test_failure"
 analyze \
   --packages="$FLUTTER_DIR/testing/smoke_test_failure/.dart_tool/package_config.json" \

ci/licenses_golden/tool_signature

@@ -1,2 +1,2 @@
-Signature: a62660b42e470ade57e05b076bff1222
+Signature: 8e5901632fdea212b4f49528f19c5f74
 

testing/benchmark/bin/parse_and_send.dart

@@ -3,38 +3,63 @@
 // found in the LICENSE file.
 
 // @dart = 2.6
+
 import 'dart:convert';
 import 'dart:io';
 
-import 'package:git/git.dart';
 import 'package:metrics_center/metrics_center.dart';
+import 'package:path/path.dart' as p;
 
-Future<String> _getGitRevision() async {
-  final GitDir gitDir = await GitDir.fromExisting('../../');
+Future<ProcessResult> runGit(
+  List<String> args, {
+  String processWorkingDir,
+}) async {
+  return Process.run(
+    'git',
+    args,
+    workingDirectory: processWorkingDir,
+    runInShell: true,
+  );
+}
+
+Future<List<String>> getGitLog() async {
+  final String gitRoot = p.absolute('../..');
   // Somehow gitDir.currentBranch() doesn't work in Cirrus with "fatal: 'HEAD' -
   // not a valid ref". Therefore, we use "git log" to get the revision manually.
-  final ProcessResult logResult =
-      await gitDir.runCommand(<String>['log', '--pretty=format:%H', '-n', '1']);
+  final ProcessResult logResult = await runGit(
+    <String>['log', '--pretty=format:%H %ct', '-n', '1'],
+    processWorkingDir: gitRoot,
+  );
   if (logResult.exitCode != 0) {
     throw 'Unexpected exit code ${logResult.exitCode}';
   }
-  return logResult.stdout.toString();
+  return logResult.stdout.toString().trim().split(' ');
 }
 
-Future<List<FlutterEngineMetricPoint>> _parse(String jsonFileName) async {
-  final String gitRevision = await _getGitRevision();
-  final List<MetricPoint> rawPoints =
-      await GoogleBenchmarkParser.parse(jsonFileName);
+class PointsAndDate {
+  PointsAndDate(this.points, this.date);
+
+  final List<FlutterEngineMetricPoint> points;
+  final String date;
+}
+
+Future<PointsAndDate> parse(String jsonFileName) async {
+  final List<String> gitLog = await getGitLog();
+  final String gitRevision = gitLog[0];
+  final String gitCommitDate = gitLog[1];
+  final List<MetricPoint> rawPoints = await GoogleBenchmarkParser.parse(
+    jsonFileName,
+  );
   final List<FlutterEngineMetricPoint> points = <FlutterEngineMetricPoint>[];
-  for (MetricPoint rawPoint in rawPoints) {
+  for (final MetricPoint rawPoint in rawPoints) {
     points.add(FlutterEngineMetricPoint(
       rawPoint.tags[kNameKey],
       rawPoint.value,
       gitRevision,
       moreTags: rawPoint.tags,
     ));
   }
-  return points;
+  return PointsAndDate(points, gitCommitDate);
 }
 
 Future<FlutterDestination> connectFlutterDestination() async {
@@ -49,7 +74,7 @@ Future<FlutterDestination> connectFlutterDestination() async {
       isTesting: isTesting,
     );
   }
-  return await FlutterDestination.makeFromCredentialsJson(
+  return FlutterDestination.makeFromCredentialsJson(
     jsonDecode(Platform.environment['BENCHMARK_GCP_CREDENTIALS'])
         as Map<String, dynamic>,
     isTesting: isTesting,
@@ -60,13 +85,19 @@ Future<void> main(List<String> args) async {
   if (args.length != 1) {
     throw 'Must have one argument: <benchmark_json_file>';
   }
-  final List<FlutterEngineMetricPoint> points = await _parse(args[0]);
+  final PointsAndDate pointsAndDate = await parse(args[0]);
 
   // The data will be sent to the Datastore of the GCP project specified through
   // environment variable BENCHMARK_GCP_CREDENTIALS, or TOKEN_PATH/GCP_PROJECT.
   // The engine Cirrus job has currently configured the GCP project to
   // flutter-cirrus for test. We'll eventually migrate to flutter-infra project
   // once the test is done.
   final FlutterDestination destination = await connectFlutterDestination();
-  await destination.update(points);
+  await destination.update(
+    pointsAndDate.points,
+    DateTime.fromMillisecondsSinceEpoch(
+      int.parse(pointsAndDate.date) * 1000,
+      isUtc: true,
+    ),
+  );
 }

testing/benchmark/pubspec.yaml

@@ -1,13 +1,72 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 name: flutter_engine_benchmark
+publish_to: none
+environment:
+  sdk: ">=2.2.2 <3.0.0"
 
-dependencies:
-  git: any
+# Do not add any dependencies that require more than what is provided in
+# //third_party/pkg, //third_party/dart/pkg, or
+# //third_party/dart/third_party/pkg. In particular, package:test is not usable
+# here.
 
-  metrics_center: 0.0.9
+# If you do add packages here, make sure you can run `pub get --offline`, and
+# check the .packages and .package_config to make sure all the paths are
+# relative to this directory into //third_party/dart
+
+dependencies:
+  metrics_center: any
+  path: any
 
 dev_dependencies:
-  test: any
-  pedantic: ^1.8.0
+  litetest: any
 
-environment:
-  sdk: ">=2.2.2 <3.0.0"
+dependency_overrides:
+  _discoveryapis_commons:
+    path: ../../../third_party/pkg/googleapis/discoveryapis_commons
+  async_helper:
+    path: ../../../third_party/dart/pkg/async_helper
+  async:
+    path: ../../../third_party/dart/third_party/pkg/async
+  charcode:
+    path: ../../../third_party/dart/third_party/pkg/charcode
+  collection:
+    path: ../../../third_party/dart/third_party/pkg/collection
+  convert:
+    path: ../../../third_party/dart/third_party/pkg/convert
+  crypto :
+    path: ../../../third_party/dart/third_party/pkg/crypto
+  equatable:
+    path: ../../../third_party/pkg/equatable
+  expect:
+    path: ../../../third_party/dart/pkg/expect
+  gcloud:
+    path: ../../../third_party/pkg/gcloud
+  googleapis:
+    path: ../../../third_party/pkg/googleapis/generated/googleapis
+  googleapis_auth:
+    path: ../../../third_party/pkg/googleapis/googleapis_auth
+  http:
+    path: ../../../third_party/dart/third_party/pkg/http
+  http_parser:
+    path: ../../../third_party/dart/third_party/pkg/http_parser
+  litetest:
+    path: ../litetest
+  meta:
+    path: ../../../third_party/dart/pkg/meta
+  metrics_center:
+    path: ../../../third_party/pkg/flutter_packages/packages/metrics_center
+  path:
+    path: ../../../third_party/dart/third_party/pkg/path
+  pedantic:
+    path: ../../../third_party/dart/third_party/pkg/pedantic
+  source_span:
+    path: ../../../third_party/dart/third_party/pkg/source_span
+  string_scanner:
+    path: ../../../third_party/dart/third_party/pkg/string_scanner
+  term_glyph:
+    path: ../../../third_party/dart/third_party/pkg/term_glyph
+  typed_data:
+    path: ../../../third_party/dart/third_party/pkg/typed_data

testing/benchmark/test/parse_and_send_test.dart

@@ -3,50 +3,52 @@
 // found in the LICENSE file.
 
 // @dart = 2.6
-import 'dart:convert';
+
 import 'dart:io';
 
-import 'package:gcloud/src/datastore_impl.dart';
-import 'package:googleapis_auth/auth_io.dart';
-import 'package:path/path.dart' as path;
-import 'package:test/test.dart';
+import 'package:litetest/litetest.dart';
+import 'package:metrics_center/metrics_center.dart';
+import 'package:path/path.dart' as p;
+
+import '../bin/parse_and_send.dart' as pas;
 
 void main() {
-  // In order to run these tests, one should download a service account
-  // credentials json from a test GCP project, and put that json as
-  // `secret/test_gcp_credentials.json`. There's a `flutter-test` project for
-  // Flutter team members.
-  test('parse_and_send with example json does not crash.', () async {
-    final String testCred =
-        File('secret/test_gcp_credentials.json').readAsStringSync();
-    Process.runSync('dart', <String>[
-      'bin/parse_and_send.dart',
-      'example/txt_benchmarks.json',
-    ], environment: <String, String>{
-      'BENCHMARK_GCP_CREDENTIALS': testCred,
-      'IS_TESTING': 'true',
-    });
+  test('runGit() succeeds', () async {
+    // Check that git --version runs successfully
+    final ProcessResult result = await pas.runGit(<String>['--version']);
+    expect(result.exitCode, equals(0));
   });
 
-  test('parse_and_send succeeds with access token.', () async {
-    final dynamic testCred =
-        jsonDecode(File('secret/test_gcp_credentials.json').readAsStringSync())
-            as Map<String, dynamic>;
-    final AutoRefreshingAuthClient client = await clientViaServiceAccount(
-      ServiceAccountCredentials.fromJson(testCred),
-      DatastoreImpl.SCOPES,
+  test('getGitLog() succeeds', () async {
+    final List<String> gitLog = await pas.getGitLog();
+
+    // Check that gitLog[0] is a hash
+    final RegExp sha1re = RegExp(r'[a-f0-9]{40}');
+    expect(sha1re.hasMatch(gitLog[0]), true);
+
+    // Check that gitLog[1] is an int
+    final int secondsSinceEpoch = int.tryParse(gitLog[1]);
+    expect(secondsSinceEpoch, isNotNull);
+
+    // Check that gitLog[1] is a sensible Unix Epoch
+    final int millisecondsSinceEpoch = secondsSinceEpoch * 1000;
+    final DateTime commitDate = DateTime.fromMillisecondsSinceEpoch(
+      millisecondsSinceEpoch,
+      isUtc: true,
     );
-    final String tokenPath =
-        path.join(Directory.systemTemp.absolute.path, 'parse_and_send_token');
-    File(tokenPath).writeAsStringSync(client.credentials.accessToken.data);
-    final ProcessResult result = Process.runSync('dart', <String>[
-      'bin/parse_and_send.dart',
-      'example/txt_benchmarks.json',
-    ], environment: <String, String>{
-      'TOKEN_PATH': tokenPath,
-      'GCP_PROJECT': testCred['project_id'] as String,
-      'IS_TESTING': 'true',
-    });
-    expect(result.exitCode, 0);
+    expect(commitDate.year > 2000, true);
+    expect(commitDate.year < 3000, true);
+  });
+
+  test('parse() succeeds', () async {
+    // Check that the results of parse() on ../example/txt_benchmarks.json
+    //   is as expeted.
+    final String exampleJson = p.join('example', 'txt_benchmarks.json');
+    final pas.PointsAndDate pad = await pas.parse(exampleJson);
+    final List<FlutterEngineMetricPoint> points = pad.points;
+
+    expect(points[0].value, equals(101.0));
+    expect(points[2].value, equals(4460.0));
+    expect(points[4].value, equals(6548.0));
   });
 }

testing/run_tests.py

@@ -520,6 +520,21 @@ def RunLitetestTests(build_dir):
       cwd=test_dir)
 
 
+def RunBenchmarkTests(build_dir):
+  test_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'benchmark')
+  dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
+  for dart_test_file in dart_tests:
+    opts = [
+      '--disable-dart-dev',
+      dart_test_file]
+    RunEngineExecutable(
+      build_dir,
+      os.path.join('dart-sdk', 'bin', 'dart'),
+      None,
+      flags=opts,
+      cwd=test_dir)
+
+
 def main():
   parser = argparse.ArgumentParser()
 
@@ -560,10 +575,10 @@ def main():
     assert not IsWindows(), "Dart tests can't be run on windows. https://github.com/flutter/flutter/issues/36301."
     dart_filter = args.dart_filter.split(',') if args.dart_filter else None
     RunDartSmokeTest(build_dir, args.verbose_dart_snapshot)
+    RunLitetestTests(build_dir)
     RunDartTests(build_dir, dart_filter, args.verbose_dart_snapshot)
     RunConstFinderTests(build_dir)
     RunFrontEndServerTests(build_dir)
-    RunLitetestTests(build_dir)
 
   if 'java' in types:
     assert not IsWindows(), "Android engine files can't be compiled on Windows."
@@ -579,6 +594,7 @@ def main():
 
   # https://github.com/flutter/flutter/issues/36300
   if 'benchmarks' in types and not IsWindows():
+    RunBenchmarkTests(build_dir)
     RunEngineBenchmarks(build_dir, engine_filter)
 
   if ('engine' in types or 'font-subset' in types) and args.variant != 'host_release':

tools/licenses/lib/main.dart

@@ -1665,13 +1665,14 @@ class _RepositoryPkgDirectory extends _RepositoryDirectory {
   @override
   bool shouldRecurse(fs.IoNode entry) {
     return entry.name != 'archive'  // contains nothing that ends up in the binary executable
+      && entry.name != 'equatable'
       && entry.name != 'file'
-      && entry.name != 'image'
-      && entry.name != 'petitparser'
+      && entry.name != 'flutter_packages'
+      && entry.name != 'gcloud'
+      && entry.name != 'googleapis'
       && entry.name != 'platform'
       && entry.name != 'process'
-      && entry.name != 'vector_math'
-      && entry.name != 'xml';
+      && entry.name != 'vector_math';
   }
 
   @override

tools/pub_get_offline.py

@@ -16,6 +16,7 @@
 
 ALL_PACKAGES = [
   os.path.join("src", "flutter", "flutter_frontend_server"),
+  os.path.join("src", "flutter", "testing", "benchmark"),
   os.path.join("src", "flutter", "testing", "litetest"),
   os.path.join("src", "flutter", "testing", "smoke_test_failure"),
   os.path.join("src", "flutter", "testing", "symbols"),