Run all microbenchmarks (flutter#154374)

Two things:

**Re-land**: Uninstall microbenchmarks before running them.
Flakes in flutter#153828 stem from adb saying the app isn't installed, but then failing to install wtih `-r`. Several other tests uninstall the app before trying to run it. Previous fix called uninstall between tests, but iOS takes 12 to 13 seconds to perform uninstall / install, which timed out the test.  Just uninstall the one time since we only care about any lingering apps with different keys.
Potential solution flutter#153828

**Make things go fast**
Instead of installing 21 different compilations of the same app to get results; compile and run them together. Locally on Mac+iOS, this should takes ~3 minutes instead of ~15 minutes.

Author: jtmcdole

dev/benchmarks/microbenchmarks/README.md

@@ -1,22 +1,17 @@
 # microbenchmarks
 
-To run these benchmarks on a device, first run `flutter logs' in one
-window to see the device logs, then, in a different window, run any of
-these:
+To run these benchmarks on a device, first run `flutter logs` in one
+window to see the device logs, then, in a different window, run:
 
 ```sh
-flutter run --release lib/gestures/velocity_tracker_bench.dart
-flutter run --release lib/gestures/gesture_detector_bench.dart
-flutter run --release lib/stocks/animation_bench.dart
-flutter run --release lib/stocks/build_bench.dart
-flutter run --release lib/stocks/layout_bench.dart
+flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart
 ```
 
 The results should be in the device logs.
 
-### Avoid changing names of the benchmarks
+## Avoid changing names of the benchmarks
 
 Each microbenchmark is identified by a name, for example,
-"catmullrom_transform_iteration". Changing the name of an existing
-microbenchmarks will effectively remove the old benchmark and create a new one,
+"catmullrom_transform_iteration". Changing the name passed to `BenchmarkResultPrinter.addResult`
+will effectively remove the old benchmark and create a new one,
 losing the historical data associated with the old benchmark in the process.

dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart

@@ -0,0 +1,23 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+
+class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding {
+  BenchmarkingBinding();
+
+  final Stopwatch drawFrameWatch = Stopwatch();
+
+  @override
+  void handleBeginFrame(Duration? rawTimeStamp) {
+    drawFrameWatch.start();
+    super.handleBeginFrame(rawTimeStamp);
+  }
+
+  @override
+  void handleDrawFrame() {
+    super.handleDrawFrame();
+    drawFrameWatch.stop();
+  }
+}

dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart

@@ -0,0 +1,107 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:flutter/material.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'benchmark_binding.dart';
+import 'foundation/all_elements_bench.dart' as all_elements_bench;
+import 'foundation/change_notifier_bench.dart' as change_notifier_bench;
+import 'foundation/clamp.dart' as clamp;
+import 'foundation/decode_and_parse_asset_manifest.dart'
+    as decode_and_parse_asset_manifest;
+import 'foundation/platform_asset_bundle.dart' as platform_asset_bundle;
+import 'foundation/standard_message_codec_bench.dart'
+    as standard_message_codec_bench;
+import 'foundation/standard_method_codec_bench.dart'
+    as standard_method_codec_bench;
+import 'foundation/timeline_bench.dart' as timeline_bench;
+import 'geometry/matrix_utils_transform_bench.dart'
+    as matrix_utils_transform_bench;
+import 'geometry/rrect_contains_bench.dart' as rrect_contains_bench;
+import 'gestures/gesture_detector_bench.dart' as gesture_detector_bench;
+import 'gestures/velocity_tracker_bench.dart' as velocity_tracker_bench;
+import 'language/compute_bench.dart' as compute_bench;
+import 'language/sync_star_bench.dart' as sync_star_bench;
+import 'language/sync_star_semantics_bench.dart' as sync_star_semantics_bench;
+import 'layout/text_intrinsic_bench.dart' as text_intrinsic_bench;
+import 'stocks/animation_bench.dart' as animation_bench;
+import 'stocks/build_bench.dart' as build_bench;
+import 'stocks/build_bench_profiled.dart' as build_bench_profiled;
+import 'stocks/layout_bench.dart' as layout_bench;
+import 'ui/image_bench.dart' as image_bench;
+
+typedef Benchmark = (String name, Future<void> Function() value);
+
+Future<void> main() async {
+  assert(false,
+      "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
+
+  // BenchmarkingBinding is used by animation_bench, providing a simple
+  // stopwatch interface over rendering. Lifting it here makes all
+  // benchmarks run together.
+  final BenchmarkingBinding binding = BenchmarkingBinding();
+  final List<Benchmark> benchmarks = <Benchmark>[
+    ('foundation/change_notifier_bench.dart', change_notifier_bench.execute),
+    ('foundation/clamp.dart', clamp.execute),
+    ('foundation/platform_asset_bundle.dart', platform_asset_bundle.execute),
+    (
+      'foundation/standard_message_codec_bench.dart',
+      standard_message_codec_bench.execute
+    ),
+    (
+      'foundation/standard_method_codec_bench.dart',
+      standard_method_codec_bench.execute
+    ),
+    ('foundation/timeline_bench.dart', timeline_bench.execute),
+    (
+      'foundation/decode_and_parse_asset_manifest.dart',
+      decode_and_parse_asset_manifest.execute
+    ),
+    (
+      'geometry/matrix_utils_transform_bench.dart',
+      matrix_utils_transform_bench.execute
+    ),
+    ('geometry/rrect_contains_bench.dart', rrect_contains_bench.execute),
+    ('gestures/gesture_detector_bench.dart', gesture_detector_bench.execute),
+    ('gestures/velocity_tracker_bench.dart', velocity_tracker_bench.execute),
+    ('language/compute_bench.dart', compute_bench.execute),
+    ('language/sync_star_bench.dart', sync_star_bench.execute),
+    (
+      'language/sync_star_semantics_bench.dart',
+      sync_star_semantics_bench.execute
+    ),
+    ('stocks/animation_bench.dart', () => animation_bench.execute(binding)),
+    ('stocks/build_bench.dart', build_bench.execute),
+    ('stocks/build_bench_profiled.dart', build_bench_profiled.execute),
+    ('stocks/layout_bench.dart', layout_bench.execute),
+    ('ui/image_bench.dart', image_bench.execute),
+    ('layout/text_intrinsic_bench.dart', text_intrinsic_bench.execute),
+    (
+      'foundation/all_elements_bench.dart',
+      () async {
+        binding.framePolicy =
+            LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
+        runApp(const SizedBox.shrink()); // ensure dispose
+        await SchedulerBinding.instance.endOfFrame;
+        all_elements_bench.execute();
+      }
+    ),
+  ];
+
+  print('╡ ••• Running microbenchmarks ••• ╞');
+
+  for (final Benchmark mark in benchmarks) {
+    // Reset the frame policy to default - each test can set it on their own.
+    binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers;
+    print('╡ ••• Running ${mark.$1} ••• ╞');
+    await mark.$2();
+  }
+
+  print('\n\n╡ ••• Done ••• ╞\n\n');
+  exit(0);
+}

dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart

@@ -11,8 +11,7 @@ import '../common.dart';
 
 const int _kNumIters = 10000;
 
-Future<void> main() async {
-  assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
+Future<void> execute() async {
   runApp(MaterialApp(
     home: Scaffold(
       body: GridView.count(

dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart

@@ -10,7 +10,7 @@ const int _kNumIterations = 65536;
 const int _kNumWarmUp = 100;
 const int _kScale = 1000;
 
-void main() {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
 
   // In the following benchmarks, we won't remove the listeners when we don't

dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart

@@ -9,7 +9,7 @@ import '../common.dart';
 const int _kBatchSize = 100000;
 const int _kNumIterations = 1000;
 
-void main() {
+Future<void> execute() async {
   assert(false,
       "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart

@@ -9,7 +9,7 @@ import '../common.dart';
 
 const int _kNumIterations = 1000;
 
-void main() async {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
 
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart

@@ -10,7 +10,7 @@ import '../common.dart';
 const int _kBatchSize = 100;
 const int _kNumIterations = 100;
 
-void main() async {
+Future<void> execute() async {
   assert(false,
       "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart

@@ -8,7 +8,7 @@ import '../common.dart';
 
 const int _kNumIterations = 100000;
 
-void main() {
+Future<void> execute() async {
   assert(false,
       "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart

@@ -8,7 +8,7 @@ import '../common.dart';
 
 const int _kNumIterations = 100000;
 
-void main() {
+Future<void> execute() async {
   assert(false,
       "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart

@@ -8,7 +8,7 @@ import '../common.dart';
 
 const int _kNumIterations = 10000;
 
-void main() {
+Future<void> execute() async {
   assert(false,
       "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart

@@ -11,7 +11,7 @@ import '../common.dart';
 const int _kNumIterations = 10000000;
 const int _kNumWarmUp = 100000;
 
-void main() {
+Future<void> execute()  async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   print('MatrixUtils.transformRect and .transformPoint benchmark...');
 

dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart

@@ -8,7 +8,7 @@ import '../common.dart';
 
 const int _kNumIters = 10000;
 
-void main() {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final Stopwatch watch = Stopwatch();
   print('RRect contains benchmark...');

dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart

@@ -10,7 +10,7 @@ import 'apps/button_matrix_app.dart' as button_matrix;
 const int _kNumWarmUpIters = 20;
 const int _kNumIters = 300;
 
-Future<void> main() async {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final Stopwatch watch = Stopwatch();
   print('GestureDetector semantics benchmark...');

dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart

@@ -17,7 +17,7 @@ class TrackerBenchmark {
   final String name;
 }
 
-Future<void> main() async {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();
   final List<TrackerBenchmark> benchmarks = <TrackerBenchmark>[

dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart

@@ -23,7 +23,7 @@ List<Data> test(int length) {
       (int index) => Data(index * index));
 }
 
-Future<void> main() async {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
 
   // Warm up lap

dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart

@@ -7,7 +7,7 @@ import '../common.dart';
 const int _kNumIterations = 1000;
 const int _kNumWarmUp = 100;
 
-void main() {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
 
   // Warm up lap

dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart

@@ -9,7 +9,7 @@ import '../common.dart';
 const int _kNumIterations = 1000;
 const int _kNumWarmUp = 100;
 
-void main() {
+Future<void> execute() async {
   final List<String> words = 'Lorem Ipsum is simply dummy text of the printing and'
     " typesetting industry. Lorem Ipsum has been the industry's"
     ' standard dummy text ever since the 1500s, when an unknown'

dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart

@@ -20,7 +20,7 @@ final Widget intrinsicTextHeight = Directionality(
   ),
 );
 
-Future<void> main() async {
+Future<void> execute() async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
 
   // We control the framePolicy below to prevent us from scheduling frames in

dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart

@@ -7,35 +7,16 @@ import 'package:flutter_test/flutter_test.dart';
 import 'package:stocks/main.dart' as stocks;
 import 'package:stocks/stock_data.dart' as stock_data;
 
+import '../benchmark_binding.dart';
 import '../common.dart';
 
 const Duration kBenchmarkTime = Duration(seconds: 15);
 
-class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding {
-  BenchmarkingBinding(this.stopwatch);
-
-  final Stopwatch stopwatch;
-
-  @override
-  void handleBeginFrame(Duration? rawTimeStamp) {
-    stopwatch.start();
-    super.handleBeginFrame(rawTimeStamp);
-  }
-
-  @override
-  void handleDrawFrame() {
-    super.handleDrawFrame();
-    stopwatch.stop();
-  }
-}
-
-Future<void> main() async {
+Future<void> execute(BenchmarkingBinding binding) async {
   assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
   stock_data.StockData.actuallyFetchData = false;
 
   final Stopwatch wallClockWatch = Stopwatch();
-  final Stopwatch cpuWatch = Stopwatch();
-  BenchmarkingBinding(cpuWatch);
 
   int totalOpenFrameElapsedMicroseconds = 0;
   int totalOpenIterationCount = 0;
@@ -52,27 +33,27 @@ Future<void> main() async {
     bool drawerIsOpen = false;
     wallClockWatch.start();
     while (wallClockWatch.elapsed < kBenchmarkTime) {
-      cpuWatch.reset();
+      binding.drawFrameWatch.reset();
       if (drawerIsOpen) {
         await tester.tapAt(const Offset(780.0, 250.0)); // Close drawer
         await tester.pump();
         totalCloseIterationCount += 1;
-        totalCloseFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
+        totalCloseFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds;
       } else {
         await tester.tapAt(const Offset(20.0, 50.0)); // Open drawer
         await tester.pump();
         totalOpenIterationCount += 1;
-        totalOpenFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
+        totalOpenFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds;
       }
       drawerIsOpen = !drawerIsOpen;
 
       // Time how long each frame takes
-      cpuWatch.reset();
+      binding.drawFrameWatch.reset();
       while (SchedulerBinding.instance.hasScheduledFrame) {
         await tester.pump();
         totalSubsequentFramesIterationCount += 1;
       }
-      totalSubsequentFramesElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
+      totalSubsequentFramesElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds;
     }
   });
 

dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart

@@ -55,7 +55,7 @@ Future<List<double>> runBuildBenchmark() async {
   return values;
 }
 
-Future<void> main() async {
+Future<void> execute() async {
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();
   printer.addResultStatistics(
     description: 'Stock build',

dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart

@@ -8,7 +8,7 @@ import 'package:flutter_test/flutter_test.dart';
 import '../common.dart';
 import 'build_bench.dart';
 
-Future<void> main() async {
+Future<void> execute() async {
   debugProfileBuildsEnabledUserWidgets = true;
   final BenchmarkResultPrinter printer = BenchmarkResultPrinter();
   printer.addResultStatistics(