Make tests more resilient to Skia gold failures and refactor flutter_goldens for extensive technical debt removal (flutter#139549)

* Remove all use of global variables.
* Always pass in all dependencies, only create them in main or in tests.
* Pass in the "print" primitive.
* Make all network traffic retry (except when run locally, when it just auto-passes).
* Enable tests to be run in random order.

Author: Hixie

dev/automated_tests/flutter_test/flutter_gold_expectation.txt

@@ -1,7 +1 @@
-[0-9]+:[0-9]+ [+]0: Local passes non-existent baseline for new test, null expectation *
- *No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.
- *Validate image output found at flutter/test/library/
-[0-9]+:[0-9]+ [+]1: Local passes non-existent baseline for new test, empty expectation *
- *No expectations provided by Skia Gold for test: library.flutter.new_golden_test.2.png. This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.
- *Validate image output found at flutter/test/library/
 [0-9]+:[0-9]+ [+]2: All tests passed! *

dev/automated_tests/flutter_test/flutter_gold_test.dart

@@ -21,6 +21,7 @@ const List<int> _kFailPngBytes = <int>[
 ];
 
 void main() {
+  final List<String> log = <String>[];
   final MemoryFileSystem fs = MemoryFileSystem();
   final Directory basedir = fs.directory('flutter/test/library/')
     ..createSync(recursive: true);
@@ -34,6 +35,7 @@ void main() {
       environment: <String, String>{'FLUTTER_ROOT': '/flutter'},
       operatingSystem: 'macos'
     ),
+    log: log.add,
   );
 
   test('Local passes non-existent baseline for new test, null expectation', () async {
@@ -44,6 +46,12 @@ void main() {
       ),
       isTrue,
     );
+    expect(log, <String>[
+      // ignore: no_adjacent_strings_in_list
+      'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. '
+      'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
+      'Validate image output found at flutter/test/library/'
+    ]);
   });
 
   test('Local passes non-existent baseline for new test, empty expectation', () async {
@@ -54,6 +62,16 @@ void main() {
       ),
       isTrue,
     );
+    expect(log, <String>[
+      // ignore: no_adjacent_strings_in_list
+      'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. '
+      'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
+      'Validate image output found at flutter/test/library/',
+      // ignore: no_adjacent_strings_in_list
+      'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.2.png. '
+      'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
+      'Validate image output found at flutter/test/library/',
+    ]);
   });
 }
 

packages/flutter/pubspec.yaml

@@ -29,6 +29,7 @@ dev_dependencies:
   async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   matcher: 0.12.16 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   path: 1.9.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -40,6 +41,7 @@ dev_dependencies:
   string_scanner: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   term_glyph: 1.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   test_api: 0.6.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   vm_service: 13.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 8088
+# PUBSPEC CHECKSUM: bbe9

packages/flutter/test_private/test/pubspec.yaml

@@ -38,8 +38,10 @@ dev_dependencies:
     sdk: flutter
   fake_async: 1.3.1
 
+  crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   platform: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   process: 5.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: db59
+# PUBSPEC CHECKSUM: 0fba

packages/flutter_goldens/README

@@ -0,0 +1,9 @@
+This package is an internal implementation detail for our testing
+infrastructure. It enables the framework to use the Skia Gold
+infrastructure for tracking golden image tests.
+
+See also:
+
+ * https://skia.org/docs/dev/testing/skiagold/
+ * https://flutter-gold.skia.org/
+ * https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter