Make tests more resilient to Skia gold failures and refactor flutter_goldens...

Make tests more resilient to Skia gold failures and refactor flutter_goldens for extensive technical debt removal (#140101)

Originally landed in https://github.com/flutter/flutter/pull/139549
Originally reverted in https://github.com/flutter/flutter/pull/140085

- Remove all use of global variables.
- Always pass in all dependencies, only create them in main or in tests.
- Pass in the "print" primitive.
- Make all network traffic retry (except when run locally, when it just auto-passes).
- Enable tests to be run in random order.
- Better error messages

dev/automated_tests/flutter_test/flutter_gold_expectation.txt

-[0-9]+:[0-9]+ [+]0: Local passes non-existent baseline for new test, null expectation *
- *No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.
- *Validate image output found at flutter/test/library/
-[0-9]+:[0-9]+ [+]1: Local passes non-existent baseline for new test, empty expectation *
- *No expectations provided by Skia Gold for test: library.flutter.new_golden_test.2.png. This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.
- *Validate image output found at flutter/test/library/
 [0-9]+:[0-9]+ [+]2: All tests passed! *

dev/automated_tests/flutter_test/flutter_gold_test.dart

@@ -21,6 +21,7 @@ const List<int> _kFailPngBytes = <int>[
 ];
 
 void main() {
+  final List<String> log = <String>[];
   final MemoryFileSystem fs = MemoryFileSystem();
   final Directory basedir = fs.directory('flutter/test/library/')
     ..createSync(recursive: true);
@@ -34,6 +35,7 @@ void main() {
       environment: <String, String>{'FLUTTER_ROOT': '/flutter'},
       operatingSystem: 'macos'
     ),
+    log: log.add,
   );
 
   test('Local passes non-existent baseline for new test, null expectation', () async {
@@ -44,6 +46,12 @@ void main() {
       ),
       isTrue,
     );
+    expect(log, <String>[
+      // ignore: no_adjacent_strings_in_list
+      'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. '
+      'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
+      'Validate image output found at flutter/test/library/'
+    ]);
   });
 
   test('Local passes non-existent baseline for new test, empty expectation', () async {
@@ -54,6 +62,16 @@ void main() {
       ),
       isTrue,
     );
+    expect(log, <String>[
+      // ignore: no_adjacent_strings_in_list
+      'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. '
+      'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
+      'Validate image output found at flutter/test/library/',
+      // ignore: no_adjacent_strings_in_list
+      'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.2.png. '
+      'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
+      'Validate image output found at flutter/test/library/',
+    ]);
   });
 }
 

packages/flutter/pubspec.yaml

@@ -28,6 +28,7 @@ dev_dependencies:
   async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   leak_tracker: 9.0.18 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   leak_tracker_testing: 1.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -41,6 +42,7 @@ dev_dependencies:
   string_scanner: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   term_glyph: 1.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   test_api: 0.6.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   vm_service: 13.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 84b7
+# PUBSPEC CHECKSUM: e519

packages/flutter/test_private/test/pubspec.yaml

@@ -38,8 +38,10 @@ dev_dependencies:
     sdk: flutter
   fake_async: 1.3.1
 
+  crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   platform: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   process: 5.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 942d
+# PUBSPEC CHECKSUM: 5a8e

packages/flutter_goldens/README

+This package is an internal implementation detail for our testing
+infrastructure. It enables the framework to use the Skia Gold
+infrastructure for tracking golden image tests.
+
+See also:
+
+ * https://skia.org/docs/dev/testing/skiagold/
+ * https://flutter-gold.skia.org/
+ * https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter

packages/flutter_goldens/dart_test.yaml

-tags:
-  # This tag tells the test framework to not shuffle the test order according to
-  # the --test-randomize-ordering-seed for the suites that have this tag.
-  no-shuffle:
-    allow_test_randomization: false

packages/flutter_goldens/pubspec.yaml

@@ -9,26 +9,24 @@ dependencies:
     sdk: flutter
   flutter_test:
     sdk: flutter
-  flutter_goldens_client:
-    path: ../flutter_goldens_client
   file: 7.0.0
   meta: 1.11.0
   platform: 3.1.3
   process: 5.0.1
+  crypto: 3.0.3
+  path: 1.9.0
 
   async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   characters: 1.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   collection: 1.18.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   fake_async: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   leak_tracker: 9.0.18 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   leak_tracker_flutter_testing: 2.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   leak_tracker_testing: 1.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   matcher: 0.12.16+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   material_color_utilities: 0.8.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  path: 1.9.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   source_span: 1.10.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   stack_trace: 1.11.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   stream_channel: 2.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

packages/flutter_goldens/test/skia_client_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io' show HttpClient, ProcessResult;
+
+import 'package:file/file.dart';
+import 'package:file/memory.dart';
+import 'package:flutter_goldens/skia_client.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:platform/platform.dart';
+import 'package:process/process.dart';
+
+void main() {
+  test('502 retry', () async {
+    final List<String> log = <String>[];
+    await runZoned(
+      zoneSpecification: ZoneSpecification(
+        print: (Zone self, ZoneDelegate parent, Zone zone, String line) {
+          fail('unexpected print: "$line"');
+        },
+        createTimer: (Zone self, ZoneDelegate parent, Zone zone, Duration duration, void Function() f) {
+          log.add('CREATED TIMER: $duration');
+          return parent.createTimer(zone, Duration.zero, f);
+        },
+      ),
+      () async {
+        final FileSystem fs;
+        final SkiaGoldClient skiaClient = SkiaGoldClient(
+          fs: fs = MemoryFileSystem(),
+          process: FakeProcessManager(log),
+          platform: FakePlatform(
+            environment: const <String, String>{
+              'GOLDCTL': 'goldctl',
+            },
+          ),
+          httpClient: FakeHttpClient(),
+          log: log.add,
+          fs.directory('/'),
+        );
+        log.add('START'); // ignore: avoid_print
+        await skiaClient.tryjobAdd('test', fs.file('golden'));
+        log.add('END'); // ignore: avoid_print
+        expect(log, <String>[
+          'START',
+          'EXEC: goldctl imgtest add --work-dir /temp --test-name t --png-file golden',
+          'Transient failure (exit code 1) from Skia Gold.',
+          '',
+          'stdout from gold:',
+          '  test resulted in a 502: 502 Bad Gateway',
+          '  ',
+          '',
+          'Retrying in 5 seconds.',
+          'CREATED TIMER: 0:00:05.000000',
+          'EXEC: goldctl imgtest add --work-dir /temp --test-name t --png-file golden',
+          'END',
+        ]);
+      },
+    );
+  });
+}
+
+class FakeProcessManager extends Fake implements ProcessManager {
+  FakeProcessManager(this.log);
+
+  final List<String> log;
+  int _index = 0;
+
+  @override
+  Future<ProcessResult> run(List<Object> command, {
+    Map<String, String>? environment,
+    bool includeParentEnvironment = true,
+    bool runInShell = false,
+    Encoding? stderrEncoding,
+    Encoding? stdoutEncoding,
+    String? workingDirectory,
+  }) async {
+    log.add('EXEC: ${command.join(' ')}');
+    _index += 1;
+    switch (_index) {
+      case 1: return ProcessResult(0, 1, 'test resulted in a 502: 502 Bad Gateway\n', '');
+      case 2: return ProcessResult(0, 0, '200', '');
+      default: throw StateError('unexpected call to run');
+    }
+  }
+}
+
+class FakeHttpClient extends Fake implements HttpClient { }

packages/flutter_goldens_client/pubspec.yaml

-name: flutter_goldens_client
-
-environment:
-  sdk: '>=3.2.0-0 <4.0.0'
-
-dependencies:
-  # To update these, use "flutter update-packages --force-upgrade".
-  crypto: 3.0.3
-  file: 7.0.0
-  platform: 3.1.3
-  process: 5.0.1
-
-  collection: 1.18.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  meta: 1.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  path: 1.9.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-
-dartdoc:
-  # Exclude this package from the hosted API docs.
-  nodoc: true
-
-# PUBSPEC CHECKSUM: 652c