Allow Flutter golden file tests to be flaky (#114450)

* allow marking a golden check as flaky

* add matchesFlutterGolden to analyze.dart; no tags for flutter_goldens/lib

* Pause

* ++

* ++

* Analyzer therapy

* Once more with feeling

* Nits

* Review feedback

* Silly oops

* Test progress

* More tests

* Finish

* Nits

* Analyzer

* Review feedback
Co-authored-by: Yegor Jbanov <yjbanov@google.com>

dev/automated_tests/flutter_test/flutter_gold_test.dart

@@ -8,7 +8,7 @@ import 'dart:typed_data';
 
 import 'package:file/file.dart';
 import 'package:file/memory.dart';
-import 'package:flutter_goldens/flutter_goldens.dart';
+import 'package:flutter_goldens/src/flutter_goldens_io.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:platform/platform.dart';
 

dev/bots/analyze.dart

@@ -418,8 +418,8 @@ Future<void> verifyNoSyncAsyncStar(String workingDirectory, {int minimumMatches
   }
 }
 
-final RegExp _findGoldenTestPattern = RegExp(r'matchesGoldenFile\(');
-final RegExp _findGoldenDefinitionPattern = RegExp(r'matchesGoldenFile\(Object');
+final RegExp _findGoldenTestPattern = RegExp(r'(matchesGoldenFile|expectFlakyGolden)\(');
+final RegExp _findGoldenDefinitionPattern = RegExp(r'(matchesGoldenFile|expectFlakyGolden)\(Object');
 final RegExp _leadingComment = RegExp(r'//');
 final RegExp _goldenTagPattern1 = RegExp(r'@Tags\(');
 final RegExp _goldenTagPattern2 = RegExp(r"'reduced-test-set'");
@@ -431,8 +431,17 @@ const String _ignoreGoldenTag = '// flutter_ignore: golden_tag (see analyze.dart
 const String _ignoreGoldenTagForFile = '// flutter_ignore_for_file: golden_tag (see analyze.dart)';
 
 Future<void> verifyGoldenTags(String workingDirectory, { int minimumMatches = 2000 }) async {
+  // Skip flutter_goldens/lib because this library uses `matchesGoldenFile`
+  // but is not itself a test that needs tags.
+  final String flutterGoldensPackageLib = path.join(flutterPackages, 'flutter_goldens', 'lib');
+  bool isWithinFlutterGoldenLib(File file) {
+    return path.isWithin(flutterGoldensPackageLib, file.path);
+  }
+
   final List<String> errors = <String>[];
-  await for (final File file in _allFiles(workingDirectory, 'dart', minimumMatches: minimumMatches)) {
+  final Stream<File> allTestFiles = _allFiles(workingDirectory, 'dart', minimumMatches: minimumMatches)
+    .where((File file) => !isWithinFlutterGoldenLib(file));
+  await for (final File file in allTestFiles) {
     bool needsTag = false;
     bool hasTagNotation = false;
     bool hasReducedTag = false;

packages/flutter/test/_goldens_web.dart → dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_ignore.dart

@@ -2,7 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:async';
+// This would fail analysis, but it is ignored
+// flutter_ignore_for_file: golden_tag (see analyze.dart)
 
-// package:flutter_goldens is not used as part of the test process for web.
-Future<void> testExecutable(FutureOr<void> Function() testMain) async => testMain();
+@Tags(<String>['some-other-tag'])
+
+import 'package:test/test.dart';
+
+import 'golden_class.dart';
+
+void main() {
+  expectFlakyGolden('key', 'String');
+}

examples/api/test/goldens_web.dart → dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart

@@ -2,7 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:async';
+// The reduced test set tag is missing. This should fail analysis.
+@Tags(<String>['some-other-tag'])
 
-// package:flutter_goldens is not used as part of the test process for web.
-Future<void> testExecutable(FutureOr<void> Function() testMain) async => testMain();
+import 'package:test/test.dart';
+
+import 'golden_class.dart';
+
+void main() {
+  expectFlakyGolden('finder', 'missing_tag.png');
+}

examples/api/test/goldens_io.dart → dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart

@@ -2,4 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable;
+// The tag is missing. This should fail analysis.
+
+import 'golden_class.dart';
+
+void main() {
+  expectFlakyGolden('key', 'missing_tag.png');
+}

dev/bots/test/analyze-test-input/root/packages/foo/golden_class.dart

@@ -7,3 +7,7 @@
 void matchesGoldenFile(Object key) {
   return;
 }
+
+void expectFlakyGolden(Object key, String string){
+  return;
+}

dev/bots/test/analyze-test-input/root/packages/foo/golden_doc.dart

@@ -35,8 +35,11 @@
 /// ```
 /// {@end-tool}
 ///
+/// expectFlakyGolden(a, b)
 
 // Other comments
 // matchesGoldenFile('comment.png');
+// expectFlakyGolden(a, b);
 
 String literal = 'matchesGoldenFile()'; // flutter_ignore: golden_tag (see analyze.dart)
+String flakyLiteral = 'expectFlakyGolden';

dev/bots/test/analyze_test.dart

@@ -79,7 +79,9 @@ void main() {
                          'at the top of the file before import statements.';
     const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'.";
     final List<String> lines = <String>[
+        '║ test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart: $noTag',
         '║ test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
+        '║ test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart: $missingTag',
         '║ test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
       ]
       .map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/'))

dev/devicelab/bin/tasks/technical_debt__cost.dart

@@ -17,6 +17,7 @@ const double todoCost = 1009.0; // about two average SWE days, in dollars
 const double ignoreCost = 2003.0; // four average SWE days, in dollars
 const double pythonCost = 3001.0; // six average SWE days, in dollars
 const double skipCost = 2473.0; // 20 hours: 5 to fix the issue we're ignoring, 15 to fix the bugs we missed because the test was off
+const double flakyGoldenCost = 2467.0; // Similar to skip cost
 const double ignoreForFileCost = 2477.0; // similar thinking as skipCost
 const double asDynamicCost = 2011.0; // a few days to refactor the code.
 const double deprecationCost = 233.0; // a few hours to remove the old code.
@@ -69,6 +70,9 @@ Future<double> findCostsForFile(File file) async {
     if (isTest && line.contains('skip:') && !line.contains('[intended]')) {
       total += skipCost;
     }
+    if (isTest && line.contains('expectFlakyGolden(')) {
+      total += flakyGoldenCost;
+    }
     if (isDart && isOptingOutOfNullSafety(line)) {
       total += fileNullSafetyMigrationCost;
     }

examples/api/test/flutter_test_config.dart

@@ -5,7 +5,7 @@
 import 'dart:async';
 
 
-import 'goldens_io.dart' if (dart.library.html) 'goldens_web.dart' as flutter_goldens;
+import 'package:flutter_goldens/flutter_goldens.dart' as flutter_goldens;
 
 Future<void> testExecutable(FutureOr<void> Function() testMain) {
   // Enable golden file testing using Skia Gold.

packages/flutter/test/_goldens_io.dart

-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable;

packages/flutter/test/cupertino/date_picker_test.dart

@@ -17,10 +17,11 @@ import 'dart:ui';
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';
+import 'package:flutter_goldens/flutter_goldens.dart' show expectFlakyGolden;
 import 'package:flutter_test/flutter_test.dart';
 
 // TODO(yjbanov): on the web text rendered with perspective produces flaky goldens: https://github.com/flutter/flutter/issues/110785
-const bool skipPerspectiveTextGoldens = isBrowser;
+const bool perspectiveTestIsFlaky = isBrowser;
 
 // A number of the hit tests below say "warnIfMissed: false". This is because
 // the way the CupertinoPicker works, the hits don't actually reach the labels,
@@ -1197,7 +1198,13 @@ void main() {
       }
 
       await tester.pumpWidget(buildApp(CupertinoDatePickerMode.time));
-      if (!skipPerspectiveTextGoldens) {
+
+      if (perspectiveTestIsFlaky) {
+        await expectFlakyGolden(
+          find.byType(CupertinoDatePicker),
+          'date_picker_test.time.initial.png',
+        );
+      } else {
         await expectLater(
           find.byType(CupertinoDatePicker),
           matchesGoldenFile('date_picker_test.time.initial.png'),
@@ -1205,7 +1212,13 @@ void main() {
       }
 
       await tester.pumpWidget(buildApp(CupertinoDatePickerMode.date));
-      if (!skipPerspectiveTextGoldens) {
+
+      if (perspectiveTestIsFlaky) {
+        await expectFlakyGolden(
+          find.byType(CupertinoDatePicker),
+          'date_picker_test.date.initial.png',
+        );
+      } else {
         await expectLater(
           find.byType(CupertinoDatePicker),
           matchesGoldenFile('date_picker_test.date.initial.png'),
@@ -1213,7 +1226,13 @@ void main() {
       }
 
       await tester.pumpWidget(buildApp(CupertinoDatePickerMode.dateAndTime));
-      if (!skipPerspectiveTextGoldens) {
+
+      if (perspectiveTestIsFlaky) {
+        await expectFlakyGolden(
+          find.byType(CupertinoDatePicker),
+          'date_picker_test.datetime.initial.png',
+        );
+      } else {
         await expectLater(
           find.byType(CupertinoDatePicker),
           matchesGoldenFile('date_picker_test.datetime.initial.png'),
@@ -1224,7 +1243,12 @@ void main() {
       await tester.drag(find.text('4'), Offset(0, _kRowOffset.dy / 2), warnIfMissed: false); // see top of file
       await tester.pump();
 
-      if (!skipPerspectiveTextGoldens) {
+      if (perspectiveTestIsFlaky) {
+        await expectFlakyGolden(
+          find.byType(CupertinoDatePicker),
+          'date_picker_test.datetime.drag.png',
+        );
+      } else {
         await expectLater(
           find.byType(CupertinoDatePicker),
           matchesGoldenFile('date_picker_test.datetime.drag.png'),
@@ -1314,7 +1338,12 @@ void main() {
       ),
     );
 
-    if (!skipPerspectiveTextGoldens) {
+    if (perspectiveTestIsFlaky) {
+      await expectFlakyGolden(
+        find.byType(CupertinoTimerPicker),
+        'timer_picker_test.datetime.initial.png',
+      );
+    } else {
       await expectLater(
         find.byType(CupertinoTimerPicker),
         matchesGoldenFile('timer_picker_test.datetime.initial.png'),
@@ -1325,7 +1354,12 @@ void main() {
     await tester.drag(find.text('59'), Offset(0, _kRowOffset.dy / 2), warnIfMissed: false); // see top of file
     await tester.pump();
 
-    if (!skipPerspectiveTextGoldens) {
+    if (perspectiveTestIsFlaky) {
+      await expectFlakyGolden(
+        find.byType(CupertinoTimerPicker),
+        'timer_picker_test.datetime.drag.png',
+      );
+    } else {
       await expectLater(
         find.byType(CupertinoTimerPicker),
         matchesGoldenFile('timer_picker_test.datetime.drag.png'),

packages/flutter/test/flutter_test_config.dart

@@ -5,11 +5,9 @@
 import 'dart:async';
 
 import 'package:flutter/rendering.dart';
+import 'package:flutter_goldens/flutter_goldens.dart' as flutter_goldens;
 import 'package:flutter_test/flutter_test.dart';
 
-import '_goldens_io.dart'
-  if (dart.library.html) '_goldens_web.dart' as flutter_goldens;
-
 Future<void> testExecutable(FutureOr<void> Function() testMain) {
   // Enable checks because there are many implementations of [RenderBox] in this
   // package can benefit from the additional validations.
@@ -22,3 +20,7 @@ Future<void> testExecutable(FutureOr<void> Function() testMain) {
   // Enable golden file testing using Skia Gold.
   return flutter_goldens.testExecutable(testMain);
 }
+
+Future<void> processBrowserCommand(dynamic command) {
+  return flutter_goldens.processBrowserCommand(command);
+}

packages/flutter/test/flutter_web_test_config.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'flutter_test_config.dart';
+
+/// A custom host configuration for browser tests that supports flaky golden
+/// checks.
+///
+/// See also [processBrowserCommand].
+Future<void> startWebTestHostConfiguration(String testUri) async {
+  testExecutable(() async {
+    final Stream<dynamic> commands = stdin
+      .transform<String>(utf8.decoder)
+      .transform<String>(const LineSplitter())
+      .map<dynamic>(jsonDecode);
+    await for (final dynamic command in commands) {
+      await processBrowserCommand(command);
+    }
+  });
+}

packages/flutter_goldens/lib/src/flaky_goldens.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// flutter_ignore_for_file: golden_tag (see analyze.dart)
+
+import 'package:flutter_test/flutter_test.dart';
+
+/// Similar to [matchesGoldenFile] but specialized for Flutter's own tests when
+/// they are flaky.
+///
+/// Asserts that a [Finder], [Future<ui.Image>], or [ui.Image] - the [key] -
+/// matches the golden image file identified by [goldenFile].
+///
+/// For the case of a [Finder], the [Finder] must match exactly one widget and
+/// the rendered image of the first [RepaintBoundary] ancestor of the widget is
+/// treated as the image for the widget. As such, you may choose to wrap a test
+/// widget in a [RepaintBoundary] to specify a particular focus for the test.
+///
+/// The [goldenFile] may be either a [Uri] or a [String] representation of a URL.
+///
+/// Flaky golden file tests are always uploaded to Skia Gold for manual
+/// inspection. This allows contributors to validate when a test is no longer
+/// flaky by visiting https://flutter-gold.skia.org/list,
+/// and clicking on the respective golden test name. The UI will show the
+/// history of generated goldens over time. Each unique golden gets a unique
+/// color. If the color is the same for all commits in the recent history, the
+/// golden is likely no longer flaky and the standard [matchesGoldenFile] can be
+/// used in the given test. If the color changes from commit to commit then it
+/// is still flaky.
+Future<void> expectFlakyGolden(Object key, String goldenFile) {
+  if (isBrowser) {
+    _setFlakyForWeb();
+  } else {
+    _setFlakyForIO();
+  }
+  return expectLater(key, matchesGoldenFile(goldenFile));
+}
+
+void _setFlakyForWeb() {
+  assert(
+    webGoldenComparator is FlakyGoldenMixin,
+    'expectFlakyGolden can only be used with a comparator with the FlakyGoldenMixin '
+    'but found ${webGoldenComparator.runtimeType}.'
+  );
+  (webGoldenComparator as FlakyGoldenMixin).enableFlakyMode();
+}
+
+void _setFlakyForIO() {
+  assert(
+    goldenFileComparator is FlakyGoldenMixin,
+    'expectFlakyGolden can only be used with a comparator with the FlakyGoldenMixin '
+    'but found ${goldenFileComparator.runtimeType}.'
+  );
+  (goldenFileComparator as FlakyGoldenMixin).enableFlakyMode();
+}
+
+/// Allows flaky test handling for the Flutter framework.
+///
+/// Mixed in with the [FlutterGoldenFileComparator] and
+/// [_FlutterWebGoldenComparator].
+mixin FlakyGoldenMixin  {
+  /// Whether this comparator allows flaky goldens.
+  ///
+  /// If set to true, concrete implementations of this class are expected to
+  /// generate the golden and submit it for review, but not fail the test.
+  bool _isFlakyModeEnabled = false;
+
+  /// Puts this comparator into flaky comparison mode.
+  ///
+  /// After calling this method the next invocation of [compare] will allow
+  /// incorrect golden to pass the check.
+  ///
+  /// Concrete implementations of [compare] must call [getAndResetFlakyMode] so
+  /// that subsequent tests can run in non-flaky mode. If a subsequent test
+  /// needs to run in a flaky mode, it must call this method again.
+  void enableFlakyMode() {
+    assert(
+      !_isFlakyModeEnabled,
+      'Test is already marked as flaky. Call `getAndResetFlakyMode` to reset the '
+      'flag before calling this method again.',
+    );
+    _isFlakyModeEnabled = true;
+  }
+
+  /// Returns whether flaky comparison mode was enabled via [enableFlakyMode],
+  /// and if it was, resets the comparator back to non-flaky mode.
+  bool getAndResetFlakyMode() {
+    if (!_isFlakyModeEnabled) {
+      // Not in flaky mode. Nothing to do.
+      return false;
+    }
+
+    // In flaky mode. Reset it and return true.
+    _isFlakyModeEnabled = false;
+    return true;
+  }
+}

packages/flutter_goldens/lib/src/flutter_goldens_web.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async' show FutureOr;
+import 'dart:convert' show json;
+import 'dart:html' as html;
+
+import 'package:flutter_test/flutter_test.dart';
+
+import 'flaky_goldens.dart';
+
+export 'package:flutter_goldens_client/skia_client.dart';
+
+/// Wraps a web test, supplying a custom comparator that supports flaky goldens.
+Future<void> testExecutable(FutureOr<void> Function() testMain, {String? namePrefix}) async {
+  webGoldenComparator = FlutterWebGoldenComparator(webTestUri);
+  await testMain();
+}
+
+/// See the io implementation of this function.
+Future<void> processBrowserCommand(dynamic command) async {
+  throw UnimplementedError('processBrowserCommand is not used inside the browser');
+}
+
+/// Same as [DefaultWebGoldenComparator] but supports flaky golden checks.
+class FlutterWebGoldenComparator extends WebGoldenComparator with FlakyGoldenMixin {
+  /// Creates a new [FlutterWebGoldenComparator] for the specified [testUri].
+  ///
+  /// Golden file keys will be interpreted as file paths relative to the
+  /// directory in which [testUri] resides.
+  ///
+  /// The [testUri] URL must represent a file.
+  FlutterWebGoldenComparator(this.testUri);
+
+  /// The test file currently being executed.
+  ///
+  /// Golden file keys will be interpreted as file paths relative to the
+  /// directory in which this file resides.
+  Uri testUri;
+
+  @override
+  Future<bool> compare(double width, double height, Uri golden) async {
+    final bool isFlaky = getAndResetFlakyMode();
+    final String key = golden.toString();
+    final html.HttpRequest request = await html.HttpRequest.request(
+      'flutter_goldens',
+      method: 'POST',
+      sendData: json.encode(<String, Object>{
+        'testUri': testUri.toString(),
+        'key': key,
+        'width': width.round(),
+        'height': height.round(),
+        'customProperties': <String, dynamic>{
+          'isFlaky': isFlaky,
+        },
+      }),
+    );
+    final String response = request.response as String;
+    if (response == 'true') {
+      return true;
+    }
+    fail(response);
+  }
+
+  @override
+  Future<void> update(double width, double height, Uri golden) async {
+    // Update is handled on the server side, just use the same logic here
+    await compare(width, height, golden);
+  }
+}

packages/flutter_goldens/test/flaky_goldens_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See also dev/automated_tests/flutter_test/flutter_gold_test.dart
+
+// flutter_ignore_for_file: golden_tag (see analyze.dart)
+
+import 'package:flutter_goldens/flutter_goldens.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'utils/fakes.dart';
+
+void main() {
+  test('Sets flaky flag', () {
+    final FakeFlakyLocalFileComparator comparator = FakeFlakyLocalFileComparator();
+    // Is not flaky
+    expect(comparator.getAndResetFlakyMode(), isFalse);
+    comparator.enableFlakyMode();
+    // Flaky was set
+    expect(comparator.getAndResetFlakyMode(), isTrue);
+    // Flaky was unset
+    expect(comparator.getAndResetFlakyMode(), isFalse);
+  });
+
+  test('Asserts when comparator is missing mixin', (){
+    final GoldenFileComparator oldComparator = goldenFileComparator;
+    goldenFileComparator = FakeLocalFileComparator();
+    expect(
+      () {
+        expect(
+          expectFlakyGolden(<int>[0, 1, 2, 3], 'golden_file.png'),
+          throwsAssertionError,
+        );
+      },
+      throwsA(
+        isA<AssertionError>().having((AssertionError error) => error.toString(),
+          'description', contains('FlakyGoldenMixin')),
+      ),
+    );
+    goldenFileComparator = oldComparator;
+  });
+
+  test('top level function sets flag', () {
+    final GoldenFileComparator oldComparator = goldenFileComparator;
+    goldenFileComparator = FakeFlakyLocalFileComparator();
+    expectFlakyGolden(<int>[0, 1, 2, 3], 'golden_file.png');
+    final bool wasFlaky = (goldenFileComparator as FakeFlakyLocalFileComparator).getAndResetFlakyMode();
+    expect(wasFlaky, isTrue);
+    goldenFileComparator = oldComparator;
+  });
+}

packages/flutter_goldens/test/utils/fakes.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See also dev/automated_tests/flutter_test/flutter_gold_test.dart
+
+import 'dart:convert';
+import 'dart:io' hide Directory;
+
+import 'package:file/file.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_goldens/src/flaky_goldens.dart';
+import 'package:flutter_goldens/src/flutter_goldens_io.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:process/process.dart';
+
+// 1x1 transparent pixel
+const List<int> kTestPngBytes = <int>[
+  137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0,
+  1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84,
+  120, 1, 99, 97, 0, 2, 0, 0, 25, 0, 5, 144, 240, 54, 245, 0, 0, 0, 0, 73, 69,
+  78, 68, 174, 66, 96, 130,
+];
+
+@immutable
+class RunInvocation {
+  const RunInvocation(this.command, this.workingDirectory);
+
+  final List<String> command;
+  final String? workingDirectory;
+
+  @override
+  int get hashCode => Object.hash(Object.hashAll(command), workingDirectory);
+
+  bool _commandEquals(List<String> other) {
+    if (other == command) {
+      return true;
+    }
+    if (other.length != command.length) {
+      return false;
+    }
+    for (int index = 0; index < other.length; index += 1) {
+      if (other[index] != command[index]) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @override
+  bool operator ==(Object other) {
+    if (other.runtimeType != runtimeType) {
+      return false;
+    }
+    return other is RunInvocation
+        && _commandEquals(other.command)
+        && other.workingDirectory == workingDirectory;
+  }
+
+  @override
+  String toString() => '$command ($workingDirectory)';
+}
+
+class FakeProcessManager extends Fake implements ProcessManager {
+  Map<RunInvocation, ProcessResult> processResults = <RunInvocation, ProcessResult>{};
+
+  /// Used if [processResults] does not contain a matching invocation.
+  ProcessResult? fallbackProcessResult;
+
+  final List<String?> workingDirectories = <String?>[];
+
+  @override
+  Future<ProcessResult> run(
+      List<Object> command, {
+        String? workingDirectory,
+        Map<String, String>? environment,
+        bool includeParentEnvironment = true,
+        bool runInShell = false,
+        Encoding? stdoutEncoding = systemEncoding,
+        Encoding? stderrEncoding = systemEncoding,
+      }) async {
+    workingDirectories.add(workingDirectory);
+    final ProcessResult? result = processResults[RunInvocation(command.cast<String>(), workingDirectory)];
+    if (result == null && fallbackProcessResult == null) {
+      printOnFailure('ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.');
+      fail('See above.');
+    }
+    return result ?? fallbackProcessResult!;
+  }
+}
+
+// See also dev/automated_tests/flutter_test/flutter_gold_test.dart
+class FakeSkiaGoldClient extends Fake implements SkiaGoldClient {
+  Map<String, String> expectationForTestValues = <String, String>{};
+  Exception? getExpectationForTestThrowable;
+  @override
+  Future<String> getExpectationForTest(String testName) async {
+    if (getExpectationForTestThrowable != null) {
+      throw getExpectationForTestThrowable!;
+    }
+    return expectationForTestValues[testName] ?? '';
+  }
+
+  @override
+  Future<void> auth() async {}
+
+  final List<String> testNames = <String>[];
+
+  int initCalls = 0;
+  int calledWithFlaky = 0;
+  @override
+  Future<void> imgtestInit({ bool isFlaky = false }) async {
+    initCalls += 1;
+    if (isFlaky) {
+      calledWithFlaky += 1;
+    }
+  }
+  @override
+  Future<bool> imgtestAdd(String testName, File goldenFile, { bool isFlaky = false }) async {
+    testNames.add(testName);
+    if (isFlaky) {
+      calledWithFlaky += 1;
+    }
+    return true;
+  }
+
+  int tryInitCalls = 0;
+  @override
+  Future<void> tryjobInit({ bool isFlaky = false }) async {
+    tryInitCalls += 1;
+    if (isFlaky) {
+      calledWithFlaky += 1;
+    }
+  }
+
+  @override
+  Future<bool> tryjobAdd(String testName, File goldenFile, { bool isFlaky = false }) async {
+    if (isFlaky) {
+      calledWithFlaky += 1;
+    }
+    return true;
+  }
+
+  Map<String, List<int>> imageBytesValues = <String, List<int>>{};
+  @override
+  Future<List<int>> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!;
+
+  Map<String, String> cleanTestNameValues = <String, String>{};
+  @override
+  String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? '';
+}
+
+class FakeFlakyLocalFileComparator extends FakeLocalFileComparator with FlakyGoldenMixin {}
+
+class FakeLocalFileComparator extends Fake implements LocalFileComparator {
+  @override
+  late Uri basedir;
+
+  @override
+  Uri getTestUri(Uri key, int? version) => Uri.parse('fake');
+
+  @override
+  @override
+  Future<bool> compare(Uint8List imageBytes, Uri golden) async => true;
+}
+
+class FakeDirectory extends Fake implements Directory {
+  late bool existsSyncValue;
+  @override
+  bool existsSync() => existsSyncValue;
+
+  @override
+  late Uri uri;
+}
+
+class FakeHttpClient extends Fake implements HttpClient {
+  late Uri lastUri;
+  late FakeHttpClientRequest request;
+
+  @override
+  Future<HttpClientRequest> getUrl(Uri url) async {
+    lastUri = url;
+    return request;
+  }
+}
+
+class FakeHttpClientRequest extends Fake implements HttpClientRequest {
+  late FakeHttpImageResponse response;
+
+  @override
+  Future<HttpClientResponse> close() async {
+    return response;
+  }
+}
+
+class FakeHttpImageResponse extends Fake implements HttpClientResponse {
+  FakeHttpImageResponse(this.response);
+
+  final List<List<int>> response;
+
+  @override
+  Future<void> forEach(void Function(List<int> element) action) async {
+    response.forEach(action);
+  }
+}

packages/flutter_goldens_client/lib/skia_client.dart

@@ -8,6 +8,7 @@ import 'dart:io' as io;
 import 'package:crypto/crypto.dart';
 import 'package:file/file.dart';
 import 'package:file/local.dart';
+import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
 import 'package:platform/platform.dart';
 import 'package:process/process.dart';
@@ -138,7 +139,7 @@ class SkiaGoldClient {
   /// The `imgtest` command collects and uploads test results to the Skia Gold
   /// backend, the `init` argument initializes the current test. Used by the
   /// [FlutterPostSubmitFileComparator].
-  Future<void> imgtestInit() async {
+  Future<void> imgtestInit({ bool isFlaky = false }) async {
     // This client has already been initialized
     if (_initialized) {
       return;
@@ -147,7 +148,7 @@ class SkiaGoldClient {
     final File keys = workDirectory.childFile('keys.json');
     final File failures = workDirectory.childFile('failures.json');
 
-    await keys.writeAsString(_getKeysJSON());
+    await keys.writeAsString(getKeysJSON(isFlaky: isFlaky));
     await failures.create();
     final String commitHash = await _getCurrentCommit();
 
@@ -199,7 +200,7 @@ class SkiaGoldClient {
   ///
   /// The [testName] and [goldenFile] parameters reference the current
   /// comparison being evaluated by the [FlutterPostSubmitFileComparator].
-  Future<bool> imgtestAdd(String testName, File goldenFile) async {
+  Future<bool> imgtestAdd(String testName, File goldenFile, { bool isFlaky = false }) async {
     final List<String> imgtestCommand = <String>[
       _goldctl,
       'imgtest', 'add',
@@ -209,7 +210,7 @@ class SkiaGoldClient {
       '--test-name', cleanTestName(testName),
       '--png-file', goldenFile.path,
       '--passfail',
-      ..._getPixelMatchingArguments(),
+      ..._getPixelMatchingArguments(isFlaky: isFlaky),
     ];
 
     final io.ProcessResult result = await process.run(imgtestCommand);
@@ -259,7 +260,7 @@ class SkiaGoldClient {
   /// The `imgtest` command collects and uploads test results to the Skia Gold
   /// backend, the `init` argument initializes the current tryjob. Used by the
   /// [FlutterPreSubmitFileComparator].
-  Future<void> tryjobInit() async {
+  Future<void> tryjobInit({ bool isFlaky = false }) async {
     // This client has already been initialized
     if (_tryjobInitialized) {
       return;
@@ -268,7 +269,7 @@ class SkiaGoldClient {
     final File keys = workDirectory.childFile('keys.json');
     final File failures = workDirectory.childFile('failures.json');
 
-    await keys.writeAsString(_getKeysJSON());
+    await keys.writeAsString(getKeysJSON(isFlaky: isFlaky));
     await failures.create();
     final String commitHash = await _getCurrentCommit();
 
@@ -323,7 +324,7 @@ class SkiaGoldClient {
   ///
   /// The [testName] and [goldenFile] parameters reference the current
   /// comparison being evaluated by the [FlutterPreSubmitFileComparator].
-  Future<void> tryjobAdd(String testName, File goldenFile) async {
+  Future<void> tryjobAdd(String testName, File goldenFile, { bool isFlaky = false}) async {
     final List<String> imgtestCommand = <String>[
       _goldctl,
       'imgtest', 'add',
@@ -332,7 +333,7 @@ class SkiaGoldClient {
         .path,
       '--test-name', cleanTestName(testName),
       '--png-file', goldenFile.path,
-      ..._getPixelMatchingArguments(),
+      ..._getPixelMatchingArguments(isFlaky: isFlaky),
     ];
 
     final io.ProcessResult result = await process.run(imgtestCommand);
@@ -362,6 +363,42 @@ class SkiaGoldClient {
     }
   }
 
+  List<String> _getPixelMatchingArguments({ required bool isFlaky }) {
+    if (isFlaky) {
+      return _getFlakyPixelMatchingArguments();
+    } else {
+      return _getNormalPixelMatchingArguments();
+    }
+  }
+
+  List<String> _getFlakyPixelMatchingArguments() {
+    // The algorithm to be used when matching images. The available options are:
+    // - "fuzzy": Allows for customizing the thresholds of pixel differences.
+    // - "sobel": Same as "fuzzy" but performs edge detection before performing
+    //            a fuzzy match.
+    const String algorithm = 'fuzzy';
+
+    // The number of pixels in this image that are allowed to differ from the
+    // baseline.
+    //
+    // The chosen number - 1 billion - indicates that a flaky test should pass
+    // no matter how many pixels are different from the master golden.
+    const int maxDifferentPixels = 1000 * 1000 * 1000;
+
+    // The maximum acceptable difference per pixel.
+    //
+    // The chosen number - 1020 - is the maximum supported pixel delta and
+    // indicates that a flaky test should pass no matter how far the new pixels
+    // deviate from the master golden.
+    const int pixelDeltaThreshold = 1020;
+
+    return <String>[
+      '--add-test-optional-key', 'image_matching_algorithm:$algorithm',
+      '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels',
+      '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold',
+    ];
+  }
+
   // Constructs arguments for `goldctl` for controlling how pixels are compared.
   //
   // For AOT and CanvasKit exact pixel matching is used. For the HTML renderer
@@ -369,7 +406,7 @@ class SkiaGoldClient {
   // because Chromium cannot exactly reproduce the same golden on all computers.
   // It seems to depend on the hardware/OS/driver combination. However, those
   // differences are very small (typically not noticeable to human eye).
-  List<String> _getPixelMatchingArguments() {
+  List<String> _getNormalPixelMatchingArguments() {
     // Only use fuzzy pixel matching in the HTML renderer.
     if (!_isBrowserTest || _isBrowserCanvasKitTest) {
       return const <String>[];
@@ -485,17 +522,17 @@ class SkiaGoldClient {
   /// Currently, the only key value pairs being tracked is the platform the
   /// image was rendered on, and for web tests, the browser the image was
   /// rendered on.
-  String _getKeysJSON() {
+  @visibleForTesting
+  String getKeysJSON({ bool isFlaky = false}) {
     final Map<String, dynamic> keys = <String, dynamic>{
       'Platform' : platform.operatingSystem,
       'CI' : 'luci',
+      'markedFlaky' : isFlaky.toString(),
     };
     if (_isBrowserTest) {
       keys['Browser'] = _browserKey;
       keys['Platform'] = '${keys['Platform']}-browser';
-      if (_isBrowserCanvasKitTest) {
-        keys['WebRenderer'] = 'canvaskit';
-      }
+      keys['WebRenderer'] = _isBrowserCanvasKitTest ? 'canvaskit' : 'html';
     }
     return json.encode(keys);
   }

packages/flutter_test/lib/src/_goldens_web.dart

@@ -41,12 +41,12 @@ Future<ComparisonResult> compareLists(List<int> test, List<int> master) async {
 ///   * [matchesGoldenFile], the function from [flutter_test] that invokes the
 ///    comparator.
 class DefaultWebGoldenComparator extends WebGoldenComparator {
-  /// Creates a new [DefaultWebGoldenComparator] for the specified [testFile].
+  /// Creates a new [DefaultWebGoldenComparator] for the specified [testUri].
   ///
   /// Golden file keys will be interpreted as file paths relative to the
-  /// directory in which [testFile] resides.
+  /// directory in which [testUri] resides.
   ///
-  /// The [testFile] URL must represent a file.
+  /// The [testUri] URL must represent a file.
   DefaultWebGoldenComparator(this.testUri);
 
   /// The test file currently being executed.

packages/flutter_test/lib/src/goldens.dart

@@ -238,6 +238,14 @@ set webGoldenComparator(WebGoldenComparator value) {
   _webGoldenComparator = value;
 }
 
+/// The URI of the test file currently being executed.
+///
+/// This variable is populated by the Flutter Tool automatically.
+///
+/// Golden file keys will be interpreted as file paths relative to the directory
+/// in which this file resides.
+late Uri webTestUri;
+
 /// Whether golden files should be automatically updated during tests rather
 /// than compared to the image bytes recorded by the tests.
 ///

packages/flutter_tools/lib/src/test/flutter_web_goldens.dart

@@ -98,14 +98,14 @@ class TestGoldenComparator {
     return _processManager.start(command, environment: environment);
   }
 
-  Future<String?> compareGoldens(Uri testUri, Uint8List bytes, Uri goldenKey, bool? updateGoldens) async {
+  Future<String?> compareGoldens(Uri testUri, Uint8List bytes, Uri goldenKey, bool? updateGoldens, Map<String, dynamic>? customProperties) async {
     final File imageFile = await (await tempDir.createTemp('image')).childFile('image').writeAsBytes(bytes);
     final TestGoldenComparatorProcess? process = await _processForTestFile(testUri);
     if (process == null) {
       return 'process was null';
     }
 
-    process.sendCommand(imageFile, goldenKey, updateGoldens);
+    process.sendCommand(imageFile, goldenKey, updateGoldens, customProperties);
 
     final Map<String, dynamic> result = await process.getResponse();
 
@@ -152,11 +152,13 @@ class TestGoldenComparatorProcess {
     await process.exitCode;
   }
 
-  void sendCommand(File imageFile, Uri? goldenKey, bool? updateGoldens) {
+  void sendCommand(File imageFile, Uri? goldenKey, bool? updateGoldens, Map<String, dynamic>? customProperties) {
     final Object command = jsonEncode(<String, dynamic>{
       'imageFile': imageFile.path,
       'key': goldenKey.toString(),
       'update': updateGoldens,
+      if (customProperties != null)
+        'customProperties': customProperties,
     });
     _logger.printTrace('Preparing to send command: $command');
     process.stdin.writeln(command);
@@ -168,7 +170,22 @@ class TestGoldenComparatorProcess {
     return streamIterator.current;
   }
 
+  /// Generates the source code for the comparator process for the test file.
+  ///
+  /// If a test configuation exists for the tested package, uses its
+  /// implementation. Otherwise, uses the default implementation.
   static String generateBootstrap(File testFile, Uri testUri, {required Logger logger}) {
+    final File? webTestConfigFile = findWebTestConfigFile(testFile, logger);
+    if (webTestConfigFile != null) {
+      return _generateBootstrapWithWebTestConfig(webTestConfigFile, testFile, testUri);
+    } else {
+      return _generateBasicBootstrap(testFile, testUri, logger: logger);
+    }
+  }
+
+  // Generates the bootstrap used by tests that either don't have a test
+  // configuration file, or don't have a `flutter_web_test_config.dart`.
+  static String _generateBasicBootstrap(File testFile, Uri testUri, {required Logger logger}) {
     final File? testConfigFile = findTestConfigFile(testFile, logger);
     // Generate comparator process for the file.
     return '''
@@ -211,6 +228,19 @@ void main() async {
     }
   }
   ${testConfigFile != null ? '});' : ''}
+}
+    ''';
+  }
+
+  // Generates the bootstrap used by tests that have a `flutter_web_test_config.dart`.
+  static String _generateBootstrapWithWebTestConfig(File webTestConfigFile, File testFile, Uri testUri) {
+    return '''
+import 'package:flutter_test/flutter_test.dart';
+import '${Uri.file(webTestConfigFile.path)}' as web_test_config;
+void main() async {
+  final String testUri = '$testUri';
+  goldenFileComparator = LocalFileComparator(Uri.parse(testUri));
+  await web_test_config.startWebTestHostConfiguration(testUri);
 }
     ''';
   }

packages/flutter_tools/lib/src/test/flutter_web_platform.dart

@@ -343,10 +343,9 @@ class FlutterWebPlatform extends PlatformPlugin {
   }
 
   Future<shelf.Response> _goldenFileHandler(shelf.Request request) async {
-    if (request.url.path.contains('flutter_goldens')) {
-      final Map<String, Object?> body = json.decode(await request.readAsString()) as Map<String, Object?>;
-      final Uri goldenKey = Uri.parse(body['key']! as String);
-      final Uri testUri = Uri.parse(body['testUri']! as String);
+    if (request.method == 'POST' && request.url.path.contains('flutter_goldens')) {
+      final String requestJson = await request.readAsString();
+      final Map<String, Object?> body = json.decode(requestJson) as Map<String, Object?>;
       final num width = body['width']! as num;
       final num height = body['height']! as num;
       Uint8List bytes;
@@ -383,7 +382,10 @@ class FlutterWebPlatform extends PlatformPlugin {
         return shelf.Response.ok('Unknown error, bytes is null');
       }
 
-      final String? errorMessage = await _testGoldenComparator.compareGoldens(testUri, bytes, goldenKey, updateGoldens);
+      final Uri goldenKey = Uri.parse(body['key']! as String);
+      final Uri testUri = Uri.parse(body['testUri']! as String);
+      final Map<String, dynamic>? customProperties = body['customProperties'] as Map<String, dynamic>?;
+      final String? errorMessage = await _testGoldenComparator.compareGoldens(testUri, bytes, goldenKey, updateGoldens, customProperties);
       return shelf.Response.ok(errorMessage ?? 'true');
     } else {
       return shelf.Response.notFound('Not Found');

packages/flutter_tools/lib/src/test/test_config.dart

@@ -9,23 +9,36 @@ import '../base/logger.dart';
 /// test harness if it exists in the project directory hierarchy.
 const String _kTestConfigFileName = 'flutter_test_config.dart';
 
+/// The name of the web test configuration file that will be discovered by the
+/// test harness if it exists in the project directory hierarchy.
+const String _kWebTestConfigFileName = 'flutter_web_test_config.dart';
+
 /// The name of the file that signals the root of the project and that will
 /// cause the test harness to stop scanning for configuration files.
 const String _kProjectRootSentinel = 'pubspec.yaml';
 
 /// Find the `flutter_test_config.dart` file for a specific test file.
 File? findTestConfigFile(File testFile, Logger logger) {
+  return _findConfigFile(testFile, _kTestConfigFileName, logger);
+}
+
+/// Find the `flutter_web_test_config.dart` file for a specific test file.
+File? findWebTestConfigFile(File testFile, Logger logger) {
+  return _findConfigFile(testFile, _kWebTestConfigFileName, logger);
+}
+
+File? _findConfigFile(File testFile, String configFileName, Logger logger) {
   File? testConfigFile;
   Directory directory = testFile.parent;
   while (directory.path != directory.parent.path) {
-    final File configFile = directory.childFile(_kTestConfigFileName);
+    final File configFile = directory.childFile(configFileName);
     if (configFile.existsSync()) {
-      logger.printTrace('Discovered $_kTestConfigFileName in ${directory.path}');
+      logger.printTrace('Discovered $configFileName in ${directory.path}');
       testConfigFile = configFile;
       break;
     }
     if (directory.childFile(_kProjectRootSentinel).existsSync()) {
-      logger.printTrace('Stopping scan for $_kTestConfigFileName; '
+      logger.printTrace('Stopping scan for $configFileName; '
           'found project root at ${directory.path}');
       break;
     }

packages/flutter_tools/lib/src/web/bootstrap.dart

@@ -224,7 +224,8 @@ String generateTestEntrypoint({
   Future<void> main() async {
     ui.debugEmulateFlutterTesterEnvironment = true;
     await ui.webOnlyInitializePlatform();
-    webGoldenComparator = DefaultWebGoldenComparator(Uri.parse('${Uri.file(absolutePath)}'));
+    webTestUri = Uri.parse('${Uri.file(absolutePath)}');
+    webGoldenComparator = DefaultWebGoldenComparator(webTestUri);
     (ui.window as dynamic).debugOverrideDevicePixelRatio(3.0);
     (ui.window as dynamic).webOnlyDebugPhysicalSizeOverride = const ui.Size(2400, 1800);
 

packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart

@@ -41,13 +41,13 @@ void main() {
       final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink;
 
       final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess, logger: BufferLogger.test());
-      process.sendCommand(imageFile, goldenKey, false);
+      process.sendCommand(imageFile, goldenKey, false, <String, String>{'additional data' : 'data'});
 
       final Map<String, dynamic> response = await process.getResponse();
       final String stringToStdin = ioSink.getAndClear();
 
       expect(response, expectedResponse);
-      expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n');
+      expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false,"customProperties":{"additional data":"data"}}\n');
     });
 
     testWithoutContext('can handle multiple requests', () async {
@@ -64,18 +64,21 @@ void main() {
       final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink;
 
       final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess, logger: BufferLogger.test());
-      process.sendCommand(imageFile, goldenKey, false);
+      process.sendCommand(imageFile, goldenKey, false, null);
 
       final Map<String, dynamic> response1 = await process.getResponse();
 
-      process.sendCommand(imageFile2, goldenKey2, true);
+      process.sendCommand(imageFile2, goldenKey2, true, null);
 
       final Map<String, dynamic> response2 = await process.getResponse();
       final String stringToStdin = ioSink.getAndClear();
 
       expect(response1, expectedResponse1);
       expect(response2, expectedResponse2);
-      expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n{"imageFile":"second_test_image_file","key":"file://second_golden_key/","update":true}\n');
+      expect(
+        stringToStdin,
+        '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n'
+        '{"imageFile":"second_test_image_file","key":"file://second_golden_key/","update":true}\n');
     });
 
     testWithoutContext('ignores anything that does not look like JSON', () async {
@@ -94,7 +97,7 @@ Other JSON data after the initial data
       final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink;
 
       final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess,logger: BufferLogger.test());
-      process.sendCommand(imageFile, goldenKey, false);
+      process.sendCommand(imageFile, goldenKey, false, null);
 
       final Map<String, dynamic> response = await process.getResponse();
       final String stringToStdin = ioSink.getAndClear();

packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart

@@ -61,7 +61,7 @@ void main() {
         webRenderer: WebRendererMode.html,
       );
 
-      final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null);
       expect(result, null);
     });
 
@@ -90,7 +90,7 @@ void main() {
         webRenderer: WebRendererMode.canvaskit,
       );
 
-      final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null);
       expect(result, 'some message');
     });
 
@@ -123,10 +123,10 @@ void main() {
         webRenderer: WebRendererMode.html,
       );
 
-      final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null);
       expect(result1, 'some message');
 
-      final String? result2 = await comparator.compareGoldens(testUri, imageBytes, goldenKey2, false);
+      final String? result2 = await comparator.compareGoldens(testUri, imageBytes, goldenKey2, false, null);
       expect(result2, 'some other message');
     });
 
@@ -168,10 +168,10 @@ void main() {
         webRenderer: WebRendererMode.canvaskit,
       );
 
-      final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null);
       expect(result1, 'some message');
 
-      final String? result2 = await comparator.compareGoldens(testUri2, imageBytes, goldenKey2, false);
+      final String? result2 = await comparator.compareGoldens(testUri2, imageBytes, goldenKey2, false, null);
       expect(result2, 'some other message');
     });
 
@@ -203,7 +203,7 @@ void main() {
         webRenderer: WebRendererMode.html,
       );
 
-      final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null);
       expect(result, null);
 
       await comparator.close();

packages/flutter_tools/test/integration.shard/forbidden_imports_test.dart

@@ -117,7 +117,7 @@ void main() {
     final List<String> allowedPaths = <String>[
       fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_platform.dart'),
       fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_web_platform.dart'),
-      fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'test_wrapper.dart'),
+      fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_goldens.dart'),
     ];
     bool isNotAllowed(FileSystemEntity entity) => allowedPaths.every((String path) => path != entity.path);