Fix screenshot testing for flutter web integration_test (#117114)

* Fix screenshot testing for flutter web integration_test

* update packages

* fix method signature and todo

* Run tests on CI

* fix type

* remove silences

* Add docs

* fix comment

* fix whitespace

* review comments

dev/bots/test.dart

@@ -1138,7 +1138,7 @@ Future<void> _runWebUnitTests(String webRenderer) async {
 /// Coarse-grained integration tests running on the Web.
 Future<void> _runWebLongRunningTests() async {
   final List<ShardRunner> tests = <ShardRunner>[
-    for (String buildMode in _kAllBuildModes)
+    for (String buildMode in _kAllBuildModes) ...<ShardRunner>[
       () => _runFlutterDriverWebTest(
         testAppDirectory: path.join('packages', 'integration_test', 'example'),
         target: path.join('test_driver', 'failure.dart'),
@@ -1148,6 +1148,21 @@ Future<void> _runWebLongRunningTests() async {
         // logs. To avoid confusion, silence browser output.
         silenceBrowserOutput: true,
       ),
+      () => _runFlutterDriverWebTest(
+        testAppDirectory: path.join('packages', 'integration_test', 'example'),
+        target: path.join('integration_test', 'example_test.dart'),
+        driver: path.join('test_driver', 'integration_test.dart'),
+        buildMode: buildMode,
+        renderer: 'canvaskit',
+      ),
+      () => _runFlutterDriverWebTest(
+        testAppDirectory: path.join('packages', 'integration_test', 'example'),
+        target: path.join('integration_test', 'extended_test.dart'),
+        driver: path.join('test_driver', 'extended_integration_test.dart'),
+        buildMode: buildMode,
+        renderer: 'canvaskit',
+      ),
+    ],
 
     // This test specifically tests how images are loaded in HTML mode, so we don't run it in CanvasKit mode.
     () => _runWebE2eTest('image_loading_integration', buildMode: 'debug', renderer: 'html'),
@@ -1281,6 +1296,7 @@ Future<void> _runFlutterDriverWebTest({
   required String buildMode,
   required String renderer,
   required String testAppDirectory,
+  String? driver,
   bool expectFailure = false,
   bool silenceBrowserOutput = false,
 }) async {
@@ -1295,6 +1311,7 @@ Future<void> _runFlutterDriverWebTest({
     <String>[
       ...flutterTestArgs,
       'drive',
+      if (driver != null) '--driver=$driver',
       '--target=$target',
       '--browser-name=chrome',
       '--no-sound-null-safety',

dev/integration_tests/web_e2e_tests/lib/screenshot_support.dart

@@ -25,7 +25,7 @@ Future<void> runTestWithScreenshots({
 
   test.integrationDriver(
     driver: driver,
-    onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
+    onScreenshot: (String screenshotName, List<int> screenshotBytes, [Map<String, Object?>? args]) async {
       // TODO(yjbanov): implement, see https://github.com/flutter/flutter/issues/86120
       return true;
     },

packages/integration_test/example/integration_test/_extended_test_web.dart

@@ -17,7 +17,8 @@ import 'package:integration_test/integration_test.dart';
 import 'package:integration_test_example/main.dart' as app;
 
 void main() {
-  final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+  final IntegrationTestWidgetsFlutterBinding binding =
+      IntegrationTestWidgetsFlutterBinding.ensureInitialized();
 
   testWidgets('verify text', (WidgetTester tester) async {
     // Build our app and trigger a frame.
@@ -27,7 +28,18 @@ void main() {
     await tester.pumpAndSettle();
 
     // Take a screenshot.
-    await binding.takeScreenshot('platform_name');
+    await binding.takeScreenshot(
+      'platform_name',
+      // The optional parameter 'args' can be used to pass values to the
+      // [integrationDriver.onScreenshot] handler
+      // (see test_driver/extended_integration_test.dart). For example, you
+      // could look up environment variables in this test that were passed to
+      // the run command via `--dart-define=`, and then pass the values to the
+      // [integrationDriver.onScreenshot] handler through this 'args' map.
+      <String, Object?>{
+        'someArgumentKey': 'someArgumentValue',
+      },
+    );
 
     // Verify that platform is retrieved.
     expect(
@@ -49,7 +61,20 @@ void main() {
     await tester.pumpAndSettle();
 
     // Multiple methods can take screenshots. Screenshots are taken with the
-    // same order the methods run.
-    await binding.takeScreenshot('platform_name_2');
+    // same order the methods run.  We pass an argument that can be looked up
+    // from the [onScreenshot] handler in
+    // [test_driver/extended_integration_test.dart].
+    await binding.takeScreenshot(
+      'platform_name_2',
+      // The optional parameter 'args' can be used to pass values to the
+      // [integrationDriver.onScreenshot] handler
+      // (see test_driver/extended_integration_test.dart). For example, you
+      // could look up environment variables in this test that were passed to
+      // the run command via `--dart-define=`, and then pass the values to the
+      // [integrationDriver.onScreenshot] handler through this 'args' map.
+      <String, Object?>{
+        'someArgumentKey': 'someArgumentValue',
+      },
+    );
   });
 }

packages/integration_test/example/test_driver/extended_integration_test.dart

@@ -9,8 +9,20 @@ Future<void> main() async {
   final FlutterDriver driver = await FlutterDriver.connect();
   await integrationDriver(
     driver: driver,
-    onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
+    onScreenshot: (
+      String screenshotName,
+      List<int> screenshotBytes, [
+      Map<String, Object?>? args,
+    ]) async {
       // Return false if the screenshot is invalid.
+      // TODO(yjbanov): implement, see https://github.com/flutter/flutter/issues/86120
+
+      // Here is an example of using an argument that was passed in via the
+      // optional 'args' Map.
+      if (args != null) {
+        final String? someArgumentValue = args['someArgumentKey'] as String?;
+        return someArgumentValue != null;
+      }
       return true;
     },
   );

packages/integration_test/lib/_callback_io.dart

@@ -86,7 +86,8 @@ class IOCallbackManager implements CallbackManager {
   }
 
   @override
-  Future<Map<String, dynamic>> takeScreenshot(String screenshot) async {
+  Future<Map<String, dynamic>> takeScreenshot(String screenshot, [Map<String, Object?>? args]) async {
+    assert(args == null, '[args] handling has not been implemented for this platform');
     if (Platform.isAndroid && !_isSurfaceRendered) {
       throw StateError('Call convertFlutterSurfaceToImage() before taking a screenshot');
     }

packages/integration_test/lib/_callback_web.dart

@@ -44,10 +44,13 @@ class WebCallbackManager implements CallbackManager {
   ///
   /// See: https://www.w3.org/TR/webdriver/#screen-capture.
   @override
-  Future<Map<String, dynamic>> takeScreenshot(String screenshotName) async {
-    await _sendWebDriverCommand(WebDriverCommand.screenshot(screenshotName));
-    // Flutter Web doesn't provide the bytes.
-    return const <String, dynamic>{'bytes': <int>[]};
+  Future<Map<String, dynamic>> takeScreenshot(String screenshotName, [Map<String, Object?>? args]) async {
+    await _sendWebDriverCommand(WebDriverCommand.screenshot(screenshotName, args));
+    return <String, dynamic>{
+      'screenshotName': screenshotName,
+      // Flutter Web doesn't provide the bytes.
+      'bytes': <int>[]
+    };
   }
 
   @override

packages/integration_test/lib/common.dart

@@ -7,17 +7,23 @@ import 'dart:convert';
 
 /// A callback to use with [integrationDriver].
 ///
-/// The callback receives the name of screenshot passed to `binding.takeScreenshot(<name>)` and
-/// a PNG byte buffer.
+/// The callback receives the name of screenshot passed to `binding.takeScreenshot(<name>)`,
+/// a PNG byte buffer representing the screenshot, and an optional `Map` of arguments.
 ///
 /// The callback returns `true` if the test passes or `false` otherwise.
 ///
 /// You can use this callback to store the bytes locally in a file or upload them to a service
 /// that compares the image against a gold or baseline version.
 ///
+/// The optional `Map` of arguments can be passed from the
+/// `binding.takeScreenshot(<name>, <args>)` callsite in the integration test,
+/// and then the arguments can be used in the `onScreenshot` handler that is defined by
+/// the Flutter driver. This `Map` should only contain values that can be serialized
+/// to JSON.
+///
 /// Since the function is executed on the host driving the test, you can access any environment
 /// variable from it.
-typedef ScreenshotCallback = Future<bool> Function(String name, List<int> image);
+typedef ScreenshotCallback = Future<bool> Function(String name, List<int> image, [Map<String, Object?>? args]);
 
 /// Classes shared between `integration_test.dart` and `flutter drive` based
 /// adoptor (ex: `integration_test_driver.dart`).
@@ -247,9 +253,12 @@ class WebDriverCommand {
         values = <String, dynamic>{};
 
   /// Constructor for [WebDriverCommandType.noop] screenshot.
-  WebDriverCommand.screenshot(String screenshotName)
+  WebDriverCommand.screenshot(String screenshotName, [Map<String, Object?>? args])
       : type = WebDriverCommandType.screenshot,
-        values = <String, dynamic>{'screenshot_name': screenshotName};
+        values = <String, dynamic>{
+          'screenshot_name': screenshotName,
+          if (args != null) 'args': args,
+        };
 
   /// Type of the [WebDriverCommand].
   ///
@@ -286,7 +295,7 @@ abstract class CallbackManager {
 
   /// Takes a screenshot of the application.
   /// Returns the data that is sent back to the host.
-   Future<Map<String, dynamic>> takeScreenshot(String screenshot);
+   Future<Map<String, dynamic>> takeScreenshot(String screenshot, [Map<String, Object?>? args]);
 
   /// Android only. Converts the Flutter surface to an image view.
   Future<void> convertFlutterSurfaceToImage();

packages/integration_test/lib/integration_test.dart

@@ -184,10 +184,10 @@ https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab
   ///
   /// On Android, you need to call `convertFlutterSurfaceToImage()`, and
   /// pump a frame before taking a screenshot.
-  Future<List<int>> takeScreenshot(String screenshotName) async {
+  Future<List<int>> takeScreenshot(String screenshotName, [Map<String, Object?>? args]) async {
     reportData ??= <String, dynamic>{};
     reportData!['screenshots'] ??= <dynamic>[];
-    final Map<String, dynamic> data = await callbackManager.takeScreenshot(screenshotName);
+    final Map<String, dynamic> data = await callbackManager.takeScreenshot(screenshotName, args);
     assert(data.containsKey('bytes'));
 
     (reportData!['screenshots']! as List<dynamic>).add(data);

packages/integration_test/lib/integration_test_driver_extended.dart

@@ -50,6 +50,8 @@ Future<void> integrationDriver(
   // error if it's used as a message for requestData.
   String jsonResponse = await driver.requestData(DriverTestMessage.pending().toString());
 
+  final Map<String, bool> onScreenshotResults = <String, bool>{};
+
   Response response = Response.fromJson(jsonResponse);
 
   // Until `integration_test` returns a [WebDriverCommandType.noop], keep
@@ -63,8 +65,10 @@ Future<void> integrationDriver(
       // Use `driver.screenshot()` method to get a screenshot of the web page.
       final List<int> screenshotImage = await driver.screenshot();
       final String screenshotName = response.data!['screenshot_name']! as String;
+      final Map<String, Object?>? args = (response.data!['args'] as Map<String, Object?>?)?.cast<String, Object?>();
 
-      final bool screenshotSuccess = await onScreenshot!(screenshotName, screenshotImage);
+      final bool screenshotSuccess = await onScreenshot!(screenshotName, screenshotImage, args);
+      onScreenshotResults[screenshotName] = screenshotSuccess;
       if (screenshotSuccess) {
         jsonResponse = await driver.requestData(DriverTestMessage.complete().toString());
       } else {
@@ -104,7 +108,8 @@ Future<void> integrationDriver(
 
       bool ok = false;
       try {
-        ok = await onScreenshot(screenshotName, screenshotBytes.cast<int>());
+        ok = onScreenshotResults[screenshotName] ??
+            await onScreenshot(screenshotName, screenshotBytes.cast<int>());
       } catch (exception) {
         throw StateError(
           'Screenshot failure:\n'