Implement screenshot test for flutter web. (#45530)

.cirrus.yml

@@ -153,7 +153,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-1-linux
@@ -162,7 +165,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-2-linux
@@ -171,7 +177,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-3-linux
@@ -180,7 +189,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-4-linux
@@ -189,7 +201,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-5-linux
@@ -198,7 +213,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-6-linux
@@ -207,7 +225,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: web_tests-7_last-linux # last Web shard must end with _last
@@ -216,7 +237,10 @@ task:
         # As of October 2019, the Web shards needed more than 6G of RAM.
         CPU: 2
         MEMORY: 8G
+        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
+        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
       script:
+        - ./dev/bots/download_goldctl.sh
         - dart --enable-asserts ./dev/bots/test.dart
 
     - name: build_tests-linux

dev/bots/test.dart

@@ -68,7 +68,6 @@ const List<String> kWebTestFileBlacklist = <String>[
   'test/widgets/selectable_text_test.dart',
   'test/widgets/color_filter_test.dart',
   'test/widgets/editable_text_cursor_test.dart',
-  'test/widgets/shadow_test.dart',
   'test/widgets/raw_keyboard_listener_test.dart',
   'test/widgets/editable_text_test.dart',
   'test/widgets/widget_inspector_test.dart',

packages/flutter/test/widgets/shadow_test.dart

@@ -33,7 +33,7 @@ void main() {
       matchesGoldenFile('shadow.BoxDecoration.enabled.png'),
     );
     debugDisableShadows = true;
-  }, skip: isBrowser);
+  });
 
   testWidgets('Shadows on ShapeDecoration', (WidgetTester tester) async {
     debugDisableShadows = false;
@@ -93,7 +93,7 @@ void main() {
       matchesGoldenFile('shadow.PhysicalModel.enabled.png'),
     );
     debugDisableShadows = true;
-  }, skip: isBrowser);
+  });
 
   testWidgets('Shadows with PhysicalShape', (WidgetTester tester) async {
     debugDisableShadows = false;

packages/flutter_goldens_client/lib/skia_client.dart

@@ -19,6 +19,7 @@ import 'package:process/process.dart';
 const String _kFlutterRootKey = 'FLUTTER_ROOT';
 const String _kGoldctlKey = 'GOLDCTL';
 const String _kServiceAccountKey = 'GOLD_SERVICE_ACCOUNT';
+const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';
 
 /// A client for uploading image tests and making baseline requests to the
 /// Flutter Gold Dashboard.
@@ -408,14 +409,16 @@ class SkiaGoldClient {
   /// Returns a JSON String with keys value pairs used to uniquely identify the
   /// configuration that generated the given golden file.
   ///
-  /// Currently, the only key value pair being tracked is the platform the image
-  /// was rendered on.
+  /// Currently, the only key value pairs being tracked is the platform the
+  /// image was rendered on, and for web tests, the browser the image was
+  /// rendered on.
   String _getKeysJSON() {
-    return json.encode(
-      <String, dynamic>{
+    final Map<String, dynamic> keys = <String, dynamic>{
       'Platform' : platform.operatingSystem,
-      }
-    );
+    };
+    if (platform.environment[_kTestBrowserKey] != null)
+      keys['Browser'] = platform.environment[_kTestBrowserKey];
+    return json.encode(keys);
   }
 
   /// Removes the file extension from the [fileName] to represent the test name
@@ -455,7 +458,7 @@ class SkiaGoldDigest {
     return SkiaGoldDigest(
       imageHash: json['digest'] as String,
       paramSet: Map<String, dynamic>.from(json['paramset'] as Map<String, dynamic> ??
-        <String, String>{'Platform': 'none'}),
+        <String, List<String>>{'Platform': <String>[]}),
       testName: json['test'] as String,
       status: json['status'] as String,
     );
@@ -477,6 +480,8 @@ class SkiaGoldDigest {
   bool isValid(Platform platform, String name, String expectation) {
     return imageHash == expectation
       && (paramSet['Platform'] as List<dynamic>).contains(platform.operatingSystem)
+      && (platform.environment[_kTestBrowserKey] == null
+         || paramSet['Browser'] == platform.environment[_kTestBrowserKey])
       && testName == name
       && status == 'positive';
     }

packages/flutter_test/lib/src/_goldens_io.dart

@@ -6,7 +6,9 @@ import 'dart:async';
 import 'dart:io';
 import 'dart:math' as math;
 import 'dart:typed_data';
+import 'dart:ui';
 
+import 'package:flutter/widgets.dart' show Element;
 import 'package:image/image.dart';
 import 'package:path/path.dart' as path;
 // ignore: deprecated_member_use
@@ -240,3 +242,16 @@ ComparisonResult compareLists(List<int> test, List<int> master) {
   }
   return ComparisonResult(passed: true);
 }
+
+/// An unsupported [WebGoldenComparator] that exists for API compatibility.
+class DefaultWebGoldenComparator extends WebGoldenComparator {
+  @override
+  Future<bool> compare(Element element, Size size, Uri golden) {
+    throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
+  }
+
+  @override
+  Future<void> update(Uri golden, Element element, Size size) {
+    throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
+  }
+}

packages/flutter_test/lib/src/_goldens_web.dart

@@ -2,11 +2,19 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
- import 'dart:typed_data';
+import 'dart:convert';
+import 'dart:html' as html;
+import 'dart:typed_data';
+import 'dart:ui';
 
- import 'goldens.dart';
+import 'package:flutter/rendering.dart';
+import 'package:flutter/widgets.dart';
+// ignore: deprecated_member_use
+import 'package:test_api/test_api.dart' as test_package show TestFailure;
 
- /// An unsupported [GoldenFileComparator] that exists for API compatibility.
+import 'goldens.dart';
+
+/// An unsupported [GoldenFileComparator] that exists for API compatibility.
 class LocalFileComparator extends GoldenFileComparator {
   @override
   Future<bool> compare(Uint8List imageBytes, Uri golden) {
@@ -19,10 +27,63 @@ class LocalFileComparator extends GoldenFileComparator {
   }
 }
 
- /// Returns whether [test] and [master] are pixel by pixel identical.
+/// Returns whether [test] and [master] are pixel by pixel identical.
 ///
 /// This method is not supported on the web and throws an [UnsupportedError]
 /// when called.
 ComparisonResult compareLists(List<int> test, List<int> master) {
   throw UnsupportedError('Golden testing is not supported on the web.');
 }
+
+/// The default [WebGoldenComparator] implementation for `flutter test`.
+///
+/// This comparator will send a request to the test server for golden comparison
+/// which will then defer the comparison to [goldenFileComparator].
+///
+/// See also:
+///
+///   * [matchesGoldenFile], the function from [flutter_test] that invokes the
+///    comparator.
+class DefaultWebGoldenComparator extends WebGoldenComparator {
+  /// Creates a new [DefaultWebGoldenComparator] for the specified [testFile].
+  ///
+  /// Golden file keys will be interpreted as file paths relative to the
+  /// directory in which [testFile] resides.
+  ///
+  /// The [testFile] URL must represent a file.
+  DefaultWebGoldenComparator(this.testUri);
+
+  /// The test file currently being executed.
+  ///
+  /// Golden file keys will be interpreted as file paths relative to the
+  /// directory in which this file resides.
+  Uri testUri;
+
+  @override
+  Future<bool> compare(Element element, Size size, Uri golden) async {
+    final String key = golden.toString();
+
+    final html.HttpRequest request = await html.HttpRequest.request(
+      'flutter_goldens',
+      method: 'POST',
+      sendData: json.encode(<String, Object>{
+        'testUri': testUri.toString(),
+        'key': key.toString(),
+        'width': size.width.round(),
+        'height': size.height.round(),
+      }),
+    );
+    final String response = request.response as String;
+    if (response == 'true') {
+      return true;
+    } else {
+      throw test_package.TestFailure(response);
+    }
+  }
+
+  @override
+  Future<void> update(Uri golden, Element element, Size size) async {
+    // Update is handled on the server side, just use the same logic here
+    await compare(element, size, golden);
+  }
+}

packages/flutter_test/lib/src/_matchers_io.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:typed_data';
+import 'dart:ui' as ui;
+
+import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
+import 'package:flutter/services.dart';
+import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports
+// ignore: deprecated_member_use
+import 'package:test_api/test_api.dart' hide TypeMatcher, isInstanceOf;
+
+import 'binding.dart';
+import 'finders.dart';
+import 'goldens.dart';
+
+/// Render the closest [RepaintBoundary] of the [element] into an image.
+///
+/// See also:
+///
+///  * [OffsetLayer.toImage] which is the actual method being called.
+Future<ui.Image> captureImage(Element element) {
+  RenderObject renderObject = element.renderObject;
+  while (!renderObject.isRepaintBoundary) {
+    renderObject = renderObject.parent as RenderObject;
+    assert(renderObject != null);
+  }
+  assert(!renderObject.debugNeedsPaint);
+  final OffsetLayer layer = renderObject.debugLayer as OffsetLayer;
+  return layer.toImage(renderObject.paintBounds);
+}
+
+/// The matcher created by [matchesGoldenFile]. This class is enabled when the
+/// test is running on a VM using conditional import.
+class MatchesGoldenFile extends AsyncMatcher {
+  /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile].
+  const MatchesGoldenFile(this.key, this.version);
+
+  /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile].
+  MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path);
+
+  /// The [key] to the golden image.
+  final Uri key;
+
+  /// The [version] of the golden image.
+  final int version;
+
+  @override
+  Future<String> matchAsync(dynamic item) async {
+    Future<ui.Image> imageFuture;
+    if (item is Future<ui.Image>) {
+      imageFuture = item;
+    } else if (item is ui.Image) {
+      imageFuture = Future<ui.Image>.value(item);
+    } else {
+      final Finder finder = item as Finder;
+      final Iterable<Element> elements = finder.evaluate();
+      if (elements.isEmpty) {
+        return 'could not be rendered because no widget was found';
+      } else if (elements.length > 1) {
+        return 'matched too many widgets';
+      }
+      imageFuture = captureImage(elements.single);
+    }
+
+    final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
+
+    final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding;
+    return binding.runAsync<String>(() async {
+      final ui.Image image = await imageFuture;
+      final ByteData bytes = await image.toByteData(format: ui.ImageByteFormat.png);
+      if (bytes == null)
+        return 'could not encode screenshot.';
+      if (autoUpdateGoldenFiles) {
+        await goldenFileComparator.update(testNameUri, bytes.buffer.asUint8List());
+        return null;
+      }
+      try {
+        final bool success = await goldenFileComparator.compare(bytes.buffer.asUint8List(), testNameUri);
+        return success ? null : 'does not match';
+      } on TestFailure catch (ex) {
+        return ex.message;
+      }
+    }, additionalTime: const Duration(minutes: 1));
+  }
+
+  @override
+  Description describe(Description description) {
+    final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
+    return description.add('one widget whose rasterized image matches golden image "$testNameUri"');
+  }
+}

packages/flutter_test/lib/src/_matchers_web.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:ui' as ui;
+
+import 'package:flutter/rendering.dart';
+import 'package:flutter/widgets.dart';
+import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports
+// ignore: deprecated_member_use
+import 'package:test_api/test_api.dart' hide TypeMatcher, isInstanceOf;
+
+import 'binding.dart';
+import 'finders.dart';
+import 'goldens.dart';
+
+/// An unsupported method that exists for API compatibility.
+Future<ui.Image> captureImage(Element element) {
+  throw UnsupportedError('captureImage is not supported on the web.');
+}
+
+/// The matcher created by [matchesGoldenFile]. This class is enabled when the
+/// test is running in a web browser using conditional import.
+class MatchesGoldenFile extends AsyncMatcher {
+  /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile].
+  const MatchesGoldenFile(this.key, this.version);
+
+  /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile].
+  MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path);
+
+  /// The [key] to the golden image.
+  final Uri key;
+
+  /// The [version] of the golden image.
+  final int version;
+
+  @override
+  Future<String> matchAsync(dynamic item) async {
+    if (item is! Finder) {
+      return 'web goldens only supports matching finders.';
+    }
+    final Finder finder = item as Finder;
+    final Iterable<Element> elements = finder.evaluate();
+    if (elements.isEmpty) {
+      return 'could not be rendered because no widget was found';
+    } else if (elements.length > 1) {
+      return 'matched too many widgets';
+    }
+    final Element element = elements.single;
+    final RenderObject renderObject = _findRepaintBoundary(element);
+    final Size size = renderObject.paintBounds.size;
+    final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding;
+    final Element e = binding.renderViewElement;
+
+    // Unlike `flutter_tester`, we don't have the ability to render an element
+    // to an image directly. Instead, we will use `window.render()` to render
+    // only the element being requested, and send a request to the test server
+    // requesting it to take a screenshot through the browser's debug interface.
+    _renderElement(binding.window, renderObject);
+    final String result = await binding.runAsync<String>(() async {
+      if (autoUpdateGoldenFiles) {
+        await webGoldenComparator.update(key, element, size);
+        return null;
+      }
+      try {
+        final bool success = await webGoldenComparator.compare(element, size, key);
+        return success ? null : 'does not match';
+      } on TestFailure catch (ex) {
+        return ex.message;
+      }
+    }, additionalTime: const Duration(seconds: 11));
+    _renderElement(binding.window, _findRepaintBoundary(e));
+    return result;
+  }
+
+  @override
+  Description describe(Description description) {
+    final Uri testNameUri = webGoldenComparator.getTestUri(key, version);
+    return description.add('one widget whose rasterized image matches golden image "$testNameUri"');
+  }
+}
+
+RenderObject _findRepaintBoundary(Element element) {
+  RenderObject renderObject = element.renderObject;
+  while (!renderObject.isRepaintBoundary) {
+    renderObject = renderObject.parent as RenderObject;
+    assert(renderObject != null);
+  }
+  return renderObject;
+}
+
+void _renderElement(ui.Window window, RenderObject renderObject) {
+  final Layer layer = renderObject.debugLayer;
+  final ui.SceneBuilder sceneBuilder = ui.SceneBuilder();
+  if (layer is OffsetLayer) {
+    sceneBuilder.pushOffset(-layer.offset.dx, -layer.offset.dy);
+  }
+  // ignore: invalid_use_of_visible_for_testing_member, invalid_use_of_protected_member
+  layer.updateSubtreeNeedsAddToScene();
+  // ignore: invalid_use_of_protected_member
+  layer.addToScene(sceneBuilder);
+  sceneBuilder.pop();
+  window.render(sceneBuilder.build());
+}

packages/flutter_test/lib/src/goldens.dart

@@ -6,6 +6,7 @@ import 'dart:async';
 import 'dart:typed_data';
 
 import 'package:flutter/foundation.dart';
+import 'package:flutter/widgets.dart';
 import 'package:path/path.dart' as path;
 import '_goldens_io.dart' if (dart.library.html) '_goldens_web.dart' as _goldens;
 
@@ -134,6 +135,115 @@ set goldenFileComparator(GoldenFileComparator value) {
   _goldenFileComparator = value;
 }
 
+/// Compares image pixels against a golden image file.
+///
+/// Instances of this comparator will be used as the backend for
+/// [matchesGoldenFile] when tests are running on Flutter Web, and will usually
+/// implemented by deferring the screenshot taking and image comparison to a
+/// test server.
+///
+/// Instances of this comparator will be invoked by the test framework in the
+/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the
+/// fake async constraints that are normally imposed on widget tests (i.e. the
+/// need or the ability to call [WidgetTester.pump] to advance the microtask
+/// queue). Prior to the invocation, the test framework will render only the
+/// [Element] to be compared on the screen.
+///
+/// See also:
+///
+///  * [GoldenFileComparator] for the comparator to be used when the test is
+///    not running in a web browser.
+///  * [DefaultWebGoldenComparator] for the default [WebGoldenComparator]
+///    implementation for `flutter test`.
+///  * [matchesGoldenFile], the function from [flutter_test] that invokes the
+///    comparator.
+abstract class WebGoldenComparator {
+  /// Compares the rendered pixels of [element] of size [size] that is being
+  /// rendered on the top left of the screen against the golden file identified
+  /// by [golden].
+  ///
+  /// The returned future completes with a boolean value that indicates whether
+  /// the pixels rendered on screen match the golden file's pixels.
+  ///
+  /// In the case of comparison mismatch, the comparator may choose to throw a
+  /// [TestFailure] if it wants to control the failure message, often in the
+  /// form of a [ComparisonResult] that provides detailed information about the
+  /// mismatch.
+  ///
+  /// The method by which [golden] is located and by which its bytes are loaded
+  /// is left up to the implementation class. For instance, some implementations
+  /// may load files from the local file system, whereas others may load files
+  /// over the network or from a remote repository.
+  Future<bool> compare(Element element, Size size, Uri golden);
+
+  /// Updates the golden file identified by [golden] with rendered pixels of
+  /// [element].
+  ///
+  /// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
+  /// is `true` (which gets set automatically by the test framework when the
+  /// user runs `flutter test --update-goldens --platform=chrome`).
+  ///
+  /// The method by which [golden] is located and by which its bytes are written
+  /// is left up to the implementation class.
+  Future<void> update(Uri golden, Element element, Size size);
+
+  /// Returns a new golden file [Uri] to incorporate any [version] number with
+  /// the [key].
+  ///
+  /// The [version] is an optional int that can be used to differentiate
+  /// historical golden files.
+  ///
+  /// Version numbers are used in golden file tests for package:flutter. You can
+  /// learn more about these tests [here](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter).
+  Uri getTestUri(Uri key, int version) {
+    if (version == null)
+      return key;
+    final String keyString = key.toString();
+    final String extension = path.extension(keyString);
+    return Uri.parse(
+      keyString
+        .split(extension)
+        .join() + '.' + version.toString() + extension
+    );
+  }
+}
+
+/// Compares pixels against those of a golden image file.
+///
+/// This comparator is used as the backend for [matchesGoldenFile] when tests
+/// are running in a web browser.
+///
+/// When using `flutter test --platform=chrome`, a comparator implemented by
+/// [DefaultWebGoldenComparator] is used if no other comparator is specified. It
+/// will send a request to the test server, which uses [goldenFileComparator]
+/// for golden file compatison.
+///
+/// When using `flutter test --update-goldens`, the [DefaultWebGoldenComparator]
+/// updates the files on disk to match the rendering.
+///
+/// When using `flutter run`, the default comparator
+/// ([_TrivialWebGoldenComparator]) is used. It prints a message to the console
+/// but otherwise does nothing. This allows tests to be developed visually on a
+/// web browser.
+///
+/// Callers may choose to override the default comparator by setting this to a
+/// custom comparator during test set-up (or using directory-level test
+/// configuration). For example, some projects may wish to install a comparator
+/// with tolerance levels for allowable differences.
+///
+/// See also:
+///
+///  * [flutter_test] for more information about how to configure tests at the
+///    directory-level.
+///  * [goldenFileComparator], the comparator used when tests are not running on
+///    a web browser.
+WebGoldenComparator get webGoldenComparator => _webGoldenComparator;
+WebGoldenComparator _webGoldenComparator = const _TrivialWebGoldenComparator._();
+set webGoldenComparator(WebGoldenComparator value) {
+  assert(value != null);
+  _webGoldenComparator = value;
+}
+
 /// Whether golden files should be automatically updated during tests rather
 /// than compared to the image bytes recorded by the tests.
 ///
@@ -185,6 +295,26 @@ class TrivialComparator implements GoldenFileComparator {
   }
 }
 
+class _TrivialWebGoldenComparator implements WebGoldenComparator {
+  const _TrivialWebGoldenComparator._();
+
+  @override
+  Future<bool> compare(Element element, Size size, Uri golden) {
+    debugPrint('Golden comparison requested for "$golden"; skipping...');
+    return Future<bool>.value(true);
+  }
+
+  @override
+  Future<void> update(Uri golden, Element element, Size size) {
+    throw StateError('webGoldenComparator has not been initialized');
+  }
+
+  @override
+  Uri getTestUri(Uri key, int version) {
+    return key;
+  }
+}
+
 /// The result of a pixel comparison test.
 ///
 /// The [ComparisonResult] will always indicate if a test has [passed]. The

packages/flutter_test/lib/src/matchers.dart

@@ -20,6 +20,7 @@ import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';
 import 'package:flutter/services.dart';
 
+import '_matchers_io.dart' if (dart.library.html) '_matchers_web.dart' show MatchesGoldenFile, captureImage;
 import 'accessibility.dart';
 import 'binding.dart';
 import 'finders.dart';
@@ -366,9 +367,9 @@ Matcher coversSameAreaAs(Path expectedPath, { @required Rect areaToCompare, int
 ///    may swap out the backend for this matcher.
 AsyncMatcher matchesGoldenFile(dynamic key, {int version}) {
   if (key is Uri) {
-    return _MatchesGoldenFile(key, version);
+    return MatchesGoldenFile(key, version);
   } else if (key is String) {
-    return _MatchesGoldenFile.forStringPath(key, version);
+    return MatchesGoldenFile.forStringPath(key, version);
   }
   throw ArgumentError('Unexpected type for golden file: ${key.runtimeType}');
 }
@@ -1636,17 +1637,6 @@ class _ColorMatcher extends Matcher {
   Description describe(Description description) => description.add('matches color $targetColor');
 }
 
-Future<ui.Image> _captureImage(Element element) {
-  RenderObject renderObject = element.renderObject;
-  while (!renderObject.isRepaintBoundary) {
-    renderObject = renderObject.parent as RenderObject;
-    assert(renderObject != null);
-  }
-  assert(!renderObject.debugNeedsPaint);
-  final OffsetLayer layer = renderObject.debugLayer as OffsetLayer;
-  return layer.toImage(renderObject.paintBounds);
-}
-
 int _countDifferentPixels(Uint8List imageA, Uint8List imageB) {
   assert(imageA.length == imageB.length);
   int delta = 0;
@@ -1681,7 +1671,7 @@ class _MatchesReferenceImage extends AsyncMatcher {
       } else if (elements.length > 1) {
         return 'matched too many widgets';
       }
-      imageFuture = _captureImage(elements.single);
+      imageFuture = captureImage(elements.single);
     }
 
     final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding;
@@ -1712,60 +1702,6 @@ class _MatchesReferenceImage extends AsyncMatcher {
   }
 }
 
-class _MatchesGoldenFile extends AsyncMatcher {
-  const _MatchesGoldenFile(this.key, this.version);
-
-  _MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path);
-
-  final Uri key;
-  final int version;
-
-  @override
-  Future<String> matchAsync(dynamic item) async {
-    Future<ui.Image> imageFuture;
-    if (item is Future<ui.Image>) {
-      imageFuture = item;
-    } else if (item is ui.Image) {
-      imageFuture = Future<ui.Image>.value(item);
-    } else {
-      final Finder finder = item as Finder;
-      final Iterable<Element> elements = finder.evaluate();
-      if (elements.isEmpty) {
-        return 'could not be rendered because no widget was found';
-      } else if (elements.length > 1) {
-        return 'matched too many widgets';
-      }
-      imageFuture = _captureImage(elements.single);
-    }
-
-    final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
-
-    final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding;
-    return binding.runAsync<String>(() async {
-      final ui.Image image = await imageFuture;
-      final ByteData bytes = await image.toByteData(format: ui.ImageByteFormat.png);
-      if (bytes == null)
-        return 'could not encode screenshot.';
-      if (autoUpdateGoldenFiles) {
-        await goldenFileComparator.update(testNameUri, bytes.buffer.asUint8List());
-        return null;
-      }
-      try {
-        final bool success = await goldenFileComparator.compare(bytes.buffer.asUint8List(), testNameUri);
-        return success ? null : 'does not match';
-      } on TestFailure catch (ex) {
-        return ex.message;
-      }
-    }, additionalTime: const Duration(minutes: 1));
-  }
-
-  @override
-  Description describe(Description description) {
-    final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
-    return description.add('one widget whose rasterized image matches golden image "$testNameUri"');
-  }
-}
-
 class _MatchesSemanticsData extends Matcher {
   _MatchesSemanticsData({
     this.label,

packages/flutter_tools/lib/src/build_runner/build_script.dart

@@ -255,6 +255,7 @@ class FlutterWebTestBootstrapBuilder implements Builder {
     final String assetPath = id.pathSegments.first == 'lib'
         ? path.url.join('packages', id.package, id.path)
         : id.path;
+    final Uri testUrl = path.toUri(path.absolute(assetPath));
     final Metadata metadata = parseMetadata(
         assetPath, contents, Runtime.builtIn.map((Runtime runtime) => runtime.name).toSet());
 
@@ -265,6 +266,7 @@ import 'dart:html';
 import 'dart:js';
 
 import 'package:stream_channel/stream_channel.dart';
+import 'package:flutter_test/flutter_test.dart';
 import 'package:test_api/src/backend/stack_trace_formatter.dart'; // ignore: implementation_imports
 import 'package:test_api/src/util/stack_trace_mapper.dart'; // ignore: implementation_imports
 import 'package:test_api/src/remote_listener.dart'; // ignore: implementation_imports
@@ -279,6 +281,7 @@ Future<void> main() async {
   // this stuff in.
   ui.debugEmulateFlutterTesterEnvironment = true;
   await ui.webOnlyInitializePlatform();
+  webGoldenComparator = DefaultWebGoldenComparator(Uri.parse('$testUrl'));
   // TODO(flutterweb): remove need for dynamic cast.
   (ui.window as dynamic).debugOverrideDevicePixelRatio(3.0);
   (ui.window as dynamic).webOnlyDebugPhysicalSizeOverride = const ui.Size(2400, 1800);

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -29,6 +29,7 @@ import '../globals.dart';
 import '../project.dart';
 import '../vmservice.dart';
 import 'test_compiler.dart';
+import 'test_config.dart';
 import 'watcher.dart';
 
 /// The timeout we give the test process to connect to the test harness
@@ -55,14 +56,6 @@ const Duration _kTestProcessTimeout = Duration(minutes: 5);
 /// hold that against the test.
 const String _kStartTimeoutTimerMessage = 'sky_shell test process has entered main method';
 
-/// The name of the test configuration file that will be discovered by the
-/// test harness if it exists in the project directory hierarchy.
-const String _kTestConfigFileName = 'flutter_test_config.dart';
-
-/// The name of the file that signals the root of the project and that will
-/// cause the test harness to stop scanning for configuration files.
-const String _kProjectRootSentinel = 'pubspec.yaml';
-
 /// The address at which our WebSocket server resides and at which the sky_shell
 /// processes will host the Observatory server.
 final Map<InternetAddressType, InternetAddress> _kHosts = <InternetAddressType, InternetAddress>{
@@ -743,25 +736,9 @@ class FlutterPlatform extends PlatformPlugin {
     Uri testUrl,
   }) {
     assert(testUrl.scheme == 'file');
-    File testConfigFile;
-    Directory directory = fs.file(testUrl).parent;
-    while (directory.path != directory.parent.path) {
-      final File configFile = directory.childFile(_kTestConfigFileName);
-      if (configFile.existsSync()) {
-        printTrace('Discovered $_kTestConfigFileName in ${directory.path}');
-        testConfigFile = configFile;
-        break;
-      }
-      if (directory.childFile(_kProjectRootSentinel).existsSync()) {
-        printTrace('Stopping scan for $_kTestConfigFileName; '
-            'found project root at ${directory.path}');
-        break;
-      }
-      directory = directory.parent;
-    }
     return generateTestBootstrap(
       testUrl: testUrl,
-      testConfigFile: testConfigFile,
+      testConfigFile: findTestConfigFile(fs.file(testUrl)),
       host: host,
       updateGoldens: updateGoldens,
     );

packages/flutter_tools/lib/src/test/runner.dart

@@ -48,6 +48,12 @@ Future<int> runTests(
   Directory coverageDirectory,
   bool web = false,
 }) async {
+  // Configure package:test to use the Flutter engine for child processes.
+  final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester);
+  if (!processManager.canRun(shellPath)) {
+    throwToolExit('Cannot execute Flutter tester at $shellPath');
+  }
+
   // Compute the command-line arguments for package:test.
   final List<String> testArgs = <String>[
     if (!terminal.supportsColor)
@@ -86,7 +92,12 @@ Future<int> runTests(
     hack.registerPlatformPlugin(
       <Runtime>[Runtime.chrome],
       () {
-        return FlutterWebPlatform.start(flutterProject.directory.path);
+        return FlutterWebPlatform.start(
+          flutterProject.directory.path,
+          updateGoldens: updateGoldens,
+          shellPath: shellPath,
+          flutterProject: flutterProject,
+        );
       },
     );
     await test.main(testArgs);
@@ -97,12 +108,6 @@ Future<int> runTests(
     ..add('--')
     ..addAll(testFiles);
 
-  // Configure package:test to use the Flutter engine for child processes.
-  final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester);
-  if (!processManager.canRun(shellPath)) {
-    throwToolExit('Cannot find Flutter shell at $shellPath');
-  }
-
   final InternetAddressType serverType =
       ipv6 ? InternetAddressType.IPv6 : InternetAddressType.IPv4;
 

packages/flutter_tools/lib/src/test/test_config.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import '../base/file_system.dart';
+import '../globals.dart';
+
+/// The name of the test configuration file that will be discovered by the
+/// test harness if it exists in the project directory hierarchy.
+const String _kTestConfigFileName = 'flutter_test_config.dart';
+
+/// The name of the file that signals the root of the project and that will
+/// cause the test harness to stop scanning for configuration files.
+const String _kProjectRootSentinel = 'pubspec.yaml';
+
+/// Find the `flutter_test_config.dart` file for a specific test file.
+File findTestConfigFile(File testFile) {
+  File testConfigFile;
+  Directory directory = testFile.parent;
+  while (directory.path != directory.parent.path) {
+    final File configFile = directory.childFile(_kTestConfigFileName);
+    if (configFile.existsSync()) {
+      printTrace('Discovered $_kTestConfigFileName in ${directory.path}');
+      testConfigFile = configFile;
+      break;
+    }
+    if (directory.childFile(_kProjectRootSentinel).existsSync()) {
+      printTrace('Stopping scan for $_kTestConfigFileName; '
+          'found project root at ${directory.path}');
+      break;
+    }
+    directory = directory.parent;
+  }
+  return testConfigFile;
+}

packages/flutter_tools/lib/src/web/chrome.dart

@@ -137,6 +137,7 @@ class ChromeLauncher {
       '--no-default-browser-check',
       '--disable-default-apps',
       '--disable-translate',
+      '--window-size=2400,1800',
       if (headless)
         ...<String>['--headless', '--disable-gpu', '--no-sandbox'],
       url,
@@ -174,6 +175,7 @@ class ChromeLauncher {
     return _connect(Chrome._(
       port,
       ChromeConnection('localhost', port),
+      url: url,
       process: process,
       remoteDebuggerUri: remoteDebuggerUri,
     ), skipCheck);
@@ -225,10 +227,12 @@ class Chrome {
   Chrome._(
     this.debugPort,
     this.chromeConnection, {
+    this.url,
     Process process,
     this.remoteDebuggerUri,
   })  : _process = process;
 
+  final String url;
   final int debugPort;
   final Process _process;
   final ChromeConnection chromeConnection;

packages/flutter_tools/static/index.html

@@ -4,23 +4,31 @@ Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file. -->
 <html>
 <head>
-  <title>test Browser Host</title>
+  <title>Flutter Test Browser Host</title>
+  <style>
+    /* Configure so that the test application takes up the whole screen */
+    body {
+      margin: 0;
+      padding: 0;
+      position: fixed;
+      top: 0px;
+      left: 0px;
+      overflow: hidden;
+    }
+
+    iframe {
+      border: none;
+      width: 2400px;
+      height: 1800px;
+      position: fixed;
+      top: 0px;
+      left: 0px;
+      overflow: hidden;
+    }
+  </style>
 </head>
 <body>
-  <svg id="dart" version="1.1" x="0px" y="0px" width="400px" height="400px" viewBox="0 0 400 400">
-    <path id="right-flank" fill="#0083C9" d="M249.379,226.486l-6.676,15.572L166.174,166h58.82c0,0,2.807-0.409,3.645,1.966L249.379,226.486z"/>
-    <path id="right-ear"   fill="#00D2B8" d="M201.84,141.906L166.174,166h58.82c0,0,2.168-0.25,2.645,0.566l-2.694-8.848l-15.024-14.68C207.555,140.329,203.578,140.744,201.84,141.906z"/>
-    <path id="left-flank"  fill="#00D2B8" d="M242.616,241.856l-15.022,6.799l-60.493-21.429c-1.035-0.395-1.101-3.696-1.101-3.696v-57.932L242.616,241.856z"/>
-    <path id="left-paw"    fill="#55DECA" d="M167.003,227.098l60.636,21.558l15.064-6.799L237.224,259h-43.856c0,0-14.077-13.929-18.141-17.993C171.162,236.943,169.162,233.989,167.003,227.098z"/>
-    <path id="right-paw"   fill="#00A4E4" d="M227.676,166.365c0.963,1.401,1.361,2.473,1.361,2.473l20.352,57.648l-6.711,15.37L259,236.463v-44.854c0,0-13.678-13.965-17.741-17.882C237.193,169.811,231.466,166.319,227.676,166.365z"/>
-    <path id="left-ear"    fill="#0083C9" d="M166.769,227.098c0,0-0.769-1.104-0.769-4.355v-57.144l-23.115,34.877c-1.626,1.774-1.567,6.538,1.595,9.755l13.636,13.892L166.769,227.098z"/>
-  </svg>
   <div id="dark"></div>
-  <svg id="play" version="1.1" x="0px" y="0px" width="80px" height="80px" viewBox="0 0 25 25">
-    <defs><filter id="blur"><feGaussianBlur stdDeviation="0.3" id="feGaussianBlur5097" /></filter></defs>
-    <path d="M 3.777014,1.3715789 A 1.1838119,1.1838119 0 0 0 2.693923,2.5488509 V 22.444746 a 1.1838119,1.1838119 0 0 0 1.765908,1.035999 l 17.235259,-9.95972 a 1.1838119,1.1838119 0 0 0 0,-2.071998 L 4.459831,1.5128519 A 1.1838119,1.1838119 0 0 0 3.777014,1.3715789 z" style="opacity:0.5;stroke:#000000;stroke-width:1;filter:url(#blur)" />
-    <path style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1.32722104" d="M 3.4770491,1.0714664 A 1.1838119,1.1838119 0 0 0 2.3939589,2.2487382 V 22.144633 a 1.1838119,1.1838119 0 0 0 1.7659079,1.035999 l 17.2352602,-9.95972 a 1.1838119,1.1838119 0 0 0 0,-2.071998 L 4.1598668,1.2127389 A 1.1838119,1.1838119 0 0 0 3.4770491,1.0714664 z" />
-  </svg>
   <script src="host.dart.js"></script>
 </body>
 </html>

packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/test/flutter_web_platform.dart';
+
+import '../../src/common.dart';
+import '../../src/mocks.dart';
+import '../../src/testbed.dart';
+
+void main() {
+  final Testbed testbed = Testbed();
+
+  group('Test that TestGoldenComparatorProcess', () {
+    File imageFile;
+    Uri goldenKey;
+    File imageFile2;
+    Uri goldenKey2;
+    MockProcess Function(String) createMockProcess;
+
+    setUpAll(() {
+      imageFile = fs.file('test_image_file');
+      goldenKey = Uri.parse('file://golden_key');
+      imageFile2 = fs.file('second_test_image_file');
+      goldenKey2 = Uri.parse('file://second_golden_key');
+      createMockProcess = (String stdout) => MockProcess(
+        exitCode: Future<int>.value(0),
+        stdout: stdoutFromString(stdout),
+      );
+    });
+
+    test('can pass data', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse = <String, dynamic>{
+        'success': true,
+        'message': 'some message',
+      };
+
+      final MockProcess mockProcess = createMockProcess(jsonEncode(expectedResponse) + '\n');
+      final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink;
+
+      final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess);
+      process.sendCommand(imageFile, goldenKey, false);
+
+      final Map<String, dynamic> response = await process.getResponse();
+      final String stringToStdin = stringFromMemoryIOSink(ioSink);
+
+      expect(response, expectedResponse);
+      expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n');
+    }));
+
+    test('can handle multiple requests', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse1 = <String, dynamic>{
+        'success': true,
+        'message': 'some message',
+      };
+      final Map<String, dynamic> expectedResponse2 = <String, dynamic>{
+        'success': false,
+        'message': 'some other message',
+      };
+
+      final MockProcess mockProcess = createMockProcess(jsonEncode(expectedResponse1) + '\n' + jsonEncode(expectedResponse2) + '\n');
+      final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink;
+
+      final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess);
+      process.sendCommand(imageFile, goldenKey, false);
+
+      final Map<String, dynamic> response1 = await process.getResponse();
+
+      process.sendCommand(imageFile2, goldenKey2, true);
+
+      final Map<String, dynamic> response2 = await process.getResponse();
+      final String stringToStdin = stringFromMemoryIOSink(ioSink);
+
+      expect(response1, expectedResponse1);
+      expect(response2, expectedResponse2);
+      expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n{"imageFile":"second_test_image_file","key":"file://second_golden_key/","update":true}\n');
+    }));
+
+    test('ignores anything that does not look like JSON', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse = <String, dynamic>{
+        'success': true,
+        'message': 'some message',
+      };
+
+      final MockProcess mockProcess = createMockProcess('''
+Some random data including {} curly bracket
+  {} curly bracket that is not on the beginning of the line
+${jsonEncode(expectedResponse)}
+{"success": false}
+Other JSON data after the initial data
+''');
+      final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink;
+
+      final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess);
+      process.sendCommand(imageFile, goldenKey, false);
+
+      final Map<String, dynamic> response = await process.getResponse();
+      final String stringToStdin = stringFromMemoryIOSink(ioSink);
+
+      expect(response, expectedResponse);
+      expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n');
+    }));
+  });
+}
+
+Stream<List<int>> stdoutFromString(String string) => Stream<List<int>>.fromIterable(<List<int>>[
+  utf8.encode(string),
+]);
+
+String stringFromMemoryIOSink(MemoryIOSink ioSink) => utf8.decode(ioSink.writes.expand((List<int> l) => l).toList());

packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:typed_data';
+
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/test/flutter_web_platform.dart';
+import 'package:flutter_tools/src/test/test_compiler.dart';
+import 'package:mockito/mockito.dart';
+import 'package:process/process.dart';
+
+import '../../src/common.dart';
+import '../../src/mocks.dart';
+import '../../src/testbed.dart';
+
+void main() {
+
+  group('Test that TestGoldenComparator', () {
+    Testbed testbed;
+    Uri goldenKey;
+    Uri goldenKey2;
+    Uri testUri;
+    Uri testUri2;
+    Uint8List imageBytes;
+    MockProcessManager mockProcessManager;
+    MockTestCompiler mockCompiler;
+
+    setUp(() {
+      goldenKey = Uri.parse('file://golden_key');
+      goldenKey2 = Uri.parse('file://second_golden_key');
+      testUri = Uri.parse('file://test_uri');
+      testUri2 = Uri.parse('file://second_test_uri');
+      imageBytes = Uint8List.fromList(<int>[1,2,3,4,5]);
+      mockProcessManager = MockProcessManager();
+      mockCompiler = MockTestCompiler();
+      when(mockCompiler.compile(any)).thenAnswer((_) => Future<String>.value('compiler_output'));
+
+      testbed = Testbed(overrides: <Type, Generator>{
+        ProcessManager: () {
+          print('in get process manager');
+          return mockProcessManager;
+        }
+      });
+    });
+
+    test('succeed when golden comparison succeed', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse = <String, dynamic>{
+        'success': true,
+        'message': 'some message',
+      };
+
+      when(mockProcessManager.start(any, environment: anyNamed('environment')))
+        .thenAnswer((Invocation invocation) async {
+          return FakeProcess(
+            exitCode: Future<int>.value(0),
+            stdout: stdoutFromString(jsonEncode(expectedResponse) + '\n'),
+          );
+      });
+
+      final TestGoldenComparator comparator = TestGoldenComparator(
+        'shell',
+        () => mockCompiler,
+      );
+
+      final String result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      expect(result, null);
+    }));
+
+    test('fail with error message when golden comparison failed', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse = <String, dynamic>{
+        'success': false,
+        'message': 'some message',
+      };
+
+      when(mockProcessManager.start(any, environment: anyNamed('environment')))
+        .thenAnswer((Invocation invocation) async {
+          return FakeProcess(
+            exitCode: Future<int>.value(0),
+            stdout: stdoutFromString(jsonEncode(expectedResponse) + '\n'),
+          );
+      });
+
+      final TestGoldenComparator comparator = TestGoldenComparator(
+        'shell',
+        () => mockCompiler,
+      );
+
+      final String result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      expect(result, 'some message');
+    }));
+
+    test('reuse the process for the same test file', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse1 = <String, dynamic>{
+        'success': false,
+        'message': 'some message',
+      };
+      final Map<String, dynamic> expectedResponse2 = <String, dynamic>{
+        'success': false,
+        'message': 'some other message',
+      };
+
+      when(mockProcessManager.start(any, environment: anyNamed('environment')))
+        .thenAnswer((Invocation invocation) async {
+          return FakeProcess(
+            exitCode: Future<int>.value(0),
+            stdout: stdoutFromString(jsonEncode(expectedResponse1) + '\n' + jsonEncode(expectedResponse2) + '\n'),
+          );
+      });
+
+      final TestGoldenComparator comparator = TestGoldenComparator(
+        'shell',
+        () => mockCompiler,
+      );
+
+      final String result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      expect(result1, 'some message');
+      final String result2 = await comparator.compareGoldens(testUri, imageBytes, goldenKey2, false);
+      expect(result2, 'some other message');
+      verify(mockProcessManager.start(any, environment: anyNamed('environment'))).called(1);
+    }));
+
+    test('does not reuse the process for different test file', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse1 = <String, dynamic>{
+        'success': false,
+        'message': 'some message',
+      };
+      final Map<String, dynamic> expectedResponse2 = <String, dynamic>{
+        'success': false,
+        'message': 'some other message',
+      };
+
+      when(mockProcessManager.start(any, environment: anyNamed('environment')))
+        .thenAnswer((Invocation invocation) async {
+          return FakeProcess(
+            exitCode: Future<int>.value(0),
+            stdout: stdoutFromString(jsonEncode(expectedResponse1) + '\n' + jsonEncode(expectedResponse2) + '\n'),
+          );
+      });
+
+      final TestGoldenComparator comparator = TestGoldenComparator(
+        'shell',
+        () => mockCompiler,
+      );
+
+      final String result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      expect(result1, 'some message');
+      final String result2 = await comparator.compareGoldens(testUri2, imageBytes, goldenKey2, false);
+      expect(result2, 'some message');
+      verify(mockProcessManager.start(any, environment: anyNamed('environment'))).called(2);
+    }));
+
+    test('removes all temporary files when closed', () => testbed.run(() async {
+      final Map<String, dynamic> expectedResponse = <String, dynamic>{
+        'success': true,
+        'message': 'some message',
+      };
+
+      when(mockProcessManager.start(any, environment: anyNamed('environment')))
+        .thenAnswer((Invocation invocation) async {
+          return FakeProcess(
+            exitCode: Future<int>.value(0),
+            stdout: stdoutFromString(jsonEncode(expectedResponse) + '\n'),
+          );
+      });
+
+      final TestGoldenComparator comparator = TestGoldenComparator(
+        'shell',
+        () => mockCompiler,
+      );
+
+      final String result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false);
+      expect(result, null);
+
+      await comparator.close();
+      expect(fs.systemTempDirectory.listSync(recursive: true), isEmpty);
+    }));
+  });
+}
+
+Stream<List<int>> stdoutFromString(String string) => Stream<List<int>>.fromIterable(<List<int>>[
+  utf8.encode(string),
+]);
+
+class MockProcessManager extends Mock implements ProcessManager {}
+class MockTestCompiler extends Mock implements TestCompiler {}