goldens.dart 14.7 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:typed_data';
6
import 'dart:ui';
7 8

import 'package:path/path.dart' as path;
9 10
import 'package:test_api/test_api.dart'; // ignore: deprecated_member_use

11
import '_goldens_io.dart' if (dart.library.html) '_goldens_web.dart' as goldens;
12

13
/// Compares image pixels against a golden image file.
14 15 16 17 18 19 20 21 22
///
/// Instances of this comparator will be used as the backend for
/// [matchesGoldenFile].
///
/// Instances of this comparator will be invoked by the test framework in the
/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the
/// fake async constraints that are normally imposed on widget tests (i.e. the
/// need or the ability to call [WidgetTester.pump] to advance the microtask
/// queue).
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
///
/// ## What is Golden File Testing?
///
/// The term __golden file__ refers to a master image that is considered the true
/// rendering of a given widget, state, application, or other visual
/// representation you have chosen to capture.
///
/// By keeping a master reference of visual aspects of your application, you can
/// prevent unintended changes as you develop by testing against them.
///
/// Here, a minor code change has altered the appearance of a widget. A golden
/// file test has compared the image generated at the time of the test to the
/// golden master file that was generated earlier. The test has identified the
/// change, preventing unintended modifications.
///
/// |  Sample                        |  Image |
/// |--------------------------------|--------|
/// |  Golden Master Image           | ![A golden master image](https://flutter.github.io/assets-for-api-docs/assets/flutter-test/goldens/widget_masterImage.png)  |
/// |  Difference                    | ![The pixel difference](https://flutter.github.io/assets-for-api-docs/assets/flutter-test/goldens/widget_isolatedDiff.png)  |
/// |  Test image after modification | ![Test image](https://flutter.github.io/assets-for-api-docs/assets/flutter-test/goldens/widget_testImage.png) |
///
44 45
/// {@macro flutter.flutter_test.matchesGoldenFile.custom_fonts}
///
46 47 48 49 50 51
/// See also:
///
///  * [LocalFileComparator] for the default [GoldenFileComparator]
///    implementation for `flutter test`.
///  * [matchesGoldenFile], the function from [flutter_test] that invokes the
///    comparator.
52
abstract class GoldenFileComparator {
53 54
  /// Compares the pixels of decoded png [imageBytes] against the golden file
  /// identified by [golden].
55 56
  ///
  /// The returned future completes with a boolean value that indicates whether
57
  /// the pixels decoded from [imageBytes] match the golden file's pixels.
58 59
  ///
  /// In the case of comparison mismatch, the comparator may choose to throw a
60 61 62
  /// [TestFailure] if it wants to control the failure message, often in the
  /// form of a [ComparisonResult] that provides detailed information about the
  /// mismatch.
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
  ///
  /// The method by which [golden] is located and by which its bytes are loaded
  /// is left up to the implementation class. For instance, some implementations
  /// may load files from the local file system, whereas others may load files
  /// over the network or from a remote repository.
  Future<bool> compare(Uint8List imageBytes, Uri golden);

  /// Updates the golden file identified by [golden] with [imageBytes].
  ///
  /// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
  /// is `true` (which gets set automatically by the test framework when the
  /// user runs `flutter test --update-goldens`).
  ///
  /// The method by which [golden] is located and by which its bytes are written
  /// is left up to the implementation class.
  Future<void> update(Uri golden, Uint8List imageBytes);
79 80 81 82 83 84 85 86 87

  /// Returns a new golden file [Uri] to incorporate any [version] number with
  /// the [key].
  ///
  /// The [version] is an optional int that can be used to differentiate
  /// historical golden files.
  ///
  /// Version numbers are used in golden file tests for package:flutter. You can
  /// learn more about these tests [here](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter).
88
  Uri getTestUri(Uri key, int? version) {
89 90 91 92
    if (version == null)
      return key;
    final String keyString = key.toString();
    final String extension = path.extension(keyString);
93
    return Uri.parse('${keyString.split(extension).join()}.$version$extension');
94
  }
95 96 97

  /// Returns a [ComparisonResult] to describe the pixel differential of the
  /// [test] and [master] image bytes provided.
98
  static Future<ComparisonResult> compareLists(List<int> test, List<int> master) {
99
    return goldens.compareLists(test, master);
100
  }
101 102
}

103
/// Compares pixels against those of a golden image file.
104 105 106
///
/// This comparator is used as the backend for [matchesGoldenFile].
///
107 108
/// When using `flutter test`, a comparator implemented by [LocalFileComparator]
/// is used if no other comparator is specified. It treats the golden key as
109
/// a relative path from the test file's directory. It will then load the
110 111
/// golden file's bytes from disk and perform a pixel-for-pixel comparison of
/// the decoded PNGs, returning true only if there's an exact match.
112
///
113 114 115
/// When using `flutter test --update-goldens`, the [LocalFileComparator]
/// updates the files on disk to match the rendering.
///
116 117 118
/// When using `flutter run`, the default comparator ([TrivialComparator])
/// is used. It prints a message to the console but otherwise does nothing. This
/// allows tests to be developed visually on a real device.
119
///
120
/// Callers may choose to override the default comparator by setting this to a
121
/// custom comparator during test set-up (or using directory-level test
122 123
/// configuration). For example, some projects may wish to install a comparator
/// with tolerance levels for allowable differences.
124 125 126 127 128
///
/// See also:
///
///  * [flutter_test] for more information about how to configure tests at the
///    directory-level.
129 130 131 132
GoldenFileComparator get goldenFileComparator => _goldenFileComparator;
GoldenFileComparator _goldenFileComparator = const TrivialComparator._();
set goldenFileComparator(GoldenFileComparator value) {
  _goldenFileComparator = value;
133
}
134

135 136 137 138 139 140 141 142 143 144 145 146
/// Compares image pixels against a golden image file.
///
/// Instances of this comparator will be used as the backend for
/// [matchesGoldenFile] when tests are running on Flutter Web, and will usually
/// implemented by deferring the screenshot taking and image comparison to a
/// test server.
///
/// Instances of this comparator will be invoked by the test framework in the
/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the
/// fake async constraints that are normally imposed on widget tests (i.e. the
/// need or the ability to call [WidgetTester.pump] to advance the microtask
/// queue). Prior to the invocation, the test framework will render only the
Janice Collins's avatar
Janice Collins committed
147
/// [widgets.Element] to be compared on the screen.
148 149 150 151 152 153 154 155 156 157
///
/// See also:
///
///  * [GoldenFileComparator] for the comparator to be used when the test is
///    not running in a web browser.
///  * [DefaultWebGoldenComparator] for the default [WebGoldenComparator]
///    implementation for `flutter test`.
///  * [matchesGoldenFile], the function from [flutter_test] that invokes the
///    comparator.
abstract class WebGoldenComparator {
158
  /// Compares the rendered pixels of size [width]x[height] that is being
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
  /// rendered on the top left of the screen against the golden file identified
  /// by [golden].
  ///
  /// The returned future completes with a boolean value that indicates whether
  /// the pixels rendered on screen match the golden file's pixels.
  ///
  /// In the case of comparison mismatch, the comparator may choose to throw a
  /// [TestFailure] if it wants to control the failure message, often in the
  /// form of a [ComparisonResult] that provides detailed information about the
  /// mismatch.
  ///
  /// The method by which [golden] is located and by which its bytes are loaded
  /// is left up to the implementation class. For instance, some implementations
  /// may load files from the local file system, whereas others may load files
  /// over the network or from a remote repository.
174
  Future<bool> compare(double width, double height, Uri golden);
175 176

  /// Updates the golden file identified by [golden] with rendered pixels of
177
  /// [width]x[height].
178 179 180 181 182 183 184
  ///
  /// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
  /// is `true` (which gets set automatically by the test framework when the
  /// user runs `flutter test --update-goldens --platform=chrome`).
  ///
  /// The method by which [golden] is located and by which its bytes are written
  /// is left up to the implementation class.
185
  Future<void> update(double width, double height, Uri golden);
186 187 188 189 190 191 192 193 194

  /// Returns a new golden file [Uri] to incorporate any [version] number with
  /// the [key].
  ///
  /// The [version] is an optional int that can be used to differentiate
  /// historical golden files.
  ///
  /// Version numbers are used in golden file tests for package:flutter. You can
  /// learn more about these tests [here](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter).
195
  Uri getTestUri(Uri key, int? version) {
196 197 198 199
    if (version == null)
      return key;
    final String keyString = key.toString();
    final String extension = path.extension(keyString);
200
    return Uri.parse('${keyString.split(extension).join()}.$version$extension');
201 202 203 204 205 206 207 208 209 210 211
  }
}

/// Compares pixels against those of a golden image file.
///
/// This comparator is used as the backend for [matchesGoldenFile] when tests
/// are running in a web browser.
///
/// When using `flutter test --platform=chrome`, a comparator implemented by
/// [DefaultWebGoldenComparator] is used if no other comparator is specified. It
/// will send a request to the test server, which uses [goldenFileComparator]
212
/// for golden file comparison.
213 214 215 216 217
///
/// When using `flutter test --update-goldens`, the [DefaultWebGoldenComparator]
/// updates the files on disk to match the rendering.
///
/// When using `flutter run`, the default comparator
218
/// (`_TrivialWebGoldenComparator`) is used. It prints a message to the console
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
/// but otherwise does nothing. This allows tests to be developed visually on a
/// web browser.
///
/// Callers may choose to override the default comparator by setting this to a
/// custom comparator during test set-up (or using directory-level test
/// configuration). For example, some projects may wish to install a comparator
/// with tolerance levels for allowable differences.
///
/// See also:
///
///  * [flutter_test] for more information about how to configure tests at the
///    directory-level.
///  * [goldenFileComparator], the comparator used when tests are not running on
///    a web browser.
WebGoldenComparator get webGoldenComparator => _webGoldenComparator;
WebGoldenComparator _webGoldenComparator = const _TrivialWebGoldenComparator._();
set webGoldenComparator(WebGoldenComparator value) {
  _webGoldenComparator = value;
}

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
/// Whether golden files should be automatically updated during tests rather
/// than compared to the image bytes recorded by the tests.
///
/// When this is `true`, [matchesGoldenFile] will always report a successful
/// match, because the bytes being tested implicitly become the new golden.
///
/// The Flutter tool will automatically set this to `true` when the user runs
/// `flutter test --update-goldens`, so callers should generally never have to
/// explicitly modify this value.
///
/// See also:
///
///   * [goldenFileComparator]
bool autoUpdateGoldenFiles = false;

254 255 256
/// Placeholder comparator that is set as the value of [goldenFileComparator]
/// when the initialization that happens in the test bootstrap either has not
/// yet happened or has been bypassed.
257
///
258
/// The test bootstrap file that gets generated by the Flutter tool when the
259
/// user runs `flutter test` is expected to set [goldenFileComparator] to
260 261 262 263 264 265 266
/// a comparator that resolves golden file references relative to the test
/// directory. From there, the caller may choose to override the comparator by
/// setting it to another value during test initialization. The only case
/// where we expect it to remain uninitialized is when the user runs a test
/// via `flutter run`. In this case, the [compare] method will just print a
/// message that it would have otherwise run a real comparison, and it will
/// return trivial success.
267 268 269 270 271
///
/// This class can't be constructed. It represents the default value of
/// [goldenFileComparator].
class TrivialComparator implements GoldenFileComparator {
  const TrivialComparator._();
272 273 274

  @override
  Future<bool> compare(Uint8List imageBytes, Uri golden) {
275 276 277 278
    // Ideally we would use markTestSkipped here but in some situations,
    // comparators are called outside of tests.
    // See also: https://github.com/flutter/flutter/issues/91285
    // ignore: avoid_print
279
    print('Golden file comparison requested for "$golden"; skipping...');
280
    return Future<bool>.value(true);
281 282 283 284
  }

  @override
  Future<void> update(Uri golden, Uint8List imageBytes) {
285
    throw StateError('goldenFileComparator has not been initialized');
286 287 288
  }

  @override
289
  Uri getTestUri(Uri key, int? version) {
290 291 292 293 294 295 296 297
    return key;
  }
}

class _TrivialWebGoldenComparator implements WebGoldenComparator {
  const _TrivialWebGoldenComparator._();

  @override
298
  Future<bool> compare(double width, double height, Uri golden) {
299 300 301 302
    // Ideally we would use markTestSkipped here but in some situations,
    // comparators are called outside of tests.
    // See also: https://github.com/flutter/flutter/issues/91285
    // ignore: avoid_print
303
    print('Golden comparison requested for "$golden"; skipping...');
304 305 306 307
    return Future<bool>.value(true);
  }

  @override
308
  Future<void> update(double width, double height, Uri golden) {
309
    throw StateError('webGoldenComparator has not been initialized');
310
  }
311 312

  @override
313
  Uri getTestUri(Uri key, int? version) {
314 315
    return key;
  }
316 317
}

318
/// The result of a pixel comparison test.
319
///
320 321 322 323 324 325
/// The [ComparisonResult] will always indicate if a test has [passed]. The
/// optional [error] and [diffs] parameters provide further information about
/// the result of a failing test.
class ComparisonResult {
  /// Creates a new [ComparisonResult] for the current test.
  ComparisonResult({
326
    required this.passed,
327
    required this.diffPercent,
328 329
    this.error,
    this.diffs,
330
  });
331

332
  /// Indicates whether or not a pixel comparison test has failed.
333
  ///
334 335
  /// This value cannot be null.
  final bool passed;
336

337
  /// Error message used to describe the cause of the pixel comparison failure.
338
  final String? error;
339

340 341
  /// Map containing differential images to illustrate found variants in pixel
  /// values in the execution of the pixel test.
342
  final Map<String, Image>? diffs;
343 344 345

  /// The calculated percentage of pixel difference between two images.
  final double diffPercent;
346
}