flutter_goldens.dart 21.2 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5 6 7
import 'dart:async' show FutureOr;
import 'dart:io' as io show OSError, SocketException;
import 'dart:typed_data' show Uint8List;
8 9 10

import 'package:file/file.dart';
import 'package:file/local.dart';
11
import 'package:flutter/foundation.dart';
12
import 'package:flutter_goldens_client/skia_client.dart';
13
import 'package:flutter_test/flutter_test.dart';
14
import 'package:platform/platform.dart';
15

16
export 'package:flutter_goldens_client/skia_client.dart';
17

18 19 20 21 22 23
// If you are here trying to figure out how to use golden files in the Flutter
// repo itself, consider reading this wiki page:
// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter

const String _kFlutterRootKey = 'FLUTTER_ROOT';

24 25
/// Main method that can be used in a `flutter_test_config.dart` file to set
/// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that
26 27
/// works for the current test. _Which_ FlutterGoldenFileComparator is
/// instantiated is based on the current testing environment.
28 29 30
///
/// When set, the `namePrefix` is prepended to the names of all gold images.
Future<void> testExecutable(FutureOr<void> Function() testMain, {String? namePrefix}) async {
31
  const Platform platform = LocalPlatform();
32
  if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) {
33
    goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix);
34
  } else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) {
35
    goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix);
36 37
  } else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) {
    goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator(
38 39
      'Golden file testing is not executed on Cirrus, or LUCI environments outside of flutter/flutter.',
        namePrefix: namePrefix
40
    );
41 42
  } else {
    goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform);
43
  }
44

45 46 47
  await testMain();
}

48 49
/// Abstract base class golden file comparator specific to the `flutter/flutter`
/// repository.
50 51 52 53 54
///
/// Golden file testing for the `flutter/flutter` repository is handled by three
/// different [FlutterGoldenFileComparator]s, depending on the current testing
/// environment.
///
55
///   * The [FlutterPostSubmitFileComparator] is utilized during post-submit
56 57 58 59 60 61 62
///     testing, after a pull request has landed on the master branch. This
///     comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload
///     tests to the [Flutter Gold dashboard](https://flutter-gold.skia.org).
///     Flutter Gold manages the master golden files for the `flutter/flutter`
///     repository.
///
///   * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing,
63
///     before a pull request lands on the master branch. This
64 65 66
///     comparator uses the [SkiaGoldClient] to execute tryjobs, allowing
///     contributors to view and check in visual differences before landing the
///     change.
67
///
68
///   * The [FlutterLocalFileComparator] is used for local development testing.
69 70
///     This comparator will use the [SkiaGoldClient] to request baseline images
///     from [Flutter Gold](https://flutter-gold.skia.org) and manually compare
71
///     pixels. If a difference is detected, this comparator will
72 73 74
///     generate failure output illustrating the found difference. If a baseline
///     is not found for a given test image, it will consider it a new test and
///     output the new image for verification.
75 76
///
///  The [FlutterSkippingFileComparator] is utilized to skip tests outside
77 78 79 80
///  of the appropriate environments described above. Currently, some Luci
///  environments do not execute golden file testing, and as such do not require
///  a comparator. This comparator is also used when an internet connection is
///  unavailable.
81
abstract class FlutterGoldenFileComparator extends GoldenFileComparator {
82
  /// Creates a [FlutterGoldenFileComparator] that will resolve golden file
83 84 85 86 87
  /// URIs relative to the specified [basedir], and retrieve golden baselines
  /// using the [skiaClient]. The [basedir] is used for writing and accessing
  /// information and files for interacting with the [skiaClient]. When testing
  /// locally, the [basedir] will also contain any diffs from failed tests, or
  /// goldens generated from newly introduced tests.
88
  ///
89 90
  /// The [fs] and [platform] parameters are useful in tests, where the default
  /// file system and platform can be replaced by mock instances.
91 92
  @visibleForTesting
  FlutterGoldenFileComparator(
93 94
    this.basedir,
    this.skiaClient, {
95
    this.fs = const LocalFileSystem(),
96
    this.platform = const LocalPlatform(),
97
    this.namePrefix,
98
  });
99

100
  /// The directory to which golden file URIs will be resolved in [compare] and
101
  /// [update], cannot be null.
102
  final Uri basedir;
103

104 105 106 107
  /// A client for uploading image tests and making baseline requests to the
  /// Flutter Gold Dashboard, cannot be null.
  final SkiaGoldClient skiaClient;

108 109
  /// The file system used to perform file access.
  @visibleForTesting
110 111
  final FileSystem fs;

112 113 114 115
  /// A wrapper for the [dart:io.Platform] API.
  @visibleForTesting
  final Platform platform;

116 117 118
  /// The prefix that is added to all golden names.
  final String? namePrefix;

119 120 121 122 123 124 125
  @override
  Future<void> update(Uri golden, Uint8List imageBytes) async {
    final File goldenFile = getGoldenFile(golden);
    await goldenFile.parent.create(recursive: true);
    await goldenFile.writeAsBytes(imageBytes, flush: true);
  }

126
  @override
127
  Uri getTestUri(Uri key, int? version) => key;
128

129
  /// Calculate the appropriate basedir for the current test context.
130 131
  ///
  /// The optional [suffix] argument is used by the
132
  /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator].
133 134
  /// These [FlutterGoldenFileComparators] randomize their base directories to
  /// maintain thread safety while using the `goldctl` tool.
135 136
  @protected
  @visibleForTesting
137 138 139
  static Directory getBaseDirectory(
    LocalFileComparator defaultComparator,
    Platform platform, {
140
    String? suffix,
141
  }) {
142 143
    const FileSystem fs = LocalFileSystem();
    final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]);
144 145
    Directory comparisonRoot;

146 147
    if (suffix != null) {
      comparisonRoot = fs.systemTempDirectory.createTempSync(suffix);
148 149 150 151 152 153 154 155 156 157 158
    } else {
      comparisonRoot = flutterRoot.childDirectory(
        fs.path.join(
          'bin',
          'cache',
          'pkg',
          'skia_goldens',
        )
      );
    }

159
    final Directory testDirectory = fs.directory(defaultComparator.basedir);
160 161 162 163 164
    final String testDirectoryRelativePath = fs.path.relative(
      testDirectory.path,
      from: flutterRoot.path,
    );
    return comparisonRoot.childDirectory(testDirectoryRelativePath);
165 166 167 168 169 170 171 172
  }

  /// Returns the golden [File] identified by the given [Uri].
  @protected
  File getGoldenFile(Uri uri) {
    final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path);
    return goldenFile;
  }
173

174
  /// Prepends the golden URL with the library name that encloses the current
175 176
  /// test.
  Uri _addPrefix(Uri golden) {
177 178 179 180 181 182
    // Ensure the Uri ends in .png as the SkiaClient expects
    assert(
      golden.toString().split('.').last == 'png',
      'Golden files in the Flutter framework must end with the file extension '
      '.png.'
    );
183 184 185 186 187 188
    return Uri.parse(<String>[
      if (namePrefix != null)
        namePrefix!,
      basedir.pathSegments[basedir.pathSegments.length - 2],
      golden.toString(),
    ].join('.'));
189
  }
190 191
}

192 193
/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in
/// post-submit.
194
///
195
/// For testing across all platforms, the [SkiaGoldClient] is used to upload
196
/// images for framework-related golden tests and process results.
197 198 199 200 201
///
/// See also:
///
///  * [GoldenFileComparator], the abstract class that
///    [FlutterGoldenFileComparator] implements.
202 203 204 205 206 207
///  * [FlutterPreSubmitFileComparator], another
///    [FlutterGoldenFileComparator] that tests golden images before changes are
///    merged into the master branch.
///  * [FlutterLocalFileComparator], another
///    [FlutterGoldenFileComparator] that tests golden images locally on your
///    current machine.
208 209
class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator {
  /// Creates a [FlutterPostSubmitFileComparator] that will test golden file
210
  /// images against Skia Gold.
211
  ///
212 213
  /// The [fs] and [platform] parameters are useful in tests, where the default
  /// file system and platform can be replaced by mock instances.
214
  FlutterPostSubmitFileComparator(
215 216 217 218
    final Uri basedir,
    final SkiaGoldClient skiaClient, {
    final FileSystem fs = const LocalFileSystem(),
    final Platform platform = const LocalPlatform(),
219
    String? namePrefix,
220 221
  }) : super(
    basedir,
222
    skiaClient,
223 224
    fs: fs,
    platform: platform,
225
    namePrefix: namePrefix,
226 227
  );

228
  /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative
229
  /// path resolution of the default [goldenFileComparator].
230 231
  ///
  /// The [goldens] and [defaultComparator] parameters are visible for testing
232
  /// purposes only.
233
  static Future<FlutterPostSubmitFileComparator> fromDefaultComparator(
234
    final Platform platform, {
235 236
    SkiaGoldClient? goldens,
    LocalFileComparator? defaultComparator,
237
    String? namePrefix,
238
  }) async {
239

240
    defaultComparator ??= goldenFileComparator as LocalFileComparator;
241 242 243
    final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory(
      defaultComparator,
      platform,
244
      suffix: 'flutter_goldens_postsubmit.',
245
    );
246
    baseDirectory.createSync(recursive: true);
247

248
    goldens ??= SkiaGoldClient(baseDirectory);
249
    await goldens.auth();
250
    return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens, namePrefix: namePrefix);
251 252 253 254
  }

  @override
  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
255
    await skiaClient.imgtestInit();
256 257
    golden = _addPrefix(golden);
    await update(golden, imageBytes);
258
    final File goldenFile = getGoldenFile(golden);
259 260

    return skiaClient.imgtestAdd(golden.path, goldenFile);
261 262
  }

263 264
  /// Decides based on the current environment if goldens tests should be
  /// executed through Skia Gold.
265
  static bool isAvailableForEnvironment(Platform platform) {
266 267 268 269 270
    final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID')
      && platform.environment.containsKey('GOLDCTL')
      // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator].
      && !platform.environment.containsKey('GOLD_TRYJOB');

271
    return luciPostSubmit;
272
  }
273 274
}

275
/// A [FlutterGoldenFileComparator] for testing golden images before changes are
276 277
/// merged into the master branch. The comparator executes tryjobs using the
/// [SkiaGoldClient].
278 279 280 281 282
///
/// See also:
///
///  * [GoldenFileComparator], the abstract class that
///    [FlutterGoldenFileComparator] implements.
283
///  * [FlutterPostSubmitFileComparator], another
284
///    [FlutterGoldenFileComparator] that uploads tests to the Skia Gold
285
///    dashboard in post-submit.
286 287 288 289 290 291
///  * [FlutterLocalFileComparator], another
///    [FlutterGoldenFileComparator] that tests golden images locally on your
///    current machine.
class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator {
  /// Creates a [FlutterPreSubmitFileComparator] that will test golden file
  /// images against baselines requested from Flutter Gold.
292
  ///
293 294 295
  /// The [fs] and [platform] parameters are useful in tests, where the default
  /// file system and platform can be replaced by mock instances.
  FlutterPreSubmitFileComparator(
296
    final Uri basedir,
297 298 299
    final SkiaGoldClient skiaClient, {
    final FileSystem fs = const LocalFileSystem(),
    final Platform platform = const LocalPlatform(),
300
    final String? namePrefix,
301 302
  }) : super(
    basedir,
303
    skiaClient,
304 305
    fs: fs,
    platform: platform,
306
    namePrefix: namePrefix,
307 308
  );

309 310
  /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the
  /// relative path resolution of the default [goldenFileComparator].
311 312 313
  ///
  /// The [goldens] and [defaultComparator] parameters are visible for testing
  /// purposes only.
314 315
  static Future<FlutterGoldenFileComparator> fromDefaultComparator(
    final Platform platform, {
316 317 318
    SkiaGoldClient? goldens,
    LocalFileComparator? defaultComparator,
    Directory? testBasedir,
319
    String? namePrefix,
320
  }) async {
321

322
    defaultComparator ??= goldenFileComparator as LocalFileComparator;
323
    final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory(
324 325
      defaultComparator,
      platform,
326
      suffix: 'flutter_goldens_presubmit.',
327
    );
328 329 330

    if (!baseDirectory.existsSync())
      baseDirectory.createSync(recursive: true);
331

332
    goldens ??= SkiaGoldClient(baseDirectory);
333

334
    await goldens.auth();
335 336 337
    return FlutterPreSubmitFileComparator(
      baseDirectory.uri,
      goldens, platform: platform,
338
      namePrefix: namePrefix,
339 340
    );
  }
341 342 343

  @override
  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
344
    await skiaClient.tryjobInit();
345 346 347 348
    golden = _addPrefix(golden);
    await update(golden, imageBytes);
    final File goldenFile = getGoldenFile(golden);

349 350 351 352 353
    await skiaClient.tryjobAdd(golden.path, goldenFile);

    // This will always return true since golden file test failures are managed
    // in pre-submit checks by the flutter-gold status check.
    return true;
354 355
  }

356 357 358 359 360 361 362
  /// Decides based on the current environment if goldens tests should be
  /// executed as pre-submit tests with Skia Gold.
  static bool isAvailableForEnvironment(Platform platform) {
    final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID')
      && platform.environment.containsKey('GOLDCTL')
      && platform.environment.containsKey('GOLD_TRYJOB');
    return luciPreSubmit;
363 364 365
  }
}

366 367
/// A [FlutterGoldenFileComparator] for testing conditions that do not execute
/// golden file tests.
368
///
369 370
/// Currently, this comparator is used on Cirrus, or in Luci environments when executing tests
/// outside of the flutter/flutter repository.
371 372 373
///
/// See also:
///
374
///  * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator]
375 376 377 378 379 380 381
///    that tests golden images through Skia Gold.
///  * [FlutterPreSubmitFileComparator], another
///    [FlutterGoldenFileComparator] that tests golden images before changes are
///    merged into the master branch.
///  * [FlutterLocalFileComparator], another
///    [FlutterGoldenFileComparator] that tests golden images locally on your
///    current machine.
382 383
class FlutterSkippingFileComparator extends FlutterGoldenFileComparator {
  /// Creates a [FlutterSkippingFileComparator] that will skip tests that
384
  /// are not in the right environment for golden file testing.
385
  FlutterSkippingFileComparator(
386 387
    final Uri basedir,
    final SkiaGoldClient skiaClient,
388 389 390
    this.reason, {
    String? namePrefix,
  }) : super(basedir, skiaClient, namePrefix: namePrefix);
391

392
  /// Describes the reason for using the [FlutterSkippingFileComparator].
393 394 395
  ///
  /// Cannot be null.
  final String reason;
396

397
  /// Creates a new [FlutterSkippingFileComparator] that mirrors the
398
  /// relative path resolution of the default [goldenFileComparator].
399
  static FlutterSkippingFileComparator fromDefaultComparator(
400
    String reason, {
401
    LocalFileComparator? defaultComparator,
402
    String? namePrefix,
403
  }) {
404
    defaultComparator ??= goldenFileComparator as LocalFileComparator;
405 406
    const FileSystem fs = LocalFileSystem();
    final Uri basedir = defaultComparator.basedir;
407
    final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir));
408
    return FlutterSkippingFileComparator(basedir, skiaClient, reason, namePrefix: namePrefix);
409 410 411 412
  }

  @override
  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
413 414 415 416 417
    // Ideally we would use markTestSkipped here but in some situations,
    // comparators are called outside of tests.
    // See also: https://github.com/flutter/flutter/issues/91285
    // ignore: avoid_print
    print('Skipping "$golden" test: $reason');
418 419 420 421
    return true;
  }

  @override
422
  Future<void> update(Uri golden, Uint8List imageBytes) async {}
423

424
  /// Decides, based on the current environment, if this comparator should be
425
  /// used.
426
  ///
427
  /// If we are in a CI environment, LUCI or Cirrus, but are not using the other
428
  /// comparators, we skip.
429
  static bool isAvailableForEnvironment(Platform platform) {
430
    return platform.environment.containsKey('SWARMING_TASK_ID')
431
      // Some builds are still being run on Cirrus, we should skip these.
432
      || platform.environment.containsKey('CIRRUS_CI');
433
  }
434
}
435

436 437 438 439
/// A [FlutterGoldenFileComparator] for testing golden images locally on your
/// current machine.
///
/// This comparator utilizes the [SkiaGoldClient] to request baseline images for
440 441 442 443 444 445 446 447
/// the given device under test for comparison. This comparator is initialized
/// when conditions for all other [FlutterGoldenFileComparators] have not been
/// met, see the `isAvailableForEnvironment` method for each one listed below.
///
/// The [FlutterLocalFileComparator] is intended to run on local machines and
/// serve as a smoke test during development. As such, it will not be able to
/// detect unintended changes on environments other than the currently executing
/// machine, until they are tested using the [FlutterPreSubmitFileComparator].
448 449 450 451 452
///
/// See also:
///
///  * [GoldenFileComparator], the abstract class that
///    [FlutterGoldenFileComparator] implements.
453
///  * [FlutterPostSubmitFileComparator], another
454 455 456 457 458
///    [FlutterGoldenFileComparator] that uploads tests to the Skia Gold
///    dashboard.
///  * [FlutterPreSubmitFileComparator], another
///    [FlutterGoldenFileComparator] that tests golden images before changes are
///    merged into the master branch.
459
///  * [FlutterSkippingFileComparator], another
460 461
///    [FlutterGoldenFileComparator] that controls post-submit testing
///    conditions that do not execute golden file tests.
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput {
  /// Creates a [FlutterLocalFileComparator] that will test golden file
  /// images against baselines requested from Flutter Gold.
  ///
  /// The [fs] and [platform] parameters are useful in tests, where the default
  /// file system and platform can be replaced by mock instances.
  FlutterLocalFileComparator(
    final Uri basedir,
    final SkiaGoldClient skiaClient, {
    final FileSystem fs = const LocalFileSystem(),
    final Platform platform = const LocalPlatform(),
  }) : super(
    basedir,
    skiaClient,
    fs: fs,
    platform: platform,
  );

  /// Creates a new [FlutterLocalFileComparator] that mirrors the
  /// relative path resolution of the default [goldenFileComparator].
  ///
483 484
  /// The [goldens], [defaultComparator], and [baseDirectory] parameters are
  /// visible for testing purposes only.
485 486
  static Future<FlutterGoldenFileComparator> fromDefaultComparator(
    final Platform platform, {
487 488 489
    SkiaGoldClient? goldens,
    LocalFileComparator? defaultComparator,
    Directory? baseDirectory,
490
  }) async {
491
    defaultComparator ??= goldenFileComparator as LocalFileComparator;
492
    baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory(
493 494 495 496 497 498 499 500
      defaultComparator,
      platform,
    );

    if(!baseDirectory.existsSync()) {
      baseDirectory.createSync(recursive: true);
    }

501
    goldens ??= SkiaGoldClient(baseDirectory);
502
    try {
503 504
      // Check if we can reach Gold.
      await goldens.getExpectationForTest('');
505
    } on io.OSError catch (_) {
506
      return FlutterSkippingFileComparator(
507 508
        baseDirectory.uri,
        goldens,
509 510 511 512
        'OSError occurred, could not reach Gold. '
          'Switching to FlutterSkippingGoldenFileComparator.',
      );
    } on io.SocketException catch (_) {
513
      return FlutterSkippingFileComparator(
514 515 516 517
        baseDirectory.uri,
        goldens,
        'SocketException occurred, could not reach Gold. '
          'Switching to FlutterSkippingGoldenFileComparator.',
518 519
      );
    }
520 521 522 523 524 525 526 527

    return FlutterLocalFileComparator(baseDirectory.uri, goldens);
  }

  @override
  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
    golden = _addPrefix(golden);
    final String testName = skiaClient.cleanTestName(golden.path);
528 529 530
    late String? testExpectation;
    testExpectation = await skiaClient.getExpectationForTest(testName);

531
    if (testExpectation == null || testExpectation.isEmpty) {
532 533 534 535 536 537 538
      // There is no baseline for this test.
      // Ideally we would use markTestSkipped here but in some situations,
      // comparators are called outside of tests.
      // See also: https://github.com/flutter/flutter/issues/91285
      // ignore: avoid_print
      print(
        'No expectations provided by Skia Gold for test: $golden. '
539 540 541 542 543 544 545 546 547
        'This may be a new test. If this is an unexpected result, check '
        'https://flutter-gold.skia.org.\n'
        'Validate image output found at $basedir'
      );
      update(golden, imageBytes);
      return true;
    }

    ComparisonResult result;
548
    final List<int> goldenBytes = await skiaClient.getImageBytes(testExpectation);
549

550 551 552 553
    result = await GoldenFileComparator.compareLists(
      imageBytes,
      goldenBytes,
    );
554

555 556
    if (result.passed)
      return true;
557

558 559
    final String error = await generateFailureOutput(result, golden, basedir);
    throw FlutterError(error);
560 561
  }
}