Skip to content

F
Front-End
Unverified Commit 62f15949 authored by Ian Hickson's avatar Ian Hickson Committed by GitHub
Browse files
Options

Make tests more resilient to Skia gold failures and refactor flutter_goldens... 

Make tests more resilient to Skia gold failures and refactor flutter_goldens for extensive technical debt removal (#140101)

Originally landed in https://github.com/flutter/flutter/pull/139549
Originally reverted in https://github.com/flutter/flutter/pull/140085

- Remove all use of global variables.
- Always pass in all dependencies, only create them in main or in tests.
- Pass in the "print" primitive.
- Make all network traffic retry (except when run locally, when it just auto-passes).
- Enable tests to be run in random order.
- Better error messages
parent 9e104eb7
Show whitespace changes
Inline Side-by-side
Showing with 1707 additions and 1484 deletions
dev/automated_tests/flutter_test/flutter_gold_expectation.txt
View file @ 62f15949
[0-9]+:[0-9]+ [+]0: Local passes non-existent baseline for new test, null expectation *
*No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.
*Validate image output found at flutter/test/library/
[0-9]+:[0-9]+ [+]1: Local passes non-existent baseline for new test, empty expectation *
*No expectations provided by Skia Gold for test: library.flutter.new_golden_test.2.png. This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.
*Validate image output found at flutter/test/library/
[0-9]+:[0-9]+ [+]2: All tests passed! * [0-9]+:[0-9]+ [+]2: All tests passed! *
dev/automated_tests/flutter_test/flutter_gold_test.dart
View file @ 62f15949
...@@ -21,6 +21,7 @@ const List<int> _kFailPngBytes = <int>[ ...@@ -21,6 +21,7 @@ const List<int> _kFailPngBytes = <int>[
]; ];
void main() { void main() {
final List<String> log = <String>[];
final MemoryFileSystem fs = MemoryFileSystem(); final MemoryFileSystem fs = MemoryFileSystem();
final Directory basedir = fs.directory('flutter/test/library/') final Directory basedir = fs.directory('flutter/test/library/')
..createSync(recursive: true); ..createSync(recursive: true);
...@@ -34,6 +35,7 @@ void main() { ...@@ -34,6 +35,7 @@ void main() {
environment: <String, String>{'FLUTTER_ROOT': '/flutter'}, environment: <String, String>{'FLUTTER_ROOT': '/flutter'},
operatingSystem: 'macos' operatingSystem: 'macos'
), ),
log: log.add,
); );
test('Local passes non-existent baseline for new test, null expectation', () async { test('Local passes non-existent baseline for new test, null expectation', () async {
...@@ -44,6 +46,12 @@ void main() { ...@@ -44,6 +46,12 @@ void main() {
), ),
isTrue, isTrue,
); );
expect(log, <String>[
// ignore: no_adjacent_strings_in_list
'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. '
'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
'Validate image output found at flutter/test/library/'
]);
}); });
test('Local passes non-existent baseline for new test, empty expectation', () async { test('Local passes non-existent baseline for new test, empty expectation', () async {
...@@ -54,6 +62,16 @@ void main() { ...@@ -54,6 +62,16 @@ void main() {
), ),
isTrue, isTrue,
); );
expect(log, <String>[
// ignore: no_adjacent_strings_in_list
'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.1.png. '
'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
'Validate image output found at flutter/test/library/',
// ignore: no_adjacent_strings_in_list
'No expectations provided by Skia Gold for test: library.flutter.new_golden_test.2.png. '
'This may be a new test. If this is an unexpected result, check https://flutter-gold.skia.org.\n'
'Validate image output found at flutter/test/library/',
]);
}); });
} }
......
packages/flutter/pubspec.yaml
View file @ 62f15949
...@@ -28,6 +28,7 @@ dev_dependencies: ...@@ -28,6 +28,7 @@ dev_dependencies:
async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
leak_tracker: 9.0.18 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" leak_tracker: 9.0.18 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
leak_tracker_testing: 1.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" leak_tracker_testing: 1.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
...@@ -41,6 +42,7 @@ dev_dependencies: ...@@ -41,6 +42,7 @@ dev_dependencies:
string_scanner: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" string_scanner: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
term_glyph: 1.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" term_glyph: 1.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
test_api: 0.6.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" test_api: 0.6.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
vm_service: 13.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" vm_service: 13.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
# PUBSPEC CHECKSUM: 84b7 # PUBSPEC CHECKSUM: e519
packages/flutter/test_private/test/pubspec.yaml
View file @ 62f15949
...@@ -38,8 +38,10 @@ dev_dependencies: ...@@ -38,8 +38,10 @@ dev_dependencies:
sdk: flutter sdk: flutter
fake_async: 1.3.1 fake_async: 1.3.1
crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" file: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
platform: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" platform: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
process: 5.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" process: 5.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
# PUBSPEC CHECKSUM: 942d # PUBSPEC CHECKSUM: 5a8e
packages/flutter_goldens/README 0 → 100644
View file @ 62f15949
This package is an internal implementation detail for our testing
infrastructure. It enables the framework to use the Skia Gold
infrastructure for tracking golden image tests.
See also:
* https://skia.org/docs/dev/testing/skiagold/
* https://flutter-gold.skia.org/
* https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter
packages/flutter_goldens/dart_test.yaml deleted 100644 → 0
View file @ 9e104eb7
tags:
# This tag tells the test framework to not shuffle the test order according to
# the --test-randomize-ordering-seed for the suites that have this tag.
no-shuffle:
allow_test_randomization: false
packages/flutter_goldens/lib/flutter_goldens.dart
View file @ 62f15949
...@@ -3,23 +3,28 @@ ...@@ -3,23 +3,28 @@
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:async' show FutureOr; import 'dart:async' show FutureOr;
import 'dart:io' as io show OSError, SocketException; import 'dart:io' as io;
import 'package:file/file.dart'; import 'package:file/file.dart';
import 'package:file/local.dart'; import 'package:file/local.dart';
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:flutter_goldens_client/skia_client.dart';
import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test/flutter_test.dart';
import 'package:platform/platform.dart'; import 'package:platform/platform.dart';
import 'package:process/process.dart';
export 'package:flutter_goldens_client/skia_client.dart'; import 'skia_client.dart';
export 'skia_client.dart';
// If you are here trying to figure out how to use golden files in the Flutter // If you are here trying to figure out how to use golden files in the Flutter
// repo itself, consider reading this wiki page: // repo itself, consider reading this wiki page:
// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter // https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter
const String _kFlutterRootKey = 'FLUTTER_ROOT'; const String _kFlutterRootKey = 'FLUTTER_ROOT';
final RegExp _kMainBranch = RegExp(r'master|main');
bool _isMainBranch(String? branch) {
return branch == 'main'
|| branch == 'master';
}
/// Main method that can be used in a `flutter_test_config.dart` file to set /// Main method that can be used in a `flutter_test_config.dart` file to set
/// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that /// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that
...@@ -28,18 +33,51 @@ final RegExp _kMainBranch = RegExp(r'master|main'); ...@@ -28,18 +33,51 @@ final RegExp _kMainBranch = RegExp(r'master|main');
/// ///
/// When set, the `namePrefix` is prepended to the names of all gold images. /// When set, the `namePrefix` is prepended to the names of all gold images.
Future<void> testExecutable(FutureOr<void> Function() testMain, {String? namePrefix}) async { Future<void> testExecutable(FutureOr<void> Function() testMain, {String? namePrefix}) async {
assert(goldenFileComparator is LocalFileComparator);
const Platform platform = LocalPlatform(); const Platform platform = LocalPlatform();
if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) { const FileSystem fs = LocalFileSystem();
goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix); const ProcessManager process = LocalProcessManager();
} else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) { final io.HttpClient httpClient = io.HttpClient();
goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix); if (FlutterPostSubmitFileComparator.isRecommendedForEnvironment(platform)) {
} else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) { goldenFileComparator = await FlutterPostSubmitFileComparator.fromLocalFileComparator(
goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator( localFileComparator: goldenFileComparator as LocalFileComparator,
'Golden file testing is not executed on Cirrus, or LUCI environments outside of flutter/flutter.', namePrefix: namePrefix,
namePrefix: namePrefix platform: platform,
fs: fs,
process: process,
httpClient: httpClient,
log: print,
);
} else if (FlutterPreSubmitFileComparator.isRecommendedForEnvironment(platform)) {
goldenFileComparator = await FlutterPreSubmitFileComparator.fromLocalFileComparator(
localFileComparator: goldenFileComparator as LocalFileComparator,
namePrefix: namePrefix,
platform: platform,
fs: fs,
process: process,
httpClient: httpClient,
log: print,
);
} else if (FlutterSkippingFileComparator.isRecommendedForEnvironment(platform)) {
goldenFileComparator = FlutterSkippingFileComparator.fromLocalFileComparator(
localFileComparator: goldenFileComparator as LocalFileComparator,
namePrefix: namePrefix,
reason: 'Golden file testing is not executed on Cirrus, or LUCI environments outside of flutter/flutter.',
platform: platform,
fs: fs,
process: process,
httpClient: httpClient,
log: print,
); );
} else { } else {
goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform); goldenFileComparator = await FlutterLocalFileComparator.fromLocalFileComparator(
localFileComparator: goldenFileComparator as LocalFileComparator,
platform: platform,
fs: fs,
process: process,
httpClient: httpClient,
log: print,
);
} }
await testMain(); await testMain();
...@@ -88,12 +126,12 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { ...@@ -88,12 +126,12 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator {
/// ///
/// The [fs] and [platform] parameters are useful in tests, where the default /// The [fs] and [platform] parameters are useful in tests, where the default
/// file system and platform can be replaced by mock instances. /// file system and platform can be replaced by mock instances.
@visibleForTesting
FlutterGoldenFileComparator( FlutterGoldenFileComparator(
this.basedir, this.basedir,
this.skiaClient, { this.skiaClient, {
this.fs = const LocalFileSystem(), required this.fs,
this.platform = const LocalPlatform(), required this.platform,
required this.log,
this.namePrefix, this.namePrefix,
}); });
...@@ -113,6 +151,9 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { ...@@ -113,6 +151,9 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator {
@visibleForTesting @visibleForTesting
final Platform platform; final Platform platform;
/// The logging function to use when reporting messages to the console.
final LogCallback log;
/// The prefix that is added to all golden names. /// The prefix that is added to all golden names.
final String? namePrefix; final String? namePrefix;
...@@ -135,11 +176,11 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator { ...@@ -135,11 +176,11 @@ abstract class FlutterGoldenFileComparator extends GoldenFileComparator {
@protected @protected
@visibleForTesting @visibleForTesting
static Directory getBaseDirectory( static Directory getBaseDirectory(
LocalFileComparator defaultComparator, LocalFileComparator defaultComparator, {
Platform platform, {
String? suffix, String? suffix,
required Platform platform,
required FileSystem fs,
}) { }) {
const FileSystem fs = LocalFileSystem();
final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]);
Directory comparisonRoot; Directory comparisonRoot;
...@@ -214,34 +255,48 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { ...@@ -214,34 +255,48 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator {
FlutterPostSubmitFileComparator( FlutterPostSubmitFileComparator(
super.basedir, super.basedir,
super.skiaClient, { super.skiaClient, {
super.fs, required super.fs,
super.platform, required super.platform,
required super.log,
super.namePrefix, super.namePrefix,
}); });
/// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative
/// path resolution of the default [goldenFileComparator]. /// path resolution of the default [goldenFileComparator].
/// static Future<FlutterPostSubmitFileComparator> fromLocalFileComparator({
/// The [goldens] and [defaultComparator] parameters are visible for testing required LocalFileComparator localFileComparator,
/// purposes only.
static Future<FlutterPostSubmitFileComparator> fromDefaultComparator(
final Platform platform, {
SkiaGoldClient? goldens,
LocalFileComparator? defaultComparator,
String? namePrefix, String? namePrefix,
required Platform platform,
required FileSystem fs,
required ProcessManager process,
required io.HttpClient httpClient,
required LogCallback log,
}) async { }) async {
defaultComparator ??= goldenFileComparator as LocalFileComparator;
final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory(
defaultComparator, localFileComparator,
platform,
suffix: 'flutter_goldens_postsubmit.', suffix: 'flutter_goldens_postsubmit.',
platform: platform,
fs: fs,
); );
baseDirectory.createSync(recursive: true); baseDirectory.createSync(recursive: true);
goldens ??= SkiaGoldClient(baseDirectory); final SkiaGoldClient goldens = SkiaGoldClient(
baseDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: httpClient,
log: log,
);
await goldens.auth(); await goldens.auth();
return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens, namePrefix: namePrefix); return FlutterPostSubmitFileComparator(
baseDirectory.uri,
goldens,
fs: fs,
platform: platform,
log: log,
namePrefix: namePrefix,
);
} }
@override @override
...@@ -250,21 +305,17 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { ...@@ -250,21 +305,17 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator {
golden = _addPrefix(golden); golden = _addPrefix(golden);
await update(golden, imageBytes); await update(golden, imageBytes);
final File goldenFile = getGoldenFile(golden); final File goldenFile = getGoldenFile(golden);
await skiaClient.imgtestAdd(golden.path, goldenFile); // throws if the result is false
return skiaClient.imgtestAdd(golden.path, goldenFile); return true;
} }
/// Decides based on the current environment if goldens tests should be /// Decides based on the current environment if goldens tests should be
/// executed through Skia Gold. /// executed through Skia Gold.
static bool isAvailableForEnvironment(Platform platform) { static bool isRecommendedForEnvironment(Platform platform) {
final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID') return platform.environment.containsKey('SWARMING_TASK_ID') // Indicates LUCI environment.
&& platform.environment.containsKey('GOLDCTL') && platform.environment.containsKey('GOLDCTL') // Needed to use Gold.
// Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. && !platform.environment.containsKey('GOLD_TRYJOB') // Indicates a pre-submit environment on LUCI.
&& !platform.environment.containsKey('GOLD_TRYJOB') && _isMainBranch(platform.environment['GIT_BRANCH']);
// Only run on main branch.
&& _kMainBranch.hasMatch(platform.environment['GIT_BRANCH'] ?? '');
return luciPostSubmit;
} }
} }
...@@ -291,41 +342,49 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { ...@@ -291,41 +342,49 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator {
FlutterPreSubmitFileComparator( FlutterPreSubmitFileComparator(
super.basedir, super.basedir,
super.skiaClient, { super.skiaClient, {
super.fs, required super.fs,
super.platform, required super.platform,
required super.log,
super.namePrefix, super.namePrefix,
}); });
/// Creates a new [FlutterPreSubmitFileComparator] that mirrors the /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the
/// relative path resolution of the default [goldenFileComparator]. /// relative path resolution of the default [goldenFileComparator].
/// static Future<FlutterGoldenFileComparator> fromLocalFileComparator({
/// The [goldens] and [defaultComparator] parameters are visible for testing required LocalFileComparator localFileComparator,
/// purposes only.
static Future<FlutterGoldenFileComparator> fromDefaultComparator(
final Platform platform, {
SkiaGoldClient? goldens,
LocalFileComparator? defaultComparator,
Directory? testBasedir, Directory? testBasedir,
String? namePrefix, String? namePrefix,
required Platform platform,
required FileSystem fs,
required ProcessManager process,
required io.HttpClient httpClient,
required LogCallback log,
}) async { }) async {
defaultComparator ??= goldenFileComparator as LocalFileComparator;
final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory( final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory(
defaultComparator, localFileComparator,
platform,
suffix: 'flutter_goldens_presubmit.', suffix: 'flutter_goldens_presubmit.',
platform: platform,
fs: fs,
); );
if (!baseDirectory.existsSync()) { if (!baseDirectory.existsSync()) {
baseDirectory.createSync(recursive: true); baseDirectory.createSync(recursive: true);
} }
goldens ??= SkiaGoldClient(baseDirectory); final SkiaGoldClient goldens = SkiaGoldClient(
baseDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: httpClient,
log: log,
);
await goldens.auth(); await goldens.auth();
return FlutterPreSubmitFileComparator( return FlutterPreSubmitFileComparator(
baseDirectory.uri, baseDirectory.uri,
goldens, platform: platform, goldens,
fs: fs,
platform: platform,
log: log,
namePrefix: namePrefix, namePrefix: namePrefix,
); );
} }
...@@ -346,21 +405,19 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { ...@@ -346,21 +405,19 @@ class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator {
/// Decides based on the current environment if goldens tests should be /// Decides based on the current environment if goldens tests should be
/// executed as pre-submit tests with Skia Gold. /// executed as pre-submit tests with Skia Gold.
static bool isAvailableForEnvironment(Platform platform) { static bool isRecommendedForEnvironment(Platform platform) {
final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID') return platform.environment.containsKey('SWARMING_TASK_ID') // Indicates LUCI environment.
&& platform.environment.containsKey('GOLDCTL') && platform.environment.containsKey('GOLDCTL') // Needed to use Gold.
&& platform.environment.containsKey('GOLD_TRYJOB') && platform.environment.containsKey('GOLD_TRYJOB') // Indicates a pre-submit environment on LUCI.
// Only run on the main branch && _isMainBranch(platform.environment['GIT_BRANCH']);
&& _kMainBranch.hasMatch(platform.environment['GIT_BRANCH'] ?? '');
return luciPreSubmit;
} }
} }
/// A [FlutterGoldenFileComparator] for testing conditions that do not execute /// A [FlutterGoldenFileComparator] for testing conditions that do not execute
/// golden file tests. /// golden file tests.
/// ///
/// Currently, this comparator is used on Cirrus, or in Luci environments when executing tests /// Currently, this comparator is used on Cirrus, or in Luci environments when
/// outside of the flutter/flutter repository. /// executing tests outside of the flutter/flutter repository.
/// ///
/// See also: /// See also:
/// ///
...@@ -379,6 +436,9 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { ...@@ -379,6 +436,9 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator {
super.basedir, super.basedir,
super.skiaClient, super.skiaClient,
this.reason, { this.reason, {
required super.fs,
required super.platform,
required super.log,
super.namePrefix, super.namePrefix,
}); });
...@@ -387,25 +447,39 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { ...@@ -387,25 +447,39 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator {
/// Creates a new [FlutterSkippingFileComparator] that mirrors the /// Creates a new [FlutterSkippingFileComparator] that mirrors the
/// relative path resolution of the default [goldenFileComparator]. /// relative path resolution of the default [goldenFileComparator].
static FlutterSkippingFileComparator fromDefaultComparator( static FlutterSkippingFileComparator fromLocalFileComparator({
String reason, { required LocalFileComparator localFileComparator,
LocalFileComparator? defaultComparator,
String? namePrefix, String? namePrefix,
required String reason,
required FileSystem fs,
required ProcessManager process,
required Platform platform,
required io.HttpClient httpClient,
required LogCallback log,
}) { }) {
defaultComparator ??= goldenFileComparator as LocalFileComparator; final Uri basedir = localFileComparator.basedir;
const FileSystem fs = LocalFileSystem(); final SkiaGoldClient skiaClient = SkiaGoldClient(
final Uri basedir = defaultComparator.basedir; fs.directory(basedir),
final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir)); fs: fs,
return FlutterSkippingFileComparator(basedir, skiaClient, reason, namePrefix: namePrefix); process: process,
platform: platform,
httpClient: httpClient,
log: log,
);
return FlutterSkippingFileComparator(
basedir,
skiaClient,
reason,
fs: fs,
platform: platform,
log: log,
namePrefix: namePrefix,
);
} }
@override @override
Future<bool> compare(Uint8List imageBytes, Uri golden) async { Future<bool> compare(Uint8List imageBytes, Uri golden) async {
// Ideally we would use markTestSkipped here but in some situations, log('Auto-passing "$golden" test without checking: $reason');
// comparators are called outside of tests.
// See also: https://github.com/flutter/flutter/issues/91285
// ignore: avoid_print
print('Skipping "$golden" test: $reason');
return true; return true;
} }
...@@ -416,13 +490,10 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { ...@@ -416,13 +490,10 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator {
/// used. /// used.
/// ///
/// If we are in a CI environment, LUCI or Cirrus, but are not using the other /// If we are in a CI environment, LUCI or Cirrus, but are not using the other
/// comparators, we skip. /// comparators (determined by checking this after the others), we skip.
static bool isAvailableForEnvironment(Platform platform) { static bool isRecommendedForEnvironment(Platform platform) {
return (platform.environment.containsKey('SWARMING_TASK_ID') return platform.environment.containsKey('SWARMING_TASK_ID') // Indicates LUCI environment.
// Some builds are still being run on Cirrus, we should skip these. || platform.environment.containsKey('CIRRUS_CI'); // Indicates Cirrus environment.
|| platform.environment.containsKey('CIRRUS_CI'))
// If we are in CI, skip on branches that are not main.
&& !_kMainBranch.hasMatch(platform.environment['GIT_BRANCH'] ?? '');
} }
} }
...@@ -432,7 +503,7 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { ...@@ -432,7 +503,7 @@ class FlutterSkippingFileComparator extends FlutterGoldenFileComparator {
/// This comparator utilizes the [SkiaGoldClient] to request baseline images for /// This comparator utilizes the [SkiaGoldClient] to request baseline images for
/// the given device under test for comparison. This comparator is initialized /// the given device under test for comparison. This comparator is initialized
/// when conditions for all other [FlutterGoldenFileComparators] have not been /// when conditions for all other [FlutterGoldenFileComparators] have not been
/// met, see the `isAvailableForEnvironment` method for each one listed below. /// met, see the `isRecommendedForEnvironment` method for each one listed below.
/// ///
/// The [FlutterLocalFileComparator] is intended to run on local machines and /// The [FlutterLocalFileComparator] is intended to run on local machines and
/// serve as a smoke test during development. As such, it will not be able to /// serve as a smoke test during development. As such, it will not be able to
...@@ -461,68 +532,56 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC ...@@ -461,68 +532,56 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
FlutterLocalFileComparator( FlutterLocalFileComparator(
super.basedir, super.basedir,
super.skiaClient, { super.skiaClient, {
super.fs, required super.fs,
super.platform, required super.platform,
required super.log,
}); });
/// Creates a new [FlutterLocalFileComparator] that mirrors the /// Creates a new [FlutterLocalFileComparator] that mirrors the
/// relative path resolution of the default [goldenFileComparator]. /// relative path resolution of the default [goldenFileComparator].
/// static Future<FlutterGoldenFileComparator> fromLocalFileComparator({
/// The [goldens], [defaultComparator], and [baseDirectory] parameters are required LocalFileComparator localFileComparator,
/// visible for testing purposes only. required Platform platform,
static Future<FlutterGoldenFileComparator> fromDefaultComparator( required FileSystem fs,
final Platform platform, { required ProcessManager process,
SkiaGoldClient? goldens, required io.HttpClient httpClient,
LocalFileComparator? defaultComparator, required LogCallback log,
Directory? baseDirectory,
}) async { }) async {
defaultComparator ??= goldenFileComparator as LocalFileComparator; final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory(
baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( localFileComparator,
defaultComparator, platform: platform,
platform, fs: fs,
); );
if (!baseDirectory.existsSync()) { if (!baseDirectory.existsSync()) {
baseDirectory.createSync(recursive: true); baseDirectory.createSync(recursive: true);
} }
goldens ??= SkiaGoldClient(baseDirectory); final SkiaGoldClient goldens = SkiaGoldClient(
try { baseDirectory,
// Check if we can reach Gold. fs: fs,
await goldens.getExpectationForTest(''); process: process,
} on io.OSError catch (_) { platform: platform,
return FlutterSkippingFileComparator( httpClient: httpClient,
baseDirectory.uri, log: log,
goldens,
'OSError occurred, could not reach Gold. '
'Switching to FlutterSkippingGoldenFileComparator.',
); );
} on io.SocketException catch (_) { return FlutterLocalFileComparator(
return FlutterSkippingFileComparator(
baseDirectory.uri, baseDirectory.uri,
goldens, goldens,
'SocketException occurred, could not reach Gold. ' fs: fs,
'Switching to FlutterSkippingGoldenFileComparator.', platform: platform,
log: log,
); );
} }
return FlutterLocalFileComparator(baseDirectory.uri, goldens);
}
@override @override
Future<bool> compare(Uint8List imageBytes, Uri golden) async { Future<bool> compare(Uint8List imageBytes, Uri golden) async {
golden = _addPrefix(golden); golden = _addPrefix(golden);
final String testName = skiaClient.cleanTestName(golden.path); final String testName = skiaClient.cleanTestName(golden.path);
late String? testExpectation; late String? testExpectation;
try {
testExpectation = await skiaClient.getExpectationForTest(testName); testExpectation = await skiaClient.getExpectationForTest(testName);
if (testExpectation == null || testExpectation.isEmpty) { if (testExpectation == null || testExpectation.isEmpty) {
// There is no baseline for this test. log(
// Ideally we would use markTestSkipped here but in some situations,
// comparators are called outside of tests.
// See also: https://github.com/flutter/flutter/issues/91285
// ignore: avoid_print
print(
'No expectations provided by Skia Gold for test: $golden. ' 'No expectations provided by Skia Gold for test: $golden. '
'This may be a new test. If this is an unexpected result, check ' 'This may be a new test. If this is an unexpected result, check '
'https://flutter-gold.skia.org.\n' 'https://flutter-gold.skia.org.\n'
...@@ -531,6 +590,14 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC ...@@ -531,6 +590,14 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
update(golden, imageBytes); update(golden, imageBytes);
return true; return true;
} }
} on Exception catch (error) {
if (error is! io.SocketException &&
error is! io.OSError) {
rethrow; // "uncaught error"
}
log('Auto-passing "$golden" test, ignoring network error when contacting Skia.');
return true;
}
ComparisonResult result; ComparisonResult result;
final List<int> goldenBytes = await skiaClient.getImageBytes(testExpectation); final List<int> goldenBytes = await skiaClient.getImageBytes(testExpectation);
......
packages/flutter_goldens_client/lib/skia_client.dart → packages/flutter_goldens/lib/skia_client.dart
View file @ 62f15949
...@@ -7,7 +7,6 @@ import 'dart:io' as io; ...@@ -7,7 +7,6 @@ import 'dart:io' as io;
import 'package:crypto/crypto.dart'; import 'package:crypto/crypto.dart';
import 'package:file/file.dart'; import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:platform/platform.dart'; import 'package:platform/platform.dart';
import 'package:process/process.dart'; import 'package:process/process.dart';
...@@ -21,6 +20,17 @@ const String _kGoldctlKey = 'GOLDCTL'; ...@@ -21,6 +20,17 @@ const String _kGoldctlKey = 'GOLDCTL';
const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER'; const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';
const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER'; const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER';
/// Signature of callbacks used to inject [print] replacements.
typedef LogCallback = void Function(String);
/// Signature of callbacks used to determine if a Skia Gold command succeeded,
/// and if not, what the error message should be.
///
/// Return null if the given arguments indicate success.
///
/// Otherwise, return the error message to show.
typedef SkiaErrorCallback = String? Function(int exitCode, String stdout, String stderr);
/// Exception thrown when an error is returned from the [SkiaClient]. /// Exception thrown when an error is returned from the [SkiaClient].
class SkiaException implements Exception { class SkiaException implements Exception {
/// Creates a new `SkiaException` with a required error [message]. /// Creates a new `SkiaException` with a required error [message].
...@@ -40,16 +50,14 @@ class SkiaException implements Exception { ...@@ -40,16 +50,14 @@ class SkiaException implements Exception {
/// Flutter Gold Dashboard. /// Flutter Gold Dashboard.
class SkiaGoldClient { class SkiaGoldClient {
/// Creates a [SkiaGoldClient] with the given [workDirectory]. /// Creates a [SkiaGoldClient] with the given [workDirectory].
///
/// All other parameters are optional. They may be provided in tests to
/// override the defaults for [fs], [process], [platform], and [httpClient].
SkiaGoldClient( SkiaGoldClient(
this.workDirectory, { this.workDirectory, {
this.fs = const LocalFileSystem(), required this.fs,
this.process = const LocalProcessManager(), required this.process,
this.platform = const LocalPlatform(), required this.platform,
io.HttpClient? httpClient, required this.httpClient,
}) : httpClient = httpClient ?? io.HttpClient(); required this.log,
});
/// The file system to use for storing the local clone of the repository. /// The file system to use for storing the local clone of the repository.
/// ///
...@@ -57,12 +65,6 @@ class SkiaGoldClient { ...@@ -57,12 +65,6 @@ class SkiaGoldClient {
/// replaced by a memory file system. /// replaced by a memory file system.
final FileSystem fs; final FileSystem fs;
/// A wrapper for the [dart:io.Platform] API.
///
/// This is useful in tests, where the system platform (the default) can be
/// replaced by a mock platform instance.
final Platform platform;
/// A controller for launching sub-processes. /// A controller for launching sub-processes.
/// ///
/// This is useful in tests, where the real process manager (the default) can /// This is useful in tests, where the real process manager (the default) can
...@@ -70,9 +72,18 @@ class SkiaGoldClient { ...@@ -70,9 +72,18 @@ class SkiaGoldClient {
/// sub-processes. /// sub-processes.
final ProcessManager process; final ProcessManager process;
/// A wrapper for the [dart:io.Platform] API.
///
/// This is useful in tests, where the system platform (the default) can be
/// replaced by a mock platform instance.
final Platform platform;
/// A client for making Http requests to the Flutter Gold dashboard. /// A client for making Http requests to the Flutter Gold dashboard.
final io.HttpClient httpClient; final io.HttpClient httpClient;
/// The logging function to use when reporting messages to the console.
final void Function(String message) log;
/// The local [Directory] within the [comparisonRoot] for the current test /// The local [Directory] within the [comparisonRoot] for the current test
/// context. In this directory, the client will create image and JSON files /// context. In this directory, the client will create image and JSON files
/// for the goldctl tool to use. /// for the goldctl tool to use.
...@@ -91,38 +102,117 @@ class SkiaGoldClient { ...@@ -91,38 +102,117 @@ class SkiaGoldClient {
/// Uses the [platform] environment in this implementation. /// Uses the [platform] environment in this implementation.
String get _goldctl => platform.environment[_kGoldctlKey]!; String get _goldctl => platform.environment[_kGoldctlKey]!;
static void _indent(LogCallback writeln, String text) {
if (text.isEmpty) {
writeln(' <empty>');
} else {
for (final String line in text.split('\n')) {
writeln(' $line');
}
}
}
static void _dump(LogCallback writeln, String data, String label) {
if (data.isNotEmpty) {
writeln('');
writeln('$label:');
_indent(writeln, data);
}
}
Future<void> _retry({
required Future<io.ProcessResult> Function() task,
required String taskName,
SkiaErrorCallback? errorMessage,
}) async {
Duration delay = const Duration(seconds: 5);
while (true) {
final io.ProcessResult result = await task();
final String resultStdout = result.stdout as String;
final String resultStderr = result.stderr as String;
if (result.exitCode != 0 && resultStdout.contains('resulted in a 502: 502 Bad Gateway')) {
// Probably a transient error, try again.
// (See https://issues.skia.org/issues/40044713)
//
// This could have false-positives, because there's no standard format
// for the error messages from Skia gold. Maybe the test name is output
// and the test name contains the string above, who knows. For now it
// seems more likely that the server is flaking than that there's a
// false positive, and false positives seem less likely to be flaky so
// we're likely to catch them when they happen.
log('Transient failure (exit code ${result.exitCode}) from Skia Gold.');
_dump(log, resultStdout, 'stdout from gold');
_dump(log, resultStderr, 'stderr from gold');
log('');
log('Retrying in ${delay.inSeconds} seconds.');
await Future<void>.delayed(delay);
delay *= 2;
continue; // retry
}
String? message;
if (errorMessage != null) {
message = errorMessage(result.exitCode, resultStdout, resultStderr);
if (message == null) {
return; // success
}
} else {
if (result.exitCode == 0) {
return; // success
}
}
final StringBuffer buffer = StringBuffer();
if (message != null) {
buffer.writeln(message);
buffer.writeln();
}
buffer.writeln('$taskName failed with exit code ${result.exitCode}.');
_dump(buffer.writeln, resultStdout, 'stdout from gold');
_dump(buffer.writeln, resultStderr, 'stderr from gold');
final File resultFile = workDirectory.childFile('result-state.json');
if (await resultFile.exists()) {
_dump(buffer.writeln, resultFile.readAsStringSync(), 'result-state.json contents');
}
throw SkiaException(buffer.toString()); // failure
}
}
/// Prepares the local work space for golden file testing and calls the /// Prepares the local work space for golden file testing and calls the
/// goldctl `auth` command. /// goldctl `auth` command.
/// ///
/// This ensures that the goldctl tool is authorized and ready for testing. /// This ensures that the goldctl tool is authorized and ready for testing.
/// Used by the [FlutterPostSubmitFileComparator] and the /// Used by the [FlutterPostSubmitFileComparator] and the
/// [FlutterPreSubmitFileComparator]. /// [FlutterPreSubmitFileComparator].
///
/// Does nothing if [clientIsAuthorized] returns true.
Future<void> auth() async { Future<void> auth() async {
if (await clientIsAuthorized()) { if (await clientIsAuthorized()) {
return; return;
} }
final List<String> authCommand = <String>[
await _retry(
task: () => process.run(<String>[
_goldctl, _goldctl,
'auth', 'auth',
'--work-dir', workDirectory '--work-dir', workDirectory
.childDirectory('temp') .childDirectory('temp')
.path, .path,
'--luci', '--luci',
]; ]),
taskName: 'auth',
final io.ProcessResult result = await process.run(authCommand); errorMessage: (int exitCode, String resultStdout, String resultStderr) {
if (exitCode == 0) {
if (result.exitCode != 0) { return null;
final StringBuffer buf = StringBuffer() }
..writeln('Skia Gold authorization failed.') return 'Skia Gold authorization failed.\n'
..writeln('Luci environments authenticate using the file provided ' '\n'
'by LUCI_CONTEXT. There may be an error with this file or Gold ' 'Luci environments authenticate using the file provided by '
'authentication.') 'LUCI_CONTEXT. There may be an error with this file or Gold '
..writeln('Debug information for Gold --------------------------------') 'authentication.';
..writeln('stdout: ${result.stdout}') },
..writeln('stderr: ${result.stderr}'); );
throw SkiaException(buf.toString());
}
} }
/// Signals if this client is initialized for uploading images to the Gold /// Signals if this client is initialized for uploading images to the Gold
...@@ -138,6 +228,8 @@ class SkiaGoldClient { ...@@ -138,6 +228,8 @@ class SkiaGoldClient {
/// The `imgtest` command collects and uploads test results to the Skia Gold /// The `imgtest` command collects and uploads test results to the Skia Gold
/// backend, the `init` argument initializes the current test. Used by the /// backend, the `init` argument initializes the current test. Used by the
/// [FlutterPostSubmitFileComparator]. /// [FlutterPostSubmitFileComparator].
///
/// This function is idempotent.
Future<void> imgtestInit() async { Future<void> imgtestInit() async {
// This client has already been initialized // This client has already been initialized
if (_initialized) { if (_initialized) {
...@@ -173,20 +265,16 @@ class SkiaGoldClient { ...@@ -173,20 +265,16 @@ class SkiaGoldClient {
throw SkiaException(buf.toString()); throw SkiaException(buf.toString());
} }
final io.ProcessResult result = await process.run(imgtestInitCommand); await _retry(
task: () => process.run(imgtestInitCommand),
if (result.exitCode != 0) { taskName: 'imgtest init',
_initialized = false; errorMessage: (int exitCode, String resultStdout, String resultStderr) {
final StringBuffer buf = StringBuffer() if (exitCode == 0) {
..writeln('Skia Gold imgtest init failed.') return null;
..writeln('An error occurred when initializing golden file test with ')
..writeln('goldctl.')
..writeln()
..writeln('Debug information for Gold --------------------------------')
..writeln('stdout: ${result.stdout}')
..writeln('stderr: ${result.stderr}');
throw SkiaException(buf.toString());
} }
return 'An error occurred when initializing golden file test with goldctl.';
},
);
_initialized = true; _initialized = true;
} }
...@@ -197,10 +285,14 @@ class SkiaGoldClient { ...@@ -197,10 +285,14 @@ class SkiaGoldClient {
/// returned from the invocation of this command that indicates a pass or fail /// returned from the invocation of this command that indicates a pass or fail
/// result. /// result.
/// ///
/// If an unapproved image has made it to post-submit, this throws, to close
/// the tree.
///
/// The [testName] and [goldenFile] parameters reference the current /// The [testName] and [goldenFile] parameters reference the current
/// comparison being evaluated by the [FlutterPostSubmitFileComparator]. /// comparison being evaluated by the [FlutterPostSubmitFileComparator].
Future<bool> imgtestAdd(String testName, File goldenFile) async { Future<void> imgtestAdd(String testName, File goldenFile) async {
final List<String> imgtestCommand = <String>[ await _retry(
task: () => process.run(<String>[
_goldctl, _goldctl,
'imgtest', 'add', 'imgtest', 'add',
'--work-dir', workDirectory '--work-dir', workDirectory
...@@ -210,40 +302,26 @@ class SkiaGoldClient { ...@@ -210,40 +302,26 @@ class SkiaGoldClient {
'--png-file', goldenFile.path, '--png-file', goldenFile.path,
'--passfail', '--passfail',
..._getPixelMatchingArguments(), ..._getPixelMatchingArguments(),
]; ]),
taskName: 'imgtest add',
final io.ProcessResult result = await process.run(imgtestCommand); errorMessage: (int exitCode, String resultStdout, String resultStderr) {
if (exitCode == 0) {
if (result.exitCode != 0) { return null;
// If an unapproved image has made it to post-submit, throw to close the }
// tree. if (resultStdout.contains('Untriaged') ||
String? resultContents; resultStdout.contains('negative image')) {
final File resultFile = workDirectory.childFile(fs.path.join( return 'Skia Gold received an unapproved image in post-submit '
'result-state.json', 'testing. Golden file images in flutter/flutter are triaged '
)); 'in pre-submit during code review for the given PR.\n'
if (await resultFile.exists()) { '\n'
resultContents = await resultFile.readAsString(); 'Visit https://flutter-gold.skia.org/ to view and approve '
} 'the image(s), or revert the associated change. For more '
'information, visit the wiki:\n'
final StringBuffer buf = StringBuffer() ' https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter';
..writeln('Skia Gold received an unapproved image in post-submit ') }
..writeln('testing. Golden file images in flutter/flutter are triaged ') return 'Golden test for "$testName" failed for a reason unrelated to pixel comparison.';
..writeln('in pre-submit during code review for the given PR.') },
..writeln() );
..writeln('Visit https://flutter-gold.skia.org/ to view and approve ')
..writeln('the image(s), or revert the associated change. For more ')
..writeln('information, visit the wiki: ')
..writeln('https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter')
..writeln()
..writeln('Debug information for Gold --------------------------------')
..writeln('stdout: ${result.stdout}')
..writeln('stderr: ${result.stderr}')
..writeln()
..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
throw SkiaException(buf.toString());
}
return true;
} }
/// Signals if this client is initialized for uploading tryjobs to the Gold /// Signals if this client is initialized for uploading tryjobs to the Gold
...@@ -259,6 +337,8 @@ class SkiaGoldClient { ...@@ -259,6 +337,8 @@ class SkiaGoldClient {
/// The `imgtest` command collects and uploads test results to the Skia Gold /// The `imgtest` command collects and uploads test results to the Skia Gold
/// backend, the `init` argument initializes the current tryjob. Used by the /// backend, the `init` argument initializes the current tryjob. Used by the
/// [FlutterPreSubmitFileComparator]. /// [FlutterPreSubmitFileComparator].
///
/// This function is idempotent.
Future<void> tryjobInit() async { Future<void> tryjobInit() async {
// This client has already been initialized // This client has already been initialized
if (_tryjobInitialized) { if (_tryjobInitialized) {
...@@ -297,20 +377,16 @@ class SkiaGoldClient { ...@@ -297,20 +377,16 @@ class SkiaGoldClient {
throw SkiaException(buf.toString()); throw SkiaException(buf.toString());
} }
final io.ProcessResult result = await process.run(imgtestInitCommand); await _retry(
task: () => process.run(imgtestInitCommand),
if (result.exitCode != 0) { taskName: 'imgtest init',
_tryjobInitialized = false; errorMessage: (int exitCode, String resultStdout, String resultStderr) {
final StringBuffer buf = StringBuffer() if (exitCode == 0) {
..writeln('Skia Gold tryjobInit failure.') return null;
..writeln('An error occurred when initializing golden file tryjob with ')
..writeln('goldctl.')
..writeln()
..writeln('Debug information for Gold --------------------------------')
..writeln('stdout: ${result.stdout}')
..writeln('stderr: ${result.stderr}');
throw SkiaException(buf.toString());
} }
return 'An error occurred when initializing golden file tryjob with goldctl.';
},
);
_tryjobInitialized = true; _tryjobInitialized = true;
} }
...@@ -324,42 +400,25 @@ class SkiaGoldClient { ...@@ -324,42 +400,25 @@ class SkiaGoldClient {
/// The [testName] and [goldenFile] parameters reference the current /// The [testName] and [goldenFile] parameters reference the current
/// comparison being evaluated by the [FlutterPreSubmitFileComparator]. /// comparison being evaluated by the [FlutterPreSubmitFileComparator].
Future<void> tryjobAdd(String testName, File goldenFile) async { Future<void> tryjobAdd(String testName, File goldenFile) async {
final List<String> imgtestCommand = <String>[ await _retry(
task: () => process.run(<String>[
_goldctl, _goldctl,
'imgtest', 'add', 'imgtest', 'add',
'--work-dir', workDirectory '--work-dir', workDirectory.childDirectory('temp').path,
.childDirectory('temp')
.path,
'--test-name', cleanTestName(testName), '--test-name', cleanTestName(testName),
'--png-file', goldenFile.path, '--png-file', goldenFile.path,
..._getPixelMatchingArguments(), ..._getPixelMatchingArguments(),
]; ]),
taskName: 'imgtest add',
final io.ProcessResult result = await process.run(imgtestCommand); errorMessage: (int exitCode, String resultStdout, String resultStderr) {
if (exitCode == 0 ||
final String/*!*/ resultStdout = result.stdout.toString(); resultStdout.contains('Untriaged') ||
if (result.exitCode != 0 && resultStdout.contains('negative image')) {
!(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) { return null;
String? resultContents; }
final File resultFile = workDirectory.childFile(fs.path.join( return 'Golden test for "$testName" failed for a reason unrelated to pixel comparison.';
'result-state.json', },
)); );
if (await resultFile.exists()) {
resultContents = await resultFile.readAsString();
}
final StringBuffer buf = StringBuffer()
..writeln('Unexpected Gold tryjobAdd failure.')
..writeln('Tryjob execution for golden file test $testName failed for')
..writeln('a reason unrelated to pixel comparison.')
..writeln()
..writeln('Debug information for Gold --------------------------------')
..writeln('stdout: ${result.stdout}')
..writeln('stderr: ${result.stderr}')
..writeln()
..writeln()
..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
throw SkiaException(buf.toString());
}
} }
// Constructs arguments for `goldctl` for controlling how pixels are compared. // Constructs arguments for `goldctl` for controlling how pixels are compared.
...@@ -408,15 +467,13 @@ class SkiaGoldClient { ...@@ -408,15 +467,13 @@ class SkiaGoldClient {
} }
/// Returns the latest positive digest for the given test known to Flutter /// Returns the latest positive digest for the given test known to Flutter
/// Gold at head. /// Gold at head. Throws without retrying if there's a network failure.
Future<String?> getExpectationForTest(String testName) async { Future<String?> getExpectationForTest(String testName) async {
late String? expectation;
final String traceID = getTraceID(testName); final String traceID = getTraceID(testName);
await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
final Uri requestForExpectations = Uri.parse( final Uri requestForExpectations = Uri.parse(
'https://flutter-gold.skia.org/json/v2/latestpositivedigest/$traceID' 'https://flutter-gold.skia.org/json/v2/latestpositivedigest/$traceID'
); );
late String rawResponse; String? rawResponse;
try { try {
final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations); final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
final io.HttpClientResponse response = await request.close(); final io.HttpClientResponse response = await request.close();
...@@ -425,41 +482,30 @@ class SkiaGoldClient { ...@@ -425,41 +482,30 @@ class SkiaGoldClient {
if (jsonResponse is! Map<String, dynamic>) { if (jsonResponse is! Map<String, dynamic>) {
throw const FormatException('Skia gold expectations do not match expected format.'); throw const FormatException('Skia gold expectations do not match expected format.');
} }
expectation = jsonResponse['digest'] as String?; return jsonResponse['digest'] as String?; // success
} on FormatException catch (error) { } on FormatException catch (error) {
// Ideally we'd use something like package:test's printOnError, but best reliability log(
// in getting logs on CI for now we're just using print.
// See also: https://github.com/flutter/flutter/issues/91285
print( // ignore: avoid_print
'Formatting error detected requesting expectations from Flutter Gold.\n' 'Formatting error detected requesting expectations from Flutter Gold.\n'
'error: $error\n' 'error: $error\n'
'url: $requestForExpectations\n' 'url: $requestForExpectations\n'
'response: $rawResponse' 'response: $rawResponse'
); );
rethrow; rethrow; // fail
} }
},
SkiaGoldHttpOverrides(),
);
return expectation;
} }
/// Returns a list of bytes representing the golden image retrieved from the /// Returns a list of bytes representing the golden image retrieved from the
/// Flutter Gold dashboard. /// Flutter Gold dashboard.
/// ///
/// The provided image hash represents an expectation from Flutter Gold. /// The provided image hash represents an expectation from Flutter Gold.
Future<List<int>>getImageBytes(String imageHash) async { Future<List<int>> getImageBytes(String imageHash) async {
final List<int> imageBytes = <int>[]; final List<int> imageBytes = <int>[];
await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
final Uri requestForImage = Uri.parse( final Uri requestForImage = Uri.parse(
'https://flutter-gold.skia.org/img/images/$imageHash.png', 'https://flutter-gold.skia.org/img/images/$imageHash.png',
); );
final io.HttpClientRequest request = await httpClient.getUrl(requestForImage); final io.HttpClientRequest request = await httpClient.getUrl(requestForImage);
final io.HttpClientResponse response = await request.close(); final io.HttpClientResponse response = await request.close();
await response.forEach((List<int> bytes) => imageBytes.addAll(bytes)); await response.forEach((List<int> bytes) => imageBytes.addAll(bytes));
},
SkiaGoldHttpOverrides(),
);
return imageBytes; return imageBytes;
} }
...@@ -512,7 +558,7 @@ class SkiaGoldClient { ...@@ -512,7 +558,7 @@ class SkiaGoldClient {
final File authFile = workDirectory.childFile(fs.path.join( final File authFile = workDirectory.childFile(fs.path.join(
'temp', 'temp',
'auth_opt.json', 'auth_opt.json',
))/*!*/; ));
if (await authFile.exists()) { if (await authFile.exists()) {
final String contents = await authFile.readAsString(); final String contents = await authFile.readAsString();
...@@ -568,6 +614,3 @@ class SkiaGoldClient { ...@@ -568,6 +614,3 @@ class SkiaGoldClient {
return md5Sum; return md5Sum;
} }
} }
/// Used to make HttpRequests during testing.
class SkiaGoldHttpOverrides extends io.HttpOverrides { }
packages/flutter_goldens/pubspec.yaml
View file @ 62f15949
...@@ -9,26 +9,24 @@ dependencies: ...@@ -9,26 +9,24 @@ dependencies:
sdk: flutter sdk: flutter
flutter_test: flutter_test:
sdk: flutter sdk: flutter
flutter_goldens_client:
path: ../flutter_goldens_client
file: 7.0.0 file: 7.0.0
meta: 1.11.0 meta: 1.11.0
platform: 3.1.3 platform: 3.1.3
process: 5.0.1 process: 5.0.1
crypto: 3.0.3
path: 1.9.0
async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" async: 2.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
characters: 1.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" characters: 1.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" clock: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
collection: 1.18.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" collection: 1.18.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
crypto: 3.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
fake_async: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" fake_async: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
leak_tracker: 9.0.18 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" leak_tracker: 9.0.18 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
leak_tracker_flutter_testing: 2.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" leak_tracker_flutter_testing: 2.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
leak_tracker_testing: 1.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" leak_tracker_testing: 1.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
matcher: 0.12.16+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" matcher: 0.12.16+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
material_color_utilities: 0.8.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" material_color_utilities: 0.8.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
path: 1.9.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
source_span: 1.10.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" source_span: 1.10.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
stack_trace: 1.11.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" stack_trace: 1.11.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
stream_channel: 2.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" stream_channel: 2.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
......
packages/flutter_goldens/test/comparator_selection_test.dart 0 → 100644
View file @ 62f15949
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_goldens/flutter_goldens.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:platform/platform.dart';
enum _Comparator { post, pre, skip, local }
_Comparator _testRecommendations({
bool hasFlutterRoot = false,
bool hasLuci = false,
bool hasCirrus = false,
bool hasGold = false,
bool hasTryJob = false,
String branch = 'main',
String os = 'macos',
}) {
final Platform platform = FakePlatform(
environment: <String, String>{
if (hasFlutterRoot)
'FLUTTER_ROOT': '/flutter',
if (hasLuci)
'SWARMING_TASK_ID': '8675309',
if (hasCirrus)
'CIRRUS_CI': 'true',
if (hasCirrus)
'CIRRUS_PR': '',
if (hasCirrus)
'CIRRUS_BRANCH': branch,
if (hasGold)
'GOLDCTL': 'goldctl',
if (hasGold && hasCirrus)
'GOLD_SERVICE_ACCOUNT': 'service account...',
if (hasTryJob)
'GOLD_TRYJOB': 'git/ref/12345/head',
'GIT_BRANCH': branch,
},
operatingSystem: os,
);
if (FlutterPostSubmitFileComparator.isRecommendedForEnvironment(platform)) {
return _Comparator.post;
}
if (FlutterPreSubmitFileComparator.isRecommendedForEnvironment(platform)) {
return _Comparator.pre;
}
if (FlutterSkippingFileComparator.isRecommendedForEnvironment(platform)) {
return _Comparator.skip;
}
return _Comparator.local;
}
void main() {
test('Comparator recommendations - main branch', () {
// If we're running locally (no CI), use a local comparator.
expect(_testRecommendations(), _Comparator.local);
expect(_testRecommendations(hasFlutterRoot: true), _Comparator.local);
expect(_testRecommendations(hasGold: true), _Comparator.local);
expect(_testRecommendations(hasFlutterRoot: true, hasGold: true), _Comparator.local);
// If we don't have gold but are on CI, we skip regardless.
expect(_testRecommendations(hasLuci: true), _Comparator.skip);
expect(_testRecommendations(hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(hasFlutterRoot: true, hasLuci: true), _Comparator.skip);
expect(_testRecommendations(hasFlutterRoot: true, hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(hasFlutterRoot: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(hasFlutterRoot: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(hasFlutterRoot: true, hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(hasFlutterRoot: true, hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
// On Luci, with Gold, post-submit. Flutter root and Cirrus variables should have no effect.
expect(_testRecommendations(hasGold: true, hasLuci: true), _Comparator.post);
expect(_testRecommendations(hasGold: true, hasLuci: true, hasCirrus: true), _Comparator.post);
expect(_testRecommendations(hasGold: true, hasLuci: true, hasFlutterRoot: true), _Comparator.post);
expect(_testRecommendations(hasGold: true, hasLuci: true, hasFlutterRoot: true, hasCirrus: true), _Comparator.post);
// On Luci, with Gold, pre-submit. Flutter root and Cirrus variables should have no effect.
expect(_testRecommendations(hasGold: true, hasLuci: true, hasTryJob: true), _Comparator.pre);
expect(_testRecommendations(hasGold: true, hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.pre);
expect(_testRecommendations(hasGold: true, hasLuci: true, hasFlutterRoot: true, hasTryJob: true), _Comparator.pre);
expect(_testRecommendations(hasGold: true, hasLuci: true, hasFlutterRoot: true, hasCirrus: true, hasTryJob: true), _Comparator.pre);
// On Cirrus (with Gold and not on Luci), we skip regardless.
expect(_testRecommendations(hasCirrus: true, hasGold: true, hasFlutterRoot: true), _Comparator.skip);
expect(_testRecommendations(hasCirrus: true, hasGold: true, hasFlutterRoot: true, hasTryJob: true), _Comparator.skip);
});
test('Comparator recommendations - release branch', () {
// If we're running locally (no CI), use a local comparator.
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0'), _Comparator.local);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true), _Comparator.local);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true), _Comparator.local);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasGold: true), _Comparator.local);
// If we don't have gold but are on CI, we skip regardless.
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasLuci: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasFlutterRoot: true, hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
// On Luci, with Gold, post-submit. Flutter root and Cirrus variables should have no effect. Branch should make us skip.
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasFlutterRoot: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasFlutterRoot: true, hasCirrus: true), _Comparator.skip);
// On Luci, with Gold, pre-submit. Flutter root and Cirrus variables should have no effect. Branch should make us skip.
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasFlutterRoot: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasGold: true, hasLuci: true, hasFlutterRoot: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
// On Cirrus (with Gold and not on Luci), we skip regardless.
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasCirrus: true, hasGold: true, hasFlutterRoot: true), _Comparator.skip);
expect(_testRecommendations(branch: 'flutter-3.16-candidate.0', hasCirrus: true, hasGold: true, hasFlutterRoot: true, hasTryJob: true), _Comparator.skip);
});
test('Comparator recommendations - Linux', () {
// If we're running locally (no CI), use a local comparator.
expect(_testRecommendations(os: 'linux'), _Comparator.local);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true), _Comparator.local);
expect(_testRecommendations(os: 'linux', hasGold: true), _Comparator.local);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasGold: true), _Comparator.local);
// If we don't have gold but are on CI, we skip regardless.
expect(_testRecommendations(os: 'linux', hasLuci: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasLuci: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasLuci: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasLuci: true, hasCirrus: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasFlutterRoot: true, hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.skip);
// On Luci, with Gold, post-submit. Flutter root and Cirrus variables should have no effect.
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true), _Comparator.post);
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasCirrus: true), _Comparator.post);
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasFlutterRoot: true), _Comparator.post);
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasFlutterRoot: true, hasCirrus: true), _Comparator.post);
// On Luci, with Gold, pre-submit. Flutter root and Cirrus variables should have no effect.
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasTryJob: true), _Comparator.pre);
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasCirrus: true, hasTryJob: true), _Comparator.pre);
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasFlutterRoot: true, hasTryJob: true), _Comparator.pre);
expect(_testRecommendations(os: 'linux', hasGold: true, hasLuci: true, hasFlutterRoot: true, hasCirrus: true, hasTryJob: true), _Comparator.pre);
// On Cirrus (with Gold and not on Luci), we skip regardless.
expect(_testRecommendations(os: 'linux', hasCirrus: true, hasGold: true, hasFlutterRoot: true), _Comparator.skip);
expect(_testRecommendations(os: 'linux', hasCirrus: true, hasGold: true, hasFlutterRoot: true, hasTryJob: true), _Comparator.skip);
});
}
packages/flutter_goldens/test/flutter_goldens_test.dart
View file @ 62f15949
...@@ -5,13 +5,14 @@ ...@@ -5,13 +5,14 @@
// See also dev/automated_tests/flutter_test/flutter_gold_test.dart // See also dev/automated_tests/flutter_test/flutter_gold_test.dart
import 'dart:convert'; import 'dart:convert';
import 'dart:io' hide Directory; import 'dart:io' as io;
import 'package:file/file.dart'; import 'package:file/file.dart';
import 'package:file/memory.dart'; import 'package:file/memory.dart';
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:flutter_goldens/flutter_goldens.dart'; import 'package:flutter_goldens/flutter_goldens.dart';
import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test/flutter_test.dart';
import 'package:path/path.dart' as path;
import 'package:platform/platform.dart'; import 'package:platform/platform.dart';
import 'package:process/process.dart'; import 'package:process/process.dart';
...@@ -27,41 +28,26 @@ const List<int> _kTestPngBytes = <int>[ ...@@ -27,41 +28,26 @@ const List<int> _kTestPngBytes = <int>[
78, 68, 174, 66, 96, 130, 78, 68, 174, 66, 96, 130,
]; ];
void main() { FileSystem createFakeFileSystem() {
late MemoryFileSystem fs; return MemoryFileSystem()
late FakePlatform platform; ..directory(_kFlutterRoot).createSync(recursive: true);
late FakeProcessManager process; }
late FakeHttpClient fakeHttpClient;
setUp(() {
fs = MemoryFileSystem();
platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
process = FakeProcessManager();
fakeHttpClient = FakeHttpClient();
fs.directory(_kFlutterRoot).createSync(recursive: true);
});
group('SkiaGoldClient', () { (FileSystem, Directory) createFakeFileSystemWithWorkDirectory() {
late SkiaGoldClient skiaClient; final FileSystem fs = createFakeFileSystem();
late Directory workDirectory; final Directory workDirectory = fs.directory('/workDirectory')..createSync(recursive: true);
return (fs, workDirectory);
}
setUp(() { (FileSystem, Directory) createFakeFileSystemWithLibDirectory() {
workDirectory = fs.directory('/workDirectory') final FileSystem fs = createFakeFileSystem();
..createSync(recursive: true); final Directory lib = fs.directory('$_kFlutterRoot/test/library/')..createSync(recursive: true);
skiaClient = SkiaGoldClient( return (fs, lib);
workDirectory, }
fs: fs,
process: process,
platform: platform,
httpClient: fakeHttpClient,
);
});
test('web HTML test', () async { void main() {
platform = FakePlatform( test('SkiaGoldClient - web HTML test', () async {
final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'GOLDCTL': 'goldctl', 'GOLDCTL': 'goldctl',
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
...@@ -70,15 +56,19 @@ void main() { ...@@ -70,15 +56,19 @@ void main() {
}, },
operatingSystem: 'macos' operatingSystem: 'macos'
); );
skiaClient = SkiaGoldClient( final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
final FakeProcessManager process = FakeProcessManager();
final io.HttpClient httpClient = ThrowingHttpClient();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: httpClient,
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') final File goldenFile = workDirectory.childFile('temp/golden_file_test.png')
..createSync(recursive: true); ..createSync(recursive: true);
const RunInvocation goldctlInvocation = RunInvocation( const RunInvocation goldctlInvocation = RunInvocation(
...@@ -95,16 +85,12 @@ void main() { ...@@ -95,16 +85,12 @@ void main() {
], ],
null, null,
); );
process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); process.processResults[goldctlInvocation] = io.ProcessResult(123, 0, '', '');
await skiaClient.imgtestAdd('golden_file_test.png', goldenFile);
expect(
await skiaClient.imgtestAdd('golden_file_test.png', goldenFile),
isTrue,
);
}); });
test('web CanvasKit test', () async { test('SkiaGoldClient - web CanvasKit test', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'GOLDCTL': 'goldctl', 'GOLDCTL': 'goldctl',
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
...@@ -113,15 +99,19 @@ void main() { ...@@ -113,15 +99,19 @@ void main() {
}, },
operatingSystem: 'macos' operatingSystem: 'macos'
); );
skiaClient = SkiaGoldClient( final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
final FakeProcessManager process = FakeProcessManager();
final io.HttpClient httpClient = FakeHttpClient();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: httpClient,
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') final File goldenFile = workDirectory.childFile('temp/golden_file_test.png')
..createSync(recursive: true); ..createSync(recursive: true);
const RunInvocation goldctlInvocation = RunInvocation( const RunInvocation goldctlInvocation = RunInvocation(
...@@ -135,75 +125,101 @@ void main() { ...@@ -135,75 +125,101 @@ void main() {
], ],
null, null,
); );
process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); process.processResults[goldctlInvocation] = io.ProcessResult(123, 0, '', '');
expect( await skiaClient.imgtestAdd('golden_file_test.png', goldenFile);
await skiaClient.imgtestAdd('golden_file_test.png', goldenFile),
isTrue,
);
}); });
test('auth performs minimal work if already authorized', () async { test('SkiaGoldClient - auth performs minimal work if already authorized', () async {
final File authFile = fs.file('/workDirectory/temp/auth_opt.json') final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
final FakeProcessManager process = FakeProcessManager();
final File authFile = workDirectory.childFile('temp/auth_opt.json')
..createSync(recursive: true); ..createSync(recursive: true);
authFile.writeAsStringSync(authTemplate()); authFile.writeAsStringSync(authTemplate());
process.fallbackProcessResult = ProcessResult(123, 0, '', ''); process.fallbackProcessResult = io.ProcessResult(123, 0, '', '');
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
);
await skiaClient.auth(); await skiaClient.auth();
expect(process.workingDirectories, isEmpty); expect(process.workingDirectories, isEmpty);
}); });
test('gsutil is checked when authorization file is present', () async { test('SkiaGoldClient - gsutil is checked when authorization file is present', () async {
final File authFile = fs.file('/workDirectory/temp/auth_opt.json') final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
final ProcessManager process = FakeProcessManager();
final File authFile = workDirectory.childFile('temp/auth_opt.json')
..createSync(recursive: true); ..createSync(recursive: true);
authFile.writeAsStringSync(authTemplate(gsutil: true)); authFile.writeAsStringSync(authTemplate(gsutil: true));
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
);
expect( expect(
await skiaClient.clientIsAuthorized(), await skiaClient.clientIsAuthorized(),
isFalse, isFalse,
); );
}); });
test('throws for error state from auth', () async { test('SkiaGoldClient - throws for error state from auth', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLD_SERVICE_ACCOUNT' : 'Service Account', 'GOLD_SERVICE_ACCOUNT': 'Service Account',
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
}, },
operatingSystem: 'macos' operatingSystem: 'macos'
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final FakeProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
expect( expect(
skiaClient.auth(), skiaClient.auth(),
throwsException, throwsException,
); );
}); });
test('throws for error state from init', () { test('SkiaGoldClient - throws for error state from init', () {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
}, },
operatingSystem: 'macos' operatingSystem: 'macos'
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final FakeProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
const RunInvocation gitInvocation = RunInvocation( const RunInvocation gitInvocation = RunInvocation(
...@@ -223,9 +239,9 @@ void main() { ...@@ -223,9 +239,9 @@ void main() {
], ],
null, null,
); );
process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', ''); process.processResults[gitInvocation] = io.ProcessResult(12345678, 0, '12345678', '');
process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); process.processResults[goldctlInvocation] = io.ProcessResult(123, 1, 'Expected failure', 'Expected failure');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
expect( expect(
skiaClient.imgtestInit(), skiaClient.imgtestInit(),
...@@ -233,21 +249,23 @@ void main() { ...@@ -233,21 +249,23 @@ void main() {
); );
}); });
test('Only calls init once', () async { test('SkiaGoldClient - Only calls init once', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
}, },
operatingSystem: 'macos' operatingSystem: 'macos',
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final FakeProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
const RunInvocation gitInvocation = RunInvocation( const RunInvocation gitInvocation = RunInvocation(
...@@ -267,9 +285,9 @@ void main() { ...@@ -267,9 +285,9 @@ void main() {
], ],
null, null,
); );
process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); process.processResults[gitInvocation] = io.ProcessResult(1234, 0, '1234', '');
process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); process.processResults[goldctlInvocation] = io.ProcessResult(5678, 0, '5678', '');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
// First call // First call
await skiaClient.imgtestInit(); await skiaClient.imgtestInit();
...@@ -282,24 +300,26 @@ void main() { ...@@ -282,24 +300,26 @@ void main() {
await skiaClient.imgtestInit(); await skiaClient.imgtestInit();
}); });
test('Only calls tryjob init once', () async { test('SkiaGoldClient - Only calls tryjob init once', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
'SWARMING_TASK_ID' : '4ae997b50dfd4d11', 'SWARMING_TASK_ID': '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB' : 'refs/pull/49815/head', 'GOLD_TRYJOB': 'refs/pull/49815/head',
}, },
operatingSystem: 'macos' operatingSystem: 'macos'
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final FakeProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
const RunInvocation gitInvocation = RunInvocation( const RunInvocation gitInvocation = RunInvocation(
...@@ -324,9 +344,9 @@ void main() { ...@@ -324,9 +344,9 @@ void main() {
], ],
null, null,
); );
process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); process.processResults[gitInvocation] = io.ProcessResult(1234, 0, '1234', '');
process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); process.processResults[goldctlInvocation] = io.ProcessResult(5678, 0, '5678', '');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
// First call // First call
await skiaClient.tryjobInit(); await skiaClient.tryjobInit();
...@@ -339,23 +359,25 @@ void main() { ...@@ -339,23 +359,25 @@ void main() {
await skiaClient.tryjobInit(); await skiaClient.tryjobInit();
}); });
test('throws for error state from imgtestAdd', () { test('SkiaGoldClient - throws for error state from imgtestAdd', () {
final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
final File goldenFile = workDirectory.childFile('temp/golden_file_test.png')
..createSync(recursive: true); ..createSync(recursive: true);
platform = FakePlatform( final FakeProcessManager process = FakeProcessManager();
final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
}, },
operatingSystem: 'macos' operatingSystem: 'macos',
); );
final SkiaGoldClient skiaClient = SkiaGoldClient(
skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
const RunInvocation goldctlInvocation = RunInvocation( const RunInvocation goldctlInvocation = RunInvocation(
...@@ -369,8 +391,8 @@ void main() { ...@@ -369,8 +391,8 @@ void main() {
], ],
null, null,
); );
process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); process.processResults[goldctlInvocation] = io.ProcessResult(123, 1, 'Expected failure', 'Expected failure');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
expect( expect(
skiaClient.imgtestAdd('golden_file_test', goldenFile), skiaClient.imgtestAdd('golden_file_test', goldenFile),
...@@ -378,24 +400,26 @@ void main() { ...@@ -378,24 +400,26 @@ void main() {
); );
}); });
test('correctly inits tryjob for luci', () async { test('SkiaGoldClient - correctly inits tryjob for luci', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
'SWARMING_TASK_ID' : '4ae997b50dfd4d11', 'SWARMING_TASK_ID': '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB' : 'refs/pull/49815/head', 'GOLD_TRYJOB': 'refs/pull/49815/head',
}, },
operatingSystem: 'macos' operatingSystem: 'macos'
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final ProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
final List<String> ciArguments = skiaClient.getCIArguments(); final List<String> ciArguments = skiaClient.getCIArguments();
...@@ -412,100 +436,107 @@ void main() { ...@@ -412,100 +436,107 @@ void main() {
); );
}); });
test('Creates traceID correctly', () async { test('SkiaGoldClient - Creates traceID correctly - Linux', () async {
String traceID; final Platform platform = FakePlatform(
platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
'SWARMING_TASK_ID' : '4ae997b50dfd4d11', 'SWARMING_TASK_ID': '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB' : 'refs/pull/49815/head', 'GOLD_TRYJOB': 'refs/pull/49815/head',
}, },
operatingSystem: 'linux' operatingSystem: 'linux',
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final ProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
traceID = skiaClient.getTraceID('flutter.golden.1');
expect( expect(
traceID, skiaClient.getTraceID('flutter.golden.1'),
equals('ae18c7a6aa48e0685525dfe8fdf79003'), equals('ae18c7a6aa48e0685525dfe8fdf79003'),
); );
});
// Browser test('SkiaGoldClient - Creates traceID correctly - Linux web', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
'SWARMING_TASK_ID' : '4ae997b50dfd4d11', 'SWARMING_TASK_ID': '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', 'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB' : 'refs/pull/49815/head', 'GOLD_TRYJOB': 'refs/pull/49815/head',
'FLUTTER_TEST_BROWSER' : 'chrome', 'FLUTTER_TEST_BROWSER': 'chrome', // flips browser bit
}, },
operatingSystem: 'linux' operatingSystem: 'linux',
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final ProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
traceID = skiaClient.getTraceID('flutter.golden.1');
expect( expect(
traceID, skiaClient.getTraceID('flutter.golden.1'),
equals('e9d5c296c48e7126808520e9cc191243'), equals('e9d5c296c48e7126808520e9cc191243'),
); );
});
// Locally - should defer to luci traceID test('SkiaGoldClient - Creates traceID correctly - Linux', () async {
platform = FakePlatform( final Platform platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL': 'goldctl',
'SWARMING_TASK_ID': '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX': 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB': 'refs/pull/49815/head',
}, },
operatingSystem: 'macos' operatingSystem: 'macos', // different operating system
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final ProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
traceID = skiaClient.getTraceID('flutter.golden.1');
expect( expect(
traceID, skiaClient.getTraceID('flutter.golden.1'),
equals('9968695b9ae78cdb77cbb2be621ca2d6'), equals('9968695b9ae78cdb77cbb2be621ca2d6'),
); );
}); });
test('throws for error state from imgtestAdd', () { test('SkiaGoldClient - throws for error state from imgtestAdd', () {
final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') final Platform platform = FakePlatform(
..createSync(recursive: true);
platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
}, },
operatingSystem: 'macos' operatingSystem: 'macos',
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final File goldenFile = workDirectory.childFile('temp/golden_file_test.png')
..createSync(recursive: true);
final FakeProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
const RunInvocation goldctlInvocation = RunInvocation( const RunInvocation goldctlInvocation = RunInvocation(
...@@ -519,37 +550,47 @@ void main() { ...@@ -519,37 +550,47 @@ void main() {
], ],
null, null,
); );
process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); process.processResults[goldctlInvocation] = io.ProcessResult(123, 1, 'Expected failure', 'Expected failure');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
expect( expect(
skiaClient.imgtestAdd('golden_file_test', goldenFile), skiaClient.imgtestAdd('golden_file_test', goldenFile),
throwsA( throwsA(
isA<SkiaException>().having((SkiaException error) => error.message, isA<SkiaException>().having((SkiaException error) => error.message,
'message', 'message',
contains('result-state.json'), 'Golden test for "golden_file_test" failed for a reason unrelated to pixel comparison.\n'
'\n'
'imgtest add failed with exit code 1.\n'
'\n'
'stdout from gold:\n'
' Fallback failure\n'
'\n'
'stderr from gold:\n'
' Fallback failure\n',
), ),
), ),
); );
}); });
test('throws for error state from tryjobAdd', () { test('SkiaGoldClient - throws for error state from tryjobAdd', () {
final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') final Platform platform = FakePlatform(
..createSync(recursive: true);
platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl', 'GOLDCTL': 'goldctl',
}, },
operatingSystem: 'macos' operatingSystem: 'macos',
); );
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
skiaClient = SkiaGoldClient( final File goldenFile = workDirectory.childFile('temp/golden_file_test.png')
..createSync(recursive: true);
final FakeProcessManager process = FakeProcessManager();
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory, workDirectory,
fs: fs, fs: fs,
process: process, process: process,
platform: platform, platform: platform,
httpClient: fakeHttpClient, httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
const RunInvocation goldctlInvocation = RunInvocation( const RunInvocation goldctlInvocation = RunInvocation(
...@@ -563,27 +604,40 @@ void main() { ...@@ -563,27 +604,40 @@ void main() {
], ],
null, null,
); );
process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); process.processResults[goldctlInvocation] = io.ProcessResult(123, 1, 'Expected failure', 'Expected failure');
process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
expect( expect(
skiaClient.tryjobAdd('golden_file_test', goldenFile), skiaClient.tryjobAdd('golden_file_test', goldenFile),
throwsA( throwsA(
isA<SkiaException>().having((SkiaException error) => error.message, isA<SkiaException>().having((SkiaException error) => error.message,
'message', 'message',
contains('result-state.json'), 'Golden test for "golden_file_test" failed for a reason unrelated to pixel comparison.\n'
'\n'
'imgtest add failed with exit code 1.\n'
'\n'
'stdout from gold:\n'
' Fallback failure\n'
'\n'
'stderr from gold:\n'
' Fallback failure\n',
), ),
), ),
); );
}); });
group('Request Handling', () { test('SkiaGoldClient - Request Handling - image bytes are processed properly', () async {
const String expectation = '55109a4bed52acc780530f7a9aeff6c0'; const String expectation = '55109a4bed52acc780530f7a9aeff6c0';
final Platform platform = FakePlatform(
test('image bytes are processed properly', () async { environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
final (FileSystem fs, Directory workDirectory) = createFakeFileSystemWithWorkDirectory();
final ProcessManager process = FakeProcessManager();
final Uri imageUrl = Uri.parse( final Uri imageUrl = Uri.parse(
'https://flutter-gold.skia.org/img/images/$expectation.png' 'https://flutter-gold.skia.org/img/images/$expectation.png'
); );
final FakeHttpClient fakeHttpClient = FakeHttpClient();
final FakeHttpClientRequest fakeImageRequest = FakeHttpClientRequest(); final FakeHttpClientRequest fakeImageRequest = FakeHttpClientRequest();
final FakeHttpImageResponse fakeImageResponse = FakeHttpImageResponse( final FakeHttpImageResponse fakeImageResponse = FakeHttpImageResponse(
imageResponseTemplate() imageResponseTemplate()
...@@ -592,29 +646,26 @@ void main() { ...@@ -592,29 +646,26 @@ void main() {
fakeHttpClient.request = fakeImageRequest; fakeHttpClient.request = fakeImageRequest;
fakeImageRequest.response = fakeImageResponse; fakeImageRequest.response = fakeImageResponse;
final SkiaGoldClient skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: fakeHttpClient,
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
);
final List<int> masterBytes = await skiaClient.getImageBytes(expectation); final List<int> masterBytes = await skiaClient.getImageBytes(expectation);
expect(fakeHttpClient.lastUri, imageUrl); expect(fakeHttpClient.lastUri, imageUrl);
expect(masterBytes, equals(_kTestPngBytes)); expect(masterBytes, equals(_kTestPngBytes));
}); });
});
});
group('FlutterGoldenFileComparator', () { test('FlutterGoldenFileComparator - calculates the basedir correctly from defaultComparator for local testing', () async {
late FlutterGoldenFileComparator comparator; final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
setUp(() { operatingSystem: 'macos'
final Directory basedir = fs.directory('flutter/test/library/')
..createSync(recursive: true);
comparator = FlutterPostSubmitFileComparator(
basedir.uri,
FakeSkiaGoldClient(),
fs: fs,
platform: platform,
); );
}); final FileSystem fs = createFakeFileSystem();
test('calculates the basedir correctly from defaultComparator for local testing', () async {
final FakeLocalFileComparator defaultComparator = FakeLocalFileComparator(); final FakeLocalFileComparator defaultComparator = FakeLocalFileComparator();
final Directory flutterRoot = fs.directory(platform.environment['FLUTTER_ROOT']) final Directory flutterRoot = fs.directory(platform.environment['FLUTTER_ROOT'])
..createSync(recursive: true); ..createSync(recursive: true);
...@@ -622,7 +673,8 @@ void main() { ...@@ -622,7 +673,8 @@ void main() {
final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory( final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory(
defaultComparator, defaultComparator,
platform, platform: platform,
fs: fs,
); );
expect( expect(
basedir.uri, basedir.uri,
...@@ -630,23 +682,44 @@ void main() { ...@@ -630,23 +682,44 @@ void main() {
); );
}); });
test('ignores version number', () { test('FlutterGoldenFileComparator - ignores version number', () {
final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
final List<String> log = <String>[];
final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator(
libDirectory.uri,
FakeSkiaGoldClient(),
fs: fs,
platform: platform,
log: log.add,
);
final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1); final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1);
expect(key, Uri.parse('foo.png')); expect(key, Uri.parse('foo.png'));
expect(log, isEmpty);
}); });
test('adds namePrefix', () async { test('FlutterGoldenFileComparator - adds namePrefix', () async {
final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
const String libraryName = 'sidedishes'; const String libraryName = 'sidedishes';
const String namePrefix = 'tomatosalad'; const String namePrefix = 'tomatosalad';
const String fileName = 'lettuce.png'; const String fileName = 'lettuce.png';
final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient(); final FileSystem fs = createFakeFileSystem();
final Directory basedir = fs.directory('flutter/test/$libraryName/') final Directory basedir = fs.directory('flutter/test/$libraryName/')
..createSync(recursive: true); ..createSync(recursive: true);
final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
final List<String> log = <String>[];
final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator( final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator(
basedir.uri, basedir.uri,
fakeSkiaClient, fakeSkiaClient,
fs: fs, fs: fs,
platform: platform, platform: platform,
log: log.add,
namePrefix: namePrefix, namePrefix: namePrefix,
); );
await comparator.compare( await comparator.compare(
...@@ -654,24 +727,24 @@ void main() { ...@@ -654,24 +727,24 @@ void main() {
Uri.parse(fileName), Uri.parse(fileName),
); );
expect(fakeSkiaClient.testNames.single, '$namePrefix.$libraryName.$fileName'); expect(fakeSkiaClient.testNames.single, '$namePrefix.$libraryName.$fileName');
expect(log, isEmpty);
}); });
group('Post-Submit', () { test('FlutterGoldenFileComparator - Post-Submit - asserts .png format', () async {
late FakeSkiaGoldClient fakeSkiaClient; final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
setUp(() { operatingSystem: 'macos'
fakeSkiaClient = FakeSkiaGoldClient(); );
final Directory basedir = fs.directory('flutter/test/library/') final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
..createSync(recursive: true); final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
comparator = FlutterPostSubmitFileComparator( final List<String> log = <String>[];
basedir.uri, final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator(
libDirectory.uri,
fakeSkiaClient, fakeSkiaClient,
fs: fs, fs: fs,
platform: platform, platform: platform,
log: log.add,
); );
});
test('asserts .png format', () async {
await expectLater( await expectLater(
() async { () async {
return comparator.compare( return comparator.compare(
...@@ -689,156 +762,68 @@ void main() { ...@@ -689,156 +762,68 @@ void main() {
), ),
), ),
); );
expect(log, isEmpty);
}); });
test('calls init during compare', () { test('FlutterGoldenFileComparator - Post-Submit - calls init during compare', () async {
final Platform platform = FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos'
);
final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
final List<String> log = <String>[];
final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator(
libDirectory.uri,
fakeSkiaClient,
fs: fs,
platform: platform,
log: log.add,
);
expect(fakeSkiaClient.initCalls, 0); expect(fakeSkiaClient.initCalls, 0);
comparator.compare( await comparator.compare(
Uint8List.fromList(_kTestPngBytes), Uint8List.fromList(_kTestPngBytes),
Uri.parse('flutter.golden_test.1.png'), Uri.parse('flutter.golden_test.1.png'),
); );
expect(fakeSkiaClient.initCalls, 1); expect(fakeSkiaClient.initCalls, 1);
expect(log, isEmpty);
}); });
test('does not call init in during construction', () { test('FlutterGoldenFileComparator - Post-Submit - does not call init in during construction', () {
expect(fakeSkiaClient.initCalls, 0); final Platform platform = FakePlatform(
FlutterPostSubmitFileComparator.fromDefaultComparator(
platform,
goldens: fakeSkiaClient,
);
expect(fakeSkiaClient.initCalls, 0);
});
group('correctly determines testing environment', () {
test('returns true for configured Luci', () {
platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '12345678990', 'GOLDCTL': 'testctl',
'GOLDCTL' : 'goldctl',
'GIT_BRANCH' : 'master',
}, },
operatingSystem: 'macos', operatingSystem: 'macos'
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns false on release branches in postsubmit', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GIT_BRANCH' : 'flutter-3.16-candidate.0',
},
operatingSystem: 'macos',
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns true on master branch in postsubmit', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GIT_BRANCH' : 'master',
},
operatingSystem: 'macos',
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns true on main branch in postsubmit', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GIT_BRANCH' : 'main',
},
operatingSystem: 'macos',
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns false - GOLDCTL not present', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '12345678990',
},
operatingSystem: 'macos',
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns false - GOLD_TRYJOB active', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '12345678990',
'GOLDCTL' : 'goldctl',
'GOLD_TRYJOB' : 'git/ref/12345/head',
},
operatingSystem: 'macos',
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns false - on Cirrus', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'CIRRUS_CI': 'true',
'CIRRUS_PR': '',
'CIRRUS_BRANCH': 'master',
'GOLD_SERVICE_ACCOUNT': 'service account...',
},
operatingSystem: 'macos',
); );
expect( final FileSystem fs = createFakeFileSystem();
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform), final List<String> log = <String>[];
isFalse, FlutterPostSubmitFileComparator.fromLocalFileComparator(
localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix),
platform: platform,
fs: fs,
process: LoggingProcessManager(log),
httpClient: ThrowingHttpClient(),
log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
); );
}); expect(log, isEmpty);
});
}); });
group('Pre-Submit', () { test('FlutterGoldenFileComparator - Pre-Submit - asserts .png format', () async {
late FakeSkiaGoldClient fakeSkiaClient; final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
setUp(() { final List<String> log = <String>[];
fakeSkiaClient = FakeSkiaGoldClient(); final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator(
final Directory basedir = fs.directory('flutter/test/library/') libDirectory.uri,
..createSync(recursive: true);
comparator = FlutterPreSubmitFileComparator(
basedir.uri,
fakeSkiaClient, fakeSkiaClient,
fs: fs, fs: fs,
platform: platform, platform: FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos',
),
log: log.add,
); );
});
test('asserts .png format', () async {
await expectLater( await expectLater(
() async { () async {
return comparator.compare( return comparator.compare(
...@@ -856,259 +841,72 @@ void main() { ...@@ -856,259 +841,72 @@ void main() {
), ),
), ),
); );
expect(log, isEmpty);
}); });
test('calls init during compare', () { test('FlutterGoldenFileComparator - Pre-Submit - calls init during compare', () async {
final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
final List<String> log = <String>[];
final FlutterGoldenFileComparator comparator = FlutterPreSubmitFileComparator(
libDirectory.uri,
fakeSkiaClient,
fs: fs,
platform: FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos',
),
log: log.add,
);
expect(fakeSkiaClient.tryInitCalls, 0); expect(fakeSkiaClient.tryInitCalls, 0);
comparator.compare( await comparator.compare(
Uint8List.fromList(_kTestPngBytes), Uint8List.fromList(_kTestPngBytes),
Uri.parse('flutter.golden_test.1.png'), Uri.parse('flutter.golden_test.1.png'),
); );
expect(fakeSkiaClient.tryInitCalls, 1); expect(fakeSkiaClient.tryInitCalls, 1);
expect(log, isEmpty);
}); });
test('does not call init in during construction', () { test('FlutterGoldenFileComparator - Pre-Submit - does not call init in during construction', () async {
expect(fakeSkiaClient.tryInitCalls, 0); final Platform platform = FakePlatform(
FlutterPostSubmitFileComparator.fromDefaultComparator(
platform,
goldens: fakeSkiaClient,
);
expect(fakeSkiaClient.tryInitCalls, 0);
});
group('correctly determines testing environment', () {
test('returns false on release branches in presubmit', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GOLD_TRYJOB' : 'true',
'GIT_BRANCH' : 'flutter-3.16-candidate.0',
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns true on master branch in presubmit', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GOLD_TRYJOB' : 'true',
'GIT_BRANCH' : 'master',
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns true on main branch in presubmit', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GOLD_TRYJOB' : 'true',
'GIT_BRANCH' : 'main',
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns true for Luci', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '12345678990',
'GOLDCTL' : 'goldctl',
'GOLD_TRYJOB' : 'git/ref/12345/head',
'GIT_BRANCH' : 'master',
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns false - not on Luci', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns false - GOLDCTL missing', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '12345678990',
'GOLD_TRYJOB' : 'git/ref/12345/head',
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns false - GOLD_TRYJOB missing', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '12345678990',
'GOLDCTL' : 'goldctl',
},
operatingSystem: 'macos',
);
expect(
FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
test('returns false - on Cirrus', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'CIRRUS_CI': 'true',
'CIRRUS_PR': '',
'CIRRUS_BRANCH': 'master',
'GOLD_SERVICE_ACCOUNT': 'service account...',
},
operatingSystem: 'macos',
);
expect(
FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
isFalse,
);
});
});
});
group('Skipping', () {
group('correctly determines testing environment', () {
test('returns true on release branches in presubmit', () {
platform = FakePlatform(
environment: <String, String>{ environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot, 'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID', 'GOLDCTL': 'testctl',
'GOLDCTL' : 'some/path',
'GOLD_TRYJOB' : 'true',
'GIT_BRANCH' : 'flutter-3.16-candidate.0',
}, },
operatingSystem: 'macos', operatingSystem: 'macos',
); );
expect( final FileSystem fs = createFakeFileSystem();
FlutterSkippingFileComparator.isAvailableForEnvironment(platform), final List<String> log = <String>[];
isTrue, await FlutterPostSubmitFileComparator.fromLocalFileComparator(
); localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix),
}); platform: platform,
fs: fs,
test('returns true on release branches in postsubmit', () { process: LoggingProcessManager(log),
platform = FakePlatform( httpClient: FakeHttpClient(),
environment: <String, String>{ log: (String message) => fail('skia gold client printed unexpected output: "$message"'),
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : 'sweet task ID',
'GOLDCTL' : 'some/path',
'GIT_BRANCH' : 'flutter-3.16-candidate.0',
},
operatingSystem: 'macos',
);
expect(
FlutterSkippingFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns true on Cirrus builds', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'CIRRUS_CI' : 'yep',
},
operatingSystem: 'macos',
);
expect(
FlutterSkippingFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns true on irrelevant LUCI builds', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'SWARMING_TASK_ID' : '1234567890',
},
operatingSystem: 'macos'
);
expect(
FlutterSkippingFileComparator.isAvailableForEnvironment(platform),
isTrue,
);
});
test('returns false - not in CI', () {
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
},
operatingSystem: 'macos',
);
expect(
FlutterSkippingFileComparator.isAvailableForEnvironment(platform),
isFalse,
); );
}); expect(log, <String>['testctl auth --work-dir /.tmp_rand0/flutter_goldens_postsubmit.rand0/../temp --luci']);
});
}); });
group('Local', () { test('FlutterGoldenFileComparator - Local - asserts .png format', () async {
late FlutterLocalFileComparator comparator;
final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient(); final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
setUp(() async { final List<String> log = <String>[];
final Directory basedir = fs.directory('flutter/test/library/') final FlutterLocalFileComparator comparator = FlutterLocalFileComparator(
..createSync(recursive: true); libDirectory.uri,
comparator = FlutterLocalFileComparator(
basedir.uri,
fakeSkiaClient, fakeSkiaClient,
fs: fs, fs: fs,
platform: FakePlatform( platform: FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot}, environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos', operatingSystem: 'macos',
), ),
log: log.add,
); );
const String hash = '55109a4bed52acc780530f7a9aeff6c0'; const String hash = '55109a4bed52acc780530f7a9aeff6c0';
fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash;
fakeSkiaClient.imageBytesValues[hash] =_kTestPngBytes; fakeSkiaClient.imageBytesValues[hash] =_kTestPngBytes;
fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = 'flutter.golden_test.1'; fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = 'flutter.golden_test.1';
});
test('asserts .png format', () async {
await expectLater( await expectLater(
() async { () async {
return comparator.compare( return comparator.compare(
...@@ -1126,9 +924,28 @@ void main() { ...@@ -1126,9 +924,28 @@ void main() {
), ),
), ),
); );
expect(log, isEmpty);
}); });
test('passes when bytes match', () async { test('FlutterGoldenFileComparator - Local - passes when bytes match', () async {
final List<String> log = <String>[];
final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
final FlutterLocalFileComparator comparator = FlutterLocalFileComparator(
libDirectory.uri,
fakeSkiaClient,
fs: fs,
platform: FakePlatform(
environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
operatingSystem: 'macos',
),
log: log.add,
);
const String hash = '55109a4bed52acc780530f7a9aeff6c0';
fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash;
fakeSkiaClient.imageBytesValues[hash] =_kTestPngBytes;
fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = 'flutter.golden_test.1';
expect( expect(
await comparator.compare( await comparator.compare(
Uint8List.fromList(_kTestPngBytes), Uint8List.fromList(_kTestPngBytes),
...@@ -1136,34 +953,37 @@ void main() { ...@@ -1136,34 +953,37 @@ void main() {
), ),
isTrue, isTrue,
); );
expect(log, isEmpty);
}); });
test('returns FlutterSkippingGoldenFileComparator when network connection is unavailable', () async { test('FlutterGoldenFileComparator - Local - skips when network connection is unavailable', () async {
final FakeDirectory fakeDirectory = FakeDirectory(); final FileSystem fs = createFakeFileSystem();
fakeDirectory.existsSyncValue = true; final FakeProcessManager process = FakeProcessManager()
fakeDirectory.uri = Uri.parse('/flutter'); ..fallbackProcessResult = io.ProcessResult(123, 1, 'test resulted in a 502: 502 Bad Gateway\n', '');
final List<String> log = <String>[];
fakeSkiaClient.getExpectationForTestThrowable = const OSError("Can't reach Gold"); final FlutterGoldenFileComparator comparator = await FlutterLocalFileComparator.fromLocalFileComparator(
localFileComparator: LocalFileComparator(Uri.parse('/test'), pathStyle: path.Style.posix),
FlutterGoldenFileComparator comparator = await FlutterLocalFileComparator.fromDefaultComparator( platform: FakePlatform(
platform, environment: <String, String>{
goldens: fakeSkiaClient, 'FLUTTER_ROOT': _kFlutterRoot,
baseDirectory: fakeDirectory, },
operatingSystem: 'macos',
),
fs: fs,
process: process,
httpClient: ThrowingHttpClient(),
log: log.add,
); );
expect(comparator.runtimeType, FlutterSkippingFileComparator); expect(
await comparator.compare(
fakeSkiaClient.getExpectationForTestThrowable = const SocketException("Can't reach Gold"); Uint8List.fromList(_kTestPngBytes),
Uri.parse('flutter.golden_test.1.png'),
comparator = await FlutterLocalFileComparator.fromDefaultComparator( ),
platform, isTrue,
goldens: fakeSkiaClient,
baseDirectory: fakeDirectory,
); );
expect(comparator.runtimeType, FlutterSkippingFileComparator); expect(log, <String>[
// reset property or it will carry on to other tests 'Auto-passing "pkg.flutter.golden_test.1.png" test, ignoring network error when contacting Skia.'
fakeSkiaClient.getExpectationForTestThrowable = null; ]);
});
});
}); });
} }
...@@ -1207,28 +1027,27 @@ class RunInvocation { ...@@ -1207,28 +1027,27 @@ class RunInvocation {
} }
class FakeProcessManager extends Fake implements ProcessManager { class FakeProcessManager extends Fake implements ProcessManager {
Map<RunInvocation, ProcessResult> processResults = <RunInvocation, ProcessResult>{}; Map<RunInvocation, io.ProcessResult> processResults = <RunInvocation, io.ProcessResult>{};
/// Used if [processResults] does not contain a matching invocation. /// Used if [processResults] does not contain a matching invocation.
ProcessResult? fallbackProcessResult; io.ProcessResult? fallbackProcessResult;
final List<String?> workingDirectories = <String?>[]; final List<String?> workingDirectories = <String?>[];
@override @override
Future<ProcessResult> run( Future<io.ProcessResult> run(
List<Object> command, { List<Object> command, {
String? workingDirectory, String? workingDirectory,
Map<String, String>? environment, Map<String, String>? environment,
bool includeParentEnvironment = true, bool includeParentEnvironment = true,
bool runInShell = false, bool runInShell = false,
Encoding? stdoutEncoding = systemEncoding, Encoding? stdoutEncoding,
Encoding? stderrEncoding = systemEncoding, Encoding? stderrEncoding,
}) async { }) async {
workingDirectories.add(workingDirectory); workingDirectories.add(workingDirectory);
final ProcessResult? result = processResults[RunInvocation(command.cast<String>(), workingDirectory)]; final io.ProcessResult? result = processResults[RunInvocation(command.cast<String>(), workingDirectory)];
if (result == null && fallbackProcessResult == null) { if (result == null && fallbackProcessResult == null) {
printOnFailure('ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.'); fail('ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.');
fail('See above.');
} }
return result ?? fallbackProcessResult!; return result ?? fallbackProcessResult!;
} }
...@@ -1238,8 +1057,10 @@ class FakeProcessManager extends Fake implements ProcessManager { ...@@ -1238,8 +1057,10 @@ class FakeProcessManager extends Fake implements ProcessManager {
class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { class FakeSkiaGoldClient extends Fake implements SkiaGoldClient {
Map<String, String> expectationForTestValues = <String, String>{}; Map<String, String> expectationForTestValues = <String, String>{};
Exception? getExpectationForTestThrowable; Exception? getExpectationForTestThrowable;
@override @override
Future<String> getExpectationForTest(String testName) async { Future<String> getExpectationForTest(String testName) async {
await null; // force this to be async
if (getExpectationForTestThrowable != null) { if (getExpectationForTestThrowable != null) {
throw getExpectationForTestThrowable!; throw getExpectationForTestThrowable!;
} }
...@@ -1247,30 +1068,51 @@ class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { ...@@ -1247,30 +1068,51 @@ class FakeSkiaGoldClient extends Fake implements SkiaGoldClient {
} }
@override @override
Future<void> auth() async {} Future<void> auth() async {
await null; // force this to be async
}
final List<String> testNames = <String>[]; final List<String> testNames = <String>[];
int initCalls = 0; int initCalls = 0;
@override @override
Future<void> imgtestInit() async => initCalls += 1; Future<void> imgtestInit() async {
await null; // force this to be async
initCalls += 1;
}
@override @override
Future<bool> imgtestAdd(String testName, File goldenFile) async { Future<bool> imgtestAdd(String testName, File goldenFile) async {
await null; // force this to be async
testNames.add(testName); testNames.add(testName);
return true; return true;
} }
int tryInitCalls = 0; int tryInitCalls = 0;
@override @override
Future<void> tryjobInit() async => tryInitCalls += 1; Future<void> tryjobInit() async {
await null; // force this to be async
tryInitCalls += 1;
}
@override @override
Future<bool> tryjobAdd(String testName, File goldenFile) async => true; Future<bool> tryjobAdd(String testName, File goldenFile) async {
await null; // force this to be async
return true;
}
Map<String, List<int>> imageBytesValues = <String, List<int>>{}; Map<String, List<int>> imageBytesValues = <String, List<int>>{};
@override @override
Future<List<int>> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!; Future<List<int>> getImageBytes(String imageHash) async {
await null; // force this to be async
return imageBytesValues[imageHash]!;
}
Map<String, String> cleanTestNameValues = <String, String>{}; Map<String, String> cleanTestNameValues = <String, String>{};
@override @override
String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? ''; String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? '';
} }
...@@ -1280,36 +1122,34 @@ class FakeLocalFileComparator extends Fake implements LocalFileComparator { ...@@ -1280,36 +1122,34 @@ class FakeLocalFileComparator extends Fake implements LocalFileComparator {
late Uri basedir; late Uri basedir;
} }
class FakeDirectory extends Fake implements Directory { class ThrowingHttpClient extends Fake implements io.HttpClient {
late bool existsSyncValue;
@override @override
bool existsSync() => existsSyncValue; Future<io.HttpClientRequest> getUrl(Uri url) async {
throw const io.SocketException('test error');
@override }
late Uri uri;
} }
class FakeHttpClient extends Fake implements HttpClient { class FakeHttpClient extends Fake implements io.HttpClient {
late Uri lastUri; late Uri lastUri;
late FakeHttpClientRequest request; late FakeHttpClientRequest request;
@override @override
Future<HttpClientRequest> getUrl(Uri url) async { Future<io.HttpClientRequest> getUrl(Uri url) async {
lastUri = url; lastUri = url;
return request; return request;
} }
} }
class FakeHttpClientRequest extends Fake implements HttpClientRequest { class FakeHttpClientRequest extends Fake implements io.HttpClientRequest {
late FakeHttpImageResponse response; late FakeHttpImageResponse response;
@override @override
Future<HttpClientResponse> close() async { Future<io.HttpClientResponse> close() async {
return response; return response;
} }
} }
class FakeHttpImageResponse extends Fake implements HttpClientResponse { class FakeHttpImageResponse extends Fake implements io.HttpClientResponse {
FakeHttpImageResponse(this.response); FakeHttpImageResponse(this.response);
final List<List<int>> response; final List<List<int>> response;
...@@ -1319,3 +1159,22 @@ class FakeHttpImageResponse extends Fake implements HttpClientResponse { ...@@ -1319,3 +1159,22 @@ class FakeHttpImageResponse extends Fake implements HttpClientResponse {
response.forEach(action); response.forEach(action);
} }
} }
class LoggingProcessManager extends Fake implements ProcessManager {
LoggingProcessManager(this.log);
final List<String> log;
@override
Future<io.ProcessResult> run(List<Object> command, {
Map<String, String>? environment,
bool includeParentEnvironment = true,
bool runInShell = false,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
}) async {
log.add(command.join(' '));
return io.ProcessResult(0, 0, '200', '');
}
}
packages/flutter_goldens/test/skia_client_test.dart 0 → 100644
View file @ 62f15949
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io' show HttpClient, ProcessResult;
import 'package:file/file.dart';
import 'package:file/memory.dart';
import 'package:flutter_goldens/skia_client.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:platform/platform.dart';
import 'package:process/process.dart';
void main() {
test('502 retry', () async {
final List<String> log = <String>[];
await runZoned(
zoneSpecification: ZoneSpecification(
print: (Zone self, ZoneDelegate parent, Zone zone, String line) {
fail('unexpected print: "$line"');
},
createTimer: (Zone self, ZoneDelegate parent, Zone zone, Duration duration, void Function() f) {
log.add('CREATED TIMER: $duration');
return parent.createTimer(zone, Duration.zero, f);
},
),
() async {
final FileSystem fs;
final SkiaGoldClient skiaClient = SkiaGoldClient(
fs: fs = MemoryFileSystem(),
process: FakeProcessManager(log),
platform: FakePlatform(
environment: const <String, String>{
'GOLDCTL': 'goldctl',
},
),
httpClient: FakeHttpClient(),
log: log.add,
fs.directory('/'),
);
log.add('START'); // ignore: avoid_print
await skiaClient.tryjobAdd('test', fs.file('golden'));
log.add('END'); // ignore: avoid_print
expect(log, <String>[
'START',
'EXEC: goldctl imgtest add --work-dir /temp --test-name t --png-file golden',
'Transient failure (exit code 1) from Skia Gold.',
'',
'stdout from gold:',
' test resulted in a 502: 502 Bad Gateway',
' ',
'',
'Retrying in 5 seconds.',
'CREATED TIMER: 0:00:05.000000',
'EXEC: goldctl imgtest add --work-dir /temp --test-name t --png-file golden',
'END',
]);
},
);
});
}
class FakeProcessManager extends Fake implements ProcessManager {
FakeProcessManager(this.log);
final List<String> log;
int _index = 0;
@override
Future<ProcessResult> run(List<Object> command, {
Map<String, String>? environment,
bool includeParentEnvironment = true,
bool runInShell = false,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
}) async {
log.add('EXEC: ${command.join(' ')}');
_index += 1;
switch (_index) {
case 1: return ProcessResult(0, 1, 'test resulted in a 502: 502 Bad Gateway\n', '');
case 2: return ProcessResult(0, 0, '200', '');
default: throw StateError('unexpected call to run');
}
}
}
class FakeHttpClient extends Fake implements HttpClient { }
packages/flutter_goldens_client/pubspec.yaml deleted 100644 → 0
View file @ 9e104eb7
name: flutter_goldens_client
environment:
sdk: '>=3.2.0-0 <4.0.0'
dependencies:
# To update these, use "flutter update-packages --force-upgrade".
crypto: 3.0.3
file: 7.0.0
platform: 3.1.3
process: 5.0.1
collection: 1.18.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
meta: 1.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
path: 1.9.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
typed_data: 1.3.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
dartdoc:
# Exclude this package from the hosted API docs.
nodoc: true
# PUBSPEC CHECKSUM: 652c
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab