Unverified Commit 8c398a8d authored by Emmanuel Garcia's avatar Emmanuel Garcia Committed by GitHub

Reland: Skia gold driver test (#50160)

parent 0fc1c6f6
...@@ -10,5 +10,5 @@ import 'package:flutter_devicelab/tasks/integration_tests.dart'; ...@@ -10,5 +10,5 @@ import 'package:flutter_devicelab/tasks/integration_tests.dart';
Future<void> main() async { Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.fuchsia; deviceOperatingSystem = DeviceOperatingSystem.fuchsia;
await task(createFlutterDriverScreenshotTest()); await task(createFlutterDriverScreenshotTest(useFlutterGold: true));
} }
...@@ -107,15 +107,22 @@ TaskFunction createAndroidSplashScreenKitchenSinkTest() { ...@@ -107,15 +107,22 @@ TaskFunction createAndroidSplashScreenKitchenSinkTest() {
); );
} }
TaskFunction createFlutterDriverScreenshotTest() { /// Executes a driver test that takes a screenshot and compares it against a golden image.
/// If [useFlutterGold] is true, the golden image is served by Flutter Gold
/// (https://flutter-gold.skia.org/), otherwise the golden image is read from the disk.
TaskFunction createFlutterDriverScreenshotTest({
bool useFlutterGold = false,
}) {
return DriverTest( return DriverTest(
'${flutterDirectory.path}/dev/integration_tests/flutter_driver_screenshot_test', '${flutterDirectory.path}/dev/integration_tests/flutter_driver_screenshot_test',
'lib/main.dart', 'lib/main.dart',
extraOptions: useFlutterGold ? const <String>[
'--driver', 'test_driver/flutter_gold_main_test.dart'
] : const <String>[]
); );
} }
class DriverTest { class DriverTest {
DriverTest( DriverTest(
this.testDirectory, this.testDirectory,
this.testTarget, { this.testTarget, {
......
...@@ -4,7 +4,12 @@ This tests contains an app with a main page and sub pages. ...@@ -4,7 +4,12 @@ This tests contains an app with a main page and sub pages.
The main page contains a list of buttons; each button leads to a designated sub page when tapped on. The main page contains a list of buttons; each button leads to a designated sub page when tapped on.
Each sub page should displays some simple UIs to screenshot tested. Each sub page should displays some simple UIs to screenshot tested.
The flutter driver test runs the app and opens each page to take a screenshot. Then it compares the screenshot against a golden image stored in `test_driver/goldens/<some_test_page_name>/<device_model>.png`. The flutter driver test runs the app and opens each page to take a screenshot.
Use `test_driver/flutter_gold_main_test.dart` to test against golden files stored on Flutter Gold.
Otherwise, use `main_test.dart` to test against golden files stored on `test_driver/goldens/<some_test_page_name>/<device_model>.png`.
Note that new binaries can't be checked in the Flutter repo, so use [Flutter Gold](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter) instead.
# Add a new page to test # Add a new page to test
...@@ -16,6 +21,7 @@ The flutter driver test runs the app and opens each page to take a screenshot. T ...@@ -16,6 +21,7 @@ The flutter driver test runs the app and opens each page to take a screenshot. T
An example of a `Page` subclass can be found in `lib/image_page.dart` An example of a `Page` subclass can be found in `lib/image_page.dart`
# Experiments # Environments
The test currently only runs on device lab ["mac/ios"] which runs the app on iPhone 6s. * Device Lab which runs the app on iPhone 6s.
\ No newline at end of file * LUCI which runs the app on a Fuchsia NUC device.
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_driver/flutter_driver.dart';
import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
import 'package:flutter_test/src/buffer_matcher.dart';
Future<void> main() async {
FlutterDriver driver;
String deviceModel;
setUpAll(() async {
driver = await FlutterDriver.connect();
deviceModel = await driver.requestData('device_model');
});
tearDownAll(() => driver.close());
test('A page with an image screenshot', () async {
final SerializableFinder imagePageListTile =
find.byValueKey('image_page');
await driver.waitFor(imagePageListTile);
await driver.tap(imagePageListTile);
await driver.waitFor(find.byValueKey('red_square_image'));
await driver.waitUntilNoTransientCallbacks();
// TODO(egarciad): This is currently a no-op on LUCI.
// https://github.com/flutter/flutter/issues/49837
await expectLater(
driver.screenshot(),
bufferMatchesGoldenFile('red_square_driver_screenshot__$deviceModel.png'),
);
await driver.tap(find.byTooltip('Back'));
});
}
...@@ -8,7 +8,6 @@ import 'dart:math' as math; ...@@ -8,7 +8,6 @@ import 'dart:math' as math;
import 'dart:typed_data'; import 'dart:typed_data';
import 'dart:ui'; import 'dart:ui';
import 'package:flutter/widgets.dart' show Element;
import 'package:image/image.dart'; import 'package:image/image.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
// ignore: deprecated_member_use // ignore: deprecated_member_use
...@@ -246,12 +245,12 @@ ComparisonResult compareLists(List<int> test, List<int> master) { ...@@ -246,12 +245,12 @@ ComparisonResult compareLists(List<int> test, List<int> master) {
/// An unsupported [WebGoldenComparator] that exists for API compatibility. /// An unsupported [WebGoldenComparator] that exists for API compatibility.
class DefaultWebGoldenComparator extends WebGoldenComparator { class DefaultWebGoldenComparator extends WebGoldenComparator {
@override @override
Future<bool> compare(Element element, Size size, Uri golden) { Future<bool> compare(double width, double height, Uri golden) {
throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.'); throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
} }
@override @override
Future<void> update(Uri golden, Element element, Size size) { Future<void> update(double width, double height, Uri golden) {
throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.'); throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
} }
} }
...@@ -5,10 +5,7 @@ ...@@ -5,10 +5,7 @@
import 'dart:convert'; import 'dart:convert';
import 'dart:html' as html; import 'dart:html' as html;
import 'dart:typed_data'; import 'dart:typed_data';
import 'dart:ui';
import 'package:flutter/rendering.dart';
import 'package:flutter/widgets.dart';
// ignore: deprecated_member_use // ignore: deprecated_member_use
import 'package:test_api/test_api.dart' as test_package show TestFailure; import 'package:test_api/test_api.dart' as test_package show TestFailure;
...@@ -60,17 +57,16 @@ class DefaultWebGoldenComparator extends WebGoldenComparator { ...@@ -60,17 +57,16 @@ class DefaultWebGoldenComparator extends WebGoldenComparator {
Uri testUri; Uri testUri;
@override @override
Future<bool> compare(Element element, Size size, Uri golden) async { Future<bool> compare(double width, double height, Uri golden) async {
final String key = golden.toString(); final String key = golden.toString();
final html.HttpRequest request = await html.HttpRequest.request( final html.HttpRequest request = await html.HttpRequest.request(
'flutter_goldens', 'flutter_goldens',
method: 'POST', method: 'POST',
sendData: json.encode(<String, Object>{ sendData: json.encode(<String, Object>{
'testUri': testUri.toString(), 'testUri': testUri.toString(),
'key': key.toString(), 'key': key.toString(),
'width': size.width.round(), 'width': width.round(),
'height': size.height.round(), 'height': height.round(),
}), }),
); );
final String response = request.response as String; final String response = request.response as String;
...@@ -82,8 +78,8 @@ class DefaultWebGoldenComparator extends WebGoldenComparator { ...@@ -82,8 +78,8 @@ class DefaultWebGoldenComparator extends WebGoldenComparator {
} }
@override @override
Future<void> update(Uri golden, Element element, Size size) async { Future<void> update(double width, double height, Uri golden) async {
// Update is handled on the server side, just use the same logic here // Update is handled on the server side, just use the same logic here
await compare(element, size, golden); await compare(width, height, golden);
} }
} }
...@@ -60,11 +60,11 @@ class MatchesGoldenFile extends AsyncMatcher { ...@@ -60,11 +60,11 @@ class MatchesGoldenFile extends AsyncMatcher {
_renderElement(binding.window, renderObject); _renderElement(binding.window, renderObject);
final String result = await binding.runAsync<String>(() async { final String result = await binding.runAsync<String>(() async {
if (autoUpdateGoldenFiles) { if (autoUpdateGoldenFiles) {
await webGoldenComparator.update(key, element, size); await webGoldenComparator.update(size.width, size.height, key);
return null; return null;
} }
try { try {
final bool success = await webGoldenComparator.compare(element, size, key); final bool success = await webGoldenComparator.compare(size.width, size.height, key);
return success ? null : 'does not match'; return success ? null : 'does not match';
} on TestFailure catch (ex) { } on TestFailure catch (ex) {
return ex.message; return ex.message;
......
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:typed_data';
import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports
// ignore: deprecated_member_use
import 'package:test_api/test_api.dart' show Description, TestFailure;
import 'goldens.dart';
/// Matcher created by [bufferMatchesGoldenFile].
class _BufferGoldenMatcher extends AsyncMatcher {
/// Creates an instance of [BufferGoldenMatcher]. Called by [bufferMatchesGoldenFile].
const _BufferGoldenMatcher(this.key, this.version);
/// The [key] to the golden image.
final Uri key;
/// The [version] of the golden image.
final int version;
@override
Future<String> matchAsync(dynamic item) async {
Uint8List buffer;
if (item is List<int>) {
buffer = Uint8List.fromList(item);
} else if (item is Future<List<int>>) {
buffer = Uint8List.fromList(await item);
} else {
throw 'Expected `List<int>` or `Future<List<int>>`, instead found: ${item.runtimeType}';
}
final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
if (autoUpdateGoldenFiles) {
await goldenFileComparator.update(testNameUri, buffer);
return null;
}
try {
final bool success = await goldenFileComparator.compare(buffer, testNameUri);
return success ? null : 'does not match';
} on TestFailure catch (ex) {
return ex.message;
}
}
@override
Description describe(Description description) {
final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
return description.add('Byte buffer matches golden image "$testNameUri"');
}
}
/// Asserts that a [Future<List<int>>], or [List<int] matches the
/// golden image file identified by [key], with an optional [version] number.
///
/// The [key] is the [String] representation of a URL.
///
/// The [version] is a number that can be used to differentiate historical
/// golden files. This parameter is optional.
///
/// {@tool snippet}
/// Sample invocations of [matchesGoldenFile].
///
/// ```dart
/// await expectLater(
/// const <int>[],
/// bufferMatchesGoldenFile('sample.png'),
/// );
/// ```
/// {@end-tool}
AsyncMatcher bufferMatchesGoldenFile(String key, {int version}) {
return _BufferGoldenMatcher(Uri.parse(key), version);
}
...@@ -5,8 +5,7 @@ ...@@ -5,8 +5,7 @@
import 'dart:async'; import 'dart:async';
import 'dart:typed_data'; import 'dart:typed_data';
import 'package:flutter/foundation.dart'; import 'package:meta/meta.dart';
import 'package:flutter/widgets.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import '_goldens_io.dart' if (dart.library.html) '_goldens_web.dart' as _goldens; import '_goldens_io.dart' if (dart.library.html) '_goldens_web.dart' as _goldens;
...@@ -158,7 +157,7 @@ set goldenFileComparator(GoldenFileComparator value) { ...@@ -158,7 +157,7 @@ set goldenFileComparator(GoldenFileComparator value) {
/// * [matchesGoldenFile], the function from [flutter_test] that invokes the /// * [matchesGoldenFile], the function from [flutter_test] that invokes the
/// comparator. /// comparator.
abstract class WebGoldenComparator { abstract class WebGoldenComparator {
/// Compares the rendered pixels of [element] of size [size] that is being /// Compares the rendered pixels of size [width]x[height] that is being
/// rendered on the top left of the screen against the golden file identified /// rendered on the top left of the screen against the golden file identified
/// by [golden]. /// by [golden].
/// ///
...@@ -174,10 +173,10 @@ abstract class WebGoldenComparator { ...@@ -174,10 +173,10 @@ abstract class WebGoldenComparator {
/// is left up to the implementation class. For instance, some implementations /// is left up to the implementation class. For instance, some implementations
/// may load files from the local file system, whereas others may load files /// may load files from the local file system, whereas others may load files
/// over the network or from a remote repository. /// over the network or from a remote repository.
Future<bool> compare(Element element, Size size, Uri golden); Future<bool> compare(double width, double height, Uri golden);
/// Updates the golden file identified by [golden] with rendered pixels of /// Updates the golden file identified by [golden] with rendered pixels of
/// [element]. /// [width]x[height].
/// ///
/// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles] /// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
/// is `true` (which gets set automatically by the test framework when the /// is `true` (which gets set automatically by the test framework when the
...@@ -185,7 +184,7 @@ abstract class WebGoldenComparator { ...@@ -185,7 +184,7 @@ abstract class WebGoldenComparator {
/// ///
/// The method by which [golden] is located and by which its bytes are written /// The method by which [golden] is located and by which its bytes are written
/// is left up to the implementation class. /// is left up to the implementation class.
Future<void> update(Uri golden, Element element, Size size); Future<void> update(double width, double height, Uri golden);
/// Returns a new golden file [Uri] to incorporate any [version] number with /// Returns a new golden file [Uri] to incorporate any [version] number with
/// the [key]. /// the [key].
...@@ -280,7 +279,7 @@ class TrivialComparator implements GoldenFileComparator { ...@@ -280,7 +279,7 @@ class TrivialComparator implements GoldenFileComparator {
@override @override
Future<bool> compare(Uint8List imageBytes, Uri golden) { Future<bool> compare(Uint8List imageBytes, Uri golden) {
debugPrint('Golden file comparison requested for "$golden"; skipping...'); print('Golden file comparison requested for "$golden"; skipping...');
return Future<bool>.value(true); return Future<bool>.value(true);
} }
...@@ -299,13 +298,13 @@ class _TrivialWebGoldenComparator implements WebGoldenComparator { ...@@ -299,13 +298,13 @@ class _TrivialWebGoldenComparator implements WebGoldenComparator {
const _TrivialWebGoldenComparator._(); const _TrivialWebGoldenComparator._();
@override @override
Future<bool> compare(Element element, Size size, Uri golden) { Future<bool> compare(double width, double height, Uri golden) {
debugPrint('Golden comparison requested for "$golden"; skipping...'); print('Golden comparison requested for "$golden"; skipping...');
return Future<bool>.value(true); return Future<bool>.value(true);
} }
@override @override
Future<void> update(Uri golden, Element element, Size size) { Future<void> update(double width, double height, Uri golden) {
throw StateError('webGoldenComparator has not been initialized'); throw StateError('webGoldenComparator has not been initialized');
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment