Skip to content

F
Front-End
Unverified Commit ee69eebf authored by Kate Lovett's avatar Kate Lovett Committed by GitHub
Browse files
Options

Update Gold for new endpoint (#64982)

parent 37de94d7
Expand all Show whitespace changes
Inline Side-by-side
Showing with 191 additions and 512 deletions
packages/flutter_goldens/lib/flutter_goldens.dart
View file @ ee69eebf
......@@ -2,10 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io' as io;
import 'dart:math' as math;
import 'dart:typed_data';
import 'dart:async' show FutureOr;
import 'dart:io' as io show OSError, SocketException;
import 'dart:math' as math show Random;
import 'dart:typed_data' show Uint8List;
import 'package:file/file.dart';
import 'package:file/local.dart';
......@@ -449,13 +449,10 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar
if (await skiaClient.imgtestCheck(golden.path, goldenFile))
return true;
// We do not have a matching image, so we need to check a few things
// manually. We wait until this point to do this work so request traffic
// low.
skiaClient.getExpectations();
// We do not have a matching image hash, so we need to check manually.
final String testName = skiaClient.cleanTestName(golden.path);
final List<String>? testExpectations = skiaClient.expectations[testName];
if (testExpectations == null) {
final String? testExpectation = await skiaClient.getExpectationForTest(testName);
if (testExpectation == null) {
// This is a new test.
print('No expectations provided by Skia Gold for test: $golden. '
'This may be a new test. If this is an unexpected result, check '
......@@ -466,11 +463,26 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar
// Contributors without the proper permissions to execute a tryjob can make
// a golden file change through Gold's ignore feature instead.
String? pullRequest;
switch(skiaClient.ci) {
case ContinuousIntegrationEnvironment.cirrus:
pullRequest = platform.environment['CIRRUS_PR']!;
break;
case ContinuousIntegrationEnvironment.luci:
final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/');
pullRequest = refs[refs.length - 2];
break;
case ContinuousIntegrationEnvironment.none:
pullRequest = '';
break;
}
final bool ignoreResult = await skiaClient.testIsIgnoredForPullRequest(
platform.environment['CIRRUS_PR'] ?? '',
pullRequest,
golden.path,
);
// If true, this is an intended change.
// If true, this is an intended change and is being handled on the Flutter
// Gold dashboard: https://flutter-gold.skia.org/ignores
return ignoreResult;
}
}
......@@ -479,8 +491,7 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar
/// golden file tests.
///
/// Currently, this comparator is used in some Cirrus test shards and Luci
/// environments, as well as when an internet connection is not available for
/// contacting Gold.
/// environments.
///
/// See also:
///
......@@ -608,9 +619,9 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
}
goldens ??= SkiaGoldClient(baseDirectory, ci: ContinuousIntegrationEnvironment.none);
try {
await goldens.getExpectations();
// Check if we can reach Gold.
await goldens.getExpectationForTest('');
} on io.OSError catch (_) {
return FlutterSkippingFileComparator(
baseDirectory.uri,
......@@ -634,8 +645,10 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
Future<bool> compare(Uint8List imageBytes, Uri golden) async {
golden = _addPrefix(golden);
final String testName = skiaClient.cleanTestName(golden.path);
final List<String>? testExpectations = skiaClient.expectations[testName];
if (testExpectations == null) {
late String? testExpectation;
testExpectation = await skiaClient.getExpectationForTest(testName);
if (testExpectation == null) {
// There is no baseline for this test
print('No expectations provided by Skia Gold for test: $golden. '
'This may be a new test. If this is an unexpected result, check '
......@@ -647,25 +660,17 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
}
ComparisonResult result;
final Map<String, ComparisonResult> failureDiffs = <String, ComparisonResult>{};
for (final String expectation in testExpectations) {
final List<int> goldenBytes = await skiaClient.getImageBytes(expectation);
final List<int> goldenBytes = await skiaClient.getImageBytes(testExpectation);
result = await GoldenFileComparator.compareLists(
imageBytes,
goldenBytes,
);
if (result.passed) {
if (result.passed)
return true;
}
failureDiffs[expectation] = result;
}
for (final MapEntry<String, ComparisonResult> entry in failureDiffs.entries) {
if (await skiaClient.isValidDigestForExpectation(entry.key, golden.path))
generateFailureOutput(entry.value, golden, basedir, key: entry.key);
}
generateFailureOutput(result, golden, basedir);
return false;
}
}
packages/flutter_goldens/test/json_templates.dart
View file @ ee69eebf
......@@ -2,7 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/// JSON template for the contents of the auth_opt.json file created by goldctl.
/// Json response template for the contents of the auth_opt.json file created by
/// goldctl.
String authTemplate({
bool gsutil = false,
}) {
......@@ -15,196 +16,6 @@ String authTemplate({
''';
}
/// JSON response template for Skia Gold expectations request:
/// https://flutter-gold.skia.org/json/expectations/commit/HEAD
String rawExpectationsTemplate() {
return '''
{
"md5": "a7489b00e03a1846e43500b7c14dd7b0",
"master": {
"flutter.golden_test.1": {
"55109a4bed52acc780530f7a9aeff6c0": 1
},
"flutter.golden_test.3": {
"87cb35131e6ad4b57d4d09d59ae743c3": 1,
"dc94eb2c39c0c8ae11a4efd090b72f94": 1,
"f2583c9003978a06b7888878bdc089e2": 1
},
"flutter.golden_test.2": {
"eb03a5e3114c9ecad5e4f1178f285a49": 1,
"f14631979de24fca6e14ad247d5f2bd6": 1
}
}
}
''';
}
/// Decoded json response template for Skia Gold expectations request:
/// https://flutter-gold.skia.org/json/expectations/commit/HEAD
Map<String, List<String>> expectationsTemplate() {
return <String, List<String>>{
'flutter.golden_test.1': <String>[
'55109a4bed52acc780530f7a9aeff6c0'
],
'flutter.golden_test.3': <String>[
'87cb35131e6ad4b57d4d09d59ae743c3',
'dc94eb2c39c0c8ae11a4efd090b72f94',
'f2583c9003978a06b7888878bdc089e2',
],
'flutter.golden_test.2': <String>[
'eb03a5e3114c9ecad5e4f1178f285a49',
'f14631979de24fca6e14ad247d5f2bd6',
],
};
}
/// Same as [rawExpectationsTemplate] but with the temporary key.
String rawExpectationsTemplateWithTemporaryKey() {
return '''
{
"md5": "a7489b00e03a1846e43500b7c14dd7b0",
"master_str": {
"flutter.golden_test.1": {
"55109a4bed52acc780530f7a9aeff6c0": 1
},
"flutter.golden_test.3": {
"87cb35131e6ad4b57d4d09d59ae743c3": 1,
"dc94eb2c39c0c8ae11a4efd090b72f94": 1,
"f2583c9003978a06b7888878bdc089e2": 1
},
"flutter.golden_test.2": {
"eb03a5e3114c9ecad5e4f1178f285a49": 1,
"f14631979de24fca6e14ad247d5f2bd6": 1
}
}
}
''';
}
/// Json response template for Skia Gold digest request:
/// https://flutter-gold.skia.org/json/details?test=[testName]&digest=[expectation]
String digestResponseTemplate({
String testName = 'flutter.golden_test.1',
String expectation = '55109a4bed52acc780530f7a9aeff6c0',
String platform = 'macos',
String status = 'positive',
}) {
return '''
{
"digest": {
"test": "$testName",
"digest": "$expectation",
"status": "$status",
"paramset": {
"Platform": [
"$platform"
],
"ext": [
"png"
],
"name": [
"$testName"
],
"source_type": [
"flutter"
]
},
"traces": {
"tileSize": 200,
"traces": [
{
"data": [
{
"x": 0,
"y": 0,
"s": 0
},
{
"x": 1,
"y": 0,
"s": 0
},
{
"x": 199,
"y": 0,
"s": 0
}
],
"label": ",Platform=$platform,name=$testName,source_type=flutter,",
"params": {
"Platform": "$platform",
"ext": "png",
"name": "$testName",
"source_type": "flutter"
}
}
],
"digests": [
{
"digest": "$expectation",
"status": "$status"
}
]
},
"closestRef": "pos",
"refDiffs": {
"neg": null,
"pos": {
"numDiffPixels": 999,
"pixelDiffPercent": 0.4995,
"maxRGBADiffs": [
86,
86,
86,
0
],
"dimDiffer": false,
"diffs": {
"combined": 0.381955,
"percent": 0.4995,
"pixel": 999
},
"digest": "aa748136c70cefdda646df5be0ae189d",
"status": "positive",
"paramset": {
"Platform": [
"$platform"
],
"ext": [
"png"
],
"name": [
"$testName"
],
"source_type": [
"flutter"
]
},
"n": 197
}
}
},
"commits": [
{
"commit_time": 1568069344,
"hash": "399bb04e2de41665320d3c888f40af6d8bc734a2",
"author": "Contributor A (contributorA@getMail.com)"
},
{
"commit_time": 1568078053,
"hash": "0f365d3add253a65e5e5af1024f56c6169bf9739",
"author": "Contributor B (contributorB@getMail.com)"
},
{
"commit_time": 1569353925,
"hash": "81e693a7fe3b808cc9ae2bb3a2cbe404e67ec773",
"author": "Contributor C (contributorC@getMail.com)"
}
]
}
''';
}
/// Json response template for Skia Gold ignore request:
/// https://flutter-gold.skia.org/json/ignores
String ignoreResponseTemplate({
......
packages/flutter_goldens_client/lib/skia_client.dart
View file @ ee69eebf
......@@ -73,15 +73,6 @@ class SkiaGoldClient {
/// be null.
final Directory workDirectory;
/// A map of known golden file tests and their associated positive image
/// hashes.
///
/// This is set and used by the [FlutterLocalFileComparator] and the
/// [_UnauthorizedFlutterPreSubmitComparator] to test against golden masters
/// maintained in the Flutter Gold dashboard.
Map<String, List<String>> get expectations => _expectations;
late Map<String, List<String>> _expectations;
/// The local [Directory] where the Flutter repository is hosted.
///
/// Uses the [fs] file system.
......@@ -421,15 +412,15 @@ class SkiaGoldClient {
return result.exitCode == 0;
}
/// Requests and sets the [_expectations] known to Flutter Gold at head.
Future<void> getExpectations() async {
_expectations = <String, List<String>>{};
/// Returns the latest positive digest for the given test known to Flutter
/// Gold at head.
Future<String?> getExpectationForTest(String testName) async {
late String? expectation;
final String traceID = getTraceID(testName);
await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
final Uri requestForExpectations = Uri.parse(
'https://flutter-gold.skia.org/json/expectations/commit/HEAD'
'https://flutter-gold.skia.org/json/latestpositivedigest/$traceID'
);
const String mainKey = 'master';
const String temporaryKey = 'master_str';
late String rawResponse;
try {
final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
......@@ -438,13 +429,7 @@ class SkiaGoldClient {
final dynamic jsonResponse = json.decode(rawResponse);
if (jsonResponse is! Map<String, dynamic>)
throw const FormatException('Skia gold expectations do not match expected format.');
final Map<String, dynamic>? skiaJson = (jsonResponse[mainKey] ?? jsonResponse[temporaryKey]) as Map<String, dynamic>?;
if (skiaJson == null)
throw FormatException('Skia gold expectations are missing the "$mainKey" key (and also doesn\'t have "$temporaryKey")! Available keys: ${jsonResponse.keys.join(", ")}');
skiaJson.forEach((String key, dynamic value) {
final Map<String, dynamic> hashesMap = value as Map<String, dynamic>;
_expectations[key] = hashesMap.keys.toList();
});
expectation = jsonResponse['digest'] as String?;
} on FormatException catch (error) {
print(
'Formatting error detected requesting expectations from Flutter Gold.\n'
......@@ -457,6 +442,7 @@ class SkiaGoldClient {
},
SkiaGoldHttpOverrides(),
);
return expectation;
}
/// Returns a list of bytes representing the golden image retrieved from the
......@@ -548,44 +534,6 @@ class SkiaGoldClient {
return ignoreIsActive;
}
/// The [_expectations] retrieved from Flutter Gold do not include the
/// parameters of the given test. This function queries the Flutter Gold
/// details api to determine if the given expectation for a test matches the
/// configuration of the executing machine.
Future<bool> isValidDigestForExpectation(String expectation, String testName) async {
bool isValid = false;
testName = cleanTestName(testName);
late String rawResponse;
await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
final Uri requestForDigest = Uri.parse(
'https://flutter-gold.skia.org/json/details?test=$testName&digest=$expectation'
);
try {
final io.HttpClientRequest request = await httpClient.getUrl(requestForDigest);
final io.HttpClientResponse response = await request.close();
rawResponse = await utf8.decodeStream(response);
final Map<String, dynamic> skiaJson = json.decode(rawResponse) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
isValid = digest.isValid(platform, testName, expectation);
} on FormatException catch(_) {
if (rawResponse.contains('stream timeout')) {
final StringBuffer buf = StringBuffer()
..writeln("Stream timeout on Gold's /details api.");
throw Exception(buf.toString());
} else {
print('Formatting error detected requesting /ignores from Flutter Gold.'
'\nrawResponse: $rawResponse');
rethrow;
}
}
},
SkiaGoldHttpOverrides(),
);
return isValid;
}
/// Returns the current commit hash of the Flutter repository.
Future<String> _getCurrentCommit() async {
if (!_flutterRoot.existsSync()) {
......@@ -669,55 +617,26 @@ class SkiaGoldClient {
'--jobid', jobId,
];
}
}
/// Used to make HttpRequests during testing.
class SkiaGoldHttpOverrides extends io.HttpOverrides {}
/// A digest returned from a request to the Flutter Gold dashboard.
class SkiaGoldDigest {
const SkiaGoldDigest({
required this.imageHash,
required this.paramSet,
required this.testName,
required this.status,
});
/// Create a digest from requested JSON.
factory SkiaGoldDigest.fromJson(Map<String, dynamic> json) {
return SkiaGoldDigest(
imageHash: json['digest'] as String,
paramSet:
Map<String, dynamic>.from(
json['refDiffs']['pos']['paramset'] as Map<String, dynamic>? ??
<String, List<String>>{
'Platform': <String>[],
'Browser' : <String>[],
}),
testName: json['test'] as String,
status: json['status'] as String,
);
/// Returns a trace id based on the current testing environment to lookup
/// the latest positive digest on Flutter Gold.
///
/// Trace IDs are case sensitive and should be in alphabetical order for the
/// keys, followed by the rest of the paramset, also in alphabetical order.
/// There should also be leading and trailing commas.
///
/// Example TraceID for Flutter Gold:
/// ',CI=cirrus,Platform=linux,name=cupertino.activityIndicator.inprogress.1.0,source_type=flutter,'
String getTraceID(String testName) {
// If we are not in a CI environment, fallback on luci.
return '${platform.environment[_kTestBrowserKey] == null ? ',' : ',Browser=${platform.environment[_kTestBrowserKey]},'}'
'CI=${ci == ContinuousIntegrationEnvironment.none ? 'luci' : ci.toString().split('.').last},'
'Platform=${platform.operatingSystem},'
'name=$testName,'
'source_type=flutter,';
}
/// Unique identifier for the image associated with the digest.
final String imageHash;
/// Parameter set for the given test, e.g. Platform : Windows.
final Map<String, dynamic> paramSet;
/// Test name associated with the digest, e.g. positive or un-triaged.
final String testName;
/// Status of the given digest, e.g. positive or un-triaged.
final String status;
/// Validates a given digest against the current testing conditions.
bool isValid(Platform platform, String name, String expectation) {
return imageHash == expectation
&& (paramSet['Platform'] as List<dynamic>/*!*/).contains(platform.operatingSystem)
&& (platform.environment[_kTestBrowserKey] == null
|| paramSet['Browser'] == platform.environment[_kTestBrowserKey])
&& testName == name
&& status == 'positive';
}
}
/// Used to make HttpRequests during testing.
class SkiaGoldHttpOverrides extends io.HttpOverrides {}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab