// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:convert'; import 'dart:io' as io; import 'package:crypto/crypto.dart'; import 'package:file/file.dart'; import 'package:file/local.dart'; import 'package:path/path.dart' as path; import 'package:platform/platform.dart'; import 'package:process/process.dart'; // If you are here trying to figure out how to use golden files in the Flutter // repo itself, consider reading this wiki page: // https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter const String _kFlutterRootKey = 'FLUTTER_ROOT'; const String _kGoldctlKey = 'GOLDCTL'; const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER'; const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER'; /// Exception thrown when an error is returned from the [SkiaClient]. class SkiaException implements Exception { /// Creates a new `SkiaException` with a required error [message]. const SkiaException(this.message); /// A message describing the error. final String message; /// Returns a description of the Skia exception. /// /// The description always contains the [message]. @override String toString() => 'SkiaException: $message'; } /// A client for uploading image tests and making baseline requests to the /// Flutter Gold Dashboard. class SkiaGoldClient { /// Creates a [SkiaGoldClient] with the given [workDirectory]. /// /// All other parameters are optional. They may be provided in tests to /// override the defaults for [fs], [process], [platform], and [httpClient]. SkiaGoldClient( this.workDirectory, { this.fs = const LocalFileSystem(), this.process = const LocalProcessManager(), this.platform = const LocalPlatform(), io.HttpClient? httpClient, }) : httpClient = httpClient ?? io.HttpClient(); /// The file system to use for storing the local clone of the repository. /// /// This is useful in tests, where a local file system (the default) can be /// replaced by a memory file system. final FileSystem fs; /// A wrapper for the [dart:io.Platform] API. /// /// This is useful in tests, where the system platform (the default) can be /// replaced by a mock platform instance. final Platform platform; /// A controller for launching sub-processes. /// /// This is useful in tests, where the real process manager (the default) can /// be replaced by a mock process manager that doesn't really create /// sub-processes. final ProcessManager process; /// A client for making Http requests to the Flutter Gold dashboard. final io.HttpClient httpClient; /// The local [Directory] within the [comparisonRoot] for the current test /// context. In this directory, the client will create image and JSON files /// for the goldctl tool to use. /// /// This is informed by the [FlutterGoldenFileComparator] [basedir]. It cannot /// be null. final Directory workDirectory; /// The local [Directory] where the Flutter repository is hosted. /// /// Uses the [fs] file system. Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]); /// The path to the local [Directory] where the goldctl tool is hosted. /// /// Uses the [platform] environment in this implementation. String get _goldctl => platform.environment[_kGoldctlKey]!; /// Prepares the local work space for golden file testing and calls the /// goldctl `auth` command. /// /// This ensures that the goldctl tool is authorized and ready for testing. /// Used by the [FlutterPostSubmitFileComparator] and the /// [FlutterPreSubmitFileComparator]. Future<void> auth() async { if (await clientIsAuthorized()) { return; } final List<String> authCommand = <String>[ _goldctl, 'auth', '--work-dir', workDirectory .childDirectory('temp') .path, '--luci', ]; final io.ProcessResult result = await process.run(authCommand); if (result.exitCode != 0) { final StringBuffer buf = StringBuffer() ..writeln('Skia Gold authorization failed.') ..writeln('Luci environments authenticate using the file provided ' 'by LUCI_CONTEXT. There may be an error with this file or Gold ' 'authentication.') ..writeln('Debug information for Gold --------------------------------') ..writeln('stdout: ${result.stdout}') ..writeln('stderr: ${result.stderr}'); throw SkiaException(buf.toString()); } } /// Signals if this client is initialized for uploading images to the Gold /// service. /// /// Since Flutter framework tests are executed in parallel, and in random /// order, this will signal is this instance of the Gold client has been /// initialized. bool _initialized = false; /// Executes the `imgtest init` command in the goldctl tool. /// /// The `imgtest` command collects and uploads test results to the Skia Gold /// backend, the `init` argument initializes the current test. Used by the /// [FlutterPostSubmitFileComparator]. Future<void> imgtestInit() async { // This client has already been initialized if (_initialized) { return; } final File keys = workDirectory.childFile('keys.json'); final File failures = workDirectory.childFile('failures.json'); await keys.writeAsString(_getKeysJSON()); await failures.create(); final String commitHash = await _getCurrentCommit(); final List<String> imgtestInitCommand = <String>[ _goldctl, 'imgtest', 'init', '--instance', 'flutter', '--work-dir', workDirectory .childDirectory('temp') .path, '--commit', commitHash, '--keys-file', keys.path, '--failure-file', failures.path, '--passfail', ]; if (imgtestInitCommand.contains(null)) { final StringBuffer buf = StringBuffer() ..writeln('A null argument was provided for Skia Gold imgtest init.') ..writeln('Please confirm the settings of your golden file test.') ..writeln('Arguments provided:'); imgtestInitCommand.forEach(buf.writeln); throw SkiaException(buf.toString()); } final io.ProcessResult result = await process.run(imgtestInitCommand); if (result.exitCode != 0) { _initialized = false; final StringBuffer buf = StringBuffer() ..writeln('Skia Gold imgtest init failed.') ..writeln('An error occurred when initializing golden file test with ') ..writeln('goldctl.') ..writeln() ..writeln('Debug information for Gold --------------------------------') ..writeln('stdout: ${result.stdout}') ..writeln('stderr: ${result.stderr}'); throw SkiaException(buf.toString()); } _initialized = true; } /// Executes the `imgtest add` command in the goldctl tool. /// /// The `imgtest` command collects and uploads test results to the Skia Gold /// backend, the `add` argument uploads the current image test. A response is /// returned from the invocation of this command that indicates a pass or fail /// result. /// /// The [testName] and [goldenFile] parameters reference the current /// comparison being evaluated by the [FlutterPostSubmitFileComparator]. Future<bool> imgtestAdd(String testName, File goldenFile) async { final List<String> imgtestCommand = <String>[ _goldctl, 'imgtest', 'add', '--work-dir', workDirectory .childDirectory('temp') .path, '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, '--passfail', ..._getPixelMatchingArguments(), ]; final io.ProcessResult result = await process.run(imgtestCommand); if (result.exitCode != 0) { // If an unapproved image has made it to post-submit, throw to close the // tree. String? resultContents; final File resultFile = workDirectory.childFile(fs.path.join( 'result-state.json', )); if(await resultFile.exists()) { resultContents = await resultFile.readAsString(); } final StringBuffer buf = StringBuffer() ..writeln('Skia Gold received an unapproved image in post-submit ') ..writeln('testing. Golden file images in flutter/flutter are triaged ') ..writeln('in pre-submit during code review for the given PR.') ..writeln() ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ') ..writeln('the image(s), or revert the associated change. For more ') ..writeln('information, visit the wiki: ') ..writeln('https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter') ..writeln() ..writeln('Debug information for Gold --------------------------------') ..writeln('stdout: ${result.stdout}') ..writeln('stderr: ${result.stderr}') ..writeln() ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); throw SkiaException(buf.toString()); } return true; } /// Signals if this client is initialized for uploading tryjobs to the Gold /// service. /// /// Since Flutter framework tests are executed in parallel, and in random /// order, this will signal is this instance of the Gold client has been /// initialized for tryjobs. bool _tryjobInitialized = false; /// Executes the `imgtest init` command in the goldctl tool for tryjobs. /// /// The `imgtest` command collects and uploads test results to the Skia Gold /// backend, the `init` argument initializes the current tryjob. Used by the /// [FlutterPreSubmitFileComparator]. Future<void> tryjobInit() async { // This client has already been initialized if (_tryjobInitialized) { return; } final File keys = workDirectory.childFile('keys.json'); final File failures = workDirectory.childFile('failures.json'); await keys.writeAsString(_getKeysJSON()); await failures.create(); final String commitHash = await _getCurrentCommit(); final List<String> imgtestInitCommand = <String>[ _goldctl, 'imgtest', 'init', '--instance', 'flutter', '--work-dir', workDirectory .childDirectory('temp') .path, '--commit', commitHash, '--keys-file', keys.path, '--failure-file', failures.path, '--passfail', '--crs', 'github', '--patchset_id', commitHash, ...getCIArguments(), ]; if (imgtestInitCommand.contains(null)) { final StringBuffer buf = StringBuffer() ..writeln('A null argument was provided for Skia Gold tryjob init.') ..writeln('Please confirm the settings of your golden file test.') ..writeln('Arguments provided:'); imgtestInitCommand.forEach(buf.writeln); throw SkiaException(buf.toString()); } final io.ProcessResult result = await process.run(imgtestInitCommand); if (result.exitCode != 0) { _tryjobInitialized = false; final StringBuffer buf = StringBuffer() ..writeln('Skia Gold tryjobInit failure.') ..writeln('An error occurred when initializing golden file tryjob with ') ..writeln('goldctl.') ..writeln() ..writeln('Debug information for Gold --------------------------------') ..writeln('stdout: ${result.stdout}') ..writeln('stderr: ${result.stderr}'); throw SkiaException(buf.toString()); } _tryjobInitialized = true; } /// Executes the `imgtest add` command in the goldctl tool for tryjobs. /// /// The `imgtest` command collects and uploads test results to the Skia Gold /// backend, the `add` argument uploads the current image test. A response is /// returned from the invocation of this command that indicates a pass or fail /// result for the tryjob. /// /// The [testName] and [goldenFile] parameters reference the current /// comparison being evaluated by the [FlutterPreSubmitFileComparator]. Future<void> tryjobAdd(String testName, File goldenFile) async { final List<String> imgtestCommand = <String>[ _goldctl, 'imgtest', 'add', '--work-dir', workDirectory .childDirectory('temp') .path, '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, ..._getPixelMatchingArguments(), ]; final io.ProcessResult result = await process.run(imgtestCommand); final String/*!*/ resultStdout = result.stdout.toString(); if (result.exitCode != 0 && !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) { String? resultContents; final File resultFile = workDirectory.childFile(fs.path.join( 'result-state.json', )); if(await resultFile.exists()) { resultContents = await resultFile.readAsString(); } final StringBuffer buf = StringBuffer() ..writeln('Unexpected Gold tryjobAdd failure.') ..writeln('Tryjob execution for golden file test $testName failed for') ..writeln('a reason unrelated to pixel comparison.') ..writeln() ..writeln('Debug information for Gold --------------------------------') ..writeln('stdout: ${result.stdout}') ..writeln('stderr: ${result.stderr}') ..writeln() ..writeln() ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}'); throw SkiaException(buf.toString()); } } // Constructs arguments for `goldctl` for controlling how pixels are compared. // // For AOT and CanvasKit exact pixel matching is used. For the HTML renderer // on the web a fuzzy matching algorithm is used that allows very small deltas // because Chromium cannot exactly reproduce the same golden on all computers. // It seems to depend on the hardware/OS/driver combination. However, those // differences are very small (typically not noticeable to human eye). List<String> _getPixelMatchingArguments() { // Only use fuzzy pixel matching in the HTML renderer. if (!_isBrowserTest || _isBrowserCanvasKitTest) { return const <String>[]; } // The algorithm to be used when matching images. The available options are: // - "fuzzy": Allows for customizing the thresholds of pixel differences. // - "sobel": Same as "fuzzy" but performs edge detection before performing // a fuzzy match. const String algorithm = 'fuzzy'; // The number of pixels in this image that are allowed to differ from the // baseline. // // The chosen number - 20 - is arbitrary. Even for a small golden file, say // 50 x 50, it would be less than 1% of the total number of pixels. This // number should not grow too much. If it's growing, it is probably due to a // larger issue that needs to be addressed at the infra level. const int maxDifferentPixels = 20; // The maximum acceptable difference per pixel. // // Uses the Manhattan distance using the RGBA color components as // coordinates. The chosen number - 4 - is arbitrary. It's small enough to // both not be noticeable and not trigger test flakes due to sub-pixel // golden deltas. This number should not grow too much. If it's growing, it // is probably due to a larger issue that needs to be addressed at the infra // level. const int pixelDeltaThreshold = 4; return <String>[ '--add-test-optional-key', 'image_matching_algorithm:$algorithm', '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels', '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold', ]; } /// Returns the latest positive digest for the given test known to Flutter /// Gold at head. Future<String?> getExpectationForTest(String testName) async { late String? expectation; final String traceID = getTraceID(testName); await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async { final Uri requestForExpectations = Uri.parse( 'https://flutter-gold.skia.org/json/v2/latestpositivedigest/$traceID' ); late String rawResponse; try { final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations); final io.HttpClientResponse response = await request.close(); rawResponse = await utf8.decodeStream(response); final dynamic jsonResponse = json.decode(rawResponse); if (jsonResponse is! Map<String, dynamic>) { throw const FormatException('Skia gold expectations do not match expected format.'); } expectation = jsonResponse['digest'] as String?; } on FormatException catch (error) { // Ideally we'd use something like package:test's printOnError, but best reliability // in getting logs on CI for now we're just using print. // See also: https://github.com/flutter/flutter/issues/91285 print( // ignore: avoid_print 'Formatting error detected requesting expectations from Flutter Gold.\n' 'error: $error\n' 'url: $requestForExpectations\n' 'response: $rawResponse' ); rethrow; } }, SkiaGoldHttpOverrides(), ); return expectation; } /// Returns a list of bytes representing the golden image retrieved from the /// Flutter Gold dashboard. /// /// The provided image hash represents an expectation from Flutter Gold. Future<List<int>>getImageBytes(String imageHash) async { final List<int> imageBytes = <int>[]; await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async { final Uri requestForImage = Uri.parse( 'https://flutter-gold.skia.org/img/images/$imageHash.png', ); final io.HttpClientRequest request = await httpClient.getUrl(requestForImage); final io.HttpClientResponse response = await request.close(); await response.forEach((List<int> bytes) => imageBytes.addAll(bytes)); }, SkiaGoldHttpOverrides(), ); return imageBytes; } /// Returns the current commit hash of the Flutter repository. Future<String> _getCurrentCommit() async { if (!_flutterRoot.existsSync()) { throw SkiaException('Flutter root could not be found: $_flutterRoot\n'); } else { final io.ProcessResult revParse = await process.run( <String>['git', 'rev-parse', 'HEAD'], workingDirectory: _flutterRoot.path, ); if (revParse.exitCode != 0) { throw const SkiaException('Current commit of Flutter can not be found.'); } return (revParse.stdout as String/*!*/).trim(); } } /// Returns a JSON String with keys value pairs used to uniquely identify the /// configuration that generated the given golden file. /// /// Currently, the only key value pairs being tracked is the platform the /// image was rendered on, and for web tests, the browser the image was /// rendered on. String _getKeysJSON() { final Map<String, dynamic> keys = <String, dynamic>{ 'Platform' : platform.operatingSystem, 'CI' : 'luci', }; if (_isBrowserTest) { keys['Browser'] = _browserKey; keys['Platform'] = '${keys['Platform']}-browser'; if (_isBrowserCanvasKitTest) { keys['WebRenderer'] = 'canvaskit'; } } return json.encode(keys); } /// Removes the file extension from the [fileName] to represent the test name /// properly. String cleanTestName(String fileName) { return fileName.split(path.extension(fileName))[0]; } /// Returns a boolean value to prevent the client from re-authorizing itself /// for multiple tests. Future<bool> clientIsAuthorized() async { final File authFile = workDirectory.childFile(fs.path.join( 'temp', 'auth_opt.json', ))/*!*/; if(await authFile.exists()) { final String contents = await authFile.readAsString(); final Map<String, dynamic> decoded = json.decode(contents) as Map<String, dynamic>; return !(decoded['GSUtil'] as bool/*!*/); } return false; } /// Returns a list of arguments for initializing a tryjob based on the testing /// environment. List<String> getCIArguments() { final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last; final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/'); final String pullRequest = refs[refs.length - 2]; return <String>[ '--changelist', pullRequest, '--cis', 'buildbucket', '--jobid', jobId, ]; } bool get _isBrowserTest { return platform.environment[_kTestBrowserKey] != null; } bool get _isBrowserCanvasKitTest { return _isBrowserTest && platform.environment[_kWebRendererKey] == 'canvaskit'; } String get _browserKey { assert(_isBrowserTest); return platform.environment[_kTestBrowserKey]!; } /// Returns a trace id based on the current testing environment to lookup /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of /// the image keys. String getTraceID(String testName) { final Map<String, dynamic> keys = <String, dynamic>{ if (_isBrowserTest) 'Browser' : _browserKey, if (_isBrowserCanvasKitTest) 'WebRenderer' : 'canvaskit', 'CI' : 'luci', 'Platform' : platform.operatingSystem, 'name' : testName, 'source_type' : 'flutter', }; final String jsonTrace = json.encode(keys); final String md5Sum = md5.convert(utf8.encode(jsonTrace)).toString(); return md5Sum; } } /// Used to make HttpRequests during testing. class SkiaGoldHttpOverrides extends io.HttpOverrides { }