skia_client.dart 20 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io' as io;

8
import 'package:crypto/crypto.dart';
9 10 11 12 13 14 15 16 17 18
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import 'package:platform/platform.dart';
import 'package:process/process.dart';

// If you are here trying to figure out how to use golden files in the Flutter
// repo itself, consider reading this wiki page:
// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter

19
const String _kFlutterRootKey = 'FLUTTER_ROOT';
20
const String _kGoldctlKey = 'GOLDCTL';
21
const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';
22
const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER';
23

24 25 26
/// A client for uploading image tests and making baseline requests to the
/// Flutter Gold Dashboard.
class SkiaGoldClient {
27 28 29 30
  /// Creates a [SkiaGoldClient] with the given [workDirectory].
  ///
  /// All other parameters are optional. They may be provided in tests to
  /// override the defaults for [fs], [process], [platform], and [httpClient].
31 32 33 34 35
  SkiaGoldClient(
    this.workDirectory, {
    this.fs = const LocalFileSystem(),
    this.process = const LocalProcessManager(),
    this.platform = const LocalPlatform(),
36 37
    io.HttpClient? httpClient,
  }) : httpClient = httpClient ?? io.HttpClient();
38 39 40

  /// The file system to use for storing the local clone of the repository.
  ///
41 42
  /// This is useful in tests, where a local file system (the default) can be
  /// replaced by a memory file system.
43 44 45 46
  final FileSystem fs;

  /// A wrapper for the [dart:io.Platform] API.
  ///
47 48
  /// This is useful in tests, where the system platform (the default) can be
  /// replaced by a mock platform instance.
49 50 51 52
  final Platform platform;

  /// A controller for launching sub-processes.
  ///
53 54
  /// This is useful in tests, where the real process manager (the default) can
  /// be replaced by a mock process manager that doesn't really create
55 56 57 58 59
  /// sub-processes.
  final ProcessManager process;

  /// A client for making Http requests to the Flutter Gold dashboard.
  final io.HttpClient httpClient;
60 61

  /// The local [Directory] within the [comparisonRoot] for the current test
62
  /// context. In this directory, the client will create image and JSON files
63 64 65 66
  /// for the goldctl tool to use.
  ///
  /// This is informed by the [FlutterGoldenFileComparator] [basedir]. It cannot
  /// be null.
67 68 69 70 71 72
  final Directory workDirectory;

  /// The local [Directory] where the Flutter repository is hosted.
  ///
  /// Uses the [fs] file system.
  Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]);
73 74 75 76

  /// The path to the local [Directory] where the goldctl tool is hosted.
  ///
  /// Uses the [platform] environment in this implementation.
77
  String get _goldctl => platform.environment[_kGoldctlKey]!;
78 79 80 81

  /// Prepares the local work space for golden file testing and calls the
  /// goldctl `auth` command.
  ///
82 83
  /// This ensures that the goldctl tool is authorized and ready for testing.
  /// Used by the [FlutterPostSubmitFileComparator] and the
84
  /// [FlutterPreSubmitFileComparator].
85
  Future<void> auth() async {
86
    if (await clientIsAuthorized()) {
87
      return;
88
    }
89 90
    final List<String> authCommand = <String>[
      _goldctl,
91 92 93 94
      'auth',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
95
      '--luci',
96 97
    ];

98
    final io.ProcessResult result = await process.run(authCommand);
99 100 101

    if (result.exitCode != 0) {
      final StringBuffer buf = StringBuffer()
102 103 104 105
        ..writeln('Skia Gold authorization failed.')
        ..writeln('Luci environments authenticate using the file provided '
          'by LUCI_CONTEXT. There may be an error with this file or Gold '
          'authentication.')
106
        ..writeln('Debug information for Gold:')
107 108
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
109
      throw Exception(buf.toString());
110
    }
111 112
  }

113 114 115 116 117 118 119 120
  /// Signals if this client is initialized for uploading images to the Gold
  /// service.
  ///
  /// Since Flutter framework tests are executed in parallel, and in random
  /// order, this will signal is this instance of the Gold client has been
  /// initialized.
  bool _initialized = false;

121 122 123
  /// Executes the `imgtest init` command in the goldctl tool.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
124 125
  /// backend, the `init` argument initializes the current test. Used by the
  /// [FlutterPostSubmitFileComparator].
126
  Future<void> imgtestInit() async {
127
    // This client has already been intialized
128
    if (_initialized) {
129
      return;
130
    }
131

132 133
    final File keys = workDirectory.childFile('keys.json');
    final File failures = workDirectory.childFile('failures.json');
134 135 136 137 138

    await keys.writeAsString(_getKeysJSON());
    await failures.create();
    final String commitHash = await _getCurrentCommit();

139 140
    final List<String> imgtestInitCommand = <String>[
      _goldctl,
141 142
      'imgtest', 'init',
      '--instance', 'flutter',
143 144 145
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
146 147 148
      '--commit', commitHash,
      '--keys-file', keys.path,
      '--failure-file', failures.path,
149
      '--passfail',
150 151
    ];

152
    if (imgtestInitCommand.contains(null)) {
153
      final StringBuffer buf = StringBuffer()
154 155 156
        ..writeln('A null argument was provided for Skia Gold imgtest init.')
        ..writeln('Please confirm the settings of your golden file test.')
        ..writeln('Arguments provided:');
157
      imgtestInitCommand.forEach(buf.writeln);
158
      throw Exception(buf.toString());
159 160
    }

161
    final io.ProcessResult result = await process.run(imgtestInitCommand);
162 163

    if (result.exitCode != 0) {
164
      _initialized = false;
165 166
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold imgtest init failed.')
167
        ..writeln('An error occurred when initializing golden file test with ')
168 169 170
        ..writeln('goldctl.')
        ..writeln()
        ..writeln('Debug information for Gold:')
171 172
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
173
      throw Exception(buf.toString());
174
    }
175
    _initialized = true;
176 177 178 179 180 181 182 183 184
  }

  /// Executes the `imgtest add` command in the goldctl tool.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
  /// backend, the `add` argument uploads the current image test. A response is
  /// returned from the invocation of this command that indicates a pass or fail
  /// result.
  ///
185
  /// The [testName] and [goldenFile] parameters reference the current
186
  /// comparison being evaluated by the [FlutterPostSubmitFileComparator].
187
  Future<bool> imgtestAdd(String testName, File goldenFile) async {
188 189
    final List<String> imgtestCommand = <String>[
      _goldctl,
190
      'imgtest', 'add',
191 192 193 194
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--test-name', cleanTestName(testName),
195
      '--png-file', goldenFile.path,
196
      '--passfail',
197
      ..._getPixelMatchingArguments(),
198 199
    ];

200
    final io.ProcessResult result = await process.run(imgtestCommand);
201 202

    if (result.exitCode != 0) {
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
      // If an unapproved image has made it to post-submit, throw to close the
      // tree.
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold received an unapproved image in post-submit ')
        ..writeln('testing. Golden file images in flutter/flutter are triaged ')
        ..writeln('in pre-submit during code review for the given PR.')
        ..writeln()
        ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ')
        ..writeln('the image(s), or revert the associated change. For more ')
        ..writeln('information, visit the wiki: ')
        ..writeln('https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter')
        ..writeln()
        ..writeln('Debug information for Gold:')
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
218
      throw Exception(buf.toString());
219 220
    }

221
    return true;
222 223
  }

224 225 226 227 228 229 230 231
  /// Signals if this client is initialized for uploading tryjobs to the Gold
  /// service.
  ///
  /// Since Flutter framework tests are executed in parallel, and in random
  /// order, this will signal is this instance of the Gold client has been
  /// initialized for tryjobs.
  bool _tryjobInitialized = false;

232 233 234
  /// Executes the `imgtest init` command in the goldctl tool for tryjobs.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
235
  /// backend, the `init` argument initializes the current tryjob. Used by the
236
  /// [FlutterPreSubmitFileComparator].
237
  Future<void> tryjobInit() async {
238
    // This client has already been initialized
239
    if (_tryjobInitialized) {
240
      return;
241
    }
242

243 244 245 246 247 248 249
    final File keys = workDirectory.childFile('keys.json');
    final File failures = workDirectory.childFile('failures.json');

    await keys.writeAsString(_getKeysJSON());
    await failures.create();
    final String commitHash = await _getCurrentCommit();

250 251
    final List<String> imgtestInitCommand = <String>[
      _goldctl,
252 253 254 255 256 257 258 259 260 261 262
      'imgtest', 'init',
      '--instance', 'flutter',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--commit', commitHash,
      '--keys-file', keys.path,
      '--failure-file', failures.path,
      '--passfail',
      '--crs', 'github',
      '--patchset_id', commitHash,
263
      ...getCIArguments(),
264 265
    ];

266
    if (imgtestInitCommand.contains(null)) {
267
      final StringBuffer buf = StringBuffer()
268 269 270
        ..writeln('A null argument was provided for Skia Gold tryjob init.')
        ..writeln('Please confirm the settings of your golden file test.')
        ..writeln('Arguments provided:');
271
      imgtestInitCommand.forEach(buf.writeln);
272
      throw Exception(buf.toString());
273 274
    }

275
    final io.ProcessResult result = await process.run(imgtestInitCommand);
276 277

    if (result.exitCode != 0) {
278
      _tryjobInitialized = false;
279 280
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold tryjobInit failure.')
281
        ..writeln('An error occurred when initializing golden file tryjob with ')
282 283 284
        ..writeln('goldctl.')
        ..writeln()
        ..writeln('Debug information for Gold:')
285 286
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
287
      throw Exception(buf.toString());
288
    }
289
    _tryjobInitialized = true;
290 291 292 293 294 295 296 297 298
  }

  /// Executes the `imgtest add` command in the goldctl tool for tryjobs.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
  /// backend, the `add` argument uploads the current image test. A response is
  /// returned from the invocation of this command that indicates a pass or fail
  /// result for the tryjob.
  ///
299
  /// The [testName] and [goldenFile] parameters reference the current
300
  /// comparison being evaluated by the [FlutterPreSubmitFileComparator].
301
  Future<void> tryjobAdd(String testName, File goldenFile) async {
302 303
    final List<String> imgtestCommand = <String>[
      _goldctl,
304 305 306 307 308 309
      'imgtest', 'add',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--test-name', cleanTestName(testName),
      '--png-file', goldenFile.path,
310
      ..._getPixelMatchingArguments(),
311 312
    ];

313
    final io.ProcessResult result = await process.run(imgtestCommand);
314

315
    final String/*!*/ resultStdout = result.stdout.toString();
316 317 318 319 320 321 322 323 324 325 326 327
    if (result.exitCode != 0 &&
      !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) {
      final StringBuffer buf = StringBuffer()
        ..writeln('Unexpected Gold tryjobAdd failure.')
        ..writeln('Tryjob execution for golden file test $testName failed for')
        ..writeln('a reason unrelated to pixel comparison.')
        ..writeln()
        ..writeln('Debug information for Gold:')
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}')
        ..writeln();
      throw Exception(buf.toString());
328
    }
329 330
  }

331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
  // Constructs arguments for `goldctl` for controlling how pixels are compared.
  //
  // For AOT and CanvasKit exact pixel matching is used. For the HTML renderer
  // on the web a fuzzy matching algorithm is used that allows very small deltas
  // because Chromium cannot exactly reproduce the same golden on all computers.
  // It seems to depend on the hardware/OS/driver combination. However, those
  // differences are very small (typically not noticeable to human eye).
  List<String> _getPixelMatchingArguments() {
    // Only use fuzzy pixel matching in the HTML renderer.
    if (!_isBrowserTest || _isBrowserCanvasKitTest) {
      return const <String>[];
    }

    // The algorithm to be used when matching images. The available options are:
    // - "fuzzy": Allows for customizing the thresholds of pixel differences.
    // - "sobel": Same as "fuzzy" but performs edge detection before performing
    //            a fuzzy match.
    const String algorithm = 'fuzzy';

    // The number of pixels in this image that are allowed to differ from the
    // baseline.
    //
    // The chosen number - 20 - is arbitrary. Even for a small golden file, say
    // 50 x 50, it would be less than 1% of the total number of pixels. This
    // number should not grow too much. If it's growing, it is probably due to a
    // larger issue that needs to be addressed at the infra level.
    const int maxDifferentPixels = 20;

    // The maximum acceptable difference per pixel.
    //
    // Uses the Manhattan distance using the RGBA color components as
    // coordinates. The chosen number - 4 - is arbitrary. It's small enough to
    // both not be noticeable and not trigger test flakes due to sub-pixel
    // golden deltas. This number should not grow too much. If it's growing, it
    // is probably due to a larger issue that needs to be addressed at the infra
    // level.
    const int pixelDeltaThreshold = 4;

    return <String>[
      '--add-test-optional-key', 'image_matching_algorithm:$algorithm',
      '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels',
      '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold',
    ];
  }

376 377 378 379
  /// Returns the latest positive digest for the given test known to Flutter
  /// Gold at head.
  Future<String?> getExpectationForTest(String testName) async {
    late String? expectation;
380
    final String traceID = getTraceID(testName);
381 382
    await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
      final Uri requestForExpectations = Uri.parse(
383
        'https://flutter-gold.skia.org/json/v2/latestpositivedigest/$traceID'
384
      );
385
      late String rawResponse;
386 387 388 389
      try {
        final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
        final io.HttpClientResponse response = await request.close();
        rawResponse = await utf8.decodeStream(response);
390
        final dynamic jsonResponse = json.decode(rawResponse);
391
        if (jsonResponse is! Map<String, dynamic>) {
392
          throw const FormatException('Skia gold expectations do not match expected format.');
393
        }
394
        expectation = jsonResponse['digest'] as String?;
395
      } on FormatException catch (error) {
396 397 398 399
        // Ideally we'd use something like package:test's printOnError, but best reliabilty
        // in getting logs on CI for now we're just using print.
        // See also: https://github.com/flutter/flutter/issues/91285
        print( // ignore: avoid_print
400 401 402 403 404
          'Formatting error detected requesting expectations from Flutter Gold.\n'
          'error: $error\n'
          'url: $requestForExpectations\n'
          'response: $rawResponse'
        );
405 406 407 408 409
        rethrow;
      }
    },
      SkiaGoldHttpOverrides(),
    );
410
    return expectation;
411 412 413 414 415 416 417 418 419 420 421 422
  }

  /// Returns a list of bytes representing the golden image retrieved from the
  /// Flutter Gold dashboard.
  ///
  /// The provided image hash represents an expectation from Flutter Gold.
  Future<List<int>>getImageBytes(String imageHash) async {
    final List<int> imageBytes = <int>[];
    await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
      final Uri requestForImage = Uri.parse(
        'https://flutter-gold.skia.org/img/images/$imageHash.png',
      );
423 424 425
      final io.HttpClientRequest request = await httpClient.getUrl(requestForImage);
      final io.HttpClientResponse response = await request.close();
      await response.forEach((List<int> bytes) => imageBytes.addAll(bytes));
426 427 428 429 430 431
    },
      SkiaGoldHttpOverrides(),
    );
    return imageBytes;
  }

432 433
  /// Returns the current commit hash of the Flutter repository.
  Future<String> _getCurrentCommit() async {
434
    if (!_flutterRoot.existsSync()) {
435
      throw Exception('Flutter root could not be found: $_flutterRoot\n');
436 437 438
    } else {
      final io.ProcessResult revParse = await process.run(
        <String>['git', 'rev-parse', 'HEAD'],
439
        workingDirectory: _flutterRoot.path,
440
      );
441 442 443 444
      if (revParse.exitCode != 0) {
        throw Exception('Current commit of Flutter can not be found.');
      }
      return (revParse.stdout as String/*!*/).trim();
445 446 447 448 449 450
    }
  }

  /// Returns a JSON String with keys value pairs used to uniquely identify the
  /// configuration that generated the given golden file.
  ///
451 452 453
  /// Currently, the only key value pairs being tracked is the platform the
  /// image was rendered on, and for web tests, the browser the image was
  /// rendered on.
454
  String _getKeysJSON() {
455 456
    final Map<String, dynamic> keys = <String, dynamic>{
      'Platform' : platform.operatingSystem,
457
      'CI' : 'luci',
458
    };
459 460
    if (_isBrowserTest) {
      keys['Browser'] = _browserKey;
461
      keys['Platform'] = '${keys['Platform']}-browser';
462
      if (_isBrowserCanvasKitTest) {
463 464
        keys['WebRenderer'] = 'canvaskit';
      }
465
    }
466
    return json.encode(keys);
467 468
  }

469 470 471
  /// Removes the file extension from the [fileName] to represent the test name
  /// properly.
  String cleanTestName(String fileName) {
472
    return fileName.split(path.extension(fileName))[0];
473 474
  }

475 476
  /// Returns a boolean value to prevent the client from re-authorizing itself
  /// for multiple tests.
477
  Future<bool> clientIsAuthorized() async {
478
    final File authFile = workDirectory.childFile(fs.path.join(
479 480
      'temp',
      'auth_opt.json',
481
    ))/*!*/;
482 483 484 485

    if(await authFile.exists()) {
      final String contents = await authFile.readAsString();
      final Map<String, dynamic> decoded = json.decode(contents) as Map<String, dynamic>;
486
      return !(decoded['GSUtil'] as bool/*!*/);
487 488
    }
    return false;
489
  }
490 491 492 493

  /// Returns a list of arguments for initializing a tryjob based on the testing
  /// environment.
  List<String> getCIArguments() {
494 495 496
    final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last;
    final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/');
    final String pullRequest = refs[refs.length - 2];
497 498 499

    return <String>[
      '--changelist', pullRequest,
500
      '--cis', 'buildbucket',
501 502 503
      '--jobid', jobId,
    ];
  }
504

505 506 507 508 509 510 511 512 513 514 515 516 517
  bool get _isBrowserTest {
    return platform.environment[_kTestBrowserKey] != null;
  }

  bool get _isBrowserCanvasKitTest {
    return _isBrowserTest && platform.environment[_kWebRendererKey] == 'canvaskit';
  }

  String get _browserKey {
    assert(_isBrowserTest);
    return platform.environment[_kTestBrowserKey]!;
  }

518
  /// Returns a trace id based on the current testing environment to lookup
519 520
  /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of
  /// the image keys.
521
  String getTraceID(String testName) {
522
    final Map<String, dynamic> keys = <String, dynamic>{
523 524 525
      if (_isBrowserTest)
        'Browser' : _browserKey,
      if (_isBrowserCanvasKitTest)
526
        'WebRenderer' : 'canvaskit',
527 528 529 530 531 532
      'CI' : 'luci',
      'Platform' : platform.operatingSystem,
      'name' : testName,
      'source_type' : 'flutter',
    };
    final String jsonTrace = json.encode(keys);
533
    final String md5Sum = md5.convert(utf8.encode(jsonTrace)).toString();
534
    return md5Sum;
535 536
  }
}
537 538

/// Used to make HttpRequests during testing.
539
class SkiaGoldHttpOverrides extends io.HttpOverrides { }