skia_client.dart 21.3 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io' as io;

8
import 'package:crypto/crypto.dart';
9 10 11 12 13 14 15 16 17 18
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import 'package:platform/platform.dart';
import 'package:process/process.dart';

// If you are here trying to figure out how to use golden files in the Flutter
// repo itself, consider reading this wiki page:
// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter

19
const String _kFlutterRootKey = 'FLUTTER_ROOT';
20
const String _kGoldctlKey = 'GOLDCTL';
21
const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';
22
const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER';
23

24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
/// Exception thrown when an error is returned from the [SkiaClient].
class SkiaException implements Exception {
  /// Creates a new `SkiaException` with a required error [message].
  const SkiaException(this.message);

  /// A message describing the error.
  final String message;

  /// Returns a description of the Skia exception.
  ///
  /// The description always contains the [message].
  @override
  String toString() => 'SkiaException: $message';
}

39 40 41
/// A client for uploading image tests and making baseline requests to the
/// Flutter Gold Dashboard.
class SkiaGoldClient {
42 43 44 45
  /// Creates a [SkiaGoldClient] with the given [workDirectory].
  ///
  /// All other parameters are optional. They may be provided in tests to
  /// override the defaults for [fs], [process], [platform], and [httpClient].
46 47 48 49 50
  SkiaGoldClient(
    this.workDirectory, {
    this.fs = const LocalFileSystem(),
    this.process = const LocalProcessManager(),
    this.platform = const LocalPlatform(),
51 52
    io.HttpClient? httpClient,
  }) : httpClient = httpClient ?? io.HttpClient();
53 54 55

  /// The file system to use for storing the local clone of the repository.
  ///
56 57
  /// This is useful in tests, where a local file system (the default) can be
  /// replaced by a memory file system.
58 59 60 61
  final FileSystem fs;

  /// A wrapper for the [dart:io.Platform] API.
  ///
62 63
  /// This is useful in tests, where the system platform (the default) can be
  /// replaced by a mock platform instance.
64 65 66 67
  final Platform platform;

  /// A controller for launching sub-processes.
  ///
68 69
  /// This is useful in tests, where the real process manager (the default) can
  /// be replaced by a mock process manager that doesn't really create
70 71 72 73 74
  /// sub-processes.
  final ProcessManager process;

  /// A client for making Http requests to the Flutter Gold dashboard.
  final io.HttpClient httpClient;
75 76

  /// The local [Directory] within the [comparisonRoot] for the current test
77
  /// context. In this directory, the client will create image and JSON files
78 79 80 81
  /// for the goldctl tool to use.
  ///
  /// This is informed by the [FlutterGoldenFileComparator] [basedir]. It cannot
  /// be null.
82 83 84 85 86 87
  final Directory workDirectory;

  /// The local [Directory] where the Flutter repository is hosted.
  ///
  /// Uses the [fs] file system.
  Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]);
88 89 90 91

  /// The path to the local [Directory] where the goldctl tool is hosted.
  ///
  /// Uses the [platform] environment in this implementation.
92
  String get _goldctl => platform.environment[_kGoldctlKey]!;
93 94 95 96

  /// Prepares the local work space for golden file testing and calls the
  /// goldctl `auth` command.
  ///
97 98
  /// This ensures that the goldctl tool is authorized and ready for testing.
  /// Used by the [FlutterPostSubmitFileComparator] and the
99
  /// [FlutterPreSubmitFileComparator].
100
  Future<void> auth() async {
101
    if (await clientIsAuthorized()) {
102
      return;
103
    }
104 105
    final List<String> authCommand = <String>[
      _goldctl,
106 107 108 109
      'auth',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
110
      '--luci',
111 112
    ];

113
    final io.ProcessResult result = await process.run(authCommand);
114 115 116

    if (result.exitCode != 0) {
      final StringBuffer buf = StringBuffer()
117 118 119 120
        ..writeln('Skia Gold authorization failed.')
        ..writeln('Luci environments authenticate using the file provided '
          'by LUCI_CONTEXT. There may be an error with this file or Gold '
          'authentication.')
121
        ..writeln('Debug information for Gold --------------------------------')
122 123
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
124
      throw SkiaException(buf.toString());
125
    }
126 127
  }

128 129 130 131 132 133 134 135
  /// Signals if this client is initialized for uploading images to the Gold
  /// service.
  ///
  /// Since Flutter framework tests are executed in parallel, and in random
  /// order, this will signal is this instance of the Gold client has been
  /// initialized.
  bool _initialized = false;

136 137 138
  /// Executes the `imgtest init` command in the goldctl tool.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
139 140
  /// backend, the `init` argument initializes the current test. Used by the
  /// [FlutterPostSubmitFileComparator].
141
  Future<void> imgtestInit() async {
eggfly's avatar
eggfly committed
142
    // This client has already been initialized
143
    if (_initialized) {
144
      return;
145
    }
146

147 148
    final File keys = workDirectory.childFile('keys.json');
    final File failures = workDirectory.childFile('failures.json');
149

150
    await keys.writeAsString(_getKeysJSON());
151 152 153
    await failures.create();
    final String commitHash = await _getCurrentCommit();

154 155
    final List<String> imgtestInitCommand = <String>[
      _goldctl,
156 157
      'imgtest', 'init',
      '--instance', 'flutter',
158 159 160
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
161 162 163
      '--commit', commitHash,
      '--keys-file', keys.path,
      '--failure-file', failures.path,
164
      '--passfail',
165 166
    ];

167
    if (imgtestInitCommand.contains(null)) {
168
      final StringBuffer buf = StringBuffer()
169 170 171
        ..writeln('A null argument was provided for Skia Gold imgtest init.')
        ..writeln('Please confirm the settings of your golden file test.')
        ..writeln('Arguments provided:');
172
      imgtestInitCommand.forEach(buf.writeln);
173
      throw SkiaException(buf.toString());
174 175
    }

176
    final io.ProcessResult result = await process.run(imgtestInitCommand);
177 178

    if (result.exitCode != 0) {
179
      _initialized = false;
180 181
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold imgtest init failed.')
182
        ..writeln('An error occurred when initializing golden file test with ')
183 184
        ..writeln('goldctl.')
        ..writeln()
185
        ..writeln('Debug information for Gold --------------------------------')
186 187
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
188
      throw SkiaException(buf.toString());
189
    }
190
    _initialized = true;
191 192 193 194 195 196 197 198 199
  }

  /// Executes the `imgtest add` command in the goldctl tool.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
  /// backend, the `add` argument uploads the current image test. A response is
  /// returned from the invocation of this command that indicates a pass or fail
  /// result.
  ///
200
  /// The [testName] and [goldenFile] parameters reference the current
201
  /// comparison being evaluated by the [FlutterPostSubmitFileComparator].
202
  Future<bool> imgtestAdd(String testName, File goldenFile) async {
203 204
    final List<String> imgtestCommand = <String>[
      _goldctl,
205
      'imgtest', 'add',
206 207 208 209
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--test-name', cleanTestName(testName),
210
      '--png-file', goldenFile.path,
211
      '--passfail',
212
      ..._getPixelMatchingArguments(),
213 214
    ];

215
    final io.ProcessResult result = await process.run(imgtestCommand);
216 217

    if (result.exitCode != 0) {
218 219
      // If an unapproved image has made it to post-submit, throw to close the
      // tree.
220 221 222 223
      String? resultContents;
      final File resultFile = workDirectory.childFile(fs.path.join(
        'result-state.json',
      ));
224
      if (await resultFile.exists()) {
225 226 227
        resultContents = await resultFile.readAsString();
      }

228 229 230 231 232 233 234 235 236 237
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold received an unapproved image in post-submit ')
        ..writeln('testing. Golden file images in flutter/flutter are triaged ')
        ..writeln('in pre-submit during code review for the given PR.')
        ..writeln()
        ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ')
        ..writeln('the image(s), or revert the associated change. For more ')
        ..writeln('information, visit the wiki: ')
        ..writeln('https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter')
        ..writeln()
238
        ..writeln('Debug information for Gold --------------------------------')
239
        ..writeln('stdout: ${result.stdout}')
240 241 242 243
        ..writeln('stderr: ${result.stderr}')
        ..writeln()
        ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
      throw SkiaException(buf.toString());
244 245
    }

246
    return true;
247 248
  }

249 250 251 252 253 254 255 256
  /// Signals if this client is initialized for uploading tryjobs to the Gold
  /// service.
  ///
  /// Since Flutter framework tests are executed in parallel, and in random
  /// order, this will signal is this instance of the Gold client has been
  /// initialized for tryjobs.
  bool _tryjobInitialized = false;

257 258 259
  /// Executes the `imgtest init` command in the goldctl tool for tryjobs.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
260
  /// backend, the `init` argument initializes the current tryjob. Used by the
261
  /// [FlutterPreSubmitFileComparator].
262
  Future<void> tryjobInit() async {
263
    // This client has already been initialized
264
    if (_tryjobInitialized) {
265
      return;
266
    }
267

268 269 270
    final File keys = workDirectory.childFile('keys.json');
    final File failures = workDirectory.childFile('failures.json');

271
    await keys.writeAsString(_getKeysJSON());
272 273 274
    await failures.create();
    final String commitHash = await _getCurrentCommit();

275 276
    final List<String> imgtestInitCommand = <String>[
      _goldctl,
277 278 279 280 281 282 283 284 285 286 287
      'imgtest', 'init',
      '--instance', 'flutter',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--commit', commitHash,
      '--keys-file', keys.path,
      '--failure-file', failures.path,
      '--passfail',
      '--crs', 'github',
      '--patchset_id', commitHash,
288
      ...getCIArguments(),
289 290
    ];

291
    if (imgtestInitCommand.contains(null)) {
292
      final StringBuffer buf = StringBuffer()
293 294 295
        ..writeln('A null argument was provided for Skia Gold tryjob init.')
        ..writeln('Please confirm the settings of your golden file test.')
        ..writeln('Arguments provided:');
296
      imgtestInitCommand.forEach(buf.writeln);
297
      throw SkiaException(buf.toString());
298 299
    }

300
    final io.ProcessResult result = await process.run(imgtestInitCommand);
301 302

    if (result.exitCode != 0) {
303
      _tryjobInitialized = false;
304 305
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold tryjobInit failure.')
306
        ..writeln('An error occurred when initializing golden file tryjob with ')
307 308
        ..writeln('goldctl.')
        ..writeln()
309
        ..writeln('Debug information for Gold --------------------------------')
310 311
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
312
      throw SkiaException(buf.toString());
313
    }
314
    _tryjobInitialized = true;
315 316 317 318 319 320 321 322 323
  }

  /// Executes the `imgtest add` command in the goldctl tool for tryjobs.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
  /// backend, the `add` argument uploads the current image test. A response is
  /// returned from the invocation of this command that indicates a pass or fail
  /// result for the tryjob.
  ///
324
  /// The [testName] and [goldenFile] parameters reference the current
325
  /// comparison being evaluated by the [FlutterPreSubmitFileComparator].
326
  Future<void> tryjobAdd(String testName, File goldenFile) async {
327 328
    final List<String> imgtestCommand = <String>[
      _goldctl,
329 330 331 332 333 334
      'imgtest', 'add',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--test-name', cleanTestName(testName),
      '--png-file', goldenFile.path,
335
      ..._getPixelMatchingArguments(),
336 337
    ];

338
    final io.ProcessResult result = await process.run(imgtestCommand);
339

340
    final String/*!*/ resultStdout = result.stdout.toString();
341 342
    if (result.exitCode != 0 &&
      !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) {
343 344 345 346
      String? resultContents;
      final File resultFile = workDirectory.childFile(fs.path.join(
        'result-state.json',
      ));
347
      if (await resultFile.exists()) {
348 349
        resultContents = await resultFile.readAsString();
      }
350 351 352 353 354
      final StringBuffer buf = StringBuffer()
        ..writeln('Unexpected Gold tryjobAdd failure.')
        ..writeln('Tryjob execution for golden file test $testName failed for')
        ..writeln('a reason unrelated to pixel comparison.')
        ..writeln()
355
        ..writeln('Debug information for Gold --------------------------------')
356 357
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}')
358 359 360 361
        ..writeln()
        ..writeln()
        ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
      throw SkiaException(buf.toString());
362
    }
363 364
  }

365 366 367 368 369 370 371
  // Constructs arguments for `goldctl` for controlling how pixels are compared.
  //
  // For AOT and CanvasKit exact pixel matching is used. For the HTML renderer
  // on the web a fuzzy matching algorithm is used that allows very small deltas
  // because Chromium cannot exactly reproduce the same golden on all computers.
  // It seems to depend on the hardware/OS/driver combination. However, those
  // differences are very small (typically not noticeable to human eye).
372
  List<String> _getPixelMatchingArguments() {
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
    // Only use fuzzy pixel matching in the HTML renderer.
    if (!_isBrowserTest || _isBrowserCanvasKitTest) {
      return const <String>[];
    }

    // The algorithm to be used when matching images. The available options are:
    // - "fuzzy": Allows for customizing the thresholds of pixel differences.
    // - "sobel": Same as "fuzzy" but performs edge detection before performing
    //            a fuzzy match.
    const String algorithm = 'fuzzy';

    // The number of pixels in this image that are allowed to differ from the
    // baseline.
    //
    // The chosen number - 20 - is arbitrary. Even for a small golden file, say
    // 50 x 50, it would be less than 1% of the total number of pixels. This
    // number should not grow too much. If it's growing, it is probably due to a
    // larger issue that needs to be addressed at the infra level.
    const int maxDifferentPixels = 20;

    // The maximum acceptable difference per pixel.
    //
    // Uses the Manhattan distance using the RGBA color components as
    // coordinates. The chosen number - 4 - is arbitrary. It's small enough to
    // both not be noticeable and not trigger test flakes due to sub-pixel
    // golden deltas. This number should not grow too much. If it's growing, it
    // is probably due to a larger issue that needs to be addressed at the infra
    // level.
    const int pixelDeltaThreshold = 4;

    return <String>[
      '--add-test-optional-key', 'image_matching_algorithm:$algorithm',
      '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels',
      '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold',
    ];
  }

410 411 412 413
  /// Returns the latest positive digest for the given test known to Flutter
  /// Gold at head.
  Future<String?> getExpectationForTest(String testName) async {
    late String? expectation;
414
    final String traceID = getTraceID(testName);
415 416
    await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
      final Uri requestForExpectations = Uri.parse(
417
        'https://flutter-gold.skia.org/json/v2/latestpositivedigest/$traceID'
418
      );
419
      late String rawResponse;
420 421 422 423
      try {
        final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
        final io.HttpClientResponse response = await request.close();
        rawResponse = await utf8.decodeStream(response);
424
        final dynamic jsonResponse = json.decode(rawResponse);
425
        if (jsonResponse is! Map<String, dynamic>) {
426
          throw const FormatException('Skia gold expectations do not match expected format.');
427
        }
428
        expectation = jsonResponse['digest'] as String?;
429
      } on FormatException catch (error) {
Lioness100's avatar
Lioness100 committed
430
        // Ideally we'd use something like package:test's printOnError, but best reliability
431 432 433
        // in getting logs on CI for now we're just using print.
        // See also: https://github.com/flutter/flutter/issues/91285
        print( // ignore: avoid_print
434 435 436 437 438
          'Formatting error detected requesting expectations from Flutter Gold.\n'
          'error: $error\n'
          'url: $requestForExpectations\n'
          'response: $rawResponse'
        );
439 440 441 442 443
        rethrow;
      }
    },
      SkiaGoldHttpOverrides(),
    );
444
    return expectation;
445 446 447 448 449 450 451 452 453 454 455 456
  }

  /// Returns a list of bytes representing the golden image retrieved from the
  /// Flutter Gold dashboard.
  ///
  /// The provided image hash represents an expectation from Flutter Gold.
  Future<List<int>>getImageBytes(String imageHash) async {
    final List<int> imageBytes = <int>[];
    await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
      final Uri requestForImage = Uri.parse(
        'https://flutter-gold.skia.org/img/images/$imageHash.png',
      );
457 458 459
      final io.HttpClientRequest request = await httpClient.getUrl(requestForImage);
      final io.HttpClientResponse response = await request.close();
      await response.forEach((List<int> bytes) => imageBytes.addAll(bytes));
460 461 462 463 464 465
    },
      SkiaGoldHttpOverrides(),
    );
    return imageBytes;
  }

466 467
  /// Returns the current commit hash of the Flutter repository.
  Future<String> _getCurrentCommit() async {
468
    if (!_flutterRoot.existsSync()) {
469
      throw SkiaException('Flutter root could not be found: $_flutterRoot\n');
470 471 472
    } else {
      final io.ProcessResult revParse = await process.run(
        <String>['git', 'rev-parse', 'HEAD'],
473
        workingDirectory: _flutterRoot.path,
474
      );
475
      if (revParse.exitCode != 0) {
476
        throw const SkiaException('Current commit of Flutter can not be found.');
477 478
      }
      return (revParse.stdout as String/*!*/).trim();
479 480 481 482 483 484
    }
  }

  /// Returns a JSON String with keys value pairs used to uniquely identify the
  /// configuration that generated the given golden file.
  ///
485 486 487
  /// Currently, the only key value pairs being tracked is the platform the
  /// image was rendered on, and for web tests, the browser the image was
  /// rendered on.
488
  String _getKeysJSON() {
489 490
    final Map<String, dynamic> keys = <String, dynamic>{
      'Platform' : platform.operatingSystem,
491
      'CI' : 'luci',
492
    };
493 494
    if (_isBrowserTest) {
      keys['Browser'] = _browserKey;
495
      keys['Platform'] = '${keys['Platform']}-browser';
496 497 498
      if (_isBrowserCanvasKitTest) {
        keys['WebRenderer'] = 'canvaskit';
      }
499
    }
500
    return json.encode(keys);
501 502
  }

503 504 505
  /// Removes the file extension from the [fileName] to represent the test name
  /// properly.
  String cleanTestName(String fileName) {
506
    return fileName.split(path.extension(fileName))[0];
507 508
  }

509 510
  /// Returns a boolean value to prevent the client from re-authorizing itself
  /// for multiple tests.
511
  Future<bool> clientIsAuthorized() async {
512
    final File authFile = workDirectory.childFile(fs.path.join(
513 514
      'temp',
      'auth_opt.json',
515
    ))/*!*/;
516

517
    if (await authFile.exists()) {
518 519
      final String contents = await authFile.readAsString();
      final Map<String, dynamic> decoded = json.decode(contents) as Map<String, dynamic>;
520
      return !(decoded['GSUtil'] as bool/*!*/);
521 522
    }
    return false;
523
  }
524 525 526 527

  /// Returns a list of arguments for initializing a tryjob based on the testing
  /// environment.
  List<String> getCIArguments() {
528 529 530
    final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last;
    final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/');
    final String pullRequest = refs[refs.length - 2];
531 532 533

    return <String>[
      '--changelist', pullRequest,
534
      '--cis', 'buildbucket',
535 536 537
      '--jobid', jobId,
    ];
  }
538

539 540 541 542 543 544 545 546 547 548 549 550 551
  bool get _isBrowserTest {
    return platform.environment[_kTestBrowserKey] != null;
  }

  bool get _isBrowserCanvasKitTest {
    return _isBrowserTest && platform.environment[_kWebRendererKey] == 'canvaskit';
  }

  String get _browserKey {
    assert(_isBrowserTest);
    return platform.environment[_kTestBrowserKey]!;
  }

552
  /// Returns a trace id based on the current testing environment to lookup
553 554
  /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of
  /// the image keys.
555
  String getTraceID(String testName) {
556
    final Map<String, dynamic> keys = <String, dynamic>{
557 558 559
      if (_isBrowserTest)
        'Browser' : _browserKey,
      if (_isBrowserCanvasKitTest)
560
        'WebRenderer' : 'canvaskit',
561 562 563 564 565 566
      'CI' : 'luci',
      'Platform' : platform.operatingSystem,
      'name' : testName,
      'source_type' : 'flutter',
    };
    final String jsonTrace = json.encode(keys);
567
    final String md5Sum = md5.convert(utf8.encode(jsonTrace)).toString();
568
    return md5Sum;
569 570
  }
}
571 572

/// Used to make HttpRequests during testing.
573
class SkiaGoldHttpOverrides extends io.HttpOverrides { }