skia_client.dart 15.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io' as io;

import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import 'package:platform/platform.dart';
import 'package:process/process.dart';

// If you are here trying to figure out how to use golden files in the Flutter
// repo itself, consider reading this wiki page:
// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter

18
const String _kFlutterRootKey = 'FLUTTER_ROOT';
19
const String _kGoldctlKey = 'GOLDCTL';
20
const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';
21

22 23 24 25 26 27 28 29
/// A client for uploading image tests and making baseline requests to the
/// Flutter Gold Dashboard.
class SkiaGoldClient {
  SkiaGoldClient(
    this.workDirectory, {
    this.fs = const LocalFileSystem(),
    this.process = const LocalProcessManager(),
    this.platform = const LocalPlatform(),
30 31
    io.HttpClient? httpClient,
  }) : httpClient = httpClient ?? io.HttpClient();
32 33 34

  /// The file system to use for storing the local clone of the repository.
  ///
35 36
  /// This is useful in tests, where a local file system (the default) can be
  /// replaced by a memory file system.
37 38 39 40
  final FileSystem fs;

  /// A wrapper for the [dart:io.Platform] API.
  ///
41 42
  /// This is useful in tests, where the system platform (the default) can be
  /// replaced by a mock platform instance.
43 44 45 46
  final Platform platform;

  /// A controller for launching sub-processes.
  ///
47 48
  /// This is useful in tests, where the real process manager (the default) can
  /// be replaced by a mock process manager that doesn't really create
49 50 51 52 53
  /// sub-processes.
  final ProcessManager process;

  /// A client for making Http requests to the Flutter Gold dashboard.
  final io.HttpClient httpClient;
54 55

  /// The local [Directory] within the [comparisonRoot] for the current test
56
  /// context. In this directory, the client will create image and JSON files
57 58 59 60
  /// for the goldctl tool to use.
  ///
  /// This is informed by the [FlutterGoldenFileComparator] [basedir]. It cannot
  /// be null.
61 62 63 64 65 66
  final Directory workDirectory;

  /// The local [Directory] where the Flutter repository is hosted.
  ///
  /// Uses the [fs] file system.
  Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]);
67 68 69 70

  /// The path to the local [Directory] where the goldctl tool is hosted.
  ///
  /// Uses the [platform] environment in this implementation.
71
  String get _goldctl => platform.environment[_kGoldctlKey]!;
72 73 74 75

  /// Prepares the local work space for golden file testing and calls the
  /// goldctl `auth` command.
  ///
76 77
  /// This ensures that the goldctl tool is authorized and ready for testing.
  /// Used by the [FlutterPostSubmitFileComparator] and the
78
  /// [FlutterPreSubmitFileComparator].
79
  Future<void> auth() async {
80
    if (await clientIsAuthorized())
81
      return;
82 83 84 85 86
    final List<String> authArguments = <String>[
      'auth',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
87
      '--luci',
88 89 90 91 92 93 94 95 96
    ];

    final io.ProcessResult result = await io.Process.run(
      _goldctl,
      authArguments,
    );

    if (result.exitCode != 0) {
      final StringBuffer buf = StringBuffer()
97 98 99 100
        ..writeln('Skia Gold authorization failed.')
        ..writeln('Luci environments authenticate using the file provided '
          'by LUCI_CONTEXT. There may be an error with this file or Gold '
          'authentication.')
101
        ..writeln('Debug information for Gold:')
102 103
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
104
      throw Exception(buf.toString());
105
    }
106 107 108 109 110
  }

  /// Executes the `imgtest init` command in the goldctl tool.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
111 112
  /// backend, the `init` argument initializes the current test. Used by the
  /// [FlutterPostSubmitFileComparator].
113
  Future<void> imgtestInit() async {
114 115
    final File keys = workDirectory.childFile('keys.json');
    final File failures = workDirectory.childFile('failures.json');
116 117 118 119 120 121 122 123

    await keys.writeAsString(_getKeysJSON());
    await failures.create();
    final String commitHash = await _getCurrentCommit();

    final List<String> imgtestInitArguments = <String>[
      'imgtest', 'init',
      '--instance', 'flutter',
124 125 126
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
127 128 129 130 131 132 133
      '--commit', commitHash,
      '--keys-file', keys.path,
      '--failure-file', failures.path,
      '--passfail',
    ];

    if (imgtestInitArguments.contains(null)) {
134
      final StringBuffer buf = StringBuffer()
135 136 137
        ..writeln('A null argument was provided for Skia Gold imgtest init.')
        ..writeln('Please confirm the settings of your golden file test.')
        ..writeln('Arguments provided:');
138
      imgtestInitArguments.forEach(buf.writeln);
139
      throw Exception(buf.toString());
140 141
    }

142
    final io.ProcessResult result = await io.Process.run(
143 144 145
      _goldctl,
      imgtestInitArguments,
    );
146 147

    if (result.exitCode != 0) {
148 149
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold imgtest init failed.')
150
        ..writeln('An error occurred when initializing golden file test with ')
151 152 153
        ..writeln('goldctl.')
        ..writeln()
        ..writeln('Debug information for Gold:')
154 155
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
156
      throw Exception(buf.toString());
157
    }
158 159 160 161 162 163 164 165 166
  }

  /// Executes the `imgtest add` command in the goldctl tool.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
  /// backend, the `add` argument uploads the current image test. A response is
  /// returned from the invocation of this command that indicates a pass or fail
  /// result.
  ///
167
  /// The [testName] and [goldenFile] parameters reference the current
168
  /// comparison being evaluated by the [FlutterPostSubmitFileComparator].
169 170 171
  Future<bool> imgtestAdd(String testName, File goldenFile) async {
    final List<String> imgtestArguments = <String>[
      'imgtest', 'add',
172 173 174 175
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--test-name', cleanTestName(testName),
176 177 178
      '--png-file', goldenFile.path,
    ];

179
    final io.ProcessResult result = await io.Process.run(
180 181 182
      _goldctl,
      imgtestArguments,
    );
183 184

    if (result.exitCode != 0) {
185 186 187
      // We do not want to throw for non-zero exit codes here, as an intentional
      // change or new golden file test expect non-zero exit codes. Logging here
      // is meant to inform when an unexpected result occurs.
188 189 190 191
      print('goldctl imgtest add stdout: ${result.stdout}');
      print('goldctl imgtest add stderr: ${result.stderr}');
    }

192
    return true;
193 194
  }

195 196 197
  /// Executes the `imgtest init` command in the goldctl tool for tryjobs.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
198
  /// backend, the `init` argument initializes the current tryjob. Used by the
199
  /// [FlutterPreSubmitFileComparator].
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
  Future<void> tryjobInit() async {
    final File keys = workDirectory.childFile('keys.json');
    final File failures = workDirectory.childFile('failures.json');

    await keys.writeAsString(_getKeysJSON());
    await failures.create();
    final String commitHash = await _getCurrentCommit();

    final List<String> imgtestInitArguments = <String>[
      'imgtest', 'init',
      '--instance', 'flutter',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--commit', commitHash,
      '--keys-file', keys.path,
      '--failure-file', failures.path,
      '--passfail',
      '--crs', 'github',
      '--patchset_id', commitHash,
220
      ...getCIArguments(),
221 222 223 224
    ];

    if (imgtestInitArguments.contains(null)) {
      final StringBuffer buf = StringBuffer()
225 226 227
        ..writeln('A null argument was provided for Skia Gold tryjob init.')
        ..writeln('Please confirm the settings of your golden file test.')
        ..writeln('Arguments provided:');
228
      imgtestInitArguments.forEach(buf.writeln);
229
      throw Exception(buf.toString());
230 231 232 233 234 235 236 237 238 239
    }

    final io.ProcessResult result = await io.Process.run(
      _goldctl,
      imgtestInitArguments,
    );

    if (result.exitCode != 0) {
      final StringBuffer buf = StringBuffer()
        ..writeln('Skia Gold tryjobInit failure.')
240
        ..writeln('An error occurred when initializing golden file tryjob with ')
241 242 243
        ..writeln('goldctl.')
        ..writeln()
        ..writeln('Debug information for Gold:')
244 245
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}');
246
      throw Exception(buf.toString());
247 248 249 250 251 252 253 254 255 256
    }
  }

  /// Executes the `imgtest add` command in the goldctl tool for tryjobs.
  ///
  /// The `imgtest` command collects and uploads test results to the Skia Gold
  /// backend, the `add` argument uploads the current image test. A response is
  /// returned from the invocation of this command that indicates a pass or fail
  /// result for the tryjob.
  ///
257
  /// The [testName] and [goldenFile] parameters reference the current
258
  /// comparison being evaluated by the [FlutterPreSubmitFileComparator].
259
  Future<void> tryjobAdd(String testName, File goldenFile) async {
260 261 262 263 264 265 266 267 268 269 270 271 272 273
    final List<String> imgtestArguments = <String>[
      'imgtest', 'add',
      '--work-dir', workDirectory
        .childDirectory('temp')
        .path,
      '--test-name', cleanTestName(testName),
      '--png-file', goldenFile.path,
    ];

    final io.ProcessResult result = await io.Process.run(
      _goldctl,
      imgtestArguments,
    );

274
    final String/*!*/ resultStdout = result.stdout.toString();
275 276 277 278 279 280 281 282 283 284 285 286
    if (result.exitCode != 0 &&
      !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) {
      final StringBuffer buf = StringBuffer()
        ..writeln('Unexpected Gold tryjobAdd failure.')
        ..writeln('Tryjob execution for golden file test $testName failed for')
        ..writeln('a reason unrelated to pixel comparison.')
        ..writeln()
        ..writeln('Debug information for Gold:')
        ..writeln('stdout: ${result.stdout}')
        ..writeln('stderr: ${result.stderr}')
        ..writeln();
      throw Exception(buf.toString());
287
    }
288 289
  }

290 291 292 293 294
  /// Returns the latest positive digest for the given test known to Flutter
  /// Gold at head.
  Future<String?> getExpectationForTest(String testName) async {
    late String? expectation;
    final String traceID = getTraceID(testName);
295 296
    await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
      final Uri requestForExpectations = Uri.parse(
297
        'https://flutter-gold.skia.org/json/v1/latestpositivedigest/$traceID'
298
      );
299
      late String rawResponse;
300 301 302 303
      try {
        final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
        final io.HttpClientResponse response = await request.close();
        rawResponse = await utf8.decodeStream(response);
304 305 306
        final dynamic jsonResponse = json.decode(rawResponse);
        if (jsonResponse is! Map<String, dynamic>)
          throw const FormatException('Skia gold expectations do not match expected format.');
307
        expectation = jsonResponse['digest'] as String?;
308 309 310 311 312 313 314
      } on FormatException catch (error) {
        print(
          'Formatting error detected requesting expectations from Flutter Gold.\n'
          'error: $error\n'
          'url: $requestForExpectations\n'
          'response: $rawResponse'
        );
315 316 317 318 319
        rethrow;
      }
    },
      SkiaGoldHttpOverrides(),
    );
320
    return expectation;
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
  }

  /// Returns a list of bytes representing the golden image retrieved from the
  /// Flutter Gold dashboard.
  ///
  /// The provided image hash represents an expectation from Flutter Gold.
  Future<List<int>>getImageBytes(String imageHash) async {
    final List<int> imageBytes = <int>[];
    await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
      final Uri requestForImage = Uri.parse(
        'https://flutter-gold.skia.org/img/images/$imageHash.png',
      );

      try {
        final io.HttpClientRequest request = await httpClient.getUrl(requestForImage);
        final io.HttpClientResponse response = await request.close();
        await response.forEach((List<int> bytes) => imageBytes.addAll(bytes));

      } catch(e) {
        rethrow;
      }
    },
      SkiaGoldHttpOverrides(),
    );
    return imageBytes;
  }

348 349
  /// Returns the current commit hash of the Flutter repository.
  Future<String> _getCurrentCommit() async {
350
    if (!_flutterRoot.existsSync()) {
351
      throw Exception('Flutter root could not be found: $_flutterRoot\n');
352 353 354
    } else {
      final io.ProcessResult revParse = await process.run(
        <String>['git', 'rev-parse', 'HEAD'],
355
        workingDirectory: _flutterRoot.path,
356
      );
357 358 359 360
      if (revParse.exitCode != 0) {
        throw Exception('Current commit of Flutter can not be found.');
      }
      return (revParse.stdout as String/*!*/).trim();
361 362 363 364 365 366
    }
  }

  /// Returns a JSON String with keys value pairs used to uniquely identify the
  /// configuration that generated the given golden file.
  ///
367 368 369
  /// Currently, the only key value pairs being tracked is the platform the
  /// image was rendered on, and for web tests, the browser the image was
  /// rendered on.
370
  String _getKeysJSON() {
371 372
    final Map<String, dynamic> keys = <String, dynamic>{
      'Platform' : platform.operatingSystem,
373
      'CI' : 'luci',
374
    };
375
    if (platform.environment[_kTestBrowserKey] != null) {
376
      keys['Browser'] = platform.environment[_kTestBrowserKey];
377 378
      keys['Platform'] = keys['Platform'] + '-browser';
    }
379
    return json.encode(keys);
380 381
  }

382 383 384 385 386 387
  /// Removes the file extension from the [fileName] to represent the test name
  /// properly.
  String cleanTestName(String fileName) {
    return fileName.split(path.extension(fileName.toString()))[0];
  }

388 389
  /// Returns a boolean value to prevent the client from re-authorizing itself
  /// for multiple tests.
390
  Future<bool> clientIsAuthorized() async {
391
    final File authFile = workDirectory.childFile(fs.path.join(
392 393
      'temp',
      'auth_opt.json',
394
    ))/*!*/;
395 396 397 398

    if(await authFile.exists()) {
      final String contents = await authFile.readAsString();
      final Map<String, dynamic> decoded = json.decode(contents) as Map<String, dynamic>;
399
      return !(decoded['GSUtil'] as bool/*!*/);
400 401
    }
    return false;
402
  }
403 404 405 406

  /// Returns a list of arguments for initializing a tryjob based on the testing
  /// environment.
  List<String> getCIArguments() {
407 408 409
    final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last;
    final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/');
    final String pullRequest = refs[refs.length - 2];
410 411 412

    return <String>[
      '--changelist', pullRequest,
413
      '--cis', 'buildbucket',
414 415 416
      '--jobid', jobId,
    ];
  }
417

418 419 420 421 422 423 424 425 426 427 428
  /// Returns a trace id based on the current testing environment to lookup
  /// the latest positive digest on Flutter Gold.
  ///
  /// Trace IDs are case sensitive and should be in alphabetical order for the
  /// keys, followed by the rest of the paramset, also in alphabetical order.
  /// There should also be leading and trailing commas.
  ///
  /// Example TraceID for Flutter Gold:
  ///   ',CI=cirrus,Platform=linux,name=cupertino.activityIndicator.inprogress.1.0,source_type=flutter,'
  String getTraceID(String testName) {
    return '${platform.environment[_kTestBrowserKey] == null ? ',' : ',Browser=${platform.environment[_kTestBrowserKey]},'}'
429
      'CI=luci,'
430 431 432
      'Platform=${platform.operatingSystem},'
      'name=$testName,'
      'source_type=flutter,';
433 434
  }
}
435 436 437

/// Used to make HttpRequests during testing.
class SkiaGoldHttpOverrides extends io.HttpOverrides {}