flutter_goldens_test.dart 33.7 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
// See also dev/automated_tests/flutter_test/flutter_gold_test.dart

7
import 'dart:async';
8
import 'dart:convert';
9
import 'dart:io' hide Directory;
10 11 12 13
import 'dart:typed_data';

import 'package:file/file.dart';
import 'package:file/memory.dart';
14
import 'package:flutter/foundation.dart';
15 16 17 18 19
import 'package:flutter_goldens/flutter_goldens.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:platform/platform.dart';
import 'package:process/process.dart';

20 21
import 'json_templates.dart';

22
const String _kFlutterRoot = '/flutter';
23 24

// 1x1 transparent pixel
25 26
const List<int> _kTestPngBytes = <int>[
  137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0,
27 28
  1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84,
  120, 1, 99, 97, 0, 2, 0, 0, 25, 0, 5, 144, 240, 54, 245, 0, 0, 0, 0, 73, 69,
29 30
  78, 68, 174, 66, 96, 130,
];
31

32
void main() {
33 34 35 36
  late MemoryFileSystem fs;
  late FakePlatform platform;
  late FakeProcessManager process;
  late FakeHttpClient fakeHttpClient;
37

38
  setUp(() {
39
    fs = MemoryFileSystem();
40 41 42 43
    platform = FakePlatform(
      environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
      operatingSystem: 'macos'
    );
44 45
    process = FakeProcessManager();
    fakeHttpClient = FakeHttpClient();
46
    fs.directory(_kFlutterRoot).createSync(recursive: true);
47 48
  });

49
  group('SkiaGoldClient', () {
50 51
    late SkiaGoldClient skiaClient;
    late Directory workDirectory;
52 53

    setUp(() {
54
      workDirectory = fs.directory('/workDirectory')
55 56 57
        ..createSync(recursive: true);
      skiaClient = SkiaGoldClient(
        workDirectory,
58
        fs: fs,
59
        process: process,
60
        platform: platform,
61
        httpClient: fakeHttpClient,
62 63 64
      );
    });

65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
    test('web HTML test', () async {
      platform = FakePlatform(
        environment: <String, String>{
          'GOLDCTL': 'goldctl',
          'FLUTTER_ROOT': _kFlutterRoot,
          'FLUTTER_TEST_BROWSER': 'Chrome',
          'FLUTTER_WEB_RENDERER': 'html',
        },
        operatingSystem: 'macos'
      );
      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
        httpClient: fakeHttpClient,
      );

      final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png')
        ..createSync(recursive: true);

      const RunInvocation goldctlInvocation = RunInvocation(
        <String>[
          'goldctl',
          'imgtest', 'add',
          '--work-dir', '/workDirectory/temp',
          '--test-name', 'golden_file_test',
          '--png-file', '/workDirectory/temp/golden_file_test.png',
          '--passfail',
          '--add-test-optional-key', 'image_matching_algorithm:fuzzy',
          '--add-test-optional-key', 'fuzzy_max_different_pixels:20',
          '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:4',
        ],
        null,
      );
      process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', '');

      expect(
        await skiaClient.imgtestAdd('golden_file_test.png', goldenFile),
        isTrue,
      );
    });

    test('web CanvasKit test', () async {
      platform = FakePlatform(
        environment: <String, String>{
          'GOLDCTL': 'goldctl',
          'FLUTTER_ROOT': _kFlutterRoot,
          'FLUTTER_TEST_BROWSER': 'Chrome',
          'FLUTTER_WEB_RENDERER': 'canvaskit',
        },
        operatingSystem: 'macos'
      );
      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
        httpClient: fakeHttpClient,
      );

      final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png')
        ..createSync(recursive: true);

      const RunInvocation goldctlInvocation = RunInvocation(
        <String>[
          'goldctl',
          'imgtest', 'add',
          '--work-dir', '/workDirectory/temp',
          '--test-name', 'golden_file_test',
          '--png-file', '/workDirectory/temp/golden_file_test.png',
          '--passfail',
        ],
        null,
      );
      process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', '');

      expect(
        await skiaClient.imgtestAdd('golden_file_test.png', goldenFile),
        isTrue,
      );
    });

148
    test('auth performs minimal work if already authorized', () async {
149
      final File authFile = fs.file('/workDirectory/temp/auth_opt.json')
150
        ..createSync(recursive: true);
151
      authFile.writeAsStringSync(authTemplate());
152
      process.fallbackProcessResult = ProcessResult(123, 0, '', '');
153 154
      await skiaClient.auth();

155
      expect(process.workingDirectories, isEmpty);
156 157
    });

158 159 160 161 162 163 164 165 166 167
    test('gsutil is checked when authorization file is present', () async {
      final File authFile = fs.file('/workDirectory/temp/auth_opt.json')
        ..createSync(recursive: true);
      authFile.writeAsStringSync(authTemplate(gsutil: true));
      expect(
        await skiaClient.clientIsAuthorized(),
        isFalse,
      );
    });

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
    test('throws for error state from auth', () async {
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLD_SERVICE_ACCOUNT' : 'Service Account',
          'GOLDCTL' : 'goldctl',
        },
        operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
183
        httpClient: fakeHttpClient,
184 185
      );

186
      process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
187 188

      expect(
189
        skiaClient.auth(),
190 191 192 193
        throwsException,
      );
    });

194
    test('throws for error state from init', () {
195 196 197 198 199 200 201 202 203 204 205 206 207
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLDCTL' : 'goldctl',
        },
        operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
208
        httpClient: fakeHttpClient,
209 210
      );

211
      const RunInvocation gitInvocation = RunInvocation(
212
        <String>['git', 'rev-parse', 'HEAD'],
213 214 215
        '/flutter',
      );
      const RunInvocation goldctlInvocation = RunInvocation(
216 217 218 219 220 221 222 223 224 225
        <String>[
          'goldctl',
          'imgtest', 'init',
          '--instance', 'flutter',
          '--work-dir', '/workDirectory/temp',
          '--commit', '12345678',
          '--keys-file', '/workDirectory/keys.json',
          '--failure-file', '/workDirectory/failures.json',
          '--passfail',
        ],
226 227 228
        null,
      );
      process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', '');
229 230
      process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure');
      process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
231 232

      expect(
233
        skiaClient.imgtestInit(),
234 235
        throwsException,
      );
236 237
    });

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
    test('Only calls init once', () async {
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLDCTL' : 'goldctl',
        },
        operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
        httpClient: fakeHttpClient,
      );

      const RunInvocation gitInvocation = RunInvocation(
        <String>['git', 'rev-parse', 'HEAD'],
        '/flutter',
      );
      const RunInvocation goldctlInvocation = RunInvocation(
        <String>[
          'goldctl',
          'imgtest', 'init',
          '--instance', 'flutter',
          '--work-dir', '/workDirectory/temp',
          '--commit', '1234',
          '--keys-file', '/workDirectory/keys.json',
          '--failure-file', '/workDirectory/failures.json',
268
          '--passfail',
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
        ],
        null,
      );
      process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', '');
      process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', '');
      process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');

      // First call
      await skiaClient.imgtestInit();

      // Remove fake process result.
      // If the init call is executed again, the fallback process will throw.
      process.processResults.remove(goldctlInvocation);

      // Second call
      await skiaClient.imgtestInit();
    });

    test('Only calls tryjob init once', () async {
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLDCTL' : 'goldctl',
          'SWARMING_TASK_ID' : '4ae997b50dfd4d11',
          'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
          'GOLD_TRYJOB' : 'refs/pull/49815/head',
        },
        operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
        httpClient: fakeHttpClient,
      );

      const RunInvocation gitInvocation = RunInvocation(
        <String>['git', 'rev-parse', 'HEAD'],
        '/flutter',
      );
      const RunInvocation goldctlInvocation = RunInvocation(
        <String>[
          'goldctl',
          'imgtest', 'init',
          '--instance', 'flutter',
          '--work-dir', '/workDirectory/temp',
          '--commit', '1234',
          '--keys-file', '/workDirectory/keys.json',
          '--failure-file', '/workDirectory/failures.json',
          '--passfail',
          '--crs', 'github',
          '--patchset_id', '1234',
          '--changelist', '49815',
          '--cis', 'buildbucket',
          '--jobid', '8885996262141582672',
        ],
        null,
      );
      process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', '');
      process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', '');
      process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');

      // First call
      await skiaClient.tryjobInit();

      // Remove fake process result.
      // If the init call is executed again, the fallback process will throw.
      process.processResults.remove(goldctlInvocation);

      // Second call
      await skiaClient.tryjobInit();
    });

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
    test('throws for error state from imgtestAdd', () {
      final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png')
        ..createSync(recursive: true);
      platform = FakePlatform(
          environment: <String, String>{
            'FLUTTER_ROOT': _kFlutterRoot,
            'GOLDCTL' : 'goldctl',
          },
          operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
        httpClient: fakeHttpClient,
      );

      const RunInvocation goldctlInvocation = RunInvocation(
        <String>[
          'goldctl',
          'imgtest', 'add',
          '--work-dir', '/workDirectory/temp',
          '--test-name', 'golden_file_test',
          '--png-file', '/workDirectory/temp/golden_file_test.png',
          '--passfail',
        ],
        null,
      );
      process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure');
      process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');

      expect(
        skiaClient.imgtestAdd('golden_file_test', goldenFile),
        throwsException,
      );
    });
382

383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
    test('correctly inits tryjob for luci', () async {
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLDCTL' : 'goldctl',
          'SWARMING_TASK_ID' : '4ae997b50dfd4d11',
          'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
          'GOLD_TRYJOB' : 'refs/pull/49815/head',
        },
        operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
400
        httpClient: fakeHttpClient,
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
      );

      final List<String> ciArguments = skiaClient.getCIArguments();

      expect(
        ciArguments,
        equals(
          <String>[
            '--changelist', '49815',
            '--cis', 'buildbucket',
            '--jobid', '8885996262141582672',
          ],
        ),
      );
    });

417
    test('Creates traceID correctly', () async {
418 419 420 421 422 423 424 425 426 427 428
      String traceID;
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLDCTL' : 'goldctl',
          'SWARMING_TASK_ID' : '4ae997b50dfd4d11',
          'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
          'GOLD_TRYJOB' : 'refs/pull/49815/head',
        },
        operatingSystem: 'linux'
      );
429

430 431 432 433 434
      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
435
        httpClient: fakeHttpClient,
436
      );
437

438
      traceID = skiaClient.getTraceID('flutter.golden.1');
439 440
      expect(
        traceID,
441
        equals('ae18c7a6aa48e0685525dfe8fdf79003'),
442
      );
443

444 445 446 447 448 449 450 451 452 453 454 455
      // Browser
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
          'GOLDCTL' : 'goldctl',
          'SWARMING_TASK_ID' : '4ae997b50dfd4d11',
          'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
          'GOLD_TRYJOB' : 'refs/pull/49815/head',
          'FLUTTER_TEST_BROWSER' : 'chrome',
        },
        operatingSystem: 'linux'
      );
456

457 458 459 460 461
      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
462
        httpClient: fakeHttpClient,
463
      );
464 465

      traceID = skiaClient.getTraceID('flutter.golden.1');
466 467
      expect(
        traceID,
468
        equals('e9d5c296c48e7126808520e9cc191243'),
469
      );
470

471 472 473 474 475 476 477 478 479 480 481 482 483
      // Locally - should defer to luci traceID
      platform = FakePlatform(
        environment: <String, String>{
          'FLUTTER_ROOT': _kFlutterRoot,
        },
        operatingSystem: 'macos'
      );

      skiaClient = SkiaGoldClient(
        workDirectory,
        fs: fs,
        process: process,
        platform: platform,
484
        httpClient: fakeHttpClient,
485
      );
486 487

      traceID = skiaClient.getTraceID('flutter.golden.1');
488 489
      expect(
        traceID,
490
        equals('9968695b9ae78cdb77cbb2be621ca2d6'),
491 492 493 494
      );
    });

    group('Request Handling', () {
495
      const String expectation = '55109a4bed52acc780530f7a9aeff6c0';
496 497 498 499 500

      test('image bytes are processed properly', () async {
        final Uri imageUrl = Uri.parse(
          'https://flutter-gold.skia.org/img/images/$expectation.png'
        );
501 502
        final FakeHttpClientRequest fakeImageRequest = FakeHttpClientRequest();
        final FakeHttpImageResponse fakeImageResponse = FakeHttpImageResponse(
503 504
          imageResponseTemplate()
        );
505 506 507

        fakeHttpClient.request = fakeImageRequest;
        fakeImageRequest.response = fakeImageResponse;
508 509 510

        final List<int> masterBytes = await skiaClient.getImageBytes(expectation);

511
        expect(fakeHttpClient.lastUri, imageUrl);
512 513
        expect(masterBytes, equals(_kTestPngBytes));
      });
514 515
    });
  });
516

517
  group('FlutterGoldenFileComparator', () {
518
    late FlutterGoldenFileComparator comparator;
519 520

    setUp(() {
521 522
      final Directory basedir = fs.directory('flutter/test/library/')
        ..createSync(recursive: true);
523
      comparator = FlutterPostSubmitFileComparator(
524
        basedir.uri,
525
        FakeSkiaGoldClient(),
526 527
        fs: fs,
        platform: platform,
528 529 530
      );
    });

531
    test('calculates the basedir correctly from defaultComparator for local testing', () async {
532
      final FakeLocalFileComparator defaultComparator = FakeLocalFileComparator();
533 534
      final Directory flutterRoot = fs.directory(platform.environment['FLUTTER_ROOT'])
        ..createSync(recursive: true);
535
      defaultComparator.basedir = flutterRoot.childDirectory('baz').uri;
536 537 538 539 540 541 542 543 544 545 546 547 548 549 550

      final Directory basedir = FlutterGoldenFileComparator.getBaseDirectory(
        defaultComparator,
        platform,
      );
      expect(
        basedir.uri,
        fs.directory('/flutter/bin/cache/pkg/skia_goldens/baz').uri,
      );
    });

    test('ignores version number', () {
      final Uri key = comparator.getTestUri(Uri.parse('foo.png'), 1);
      expect(key, Uri.parse('foo.png'));
    });
551

552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
    test('adds namePrefix', () async {
      const String libraryName = 'sidedishes';
      const String namePrefix = 'tomatosalad';
      const String fileName = 'lettuce.png';
      final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
      final Directory basedir = fs.directory('flutter/test/$libraryName/')
        ..createSync(recursive: true);
      final FlutterGoldenFileComparator comparator = FlutterPostSubmitFileComparator(
        basedir.uri,
        fakeSkiaClient,
        fs: fs,
        platform: platform,
        namePrefix: namePrefix,
      );
      await comparator.compare(
        Uint8List.fromList(_kTestPngBytes),
        Uri.parse(fileName),
      );
      expect(fakeSkiaClient.testNames.single, '$namePrefix.$libraryName.$fileName');
    });

573
    group('Post-Submit', () {
574
      late FakeSkiaGoldClient fakeSkiaClient;
575 576

      setUp(() {
577
        fakeSkiaClient = FakeSkiaGoldClient();
578
        final Directory basedir = fs.directory('flutter/test/library/')
579
          ..createSync(recursive: true);
580
        comparator = FlutterPostSubmitFileComparator(
581
          basedir.uri,
582
          fakeSkiaClient,
583 584 585
          fs: fs,
          platform: platform,
        );
586
      });
587

588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
      test('asserts .png format', () async {
        await expectLater(
          () async {
            return comparator.compare(
              Uint8List.fromList(_kTestPngBytes),
              Uri.parse('flutter.golden_test.1'),
            );
          },
          throwsA(
            isA<AssertionError>().having((AssertionError error) => error.toString(),
              'description',
              contains(
                'Golden files in the Flutter framework must end with the file '
                'extension .png.'
              ),
            ),
          ),
        );
      });

608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
      test('calls init during compare', () {
        expect(fakeSkiaClient.initCalls, 0);
        comparator.compare(
          Uint8List.fromList(_kTestPngBytes),
          Uri.parse('flutter.golden_test.1.png'),
        );
        expect(fakeSkiaClient.initCalls, 1);
      });

      test('does not call init in during construction', () {
        expect(fakeSkiaClient.initCalls, 0);
        FlutterPostSubmitFileComparator.fromDefaultComparator(
          platform,
          goldens: fakeSkiaClient,
        );
        expect(fakeSkiaClient.initCalls, 0);
      });

626
      group('correctly determines testing environment', () {
627
        test('returns true for configured Luci', () {
628 629 630 631 632 633
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
              'SWARMING_TASK_ID' : '12345678990',
              'GOLDCTL' : 'goldctl',
            },
634
            operatingSystem: 'macos',
635 636 637 638 639 640 641
          );
          expect(
            FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
            isTrue,
          );
        });

642
        test('returns false - GOLDCTL not present', () {
643 644 645
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
646
              'SWARMING_TASK_ID' : '12345678990',
647
            },
648
            operatingSystem: 'macos',
649 650
          );
          expect(
651
            FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
652 653 654 655
            isFalse,
          );
        });

656
        test('returns false - GOLD_TRYJOB active', () {
657 658 659
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
660 661
              'SWARMING_TASK_ID' : '12345678990',
              'GOLDCTL' : 'goldctl',
662
              'GOLD_TRYJOB' : 'git/ref/12345/head',
663
            },
664
            operatingSystem: 'macos',
665 666
          );
          expect(
667
            FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
668 669 670 671
            isFalse,
          );
        });

672
        test('returns false - on Cirrus', () {
673 674 675 676 677
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
              'CIRRUS_CI': 'true',
              'CIRRUS_PR': '',
678
              'CIRRUS_BRANCH': 'master',
679
              'GOLD_SERVICE_ACCOUNT': 'service account...',
680
            },
681
            operatingSystem: 'macos',
682 683
          );
          expect(
684
            FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
685 686 687
            isFalse,
          );
        });
688
      });
689
    });
690

691
    group('Pre-Submit', () {
692 693 694 695 696 697 698 699 700 701 702 703 704 705
      late FakeSkiaGoldClient fakeSkiaClient;

      setUp(() {
        fakeSkiaClient = FakeSkiaGoldClient();
        final Directory basedir = fs.directory('flutter/test/library/')
          ..createSync(recursive: true);
        comparator = FlutterPreSubmitFileComparator(
          basedir.uri,
          fakeSkiaClient,
          fs: fs,
          platform: platform,
        );
      });

706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
      test('asserts .png format', () async {
        await expectLater(
          () async {
            return comparator.compare(
              Uint8List.fromList(_kTestPngBytes),
              Uri.parse('flutter.golden_test.1'),
            );
          },
          throwsA(
            isA<AssertionError>().having((AssertionError error) => error.toString(),
              'description',
              contains(
                'Golden files in the Flutter framework must end with the file '
                'extension .png.'
              ),
            ),
          ),
        );
      });

726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743
      test('calls init during compare', () {
        expect(fakeSkiaClient.tryInitCalls, 0);
        comparator.compare(
          Uint8List.fromList(_kTestPngBytes),
          Uri.parse('flutter.golden_test.1.png'),
        );
        expect(fakeSkiaClient.tryInitCalls, 1);
      });

      test('does not call init in during construction', () {
        expect(fakeSkiaClient.tryInitCalls, 0);
        FlutterPostSubmitFileComparator.fromDefaultComparator(
          platform,
          goldens: fakeSkiaClient,
        );
        expect(fakeSkiaClient.tryInitCalls, 0);
      });

744
      group('correctly determines testing environment', () {
745
        test('returns true for Luci', () {
746 747 748
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
749 750
              'SWARMING_TASK_ID' : '12345678990',
              'GOLDCTL' : 'goldctl',
751
              'GOLD_TRYJOB' : 'git/ref/12345/head',
752
            },
753
            operatingSystem: 'macos',
754 755 756 757 758 759
          );
          expect(
            FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
            isTrue,
          );
        });
760

761
        test('returns false - not on Luci', () {
762 763 764 765
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
            },
766
            operatingSystem: 'macos',
767 768 769
          );
          expect(
            FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
770
            isFalse,
771 772 773
          );
        });

774
        test('returns false - GOLDCTL missing', () {
775 776 777
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
778
              'SWARMING_TASK_ID' : '12345678990',
779
              'GOLD_TRYJOB' : 'git/ref/12345/head',
780
            },
781
            operatingSystem: 'macos',
782 783 784 785 786 787 788
          );
          expect(
            FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
            isFalse,
          );
        });

789
        test('returns false - GOLD_TRYJOB missing', () {
790 791 792
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
793 794
              'SWARMING_TASK_ID' : '12345678990',
              'GOLDCTL' : 'goldctl',
795
            },
796
            operatingSystem: 'macos',
797 798 799 800 801 802 803
          );
          expect(
            FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform),
            isFalse,
          );
        });

804
        test('returns false - on Cirrus', () {
805 806 807
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
808 809 810
              'CIRRUS_CI': 'true',
              'CIRRUS_PR': '',
              'CIRRUS_BRANCH': 'master',
811
              'GOLD_SERVICE_ACCOUNT': 'service account...',
812
            },
813
            operatingSystem: 'macos',
814 815
          );
          expect(
816
            FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform),
817 818 819
            isFalse,
          );
        });
820
      });
821
    });
822

823 824 825 826 827 828 829 830
    group('Skipping', () {
      group('correctly determines testing environment', () {
        test('returns true on Cirrus builds', () {
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
              'CIRRUS_CI' : 'yep',
            },
831
            operatingSystem: 'macos',
832
          );
833
          expect(
834
            FlutterSkippingFileComparator.isAvailableForEnvironment(platform),
835
            isTrue,
836
          );
837 838
        });

839
        test('returns true on irrelevant LUCI builds', () {
840 841 842
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
843
              'SWARMING_TASK_ID' : '1234567890',
844 845 846 847
            },
            operatingSystem: 'macos'
          );
          expect(
848
            FlutterSkippingFileComparator.isAvailableForEnvironment(platform),
849 850 851
            isTrue,
          );
        });
852

853 854 855 856 857
        test('returns false - no CI', () {
          platform = FakePlatform(
            environment: <String, String>{
              'FLUTTER_ROOT': _kFlutterRoot,
            },
858
            operatingSystem: 'macos',
859 860
          );
          expect(
861
            FlutterSkippingFileComparator.isAvailableForEnvironment(
862 863 864 865
              platform),
            isFalse,
          );
        });
866 867 868
      });
    });

869
    group('Local', () {
870
      late FlutterLocalFileComparator comparator;
871
      final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
872

873 874 875 876 877
      setUp(() async {
        final Directory basedir = fs.directory('flutter/test/library/')
          ..createSync(recursive: true);
        comparator = FlutterLocalFileComparator(
          basedir.uri,
878
          fakeSkiaClient,
879 880 881
          fs: fs,
          platform: FakePlatform(
            environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
882
            operatingSystem: 'macos',
883 884 885
          ),
        );

886 887 888 889
        const String hash = '55109a4bed52acc780530f7a9aeff6c0';
        fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash;
        fakeSkiaClient.imageBytesValues[hash] =_kTestPngBytes;
        fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = 'flutter.golden_test.1';
890 891
      });

892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911
      test('asserts .png format', () async {
        await expectLater(
          () async {
            return comparator.compare(
              Uint8List.fromList(_kTestPngBytes),
              Uri.parse('flutter.golden_test.1'),
            );
          },
          throwsA(
            isA<AssertionError>().having((AssertionError error) => error.toString(),
              'description',
              contains(
                'Golden files in the Flutter framework must end with the file '
                'extension .png.'
              ),
            ),
          ),
        );
      });

912 913 914 915 916 917 918 919 920 921
      test('passes when bytes match', () async {
        expect(
          await comparator.compare(
            Uint8List.fromList(_kTestPngBytes),
            Uri.parse('flutter.golden_test.1.png'),
          ),
          isTrue,
        );
      });

922
      test('returns FlutterSkippingGoldenFileComparator when network connection is unavailable', () async {
923 924 925 926 927
        final FakeDirectory fakeDirectory = FakeDirectory();
        fakeDirectory.existsSyncValue = true;
        fakeDirectory.uri = Uri.parse('/flutter');

        fakeSkiaClient.getExpectationForTestThrowable = const OSError("Can't reach Gold");
928 929 930

        FlutterGoldenFileComparator comparator = await FlutterLocalFileComparator.fromDefaultComparator(
          platform,
931 932
          goldens: fakeSkiaClient,
          baseDirectory: fakeDirectory,
933
        );
934
        expect(comparator.runtimeType, FlutterSkippingFileComparator);
935

936 937
        fakeSkiaClient.getExpectationForTestThrowable =  const SocketException("Can't reach Gold");

938
        comparator = await FlutterLocalFileComparator.fromDefaultComparator(
939
          platform,
940 941
          goldens: fakeSkiaClient,
          baseDirectory: fakeDirectory,
942
        );
943
        expect(comparator.runtimeType, FlutterSkippingFileComparator);
944 945
        // reset property or it will carry on to other tests
        fakeSkiaClient.getExpectationForTestThrowable = null;
946
      });
947
    });
948 949 950
  });
}

951 952 953 954 955
@immutable
class RunInvocation {
  const RunInvocation(this.command, this.workingDirectory);

  final List<String> command;
956
  final String? workingDirectory;
957 958

  @override
959
  int get hashCode => Object.hash(Object.hashAll(command), workingDirectory);
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984

  bool _commandEquals(List<String> other) {
    if (other == command) {
      return true;
    }
    if (other.length != command.length) {
      return false;
    }
    for (int index = 0; index < other.length; index += 1) {
      if (other[index] != command[index]) {
        return false;
      }
    }
    return true;
  }

  @override
  bool operator ==(Object other) {
    if (other.runtimeType != runtimeType) {
      return false;
    }
    return other is RunInvocation
        && _commandEquals(other.command)
        && other.workingDirectory == workingDirectory;
  }
985

986 987 988
  @override
  String toString() => '$command ($workingDirectory)';
}
989

990 991
class FakeProcessManager extends Fake implements ProcessManager {
  Map<RunInvocation, ProcessResult> processResults = <RunInvocation, ProcessResult>{};
992

993 994
  /// Used if [processResults] does not contain a matching invocation.
  ProcessResult? fallbackProcessResult;
995

996
  final List<String?> workingDirectories = <String?>[];
997

998 999
  @override
  Future<ProcessResult> run(
1000 1001 1002
    List<Object> command, {
    String? workingDirectory,
    Map<String, String>? environment,
1003 1004
    bool includeParentEnvironment = true,
    bool runInShell = false,
1005 1006
    Encoding? stdoutEncoding = systemEncoding,
    Encoding? stderrEncoding = systemEncoding,
1007 1008
  }) async {
    workingDirectories.add(workingDirectory);
1009
    final ProcessResult? result = processResults[RunInvocation(command.cast<String>(), workingDirectory)];
1010
    if (result == null && fallbackProcessResult == null) {
1011
      printOnFailure('ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.');
1012
      fail('See above.');
1013
    }
1014
    return result ?? fallbackProcessResult!;
1015 1016 1017
  }
}

1018
// See also dev/automated_tests/flutter_test/flutter_gold_test.dart
1019 1020
class FakeSkiaGoldClient extends Fake implements SkiaGoldClient {
  Map<String, String> expectationForTestValues = <String, String>{};
1021
  Exception? getExpectationForTestThrowable;
1022 1023 1024
  @override
  Future<String> getExpectationForTest(String testName) async {
    if (getExpectationForTestThrowable != null) {
1025
      throw getExpectationForTestThrowable!;
1026
    }
1027
    return expectationForTestValues[testName] ?? '';
1028 1029
  }

1030 1031 1032
  @override
  Future<void> auth() async {}

1033 1034
  final List<String> testNames = <String>[];

1035 1036 1037 1038
  int initCalls = 0;
  @override
  Future<void> imgtestInit() async => initCalls += 1;
  @override
1039 1040 1041 1042
  Future<bool> imgtestAdd(String testName, File goldenFile) async {
    testNames.add(testName);
    return true;
  }
1043 1044 1045 1046 1047 1048 1049

  int tryInitCalls = 0;
  @override
  Future<void> tryjobInit() async => tryInitCalls += 1;
  @override
  Future<bool> tryjobAdd(String testName, File goldenFile) async => true;

1050 1051
  Map<String, List<int>> imageBytesValues = <String, List<int>>{};
  @override
1052
  Future<List<int>> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!;
1053 1054 1055

  Map<String, String> cleanTestNameValues = <String, String>{};
  @override
1056
  String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? '';
1057 1058 1059 1060
}

class FakeLocalFileComparator extends Fake implements LocalFileComparator {
  @override
1061
  late Uri basedir;
1062 1063 1064
}

class FakeDirectory extends Fake implements Directory {
1065
  late bool existsSyncValue;
1066 1067 1068 1069
  @override
  bool existsSync() => existsSyncValue;

  @override
1070
  late Uri uri;
1071 1072 1073
}

class FakeHttpClient extends Fake implements HttpClient {
1074 1075
  late Uri lastUri;
  late FakeHttpClientRequest request;
1076 1077 1078 1079 1080 1081 1082 1083 1084

  @override
  Future<HttpClientRequest> getUrl(Uri url) async {
    lastUri = url;
    return request;
  }
}

class FakeHttpClientRequest extends Fake implements HttpClientRequest {
1085
  late FakeHttpImageResponse response;
1086 1087 1088 1089 1090 1091

  @override
  Future<HttpClientResponse> close() async {
    return response;
  }
}
1092

1093 1094
class FakeHttpClientResponse extends Fake implements HttpClientResponse {
  FakeHttpClientResponse(this.response);
1095

1096
  final List<int> response;
1097 1098

  @override
1099
  StreamSubscription<List<int>> listen(
1100
    void Function(List<int> event)? onData, {
1101
      Function? onError,
1102
      void Function()? onDone,
1103
      bool? cancelOnError,
1104
    }) {
1105
    return Stream<List<int>>.fromFuture(Future<List<int>>.value(response))
1106 1107 1108 1109
      .listen(onData, onError: onError, onDone: onDone, cancelOnError: cancelOnError);
  }
}

1110 1111
class FakeHttpImageResponse extends Fake implements HttpClientResponse {
  FakeHttpImageResponse(this.response);
1112 1113 1114 1115

  final List<List<int>> response;

  @override
1116
  Future<void> forEach(void Function(List<int> element) action) async {
1117 1118 1119
    response.forEach(action);
  }
}