test.dart 77.8 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
// Runs the tests for the flutter/flutter repository.
//
//
// By default, test output is filtered and only errors are shown. (If a
// particular test takes longer than _quietTimeout in utils.dart, the output is
// shown then also, in case something has hung.)
//
//  --verbose stops the output cleanup and just outputs everything verbatim.
//
//
// By default, errors are non-fatal; all tests are executed and the output
// ends with a summary of the errors that were detected.
//
// Exit code is 1 if there was an error.
//
//  --abort-on-error causes the script to exit immediately when hitting an error.
//
//
// By default, all tests are run. However, the tests support being split by
// shard and subshard. (Inspect the code to see what shards and subshards are
// supported.)
//
// If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
// the shard and sub-shard, by parsing it in the form shard-subshard-platform,
// ignoring the platform.
//
// For local testing you can just set the SHARD and SUBSHARD environment
// variables. For example, to run all the framework tests you can just set
// SHARD=framework_tests. Some shards support named subshards, like
// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of
// two" as in run the first half of the tests).
//
// So for example to run specifically the third subshard of the Web tests you
// would set SHARD=web_tests SUBSHARD=2 (it's zero-based).
//
// By default, where supported, tests within a shard are executed in a random
// order to (eventually) catch inter-test dependencies.
//
//  --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
//
//
// All other arguments are treated as arguments to pass to the flutter tool when
// running tests.

50
import 'dart:convert';
51
import 'dart:core' as system show print;
52
import 'dart:core' hide print;
53
import 'dart:io' as system show exit;
54
import 'dart:io' hide exit;
55
import 'dart:math' as math;
Ian Hickson's avatar
Ian Hickson committed
56
import 'dart:typed_data';
57

Ian Hickson's avatar
Ian Hickson committed
58
import 'package:archive/archive.dart';
59 60
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
61
import 'package:path/path.dart' as path;
62

63
import 'browser.dart';
64
import 'run_command.dart';
65
import 'service_worker_test.dart';
66
import 'tool_subsharding.dart';
67
import 'utils.dart';
68

69
typedef ShardRunner = Future<void> Function();
70

71 72 73 74 75 76
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
77
typedef OutputChecker = String? Function(CommandResult);
78

79 80
final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
81
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
82 83
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
84
final String pubCache = path.join(flutterRoot, '.pub-cache');
85
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
86
final String flutterPackagesVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_packages.version');
87 88

String get platformFolderName {
89
  if (Platform.isWindows) {
90
    return 'windows-x64';
91 92
  }
  if (Platform.isMacOS) {
93
    return 'darwin-x64';
94 95
  }
  if (Platform.isLinux) {
96
    return 'linux-x64';
97
  }
98 99 100
  throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.');
}
final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe');
101 102 103

/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
104
final List<String> flutterTestArgs = <String>[];
105

106 107 108 109
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};

110 111 112
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';

113 114
/// The number of Cirrus jobs that run Web tests in parallel.
///
115 116 117
/// The default is 8 shards. Typically .cirrus.yml would define the
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
///
118 119 120 121
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
122
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
123
  ? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
124
  : 8;
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

/// Tests that we don't run on Web.
///
/// In general avoid adding new tests here. If a test cannot run on the web
/// because it fails at runtime, such as when a piece of functionality is not
/// implemented or not implementable on the web, prefer using `skip` in the
/// test code. Only add tests here that cannot be skipped using `skip`. For
/// example:
///
///  * Test code cannot be compiled because it uses Dart VM-specific
///    functionality. In this case `skip` doesn't help because the code cannot
///    reach the point where it can even run the skipping logic.
///  * Migrations. It is OK to put tests here that need to be temporarily
///    disabled in certain modes because of some migration or initial bringup.
///
/// The key in the map is the renderer type that the list applies to. The value
/// is the list of tests known to fail for that renderer.
142
//
143
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String>>{
  'html': <String>[
    // These tests are not compilable on the web due to dependencies on
    // VM-specific functionality.
    'test/services/message_codecs_vm_test.dart',
    'test/examples/sector_layout_test.dart',
  ],
  'canvaskit': <String>[
    // These tests are not compilable on the web due to dependencies on
    // VM-specific functionality.
    'test/services/message_codecs_vm_test.dart',
    'test/examples/sector_layout_test.dart',

    // These tests are broken and need to be fixed.
    // TODO(yjbanov): https://github.com/flutter/flutter/issues/71604
    'test/painting/decoration_test.dart',
    'test/material/text_selection_theme_test.dart',
    'test/material/date_picker_test.dart',
    'test/rendering/layers_test.dart',
    'test/painting/text_style_test.dart',
    'test/widgets/image_test.dart',
    'test/cupertino/colors_test.dart',
    'test/cupertino/slider_test.dart',
    'test/material/text_field_test.dart',
    'test/rendering/proxy_box_test.dart',
    'test/widgets/app_overrides_test.dart',
    'test/material/calendar_date_picker_test.dart',
    'test/material/ink_paint_test.dart',
    'test/rendering/editable_test.dart',
    'test/cupertino/dialog_test.dart',
    'test/widgets/shape_decoration_test.dart',
    'test/material/time_picker_theme_test.dart',
    'test/cupertino/picker_test.dart',
    'test/material/chip_theme_test.dart',
    'test/cupertino/nav_bar_test.dart',
    'test/widgets/performance_overlay_test.dart',
    'test/widgets/html_element_view_test.dart',
    'test/cupertino/scaffold_test.dart',
    'test/rendering/platform_view_test.dart',
183
    'test/cupertino/context_menu_action_test.dart',
184 185
  ],
};
186

187
const String kTestHarnessShardName = 'test_harness_tests';
188
const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
189

190
// The seed used to shuffle tests. If not passed with
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;
String get shuffleSeed {
  if (_shuffleSeed == null) {
    // Change the seed at 7am, UTC.
    final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
    // Generates YYYYMMDD as the seed, so that testing continues to fail for a
    // day after the seed changes, and on other days the seed can be used to
    // replicate failures.
    _shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
  }
  return _shuffleSeed!;
}

206
/// When you call this, you can pass additional arguments to pass custom
207
/// arguments to flutter test. For example, you might want to call this
208
/// script with the parameter --local-engine=host_debug_unopt to
209
/// use your own build of the engine.
210
///
211
/// To run the tool_tests part, run it with SHARD=tool_tests
212
///
213
/// Examples:
214
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
215
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
216
Future<void> main(List<String> args) async {
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
  try {
    printProgress('STARTING ANALYSIS');
    for (final String arg in args) {
      if (arg.startsWith('--local-engine=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
        flutterTestArgs.add(arg);
      } else if (arg.startsWith('--local-engine-src-path=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
        flutterTestArgs.add(arg);
      } else if (arg.startsWith('--test-randomize-ordering-seed=')) {
        _shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
      } else if (arg.startsWith('--verbose')) {
        print = (Object? message) {
          system.print(message);
        };
      } else if (arg.startsWith('--abort-on-error')) {
        onError = () {
          system.exit(1);
        };
      } else {
        flutterTestArgs.add(arg);
      }
239
    }
240 241
    if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
      printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
242
    }
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
    await selectShard(<String, ShardRunner>{
      'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
      'build_tests': _runBuildTests,
      'framework_coverage': _runFrameworkCoverage,
      'framework_tests': _runFrameworkTests,
      'tool_tests': _runToolTests,
      // web_tool_tests is also used by HHH: https://dart.googlesource.com/recipes/+/refs/heads/master/recipes/dart/flutter_engine.py
      'web_tool_tests': _runWebToolTests,
      'tool_integration_tests': _runIntegrationToolTests,
      'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
      // All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
      'web_tests': _runWebHtmlUnitTests,
      // All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
      'web_canvaskit_tests': _runWebCanvasKitUnitTests,
      // All web integration tests
      'web_long_running_tests': _runWebLongRunningTests,
259
      'flutter_plugins': _runFlutterPackagesTests,
260 261 262 263 264 265 266 267 268 269 270 271
      'skp_generator': _runSkpGeneratorTests,
      kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
    });
  } catch (error, stackTrace) {
    foundError(<String>[
      'UNEXPECTED ERROR!',
      error.toString(),
      ...stackTrace.toString().split('\n'),
      'The test.dart script should be corrected to catch this error and call foundError().',
      '${yellow}Some tests are likely to have been skipped.$reset',
    ]);
    system.exit(255);
272 273
  }
  if (hasError) {
274
    reportErrorsAndExit('${bold}Test failed.$reset');
275
  }
276
  reportSuccessAndExit('${bold}Test successful.$reset');
277 278
}

279 280 281
final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
final bool _runningInDartHHHBot = _luciBotId.startsWith('luci-dart-');

282 283 284
/// Verify the Flutter Engine is the revision in
/// bin/cache/internal/engine.version.
Future<void> _validateEngineHash() async {
285
  if (_runningInDartHHHBot) {
286 287 288 289
    // The Dart HHH bots intentionally modify the local artifact cache
    // and then use this script to run Flutter's test suites.
    // Because the artifacts have been changed, this particular test will return
    // a false positive and should be skipped.
290
    print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $_luciBotId.');
291 292 293
    return;
  }
  final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
294
  final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
  if (result.flattenedStdout!.isNotEmpty) {
    foundError(<String>[
      '${red}The stdout of `$flutterTester --help` was not empty:$reset',
      ...result.flattenedStdout!.split('\n').map((String line) => ' $gray$reset $line'),
    ]);
  }
  final String actualVersion;
  try {
    actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
      return line.startsWith('Flutter Engine Version:');
    });
  } on StateError {
    foundError(<String>[
      '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset',
      ...result.flattenedStderr!.split('\n').map((String line) => ' $gray$reset $line'),
    ]);
    return;
  }
313
  if (!actualVersion.contains(expectedVersion)) {
314
    foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']);
315 316 317
  }
}

318
Future<void> _runTestHarnessTests() async {
319
  printProgress('${green}Running test harness tests...$reset');
320 321 322

  await _validateEngineHash();

323 324
  // Verify that the tests actually return failure on failure and success on
  // success.
325
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
326

327
  // We want to run these tests in parallel, because they each take some time
328 329 330
  // to run (e.g. compiling), so we don't want to run them in series, especially
  // on 20-core machines. However, we have a race condition, so for now...
  // Race condition issue: https://github.com/flutter/flutter/issues/90026
331 332
  final List<ShardRunner> tests = <ShardRunner>[
    () => _runFlutterTest(
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
      automatedTests,
      script: path.join('test_smoke_test', 'pass_test.dart'),
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'fail_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
      expectFailure: true,
      printOutput: false,
      outputChecker: (CommandResult result) {
        return result.flattenedStdout!.contains('failingPendingTimerTest')
          ? null
          : 'Failed to find the stack trace for the pending Timer.\n\n'
            'stdout:\n${result.flattenedStdout}\n\n'
            'stderr:\n${result.flattenedStderr}';
    }),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash1_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash2_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
367
    () => _runFlutterTest(
368 369 370 371 372
      automatedTests,
      script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
373
    () => _runFlutterTest(
374 375 376 377 378
      automatedTests,
      script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
379
    () => _runFlutterTest(
380 381 382 383 384
      automatedTests,
      script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
385 386 387 388
  ];

  List<ShardRunner> testsToRun;

389
  // Run all tests unless sharding is explicitly specified.
390
  final String? shardName = Platform.environment[kShardKey];
391
  if (shardName == kTestHarnessShardName) {
392 393 394 395 396 397 398
    testsToRun = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  } else {
    testsToRun = tests;
  }
  for (final ShardRunner test in testsToRun) {
    await test();
  }
399

400
  // Verify that we correctly generated the version file.
401
  final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
402
  if (versionError != null) {
403
    foundError(<String>[versionError]);
404
  }
405 406
}

407 408
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');

409
Future<void> _runGeneralToolTests() async {
410
  await _runDartTest(
411
    _toolsPath,
412 413
    testPaths: <String>[path.join('test', 'general.shard')],
    enableFlutterToolAsserts: false,
414

415
    // Detect unit test time regressions (poor time delay handling, etc).
416 417
    // This overrides the 15 minute default for tools tests.
    // See the README.md and dart_test.yaml files in the flutter_tools package.
418 419 420 421 422
    perTestTimeout: const Duration(seconds: 2),
  );
}

Future<void> _runCommandsToolTests() async {
423
  await _runDartTest(
424
    _toolsPath,
425
    forceSingleCore: true,
426
    testPaths: <String>[path.join('test', 'commands.shard')],
427 428 429
  );
}

430
Future<void> _runWebToolTests() async {
431 432 433 434 435 436 437 438
  final List<File> allFiles = Directory(path.join(_toolsPath, 'test', 'web.shard'))
      .listSync(recursive: true).whereType<File>().toList();
  final List<String> allTests = <String>[];
  for (final File file in allFiles) {
    if (file.path.endsWith('_test.dart')) {
      allTests.add(file.path);
    }
  }
439
  await _runDartTest(
440
    _toolsPath,
441
    forceSingleCore: true,
442
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
443
    includeLocalEngineEnv: true,
444 445 446
  );
}

447
Future<void> _runToolHostCrossArchTests() {
448
  return _runDartTest(
449 450 451 452 453 454 455
    _toolsPath,
    // These are integration tests
    forceSingleCore: true,
    testPaths: <String>[path.join('test', 'host_cross_arch.shard')],
  );
}

456
Future<void> _runIntegrationToolTests() async {
457
  final List<String> allTests = Directory(path.join(_toolsPath, 'test', 'integration.shard'))
458
      .listSync(recursive: true).whereType<File>()
459
      .map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
460 461
      .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();

462
  await _runDartTest(
463
    _toolsPath,
464 465
    forceSingleCore: true,
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
466
    collectMetrics: true,
467 468 469
  );
}

470
Future<void> _runToolTests() async {
471 472 473 474
  await selectSubshard(<String, ShardRunner>{
    'general': _runGeneralToolTests,
    'commands': _runCommandsToolTests,
  });
475 476
}

477 478
Future<void> runForbiddenFromReleaseTests() async {
  // Build a release APK to get the snapshot json.
479
  final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.');
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
  final List<String> command = <String>[
    'build',
    'apk',
    '--target-platform',
    'android-arm64',
    '--release',
    '--analyze-size',
    '--code-size-directory',
    tempDirectory.path,
    '-v',
  ];

  await runCommand(
    flutter,
    command,
    workingDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
  );

  // First, a smoke test.
  final List<String> smokeTestArgs = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::Widget',
  ];
  await runCommand(
    dart,
    smokeTestArgs,
    workingDirectory: flutterRoot,
    expectNonZeroExit: true,
  );

  // Actual test.
  final List<String> args = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/widget_inspector.dart::WidgetInspectorService',
518 519
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::DebugCreator',
    '--forbidden-type', 'package:flutter/src/foundation/print.dart::debugPrint',
520 521 522 523 524 525 526 527
  ];
  await runCommand(
    dart,
    args,
    workingDirectory: flutterRoot,
  );
}

528 529 530 531 532
/// Verifies that APK, and IPA (if on macOS), and native desktop builds the
/// examples apps without crashing. It does not actually launch the apps. That
/// happens later in the devicelab. This is just a smoke-test. In particular,
/// this will verify we can build when there are spaces in the path name for the
/// Flutter SDK and target app.
533 534
///
/// Also does some checking about types included in hello_world.
535
Future<void> _runBuildTests() async {
536 537 538
  final List<Directory> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync()
    // API example builds will be tested in a separate shard.
    .where((FileSystemEntity entity) => entity is Directory && path.basename(entity.path) != 'api').cast<Directory>().toList()
539
    ..add(Directory(path.join(flutterRoot, 'packages', 'integration_test', 'example')))
540 541 542 543 544 545
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'channels')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'hybrid_android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_platform_view_tests')))
546
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
547
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
548

549 550 551
  // The tests are randomly distributed into subshards so as to get a uniform
  // distribution of costs, but the seed is fixed so that issues are reproducible.
  final List<ShardRunner> tests = <ShardRunner>[
552
    for (final Directory exampleDirectory in exampleDirectories)
553 554 555 556 557 558 559 560 561 562 563 564 565
      () => _runExampleProjectBuildTests(exampleDirectory),
    ...<ShardRunner>[
      // Web compilation tests.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web'),
            path.join('lib', 'main.dart'),
          ),
      // Should not fail to compile with dart:io.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web_compile_tests'),
            path.join('lib', 'dart_io_import.dart'),
          ),
    ],
566
    runForbiddenFromReleaseTests,
567 568
  ]..shuffle(math.Random(0));

569
  await _runShardRunnerIndexOfTotalSubshard(tests);
570 571
}

572
Future<void> _runExampleProjectBuildTests(Directory exampleDirectory, [File? mainFile]) async {
573 574
  // Only verify caching with flutter gallery.
  final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
575
  final String examplePath = path.relative(exampleDirectory.path, from: Directory.current.path);
576 577 578
  final List<String> additionalArgs = <String>[
    if (mainFile != null) path.relative(mainFile.path, from: exampleDirectory.absolute.path),
  ];
579 580 581 582 583 584 585 586 587 588 589 590 591
  if (Directory(path.join(examplePath, 'android')).existsSync()) {
    await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
  } else {
    print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
  }
  if (Platform.isMacOS) {
    if (Directory(path.join(examplePath, 'ios')).existsSync()) {
      await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
    }
592
  }
593
  if (Platform.isLinux) {
594 595 596 597 598 599 600
    if (Directory(path.join(examplePath, 'linux')).existsSync()) {
      await _flutterBuildLinux(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildLinux(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no linux directory, skipping Linux');
    }
  }
601
  if (Platform.isMacOS) {
602 603 604 605 606 607 608
    if (Directory(path.join(examplePath, 'macos')).existsSync()) {
      await _flutterBuildMacOS(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildMacOS(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no macos directory, skipping macOS');
    }
  }
609
  if (Platform.isWindows) {
610
    if (Directory(path.join(examplePath, 'windows')).existsSync()) {
611 612
      await _flutterBuildWin32(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildWin32(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
613 614 615 616
    } else {
      print('Example project ${path.basename(examplePath)} has no windows directory, skipping Win32');
    }
  }
617
}
618

619
Future<void> _flutterBuildApk(String relativePathToApplication, {
620
  required bool release,
621
  bool verifyCaching = false,
622 623
  List<String> additionalArgs = const <String>[],
}) async {
624
  printProgress('${green}Testing APK ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
625 626 627 628
  await _flutterBuild(relativePathToApplication, 'APK', 'apk',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
629
  );
630 631
}

632
Future<void> _flutterBuildIpa(String relativePathToApplication, {
633
  required bool release,
634
  List<String> additionalArgs = const <String>[],
635
  bool verifyCaching = false,
636
}) async {
637
  assert(Platform.isMacOS);
638
  printProgress('${green}Testing IPA ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
639 640 641 642 643 644 645
  await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: <String>[...additionalArgs, '--no-codesign'],
  );
}

646
Future<void> _flutterBuildLinux(String relativePathToApplication, {
647
  required bool release,
648 649 650 651 652
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isLinux);
  await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
653
  printProgress('${green}Testing Linux ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
654 655 656 657 658 659 660
  await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

661
Future<void> _flutterBuildMacOS(String relativePathToApplication, {
662
  required bool release,
663 664 665 666 667
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isMacOS);
  await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
668
  printProgress('${green}Testing macOS ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
669 670 671 672 673 674 675
  await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

676
Future<void> _flutterBuildWin32(String relativePathToApplication, {
677
  required bool release,
678 679 680 681
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isWindows);
682
  printProgress('${green}Testing ${release ? 'release' : 'debug'} Windows build$reset for $cyan$relativePathToApplication$reset...');
683 684 685 686 687 688 689
  await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

690 691 692 693
Future<void> _flutterBuild(
  String relativePathToApplication,
  String platformLabel,
  String platformBuildName, {
694
  required bool release,
695 696 697
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
698
  await runCommand(flutter,
699 700
    <String>[
      'build',
701
      platformBuildName,
702 703 704 705 706 707 708
      ...additionalArgs,
      if (release)
        '--release'
      else
        '--debug',
      '-v',
    ],
709 710
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
711

712
  if (verifyCaching) {
713
    printProgress('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
714 715 716
    await runCommand(flutter,
      <String>[
        'build',
717
        platformBuildName,
718 719 720 721 722 723 724 725 726 727 728 729
        '--performance-measurement-file=perf.json',
        ...additionalArgs,
        if (release)
          '--release'
        else
          '--debug',
        '-v',
      ],
      workingDirectory: path.join(flutterRoot, relativePathToApplication),
    );
    final File file = File(path.join(flutterRoot, relativePathToApplication, 'perf.json'));
    if (!_allTargetsCached(file)) {
730 731 732 733
      foundError(<String>[
        '${red}Not all build targets cached after second run.$reset',
        'The target performance data was: ${file.readAsStringSync().replaceAll('},', '},\n')}',
      ]);
734 735 736 737 738
    }
  }
}

bool _allTargetsCached(File performanceFile) {
739 740 741 742 743
  final Map<String, Object?> data = json.decode(performanceFile.readAsStringSync())
    as Map<String, Object?>;
  final List<Map<String, Object?>> targets = (data['targets']! as List<Object?>)
    .cast<Map<String, Object?>>();
  return targets.every((Map<String, Object?> element) => element['skipped'] == true);
744 745
}

746
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
747
  printProgress('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
748 749 750 751 752 753 754
  await runCommand(flutter,
    <String>['build', 'web', '-v', '--target=$target'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: expectNonZeroExit,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
755
  );
756 757
}

758 759
Future<void> _runAddToAppLifeCycleTests() async {
  if (Platform.isMacOS) {
760
    printProgress('${green}Running add-to-app life cycle iOS integration tests$reset...');
761 762 763 764 765
    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
    await runCommand('./build_and_test.sh',
      <String>[],
      workingDirectory: addToAppDir,
    );
766 767
  } else {
    printProgress('${yellow}Skipped on this platform (only iOS has add-to-add lifecycle tests at this time).$reset');
768 769 770
  }
}

771
Future<void> _runFrameworkTests() async {
772
  final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
773

774
  Future<void> runWidgets() async {
775
    printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
776 777 778
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
779
        options: <String>[trackWidgetCreationOption],
780 781 782
        tests: <String>[ path.join('test', 'widgets') + path.separator ],
      );
    }
783
    // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
784 785 786 787
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
        options: <String>[trackWidgetCreationOption],
788
        fatalWarnings: false, // until we've migrated video_player
789 790
      );
    }
791 792 793
    // Run release mode tests (see packages/flutter/test_release/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
794
      options: <String>['--dart-define=dart.vm.product=true'],
795
      tests: <String>['test_release${path.separator}'],
796
    );
797 798 799
    // Run profile mode tests (see packages/flutter/test_profile/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
800
      options: <String>['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true'],
801 802
      tests: <String>['test_profile${path.separator}'],
    );
803 804
  }

805
  Future<void> runLibraries() async {
806
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
807
      .listSync(followLinks: false)
808
      .whereType<Directory>()
809
      .where((Directory dir) => dir.path.endsWith('widgets') == false)
810
      .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
811
      .toList();
812
    printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
813 814 815
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
816
        options: <String>[trackWidgetCreationOption],
817 818 819
        tests: tests,
      );
    }
820 821
  }

822 823 824 825 826 827 828 829 830 831 832 833 834 835
  Future<void> runExampleTests() async {
    // TODO(gspencergoog): Currently Linux LUCI bots can't run desktop Flutter applications, https://github.com/flutter/flutter/issues/90676
    if (!Platform.isLinux || ciProvider != CiProviders.luci) {
      await runCommand(
        flutter,
        <String>['config', '--enable-${Platform.operatingSystem}-desktop'],
        workingDirectory: flutterRoot,
      );
      await runCommand(
        dart,
        <String>[path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')],
        workingDirectory: path.join(flutterRoot, 'examples', 'api'),
      );
    }
836 837 838
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'api'));
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
839 840
  }

Ian Hickson's avatar
Ian Hickson committed
841 842 843 844 845 846 847 848 849 850 851 852
  Future<void> runTracingTests() async {
    final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests');

    // run the tests for debug mode
    await _runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);

    Future<List<String>> verifyTracingAppBuild({
      required String modeArgument,
      required String sourceFile,
      required Set<String> allowed,
      required Set<String> disallowed,
    }) async {
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
      try {
        await runCommand(
          flutter,
          <String>[
            'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile),
          ],
          workingDirectory: tracingDirectory,
        );
        final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync());
        final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
        final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
        final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
        await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
        final List<String> results = <String>[];
        for (final String pattern in allowed) {
          if (!libappStrings.contains(pattern)) {
            results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.');
          }
Ian Hickson's avatar
Ian Hickson committed
871
        }
872 873 874 875
        for (final String pattern in disallowed) {
          if (libappStrings.contains(pattern)) {
            results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.');
          }
Ian Hickson's avatar
Ian Hickson committed
876
        }
877 878 879 880 881 882
        return results;
      } catch (error, stackTrace) {
        return <String>[
          error.toString(),
          ...stackTrace.toString().trimRight().split('\n'),
        ];
Ian Hickson's avatar
Ian Hickson committed
883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925
      }
    }

    final List<String> results = <String>[];
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'profile',
      sourceFile: 'control.dart', // this is the control, the other two below are the actual test
      allowed: <String>{
        'TIMELINE ARGUMENTS TEST CONTROL FILE',
        'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
      },
    ));
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'profile',
      sourceFile: 'test.dart',
      allowed: <String>{
        'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls
        'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds
        // (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort)
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
        'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
        'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker
      },
    ));
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'release',
      sourceFile: 'test.dart',
      allowed: <String>{
        'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE',
        'BUILD', 'LAYOUT', 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds
        'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
        'toTimelineArguments used in non-debug build', // not included in release builds
      },
    ));
    if (results.isNotEmpty) {
926
      foundError(results);
Ian Hickson's avatar
Ian Hickson committed
927 928 929
    }
  }

930
  Future<void> runFixTests(String package) async {
931 932 933 934 935 936 937
    final List<String> args = <String>[
      'fix',
      '--compare-to-golden',
    ];
    await runCommand(
      dart,
      args,
938
      workingDirectory: path.join(flutterRoot, 'packages', package, 'test_fixes'),
939 940 941
    );
  }

942 943
  Future<void> runPrivateTests() async {
    final List<String> args = <String>[
944 945
      'run',
      'bin/test_private.dart',
946
    ];
947
    final Map<String, String> environment = <String, String>{
948
      'FLUTTER_ROOT': flutterRoot,
949 950
      if (Directory(pubCache).existsSync())
        'PUB_CACHE': pubCache,
951
    };
952
    adjustEnvironmentToEnableFlutterAsserts(environment);
953
    await runCommand(
954
      dart,
955 956
      args,
      workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'),
957
      environment: environment,
958 959 960
    );
  }

961 962 963 964 965 966 967
  // Tests that take longer than average to run. This is usually because they
  // need to compile something large or make use of the analyzer for the test.
  // These tests need to be platform agnostic as they are only run on a linux
  // machine to save on execution time and cost.
  Future<void> runSlow() async {
    printProgress('${green}Running slow package tests$reset for directories other than packages/flutter');
    await runTracingTests();
968 969
    await runFixTests('flutter');
    await runFixTests('flutter_test');
970 971 972
    await runPrivateTests();
  }

973
  Future<void> runMisc() async {
974
    printProgress('${green}Running package tests$reset for directories other than packages/flutter');
975
    await _runTestHarnessTests();
976
    await runExampleTests();
977 978 979
    await _runDartTest(path.join(flutterRoot, 'dev', 'bots'));
    await _runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
    await _runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
980
    // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed.
981
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
982
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
983 984
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
985
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
986
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
987
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
988
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')]);
989 990 991 992 993
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: <String>[
      '--enable-vmservice',
      // Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard.
      '--exclude-tags=web',
    ]);
994 995 996 997 998
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'));
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'));
999 1000 1001 1002
    const String httpClientWarning =
      'Warning: At least one test in this suite creates an HttpClient. When\n'
      'running a test suite that uses TestWidgetsFlutterBinding, all HTTP\n'
      'requests will return status code 400, and no network request will\n'
1003
      'actually be made. Any test expecting a real network connection and\n'
1004 1005 1006 1007 1008 1009 1010 1011 1012
      'status code will fail.\n'
      'To test code that needs an HttpClient, provide your own HttpClient\n'
      'implementation to the code under test, so that your test can\n'
      'consistently provide a testable response to the code under test.';
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter_test'),
      script: path.join('test', 'bindings_test_failure.dart'),
      expectFailure: true,
      printOutput: false,
1013
      outputChecker: (CommandResult result) {
1014
        final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout!);
1015
        if (matches.isEmpty || matches.length > 1) {
1016 1017 1018
          return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n'
                 'stdout:\n${result.flattenedStdout}\n\n'
                 'stderr:\n${result.flattenedStderr}';
1019 1020 1021 1022
        }
        return null;
      },
    );
1023
  }
1024

1025 1026 1027
  await selectSubshard(<String, ShardRunner>{
    'widgets': runWidgets,
    'libraries': runLibraries,
1028
    'slow': runSlow,
1029 1030
    'misc': runMisc,
  });
1031 1032
}

1033
Future<void> _runFrameworkCoverage() async {
1034 1035
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
1036 1037
    foundError(<String>[
      '${red}Coverage file not found.$reset',
1038
      'Expected to find: $cyan${coverageFile.absolute.path}$reset',
1039 1040 1041
      'This file is normally obtained by running `${green}flutter update-packages$reset`.',
    ]);
    return;
1042 1043 1044 1045 1046 1047
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
1048 1049
    foundError(<String>[
      '${red}Coverage file not found.$reset',
1050
      'Expected to find: $cyan${coverageFile.absolute.path}$reset',
1051 1052 1053
      'This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.',
    ]);
    return;
1054
  }
1055 1056
}

1057 1058 1059 1060 1061 1062 1063 1064 1065
Future<void> _runWebHtmlUnitTests() {
  return _runWebUnitTests('html');
}

Future<void> _runWebCanvasKitUnitTests() {
  return _runWebUnitTests('canvaskit');
}

Future<void> _runWebUnitTests(String webRenderer) async {
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};

  final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));

  final List<String> allTests = flutterPackageTestDirectory
    .listSync()
    .whereType<Directory>()
    .expand((Directory directory) => directory
      .listSync(recursive: true)
      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
    )
    .whereType<File>()
    .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
1080
    .where((String filePath) => !kWebTestFileKnownFailures[webRenderer]!.contains(path.split(filePath).join('/')))
1081 1082 1083 1084 1085 1086 1087
    .toList()
    // Finally we shuffle the list because we want the average cost per file to be uniformly
    // distributed. If the list is not sorted then different shards and batches may have
    // very different characteristics.
    // We use a constant seed for repeatability.
    ..shuffle(math.Random(0));

1088 1089 1090
  assert(webShardCount >= 1);
  final int testsPerShard = (allTests.length / webShardCount).ceil();
  assert(testsPerShard * webShardCount >= allTests.length);
1091 1092

  // This for loop computes all but the last shard.
1093
  for (int index = 0; index < webShardCount - 1; index += 1) {
1094
    subshards['$index'] = () => _runFlutterWebTest(
1095
      webRenderer,
1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
      flutterPackageDirectory.path,
      allTests.sublist(
        index * testsPerShard,
        (index + 1) * testsPerShard,
      ),
    );
  }

  // The last shard also runs the flutter_web_plugins tests.
  //
  // We make sure the last shard ends in _last so it's easier to catch mismatches
  // between `.cirrus.yml` and `test.dart`.
1108
  subshards['${webShardCount - 1}_last'] = () async {
1109
    await _runFlutterWebTest(
1110
      webRenderer,
1111 1112
      flutterPackageDirectory.path,
      allTests.sublist(
1113
        (webShardCount - 1) * testsPerShard,
1114 1115 1116 1117
        allTests.length,
      ),
    );
    await _runFlutterWebTest(
1118
      webRenderer,
1119 1120 1121
      path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
      <String>['test'],
    );
1122
    await _runFlutterWebTest(
1123 1124 1125
      webRenderer,
      path.join(flutterRoot, 'packages', 'flutter_driver'),
      <String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
1126
    );
1127 1128 1129 1130 1131
  };

  await selectSubshard(subshards);
}

1132 1133
/// Coarse-grained integration tests running on the Web.
Future<void> _runWebLongRunningTests() async {
1134
  final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
1135
  final List<ShardRunner> tests = <ShardRunner>[
1136
    for (String buildMode in _kAllBuildModes) ...<ShardRunner>[
1137 1138 1139 1140 1141 1142 1143 1144 1145
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('test_driver', 'failure.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
        // This test intentionally fails and prints stack traces in the browser
        // logs. To avoid confusion, silence browser output.
        silenceBrowserOutput: true,
      ),
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('integration_test', 'example_test.dart'),
        driver: path.join('test_driver', 'integration_test.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
      ),
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('integration_test', 'extended_test.dart'),
        driver: path.join('test_driver', 'extended_integration_test.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
      ),
    ],
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),

    // This test is only known to work in debug mode.
    () => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),

1185 1186 1187 1188 1189
    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'debug', renderer: 'auto'),
    () => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('capabilities_integration_html', buildMode: 'release', renderer: 'html'),

1190 1191 1192 1193 1194
    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    // CacheWidth and CacheHeight are only currently supported in CanvasKit mode, so we don't run the test in HTML mode.
    () => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'debug', renderer: 'auto'),
    () => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'profile', renderer: 'canvaskit'),

1195 1196
    () => _runWebTreeshakeTest(),

1197 1198 1199 1200
    () => _runFlutterDriverWebTest(
      testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
      target: 'test_driver/smoke_web_engine.dart',
      buildMode: 'profile',
1201
      renderer: 'auto',
1202
    ),
1203 1204 1205 1206 1207 1208
    () => _runGalleryE2eWebTest('debug'),
    () => _runGalleryE2eWebTest('debug', canvasKit: true),
    () => _runGalleryE2eWebTest('profile'),
    () => _runGalleryE2eWebTest('profile', canvasKit: true),
    () => _runGalleryE2eWebTest('release'),
    () => _runGalleryE2eWebTest('release', canvasKit: true),
1209 1210 1211
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
1212
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
1213
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
1214 1215 1216
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
1217
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
1218
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
1219
    () => runWebServiceWorkerTestWithGeneratedEntrypoint(headless: true),
1220
    () => runWebServiceWorkerTestWithBlockedServiceWorkers(headless: true),
1221 1222 1223 1224 1225 1226 1227
    () => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/stack_trace.dart'),
    () => _runWebDebugTest('lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/web_directory_loading.dart'),
1228 1229 1230 1231
    () => _runWebDebugTest('lib/web_resources_cdn_test.dart',
      additionalArguments: <String>[
        '--dart-define=TEST_FLUTTER_ENGINE_VERSION=$engineVersion',
      ]),
1232
    () => _runWebDebugTest('test/test.dart'),
1233
    () => _runWebDebugTest('lib/null_safe_main.dart'),
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
    () => _runWebDebugTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
    () => _runWebReleaseTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
1246 1247
    () => _runWebDebugTest('lib/sound_mode.dart'),
    () => _runWebReleaseTest('lib/sound_mode.dart'),
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
    () => _runFlutterWebTest(
      'html',
      path.join(flutterRoot, 'packages', 'integration_test'),
      <String>['test/web_extension_test.dart'],
    ),
    () => _runFlutterWebTest(
      'canvaskit',
      path.join(flutterRoot, 'packages', 'integration_test'),
      <String>['test/web_extension_test.dart'],
    ),
1258
  ];
1259 1260 1261 1262 1263

  // Shuffling mixes fast tests with slow tests so shards take roughly the same
  // amount of time to run.
  tests.shuffle(math.Random(0));

1264
  await _ensureChromeDriverIsRunning();
1265
  await _runShardRunnerIndexOfTotalSubshard(tests);
1266
  await _stopChromeDriver();
1267 1268
}

1269 1270 1271
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
Future<void> _runWebE2eTest(
  String name, {
1272 1273
  required String buildMode,
  required String renderer,
1274 1275 1276 1277 1278 1279 1280 1281 1282
}) async {
  await _runFlutterDriverWebTest(
    target: path.join('test_driver', '$name.dart'),
    buildMode: buildMode,
    renderer: renderer,
    testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
  );
}

1283
Future<void> _runFlutterDriverWebTest({
1284 1285 1286 1287
  required String target,
  required String buildMode,
  required String renderer,
  required String testAppDirectory,
1288
  String? driver,
1289 1290
  bool expectFailure = false,
  bool silenceBrowserOutput = false,
1291
}) async {
1292
  printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
1293 1294 1295 1296 1297 1298 1299 1300
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1301
      ...flutterTestArgs,
1302
      'drive',
1303
      if (driver != null) '--driver=$driver',
1304 1305 1306 1307 1308
      '--target=$target',
      '--browser-name=chrome',
      '-d',
      'web-server',
      '--$buildMode',
1309
      '--web-renderer=$renderer',
1310
    ],
1311
    expectNonZeroExit: expectFailure,
1312 1313 1314 1315
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
1316 1317 1318 1319 1320 1321 1322 1323 1324
    removeLine: (String line) {
      if (!silenceBrowserOutput) {
        return false;
      }
      if (line.trim().startsWith('[INFO]')) {
        return true;
      }
      return false;
    },
1325 1326 1327
  );
}

1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
// Compiles a sample web app and checks that its JS doesn't contain certain
// debug code that we expect to be tree shaken out.
//
// The app is compiled in `--profile` mode to prevent the compiler from
// minifying the symbols.
Future<void> _runWebTreeshakeTest() async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
  final String target = path.join('lib', 'treeshaking_main.dart');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--target=$target',
      '--profile',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
  final String javaScript = mainDartJs.readAsStringSync();

  // Check that we're not looking at minified JS. Otherwise this test would result in false positive.
  expect(javaScript.contains('RenderObjectToWidgetElement'), true);

  const String word = 'debugFillProperties';
  int count = 0;
  int pos = javaScript.indexOf(word);
  final int contentLength = javaScript.length;
  while (pos != -1) {
    count += 1;
    pos += word.length;
    if (pos >= contentLength || count > 100) {
      break;
    }
    pos = javaScript.indexOf(word, pos);
  }

  const int kMaxExpectedDebugFillProperties = 11;
  if (count > kMaxExpectedDebugFillProperties) {
    throw Exception(
      'Too many occurrences of "$word" in compiled JavaScript.\n'
      'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
    );
  }
}

1383
/// Returns the commit hash of the flutter/packages repository that's rolled in.
1384
///
1385
/// The flutter/packages repository is a downstream dependency, it is only used
1386
/// by flutter/flutter for testing purposes, to assure stable tests for a given
1387 1388
/// flutter commit the flutter/packages commit hash to test against is coded in
/// the bin/internal/flutter_packages.version file.
1389
///
1390 1391 1392 1393
/// The `filesystem` parameter specified filesystem to read the packages version file from.
/// The `packagesVersionFile` parameter allows specifying an alternative path for the
/// packages version file, when null [flutterPackagesVersionFile] is used.
Future<String> getFlutterPackagesVersion({
1394
  fs.FileSystem fileSystem = const LocalFileSystem(),
1395
  String? packagesVersionFile,
1396
}) async {
1397
  final File versionFile = fileSystem.file(packagesVersionFile ?? flutterPackagesVersionFile);
1398 1399 1400 1401
  final String versionFileContents = await versionFile.readAsString();
  return versionFileContents.trim();
}

1402 1403
/// Executes the test suite for the flutter/packages repo.
Future<void> _runFlutterPackagesTests() async {
1404
  Future<void> runAnalyze() async {
1405 1406
    printProgress('${green}Running analysis for flutter/packages$reset');
    final Directory checkout = Directory.systemTemp.createTempSync('flutter_packages.');
1407 1408 1409 1410 1411 1412
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'clone',
1413
        'https://github.com/flutter/packages.git',
1414
        '.',
1415 1416 1417
      ],
      workingDirectory: checkout.path,
    );
1418
    final String packagesCommit = await getFlutterPackagesVersion();
1419 1420 1421 1422 1423 1424
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'checkout',
1425
        packagesCommit,
1426 1427 1428
      ],
      workingDirectory: checkout.path,
    );
1429 1430
    // Prep the repository tooling.
    // This test does not use tool_runner.sh because in this context the test
1431 1432
    // should always run on the entire packages repo, while tool_runner.sh
    // is designed for flutter/packages CI and only analyzes changed repository
1433 1434
    // files when run for anything but master.
    final String toolDir = path.join(checkout.path, 'script', 'tool');
1435
    await runCommand(
1436
      'dart',
1437
      <String>[
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448
        'pub',
        'get',
      ],
      workingDirectory: toolDir,
    );
    final String toolScript = path.join(toolDir, 'bin', 'flutter_plugin_tools.dart');
    await runCommand(
      'dart',
      <String>[
        'run',
        toolScript,
1449
        'analyze',
1450
        '--custom-analysis=script/configs/custom_analysis.yaml',
1451 1452 1453 1454 1455 1456 1457 1458 1459
      ],
      workingDirectory: checkout.path,
    );
  }
  await selectSubshard(<String, ShardRunner>{
    'analyze': runAnalyze,
  });
}

1460 1461 1462 1463 1464 1465
/// Runs the skp_generator from the flutter/tests repo.
///
/// See also the customer_tests shard.
///
/// Generated SKPs are ditched, this just verifies that it can run without failure.
Future<void> _runSkpGeneratorTests() async {
1466
  printProgress('${green}Running skp_generator from flutter/tests$reset');
1467 1468 1469 1470 1471 1472 1473 1474
  final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.');
  await runCommand(
    'git',
    <String>[
      '-c',
      'core.longPaths=true',
      'clone',
      'https://github.com/flutter/tests.git',
1475
      '.',
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485
    ],
    workingDirectory: checkout.path,
  );
  await runCommand(
    './build.sh',
    <String>[ ],
    workingDirectory: path.join(checkout.path, 'skp_generator'),
  );
}

1486 1487 1488 1489
// The `chromedriver` process created by this test.
//
// If an existing chromedriver is already available on port 4444, the existing
// process is reused and this variable remains null.
1490
Command? _chromeDriver;
1491 1492 1493

Future<bool> _isChromeDriverRunning() async {
  try {
1494 1495 1496
    final RawSocket socket = await RawSocket.connect('localhost', 4444);
    socket.shutdown(SocketDirection.both);
    await socket.close();
1497 1498 1499 1500 1501 1502 1503 1504 1505
    return true;
  } on SocketException {
    return false;
  }
}

Future<void> _ensureChromeDriverIsRunning() async {
  // If we cannot connect to ChromeDriver, assume it is not running. Launch it.
  if (!await _isChromeDriverRunning()) {
1506
    printProgress('Starting chromedriver');
1507 1508
    // Assume chromedriver is in the PATH.
    _chromeDriver = await startCommand(
1509 1510 1511
      // TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
      // and it doesn't use most of startCommand's features; we could simplify this a lot by
      // inlining the relevant parts of startCommand here.
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
      'chromedriver',
      <String>['--port=4444'],
    );
    while (!await _isChromeDriverRunning()) {
      await Future<void>.delayed(const Duration(milliseconds: 100));
      print('Waiting for chromedriver to start up.');
    }
  }

  final HttpClient client = HttpClient();
  final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
  final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
  final HttpClientResponse response = await request.close();
1525
  final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join()) as Map<String, dynamic>;
1526
  client.close();
1527
  final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
1528 1529 1530 1531 1532 1533 1534 1535 1536
  if (!webDriverReady) {
    throw Exception('WebDriver not available.');
  }
}

Future<void> _stopChromeDriver() async {
  if (_chromeDriver == null) {
    return;
  }
1537
  print('Stopping chromedriver');
1538
  _chromeDriver!.process.kill();
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
}

/// Exercises the old gallery in a browser for a long period of time, looking
/// for memory leaks and dangling pointers.
///
/// This is not a performance test.
///
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
///
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
1551
  printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
1552 1553 1554 1555 1556 1557 1558 1559 1560
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1561
      ...flutterTestArgs,
1562 1563 1564
      'drive',
      if (canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=true',
1565 1566
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1567 1568
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
      '--driver=test_driver/transitions_perf_e2e_test.dart',
      '--target=test_driver/transitions_perf_e2e.dart',
      '--browser-name=chrome',
      '-d',
      'web-server',
      '--$buildMode',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
}

1583
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
1584
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
1585
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--$buildMode',
      '-t',
1600
      entrypoint,
1601 1602 1603 1604 1605 1606 1607 1608
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1609 1610
  final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
  final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
1611
  final String result = await evalTestAppInChrome(
1612
    appUrl: 'http://localhost:$serverPort/index.html',
1613
    appDirectory: appBuildDirectory,
1614 1615
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1616 1617
  );

1618 1619 1620 1621 1622
  if (!result.contains('--- TEST SUCCEEDED ---')) {
    foundError(<String>[
      result,
      '${red}Web stack trace integration test failed.$reset',
    ]);
1623 1624 1625
  }
}

1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
/// Run a web integration test in release mode.
Future<void> _runWebReleaseTest(String target, {
  List<String> additionalArguments = const<String>[],
}) async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1642
      ...flutterTestArgs,
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
      'build',
      'web',
      '--release',
      ...additionalArguments,
      '-t',
      target,
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1657 1658
  final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
  final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
1659
  final String result = await evalTestAppInChrome(
1660
    appUrl: 'http://localhost:$serverPort/index.html',
1661
    appDirectory: appBuildDirectory,
1662 1663
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1664 1665
  );

1666 1667 1668 1669 1670
  if (!result.contains('--- TEST SUCCEEDED ---')) {
    foundError(<String>[
      result,
      '${red}Web release mode test failed.$reset',
    ]);
1671 1672 1673
  }
}

1674 1675 1676
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
///
/// Instead, we use `flutter run --debug` and sniff out the standard output.
1677 1678 1679
Future<void> _runWebDebugTest(String target, {
  List<String> additionalArguments = const<String>[],
}) async {
1680 1681
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  bool success = false;
1682 1683 1684
  final Map<String, String> environment = <String, String>{
    'FLUTTER_WEB': 'true',
  };
1685
  adjustEnvironmentToEnableFlutterAsserts(environment);
1686
  final CommandResult result = await runCommand(
1687 1688 1689 1690 1691 1692 1693
    flutter,
    <String>[
      'run',
      '--debug',
      '-d',
      'chrome',
      '--web-run-headless',
1694
      '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1695
      '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1696
      ...additionalArguments,
1697 1698
      '-t',
      target,
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
    ],
    outputMode: OutputMode.capture,
    outputListener: (String line, Process process) {
      if (line.contains('--- TEST SUCCEEDED ---')) {
        success = true;
      }
      if (success || line.contains('--- TEST FAILED ---')) {
        process.stdin.add('q'.codeUnits);
      }
    },
    workingDirectory: testAppDirectory,
1710
    environment: environment,
1711 1712
  );

1713 1714 1715 1716 1717 1718
  if (!success) {
    foundError(<String>[
      result.flattenedStdout!,
      result.flattenedStderr!,
      '${red}Web stack trace integration test failed.$reset',
    ]);
1719 1720 1721
  }
}

1722
Future<void> _runFlutterWebTest(String webRenderer, String workingDirectory, List<String> tests) async {
1723 1724 1725 1726 1727 1728 1729 1730
  await runCommand(
    flutter,
    <String>[
      'test',
      if (ciProvider == CiProviders.cirrus)
        '--concurrency=1',  // do not parallelize on Cirrus, to reduce flakiness
      '-v',
      '--platform=chrome',
1731
      '--web-renderer=$webRenderer',
1732
      '--dart-define=DART_HHH_BOT=$_runningInDartHHHBot',
1733
      ...flutterTestArgs,
1734 1735 1736 1737 1738 1739 1740
      ...tests,
    ],
    workingDirectory: workingDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
1741 1742
}

1743 1744 1745 1746 1747
// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
1748
Future<void> _runDartTest(String workingDirectory, {
1749
  List<String>? testPaths,
1750 1751
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
1752
  String? coverage,
1753
  bool forceSingleCore = false,
1754
  Duration? perTestTimeout,
1755
  bool includeLocalEngineEnv = false,
1756
  bool ensurePrecompiledTool = true,
1757
  bool shuffleTests = true,
1758
  bool collectMetrics = false,
Dan Field's avatar
Dan Field committed
1759
}) async {
1760 1761
  int? cpus;
  final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
1762 1763 1764
  if (cpuVariable != null) {
    cpus = int.tryParse(cpuVariable, radix: 10);
    if (cpus == null) {
1765 1766 1767 1768 1769
      foundError(<String>[
        '${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset',
        'Actual value: "$cpuVariable"',
      ]);
      return;
1770 1771 1772 1773
    }
  } else {
    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
  }
1774 1775 1776 1777 1778
  // Integration tests that depend on external processes like chrome
  // can get stuck if there are multiple instances running at once.
  if (forceSingleCore) {
    cpus = 1;
  }
1779

1780 1781
  const LocalFileSystem fileSystem = LocalFileSystem();
  final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json'));
1782 1783 1784
  final List<String> args = <String>[
    'run',
    'test',
1785
    '--file-reporter=json:${metricFile.path}',
1786
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1787 1788 1789 1790 1791
    '-j$cpus',
    if (!hasColor)
      '--no-color',
    if (coverage != null)
      '--coverage=$coverage',
1792
    if (perTestTimeout != null)
1793
      '--timeout=${perTestTimeout.inMilliseconds}ms',
1794 1795 1796 1797
    if (testPaths != null)
      for (final String testPath in testPaths)
        testPath,
  ];
1798
  final Map<String, String> environment = <String, String>{
1799
    'FLUTTER_ROOT': flutterRoot,
1800 1801 1802 1803
    if (includeLocalEngineEnv)
      ...localEngineEnv,
    if (Directory(pubCache).existsSync())
      'PUB_CACHE': pubCache,
1804 1805
  };
  if (enableFlutterToolAsserts) {
1806
    adjustEnvironmentToEnableFlutterAsserts(environment);
1807
  }
1808 1809 1810 1811
  if (ensurePrecompiledTool) {
    // We rerun the `flutter` tool here just to make sure that it is compiled
    // before tests run, because the tests might time out if they have to rebuild
    // the tool themselves.
1812
    await runCommand(flutter, <String>['--version'], environment: environment);
1813
  }
1814 1815 1816 1817 1818 1819 1820
  await runCommand(
    dart,
    args,
    workingDirectory: workingDirectory,
    environment: environment,
    removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
  );
1821

1822 1823 1824 1825
  final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
  final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
  info.writeAsStringSync(json.encode(test.errors));

1826 1827 1828
  if (collectMetrics) {
    try {
      final List<String> testList = <String>[];
1829
      final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
1830 1831 1832 1833 1834
      for (final TestSpecs testSpecs in allTestSpecs.values) {
        testList.add(testSpecs.toJson());
      }
      if (testList.isNotEmpty) {
        final String testJson = json.encode(testList);
1835 1836
        final File testResults = fileSystem.file(
            path.join(flutterRoot, 'test_results.json'));
1837 1838
        testResults.writeAsStringSync(testJson);
      }
1839
    } on fs.FileSystemException catch (e) {
1840 1841 1842
      print('Failed to generate metrics: $e');
    }
  }
1843 1844
}

1845
Future<void> _runFlutterTest(String workingDirectory, {
1846
  String? script,
1847 1848
  bool expectFailure = false,
  bool printOutput = true,
1849
  OutputChecker? outputChecker,
1850
  List<String> options = const <String>[],
1851
  Map<String, String>? environment,
1852
  List<String> tests = const <String>[],
1853
  bool shuffleTests = true,
1854
  bool fatalWarnings = true,
Dan Field's avatar
Dan Field committed
1855
}) async {
1856
  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
1857

1858
  final List<String> tags = <String>[];
1859
  // Recipe-configured reduced test shards will only execute tests with the
1860
  // appropriate tag.
1861
  if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
1862 1863 1864
    tags.addAll(<String>['-t', 'reduced-test-set']);
  }

1865 1866
  final List<String> args = <String>[
    'test',
1867
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1868
    if (fatalWarnings) '--fatal-warnings',
1869
    ...options,
1870
    ...tags,
1871
    ...flutterTestArgs,
1872
  ];
Dan Field's avatar
Dan Field committed
1873

1874 1875 1876
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
1877 1878 1879 1880 1881 1882 1883 1884
      foundError(<String>[
        '${red}Could not find test$reset: $green$fullScriptPath$reset',
        'Working directory: $cyan$workingDirectory$reset',
        'Script: $green$script$reset',
        if (!printOutput)
          'This is one of the tests that does not normally print output.',
      ]);
      return;
1885
    }
1886
    args.add(script);
1887
  }
1888

1889
  args.addAll(tests);
1890

1891 1892 1893
  final OutputMode outputMode = outputChecker == null && printOutput
    ? OutputMode.print
    : OutputMode.capture;
1894

1895 1896 1897 1898 1899 1900 1901 1902
  final CommandResult result = await runCommand(
    flutter,
    args,
    workingDirectory: workingDirectory,
    expectNonZeroExit: expectFailure,
    outputMode: outputMode,
    environment: environment,
  );
1903

1904 1905 1906 1907
  if (outputChecker != null) {
    final String? message = outputChecker(result);
    if (message != null) {
      foundError(<String>[message]);
1908
    }
1909
  }
1910 1911
}

1912 1913 1914
/// This will force the next run of the Flutter tool (if it uses the provided
/// environment) to have asserts enabled, by setting an environment variable.
void adjustEnvironmentToEnableFlutterAsserts(Map<String, String> environment) {
1915 1916 1917 1918 1919 1920
  // If an existing env variable exists append to it, but only if
  // it doesn't appear to already include enable-asserts.
  String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
  if (!toolsArgs.contains('--enable-asserts')) {
    toolsArgs += ' --enable-asserts';
  }
1921
  environment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
1922 1923
}

1924 1925 1926 1927 1928
enum CiProviders {
  cirrus,
  luci,
}

1929
CiProviders? get ciProvider {
1930 1931 1932 1933 1934 1935 1936 1937
  if (Platform.environment['CIRRUS_CI'] == 'true') {
    return CiProviders.cirrus;
  }
  if (Platform.environment['LUCI_CONTEXT'] != null) {
    return CiProviders.luci;
  }
  return null;
}
1938

1939 1940 1941 1942 1943
/// Checks the given file's contents to determine if they match the allowed
/// pattern for version strings.
///
/// Returns null if the contents are good. Returns a string if they are bad.
/// The string is an error message.
1944
Future<String?> verifyVersion(File file) async {
1945 1946
  final RegExp pattern = RegExp(
    r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
1947
  if (!file.existsSync()) {
1948
    return 'The version logic failed to create the Flutter version file.';
1949
  }
1950
  final String version = await file.readAsString();
1951
  if (version == '0.0.0-unknown') {
1952
    return 'The version logic failed to determine the Flutter version.';
1953 1954
  }
  if (!version.contains(pattern)) {
1955
    return 'The version logic generated an invalid version string: "$version".';
1956
  }
1957 1958 1959
  return null;
}

1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
  // Example: "1_3" means the first (one-indexed) shard of three total shards.
1972
  final String? subshardName = Platform.environment[subshardKey];
1973 1974 1975 1976
  if (subshardName == null) {
    print('$kSubshardKey environment variable is missing, skipping sharding');
    return tests;
  }
1977
  printProgress('$bold$subshardKey=$subshardName$reset');
1978 1979

  final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
1980
  final Match? match = pattern.firstMatch(subshardName);
1981
  if (match == null || match.groupCount != 2) {
1982 1983 1984
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"',
    ]);
1985
    throw Exception('Invalid subshard name: $subshardName');
1986 1987
  }
  // One-indexed.
1988
  final int index = int.parse(match.group(1)!);
1989
  final int total = int.parse(match.group(2)!);
1990
  if (index > total) {
1991 1992 1993 1994
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.',
    ]);
    return <T>[];
1995 1996
  }

1997
  final int testsPerShard = (tests.length / total).ceil();
1998
  final int start = (index - 1) * testsPerShard;
1999
  final int end = math.min(index * testsPerShard, tests.length);
2000

2001
  print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
2002 2003 2004
  return tests.sublist(start, end);
}

2005
Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
2006 2007 2008 2009 2010 2011 2012 2013
  final List<ShardRunner> sublist = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  for (final ShardRunner test in sublist) {
    await test();
  }
}

Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
2014 2015 2016 2017

const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
2018
  String? item = Platform.environment[key];
2019
  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
2020
    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
2021 2022 2023 2024
    assert(positionInTaskName < parts.length);
    item = parts[positionInTaskName];
  }
  if (item == null) {
2025
    for (final String currentItem in items.keys) {
2026
      printProgress('$bold$key=$currentItem$reset');
2027
      await items[currentItem]!();
2028 2029
    }
  } else {
2030
    printProgress('$bold$key=$item$reset');
2031
    if (!items.containsKey(item)) {
2032 2033 2034 2035 2036
      foundError(<String>[
        '${red}Invalid $name: $item$reset',
        'The available ${name}s are: ${items.keys.join(", ")}',
      ]);
      return;
2037
    }
2038
    await items[item]!();
2039 2040
  }
}