test.dart 79.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
// Runs the tests for the flutter/flutter repository.
//
//
// By default, test output is filtered and only errors are shown. (If a
// particular test takes longer than _quietTimeout in utils.dart, the output is
// shown then also, in case something has hung.)
//
//  --verbose stops the output cleanup and just outputs everything verbatim.
//
//
// By default, errors are non-fatal; all tests are executed and the output
// ends with a summary of the errors that were detected.
//
// Exit code is 1 if there was an error.
//
//  --abort-on-error causes the script to exit immediately when hitting an error.
//
//
// By default, all tests are run. However, the tests support being split by
// shard and subshard. (Inspect the code to see what shards and subshards are
// supported.)
//
// If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
// the shard and sub-shard, by parsing it in the form shard-subshard-platform,
// ignoring the platform.
//
// For local testing you can just set the SHARD and SUBSHARD environment
// variables. For example, to run all the framework tests you can just set
// SHARD=framework_tests. Some shards support named subshards, like
// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of
// two" as in run the first half of the tests).
//
// So for example to run specifically the third subshard of the Web tests you
// would set SHARD=web_tests SUBSHARD=2 (it's zero-based).
//
// By default, where supported, tests within a shard are executed in a random
// order to (eventually) catch inter-test dependencies.
//
//  --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
//
//
// All other arguments are treated as arguments to pass to the flutter tool when
// running tests.

50
import 'dart:convert';
51
import 'dart:core' as system show print;
52
import 'dart:core' hide print;
53
import 'dart:io' as system show exit;
54
import 'dart:io' hide exit;
55
import 'dart:math' as math;
Ian Hickson's avatar
Ian Hickson committed
56
import 'dart:typed_data';
57

Ian Hickson's avatar
Ian Hickson committed
58
import 'package:archive/archive.dart';
59 60
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
61
import 'package:path/path.dart' as path;
62

63
import 'browser.dart';
64
import 'run_command.dart';
65
import 'service_worker_test.dart';
66
import 'tool_subsharding.dart';
67
import 'utils.dart';
68

69
typedef ShardRunner = Future<void> Function();
70

71 72 73 74 75 76
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
77
typedef OutputChecker = String? Function(CommandResult);
78

79 80
final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
81
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
82 83
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
84
final String pubCache = path.join(flutterRoot, '.pub-cache');
85
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
86
final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm');
87
final String flutterPackagesVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_packages.version');
88 89

String get platformFolderName {
90
  if (Platform.isWindows) {
91
    return 'windows-x64';
92 93
  }
  if (Platform.isMacOS) {
94
    return 'darwin-x64';
95 96
  }
  if (Platform.isLinux) {
97
    return 'linux-x64';
98
  }
99 100 101
  throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.');
}
final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe');
102 103 104

/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
105
final List<String> flutterTestArgs = <String>[];
106

107 108 109 110
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};

111 112 113
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';

114 115
/// The number of Cirrus jobs that run Web tests in parallel.
///
116 117 118
/// The default is 8 shards. Typically .cirrus.yml would define the
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
///
119 120 121 122
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
123
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
124
  ? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
125
  : 8;
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142

/// Tests that we don't run on Web.
///
/// In general avoid adding new tests here. If a test cannot run on the web
/// because it fails at runtime, such as when a piece of functionality is not
/// implemented or not implementable on the web, prefer using `skip` in the
/// test code. Only add tests here that cannot be skipped using `skip`. For
/// example:
///
///  * Test code cannot be compiled because it uses Dart VM-specific
///    functionality. In this case `skip` doesn't help because the code cannot
///    reach the point where it can even run the skipping logic.
///  * Migrations. It is OK to put tests here that need to be temporarily
///    disabled in certain modes because of some migration or initial bringup.
///
/// The key in the map is the renderer type that the list applies to. The value
/// is the list of tests known to fail for that renderer.
143
//
144
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String>>{
  'html': <String>[
    // These tests are not compilable on the web due to dependencies on
    // VM-specific functionality.
    'test/services/message_codecs_vm_test.dart',
    'test/examples/sector_layout_test.dart',
  ],
  'canvaskit': <String>[
    // These tests are not compilable on the web due to dependencies on
    // VM-specific functionality.
    'test/services/message_codecs_vm_test.dart',
    'test/examples/sector_layout_test.dart',

    // These tests are broken and need to be fixed.
    // TODO(yjbanov): https://github.com/flutter/flutter/issues/71604
    'test/painting/decoration_test.dart',
    'test/material/text_selection_theme_test.dart',
    'test/material/date_picker_test.dart',
    'test/rendering/layers_test.dart',
    'test/painting/text_style_test.dart',
    'test/widgets/image_test.dart',
    'test/cupertino/colors_test.dart',
    'test/cupertino/slider_test.dart',
    'test/material/text_field_test.dart',
    'test/rendering/proxy_box_test.dart',
    'test/widgets/app_overrides_test.dart',
    'test/material/calendar_date_picker_test.dart',
    'test/material/ink_paint_test.dart',
    'test/rendering/editable_test.dart',
    'test/cupertino/dialog_test.dart',
    'test/widgets/shape_decoration_test.dart',
    'test/material/time_picker_theme_test.dart',
    'test/cupertino/picker_test.dart',
    'test/material/chip_theme_test.dart',
    'test/cupertino/nav_bar_test.dart',
    'test/widgets/performance_overlay_test.dart',
    'test/widgets/html_element_view_test.dart',
    'test/cupertino/scaffold_test.dart',
    'test/rendering/platform_view_test.dart',
184
    'test/cupertino/context_menu_action_test.dart',
185 186
  ],
};
187

188
const String kTestHarnessShardName = 'test_harness_tests';
189
const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
190

191
// The seed used to shuffle tests. If not passed with
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;
String get shuffleSeed {
  if (_shuffleSeed == null) {
    // Change the seed at 7am, UTC.
    final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
    // Generates YYYYMMDD as the seed, so that testing continues to fail for a
    // day after the seed changes, and on other days the seed can be used to
    // replicate failures.
    _shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
  }
  return _shuffleSeed!;
}

207
/// When you call this, you can pass additional arguments to pass custom
208
/// arguments to flutter test. For example, you might want to call this
209
/// script with the parameter --local-engine=host_debug_unopt to
210
/// use your own build of the engine.
211
///
212
/// To run the tool_tests part, run it with SHARD=tool_tests
213
///
214
/// Examples:
215
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
216
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt --local-engine-host=host_debug_unopt
217
Future<void> main(List<String> args) async {
218 219 220 221 222 223
  try {
    printProgress('STARTING ANALYSIS');
    for (final String arg in args) {
      if (arg.startsWith('--local-engine=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
        flutterTestArgs.add(arg);
224 225 226
      } else if (arg.startsWith('--local-engine-host=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE_HOST'] = arg.substring('--local-engine-host='.length);
        flutterTestArgs.add(arg);
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
      } else if (arg.startsWith('--local-engine-src-path=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
        flutterTestArgs.add(arg);
      } else if (arg.startsWith('--test-randomize-ordering-seed=')) {
        _shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
      } else if (arg.startsWith('--verbose')) {
        print = (Object? message) {
          system.print(message);
        };
      } else if (arg.startsWith('--abort-on-error')) {
        onError = () {
          system.exit(1);
        };
      } else {
        flutterTestArgs.add(arg);
      }
243
    }
244 245
    if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
      printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
246
    }
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
    await selectShard(<String, ShardRunner>{
      'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
      'build_tests': _runBuildTests,
      'framework_coverage': _runFrameworkCoverage,
      'framework_tests': _runFrameworkTests,
      'tool_tests': _runToolTests,
      // web_tool_tests is also used by HHH: https://dart.googlesource.com/recipes/+/refs/heads/master/recipes/dart/flutter_engine.py
      'web_tool_tests': _runWebToolTests,
      'tool_integration_tests': _runIntegrationToolTests,
      'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
      // All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
      'web_tests': _runWebHtmlUnitTests,
      // All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
      'web_canvaskit_tests': _runWebCanvasKitUnitTests,
      // All web integration tests
      'web_long_running_tests': _runWebLongRunningTests,
263
      'flutter_plugins': _runFlutterPackagesTests,
264 265 266 267 268 269 270 271 272 273 274 275
      'skp_generator': _runSkpGeneratorTests,
      kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
    });
  } catch (error, stackTrace) {
    foundError(<String>[
      'UNEXPECTED ERROR!',
      error.toString(),
      ...stackTrace.toString().split('\n'),
      'The test.dart script should be corrected to catch this error and call foundError().',
      '${yellow}Some tests are likely to have been skipped.$reset',
    ]);
    system.exit(255);
276 277
  }
  if (hasError) {
278
    reportErrorsAndExit('${bold}Test failed.$reset');
279
  }
280
  reportSuccessAndExit('${bold}Test successful.$reset');
281 282
}

283
final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
284 285
final bool _runningInDartHHHBot =
    _luciBotId.startsWith('luci-dart-') || _luciBotId.startsWith('dart-tests-');
286

287 288 289
/// Verify the Flutter Engine is the revision in
/// bin/cache/internal/engine.version.
Future<void> _validateEngineHash() async {
290
  if (_runningInDartHHHBot) {
291 292 293 294
    // The Dart HHH bots intentionally modify the local artifact cache
    // and then use this script to run Flutter's test suites.
    // Because the artifacts have been changed, this particular test will return
    // a false positive and should be skipped.
295
    print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $_luciBotId.');
296 297 298
    return;
  }
  final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
299
  final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
  if (result.flattenedStdout!.isNotEmpty) {
    foundError(<String>[
      '${red}The stdout of `$flutterTester --help` was not empty:$reset',
      ...result.flattenedStdout!.split('\n').map((String line) => ' $gray$reset $line'),
    ]);
  }
  final String actualVersion;
  try {
    actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
      return line.startsWith('Flutter Engine Version:');
    });
  } on StateError {
    foundError(<String>[
      '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset',
      ...result.flattenedStderr!.split('\n').map((String line) => ' $gray$reset $line'),
    ]);
    return;
  }
318
  if (!actualVersion.contains(expectedVersion)) {
319
    foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']);
320 321 322
  }
}

323
Future<void> _runTestHarnessTests() async {
324
  printProgress('${green}Running test harness tests...$reset');
325 326 327

  await _validateEngineHash();

328 329
  // Verify that the tests actually return failure on failure and success on
  // success.
330
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
331

332
  // We want to run these tests in parallel, because they each take some time
333 334 335
  // to run (e.g. compiling), so we don't want to run them in series, especially
  // on 20-core machines. However, we have a race condition, so for now...
  // Race condition issue: https://github.com/flutter/flutter/issues/90026
336 337
  final List<ShardRunner> tests = <ShardRunner>[
    () => _runFlutterTest(
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
      automatedTests,
      script: path.join('test_smoke_test', 'pass_test.dart'),
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'fail_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
      expectFailure: true,
      printOutput: false,
      outputChecker: (CommandResult result) {
        return result.flattenedStdout!.contains('failingPendingTimerTest')
          ? null
          : 'Failed to find the stack trace for the pending Timer.\n\n'
            'stdout:\n${result.flattenedStdout}\n\n'
            'stderr:\n${result.flattenedStderr}';
    }),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash1_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash2_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
372
    () => _runFlutterTest(
373 374 375 376 377
      automatedTests,
      script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
378
    () => _runFlutterTest(
379 380 381 382 383
      automatedTests,
      script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
384
    () => _runFlutterTest(
385 386 387 388 389
      automatedTests,
      script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
390 391 392 393
  ];

  List<ShardRunner> testsToRun;

394
  // Run all tests unless sharding is explicitly specified.
395
  final String? shardName = Platform.environment[kShardKey];
396
  if (shardName == kTestHarnessShardName) {
397 398 399 400 401 402 403
    testsToRun = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  } else {
    testsToRun = tests;
  }
  for (final ShardRunner test in testsToRun) {
    await test();
  }
404

405
  // Verify that we correctly generated the version file.
406
  final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
407
  if (versionError != null) {
408
    foundError(<String>[versionError]);
409
  }
410 411
}

412 413
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');

414
Future<void> _runGeneralToolTests() async {
415
  await _runDartTest(
416
    _toolsPath,
417 418
    testPaths: <String>[path.join('test', 'general.shard')],
    enableFlutterToolAsserts: false,
419

420
    // Detect unit test time regressions (poor time delay handling, etc).
421 422
    // This overrides the 15 minute default for tools tests.
    // See the README.md and dart_test.yaml files in the flutter_tools package.
423 424 425 426 427
    perTestTimeout: const Duration(seconds: 2),
  );
}

Future<void> _runCommandsToolTests() async {
428
  await _runDartTest(
429
    _toolsPath,
430
    forceSingleCore: true,
431
    testPaths: <String>[path.join('test', 'commands.shard')],
432 433 434
  );
}

435
Future<void> _runWebToolTests() async {
436 437 438 439 440 441 442 443
  final List<File> allFiles = Directory(path.join(_toolsPath, 'test', 'web.shard'))
      .listSync(recursive: true).whereType<File>().toList();
  final List<String> allTests = <String>[];
  for (final File file in allFiles) {
    if (file.path.endsWith('_test.dart')) {
      allTests.add(file.path);
    }
  }
444
  await _runDartTest(
445
    _toolsPath,
446
    forceSingleCore: true,
447
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
448
    includeLocalEngineEnv: true,
449 450 451
  );
}

452
Future<void> _runToolHostCrossArchTests() {
453
  return _runDartTest(
454 455 456 457 458 459 460
    _toolsPath,
    // These are integration tests
    forceSingleCore: true,
    testPaths: <String>[path.join('test', 'host_cross_arch.shard')],
  );
}

461
Future<void> _runIntegrationToolTests() async {
462
  final List<String> allTests = Directory(path.join(_toolsPath, 'test', 'integration.shard'))
463
      .listSync(recursive: true).whereType<File>()
464
      .map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
465 466
      .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();

467
  await _runDartTest(
468
    _toolsPath,
469 470
    forceSingleCore: true,
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
471
    collectMetrics: true,
472 473 474
  );
}

475
Future<void> _runToolTests() async {
476 477 478 479
  await selectSubshard(<String, ShardRunner>{
    'general': _runGeneralToolTests,
    'commands': _runCommandsToolTests,
  });
480 481
}

482 483
Future<void> runForbiddenFromReleaseTests() async {
  // Build a release APK to get the snapshot json.
484
  final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.');
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
  final List<String> command = <String>[
    'build',
    'apk',
    '--target-platform',
    'android-arm64',
    '--release',
    '--analyze-size',
    '--code-size-directory',
    tempDirectory.path,
    '-v',
  ];

  await runCommand(
    flutter,
    command,
    workingDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
  );

  // First, a smoke test.
  final List<String> smokeTestArgs = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::Widget',
  ];
  await runCommand(
    dart,
    smokeTestArgs,
    workingDirectory: flutterRoot,
    expectNonZeroExit: true,
  );

  // Actual test.
  final List<String> args = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/widget_inspector.dart::WidgetInspectorService',
523 524
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::DebugCreator',
    '--forbidden-type', 'package:flutter/src/foundation/print.dart::debugPrint',
525 526 527 528 529 530 531 532
  ];
  await runCommand(
    dart,
    args,
    workingDirectory: flutterRoot,
  );
}

533 534 535 536 537
/// Verifies that APK, and IPA (if on macOS), and native desktop builds the
/// examples apps without crashing. It does not actually launch the apps. That
/// happens later in the devicelab. This is just a smoke-test. In particular,
/// this will verify we can build when there are spaces in the path name for the
/// Flutter SDK and target app.
538 539
///
/// Also does some checking about types included in hello_world.
540
Future<void> _runBuildTests() async {
541 542 543
  final List<Directory> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync()
    // API example builds will be tested in a separate shard.
    .where((FileSystemEntity entity) => entity is Directory && path.basename(entity.path) != 'api').cast<Directory>().toList()
544
    ..add(Directory(path.join(flutterRoot, 'packages', 'integration_test', 'example')))
545 546 547 548 549 550
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'channels')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'hybrid_android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_platform_view_tests')))
551
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_app_with_extensions')))
552
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
553 554
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'platform_interaction')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'spell_check')))
555
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
556

557 558 559
  // The tests are randomly distributed into subshards so as to get a uniform
  // distribution of costs, but the seed is fixed so that issues are reproducible.
  final List<ShardRunner> tests = <ShardRunner>[
560
    for (final Directory exampleDirectory in exampleDirectories)
561 562 563 564 565 566 567 568 569 570 571 572 573
      () => _runExampleProjectBuildTests(exampleDirectory),
    ...<ShardRunner>[
      // Web compilation tests.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web'),
            path.join('lib', 'main.dart'),
          ),
      // Should not fail to compile with dart:io.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web_compile_tests'),
            path.join('lib', 'dart_io_import.dart'),
          ),
    ],
574
    runForbiddenFromReleaseTests,
575 576
  ]..shuffle(math.Random(0));

577
  await _runShardRunnerIndexOfTotalSubshard(tests);
578 579
}

580
Future<void> _runExampleProjectBuildTests(Directory exampleDirectory, [File? mainFile]) async {
581 582
  // Only verify caching with flutter gallery.
  final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
583
  final String examplePath = path.relative(exampleDirectory.path, from: Directory.current.path);
584 585 586
  final List<String> additionalArgs = <String>[
    if (mainFile != null) path.relative(mainFile.path, from: exampleDirectory.absolute.path),
  ];
587 588 589 590 591 592 593 594 595 596 597 598 599
  if (Directory(path.join(examplePath, 'android')).existsSync()) {
    await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
  } else {
    print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
  }
  if (Platform.isMacOS) {
    if (Directory(path.join(examplePath, 'ios')).existsSync()) {
      await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
    }
600
  }
601
  if (Platform.isLinux) {
602 603 604 605 606 607 608
    if (Directory(path.join(examplePath, 'linux')).existsSync()) {
      await _flutterBuildLinux(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildLinux(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no linux directory, skipping Linux');
    }
  }
609
  if (Platform.isMacOS) {
610 611 612 613 614 615 616
    if (Directory(path.join(examplePath, 'macos')).existsSync()) {
      await _flutterBuildMacOS(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildMacOS(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no macos directory, skipping macOS');
    }
  }
617
  if (Platform.isWindows) {
618
    if (Directory(path.join(examplePath, 'windows')).existsSync()) {
619 620
      await _flutterBuildWin32(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildWin32(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
621 622 623 624
    } else {
      print('Example project ${path.basename(examplePath)} has no windows directory, skipping Win32');
    }
  }
625
}
626

627
Future<void> _flutterBuildApk(String relativePathToApplication, {
628
  required bool release,
629
  bool verifyCaching = false,
630 631
  List<String> additionalArgs = const <String>[],
}) async {
632
  printProgress('${green}Testing APK ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
633 634 635 636
  await _flutterBuild(relativePathToApplication, 'APK', 'apk',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
637
  );
638 639
}

640
Future<void> _flutterBuildIpa(String relativePathToApplication, {
641
  required bool release,
642
  List<String> additionalArgs = const <String>[],
643
  bool verifyCaching = false,
644
}) async {
645
  assert(Platform.isMacOS);
646
  printProgress('${green}Testing IPA ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
647 648 649 650 651 652 653
  await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: <String>[...additionalArgs, '--no-codesign'],
  );
}

654
Future<void> _flutterBuildLinux(String relativePathToApplication, {
655
  required bool release,
656 657 658 659 660
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isLinux);
  await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
661
  printProgress('${green}Testing Linux ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
662 663 664 665 666 667 668
  await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

669
Future<void> _flutterBuildMacOS(String relativePathToApplication, {
670
  required bool release,
671 672 673 674 675
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isMacOS);
  await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
676
  printProgress('${green}Testing macOS ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
677 678 679 680 681 682 683
  await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

684
Future<void> _flutterBuildWin32(String relativePathToApplication, {
685
  required bool release,
686 687 688 689
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isWindows);
690
  printProgress('${green}Testing ${release ? 'release' : 'debug'} Windows build$reset for $cyan$relativePathToApplication$reset...');
691 692 693 694 695 696 697
  await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

698 699 700 701
Future<void> _flutterBuild(
  String relativePathToApplication,
  String platformLabel,
  String platformBuildName, {
702
  required bool release,
703 704 705
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
706
  await runCommand(flutter,
707 708
    <String>[
      'build',
709
      platformBuildName,
710 711 712 713 714 715 716
      ...additionalArgs,
      if (release)
        '--release'
      else
        '--debug',
      '-v',
    ],
717 718
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
719

720
  if (verifyCaching) {
721
    printProgress('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
722 723 724
    await runCommand(flutter,
      <String>[
        'build',
725
        platformBuildName,
726 727 728 729 730 731 732 733 734 735 736 737
        '--performance-measurement-file=perf.json',
        ...additionalArgs,
        if (release)
          '--release'
        else
          '--debug',
        '-v',
      ],
      workingDirectory: path.join(flutterRoot, relativePathToApplication),
    );
    final File file = File(path.join(flutterRoot, relativePathToApplication, 'perf.json'));
    if (!_allTargetsCached(file)) {
738 739 740 741
      foundError(<String>[
        '${red}Not all build targets cached after second run.$reset',
        'The target performance data was: ${file.readAsStringSync().replaceAll('},', '},\n')}',
      ]);
742 743 744 745 746
    }
  }
}

bool _allTargetsCached(File performanceFile) {
747 748 749 750 751
  final Map<String, Object?> data = json.decode(performanceFile.readAsStringSync())
    as Map<String, Object?>;
  final List<Map<String, Object?>> targets = (data['targets']! as List<Object?>)
    .cast<Map<String, Object?>>();
  return targets.every((Map<String, Object?> element) => element['skipped'] == true);
752 753
}

754
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
755
  printProgress('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
756 757 758 759 760 761 762
  await runCommand(flutter,
    <String>['build', 'web', '-v', '--target=$target'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: expectNonZeroExit,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
763
  );
764 765
}

766 767
Future<void> _runAddToAppLifeCycleTests() async {
  if (Platform.isMacOS) {
768
    printProgress('${green}Running add-to-app life cycle iOS integration tests$reset...');
769 770 771 772 773
    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
    await runCommand('./build_and_test.sh',
      <String>[],
      workingDirectory: addToAppDir,
    );
774 775
  } else {
    printProgress('${yellow}Skipped on this platform (only iOS has add-to-add lifecycle tests at this time).$reset');
776 777 778
  }
}

779
Future<void> _runFrameworkTests() async {
780
  final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
781

782
  Future<void> runWidgets() async {
783
    printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
784 785 786
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
787
        options: <String>[trackWidgetCreationOption],
788 789 790
        tests: <String>[ path.join('test', 'widgets') + path.separator ],
      );
    }
791
    // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
792 793 794 795
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
        options: <String>[trackWidgetCreationOption],
796
        fatalWarnings: false, // until we've migrated video_player
797 798
      );
    }
799 800 801
    // Run release mode tests (see packages/flutter/test_release/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
802
      options: <String>['--dart-define=dart.vm.product=true'],
803
      tests: <String>['test_release${path.separator}'],
804
    );
805 806 807
    // Run profile mode tests (see packages/flutter/test_profile/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
808
      options: <String>['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true'],
809 810
      tests: <String>['test_profile${path.separator}'],
    );
811 812
  }

813
  Future<void> runLibraries() async {
814
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
815
      .listSync(followLinks: false)
816
      .whereType<Directory>()
817
      .where((Directory dir) => !dir.path.endsWith('widgets'))
818
      .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
819
      .toList();
820
    printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
821 822 823
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
824
        options: <String>[trackWidgetCreationOption],
825 826 827
        tests: tests,
      );
    }
828 829
  }

830 831 832 833 834 835 836 837 838 839 840 841 842 843
  Future<void> runExampleTests() async {
    // TODO(gspencergoog): Currently Linux LUCI bots can't run desktop Flutter applications, https://github.com/flutter/flutter/issues/90676
    if (!Platform.isLinux || ciProvider != CiProviders.luci) {
      await runCommand(
        flutter,
        <String>['config', '--enable-${Platform.operatingSystem}-desktop'],
        workingDirectory: flutterRoot,
      );
      await runCommand(
        dart,
        <String>[path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')],
        workingDirectory: path.join(flutterRoot, 'examples', 'api'),
      );
    }
844 845 846
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'api'));
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
847 848
  }

Ian Hickson's avatar
Ian Hickson committed
849 850 851 852 853 854 855 856 857 858 859 860
  Future<void> runTracingTests() async {
    final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests');

    // run the tests for debug mode
    await _runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);

    Future<List<String>> verifyTracingAppBuild({
      required String modeArgument,
      required String sourceFile,
      required Set<String> allowed,
      required Set<String> disallowed,
    }) async {
861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
      try {
        await runCommand(
          flutter,
          <String>[
            'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile),
          ],
          workingDirectory: tracingDirectory,
        );
        final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync());
        final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
        final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
        final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
        await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
        final List<String> results = <String>[];
        for (final String pattern in allowed) {
          if (!libappStrings.contains(pattern)) {
            results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.');
          }
Ian Hickson's avatar
Ian Hickson committed
879
        }
880 881 882 883
        for (final String pattern in disallowed) {
          if (libappStrings.contains(pattern)) {
            results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.');
          }
Ian Hickson's avatar
Ian Hickson committed
884
        }
885 886 887 888 889 890
        return results;
      } catch (error, stackTrace) {
        return <String>[
          error.toString(),
          ...stackTrace.toString().trimRight().split('\n'),
        ];
Ian Hickson's avatar
Ian Hickson committed
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
      }
    }

    final List<String> results = <String>[];
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'profile',
      sourceFile: 'control.dart', // this is the control, the other two below are the actual test
      allowed: <String>{
        'TIMELINE ARGUMENTS TEST CONTROL FILE',
        'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
      },
    ));
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'profile',
      sourceFile: 'test.dart',
      allowed: <String>{
        'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls
        'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds
        // (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort)
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
        'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
        'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker
      },
    ));
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'release',
      sourceFile: 'test.dart',
      allowed: <String>{
        'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE',
        'BUILD', 'LAYOUT', 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds
        'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
        'toTimelineArguments used in non-debug build', // not included in release builds
      },
    ));
    if (results.isNotEmpty) {
934
      foundError(results);
Ian Hickson's avatar
Ian Hickson committed
935 936 937
    }
  }

938
  Future<void> runFixTests(String package) async {
939 940 941 942 943 944 945
    final List<String> args = <String>[
      'fix',
      '--compare-to-golden',
    ];
    await runCommand(
      dart,
      args,
946
      workingDirectory: path.join(flutterRoot, 'packages', package, 'test_fixes'),
947 948 949
    );
  }

950 951
  Future<void> runPrivateTests() async {
    final List<String> args = <String>[
952 953
      'run',
      'bin/test_private.dart',
954
    ];
955
    final Map<String, String> environment = <String, String>{
956
      'FLUTTER_ROOT': flutterRoot,
957 958
      if (Directory(pubCache).existsSync())
        'PUB_CACHE': pubCache,
959
    };
960
    adjustEnvironmentToEnableFlutterAsserts(environment);
961
    await runCommand(
962
      dart,
963 964
      args,
      workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'),
965
      environment: environment,
966 967 968
    );
  }

969 970 971 972 973 974 975
  // Tests that take longer than average to run. This is usually because they
  // need to compile something large or make use of the analyzer for the test.
  // These tests need to be platform agnostic as they are only run on a linux
  // machine to save on execution time and cost.
  Future<void> runSlow() async {
    printProgress('${green}Running slow package tests$reset for directories other than packages/flutter');
    await runTracingTests();
976 977
    await runFixTests('flutter');
    await runFixTests('flutter_test');
978
    await runFixTests('integration_test');
979 980 981
    await runPrivateTests();
  }

982
  Future<void> runMisc() async {
983
    printProgress('${green}Running package tests$reset for directories other than packages/flutter');
984
    await _runTestHarnessTests();
985
    await runExampleTests();
986 987 988 989
    await _runFlutterTest(
      path.join(flutterRoot, 'dev', 'a11y_assessments'),
      tests: <String>[ 'test' ],
    );
990 991 992
    await _runDartTest(path.join(flutterRoot, 'dev', 'bots'));
    await _runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
    await _runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
993
    // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed.
994
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
995
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
996 997
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
998
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
999
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
1000
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
1001
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')]);
1002 1003 1004 1005 1006
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: <String>[
      '--enable-vmservice',
      // Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard.
      '--exclude-tags=web',
    ]);
1007 1008 1009 1010 1011
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'));
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'));
1012
    const String httpClientWarning =
1013 1014 1015 1016 1017
      'Warning: At least one test in this suite creates an HttpClient. When running a test suite that uses\n'
      'TestWidgetsFlutterBinding, all HTTP requests will return status code 400, and no network request\n'
      'will actually be made. Any test expecting a real network connection and status code will fail.\n'
      'To test code that needs an HttpClient, provide your own HttpClient implementation to the code under\n'
      'test, so that your test can consistently provide a testable response to the code under test.';
1018 1019 1020 1021 1022
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter_test'),
      script: path.join('test', 'bindings_test_failure.dart'),
      expectFailure: true,
      printOutput: false,
1023
      outputChecker: (CommandResult result) {
1024
        final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout!);
1025
        if (matches.isEmpty || matches.length > 1) {
1026 1027 1028
          return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n'
                 'stdout:\n${result.flattenedStdout}\n\n'
                 'stderr:\n${result.flattenedStderr}';
1029 1030 1031 1032
        }
        return null;
      },
    );
1033
  }
1034

1035 1036 1037
  await selectSubshard(<String, ShardRunner>{
    'widgets': runWidgets,
    'libraries': runLibraries,
1038
    'slow': runSlow,
1039 1040
    'misc': runMisc,
  });
1041 1042
}

1043
Future<void> _runFrameworkCoverage() async {
1044 1045
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
1046 1047
    foundError(<String>[
      '${red}Coverage file not found.$reset',
1048
      'Expected to find: $cyan${coverageFile.absolute.path}$reset',
1049 1050 1051
      'This file is normally obtained by running `${green}flutter update-packages$reset`.',
    ]);
    return;
1052 1053 1054 1055 1056 1057
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
1058 1059
    foundError(<String>[
      '${red}Coverage file not found.$reset',
1060
      'Expected to find: $cyan${coverageFile.absolute.path}$reset',
1061 1062 1063
      'This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.',
    ]);
    return;
1064
  }
1065 1066
}

1067 1068 1069 1070 1071 1072 1073 1074 1075
Future<void> _runWebHtmlUnitTests() {
  return _runWebUnitTests('html');
}

Future<void> _runWebCanvasKitUnitTests() {
  return _runWebUnitTests('canvaskit');
}

Future<void> _runWebUnitTests(String webRenderer) async {
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};

  final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));

  final List<String> allTests = flutterPackageTestDirectory
    .listSync()
    .whereType<Directory>()
    .expand((Directory directory) => directory
      .listSync(recursive: true)
      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
    )
    .whereType<File>()
    .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
1090
    .where((String filePath) => !kWebTestFileKnownFailures[webRenderer]!.contains(path.split(filePath).join('/')))
1091 1092 1093 1094 1095 1096 1097
    .toList()
    // Finally we shuffle the list because we want the average cost per file to be uniformly
    // distributed. If the list is not sorted then different shards and batches may have
    // very different characteristics.
    // We use a constant seed for repeatability.
    ..shuffle(math.Random(0));

1098 1099 1100
  assert(webShardCount >= 1);
  final int testsPerShard = (allTests.length / webShardCount).ceil();
  assert(testsPerShard * webShardCount >= allTests.length);
1101 1102

  // This for loop computes all but the last shard.
1103
  for (int index = 0; index < webShardCount - 1; index += 1) {
1104
    subshards['$index'] = () => _runFlutterWebTest(
1105
      webRenderer,
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
      flutterPackageDirectory.path,
      allTests.sublist(
        index * testsPerShard,
        (index + 1) * testsPerShard,
      ),
    );
  }

  // The last shard also runs the flutter_web_plugins tests.
  //
  // We make sure the last shard ends in _last so it's easier to catch mismatches
  // between `.cirrus.yml` and `test.dart`.
1118
  subshards['${webShardCount - 1}_last'] = () async {
1119
    await _runFlutterWebTest(
1120
      webRenderer,
1121 1122
      flutterPackageDirectory.path,
      allTests.sublist(
1123
        (webShardCount - 1) * testsPerShard,
1124 1125 1126 1127
        allTests.length,
      ),
    );
    await _runFlutterWebTest(
1128
      webRenderer,
1129 1130 1131
      path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
      <String>['test'],
    );
1132
    await _runFlutterWebTest(
1133 1134 1135
      webRenderer,
      path.join(flutterRoot, 'packages', 'flutter_driver'),
      <String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
1136
    );
1137 1138 1139 1140 1141
  };

  await selectSubshard(subshards);
}

1142 1143
/// Coarse-grained integration tests running on the Web.
Future<void> _runWebLongRunningTests() async {
1144
  final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
1145 1146 1147 1148
  final String engineRealm = File(engineRealmFile).readAsStringSync().trim();
  if (engineRealm.isNotEmpty) {
    return;
  }
1149
  final List<ShardRunner> tests = <ShardRunner>[
1150
    for (final String buildMode in _kAllBuildModes) ...<ShardRunner>[
1151 1152 1153 1154 1155 1156 1157 1158 1159
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('test_driver', 'failure.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
        // This test intentionally fails and prints stack traces in the browser
        // logs. To avoid confusion, silence browser output.
        silenceBrowserOutput: true,
      ),
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('integration_test', 'example_test.dart'),
        driver: path.join('test_driver', 'integration_test.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
      ),
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('integration_test', 'extended_test.dart'),
        driver: path.join('test_driver', 'extended_integration_test.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
      ),
    ],
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),

    // This test is only known to work in debug mode.
    () => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),

1199 1200 1201 1202 1203
    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'debug', renderer: 'auto'),
    () => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('capabilities_integration_html', buildMode: 'release', renderer: 'html'),

1204 1205 1206 1207 1208
    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    // CacheWidth and CacheHeight are only currently supported in CanvasKit mode, so we don't run the test in HTML mode.
    () => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'debug', renderer: 'auto'),
    () => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'profile', renderer: 'canvaskit'),

1209 1210
    () => _runWebTreeshakeTest(),

1211 1212 1213 1214
    () => _runFlutterDriverWebTest(
      testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
      target: 'test_driver/smoke_web_engine.dart',
      buildMode: 'profile',
1215
      renderer: 'auto',
1216
    ),
1217 1218 1219 1220 1221 1222
    () => _runGalleryE2eWebTest('debug'),
    () => _runGalleryE2eWebTest('debug', canvasKit: true),
    () => _runGalleryE2eWebTest('profile'),
    () => _runGalleryE2eWebTest('profile', canvasKit: true),
    () => _runGalleryE2eWebTest('release'),
    () => _runGalleryE2eWebTest('release', canvasKit: true),
1223 1224 1225
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
1226
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
1227
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
1228 1229 1230
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
1231
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
1232
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
1233
    () => runWebServiceWorkerTestWithGeneratedEntrypoint(headless: true),
1234
    () => runWebServiceWorkerTestWithBlockedServiceWorkers(headless: true),
1235 1236 1237 1238 1239 1240 1241
    () => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/stack_trace.dart'),
    () => _runWebDebugTest('lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/web_directory_loading.dart'),
1242 1243 1244 1245
    () => _runWebDebugTest('lib/web_resources_cdn_test.dart',
      additionalArguments: <String>[
        '--dart-define=TEST_FLUTTER_ENGINE_VERSION=$engineVersion',
      ]),
1246
    () => _runWebDebugTest('test/test.dart'),
1247
    () => _runWebDebugTest('lib/null_safe_main.dart'),
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
    () => _runWebDebugTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
    () => _runWebReleaseTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
1260 1261
    () => _runWebDebugTest('lib/sound_mode.dart'),
    () => _runWebReleaseTest('lib/sound_mode.dart'),
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
    () => _runFlutterWebTest(
      'html',
      path.join(flutterRoot, 'packages', 'integration_test'),
      <String>['test/web_extension_test.dart'],
    ),
    () => _runFlutterWebTest(
      'canvaskit',
      path.join(flutterRoot, 'packages', 'integration_test'),
      <String>['test/web_extension_test.dart'],
    ),
1272
  ];
1273 1274 1275 1276 1277

  // Shuffling mixes fast tests with slow tests so shards take roughly the same
  // amount of time to run.
  tests.shuffle(math.Random(0));

1278
  await _ensureChromeDriverIsRunning();
1279
  await _runShardRunnerIndexOfTotalSubshard(tests);
1280
  await _stopChromeDriver();
1281 1282
}

1283 1284 1285
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
Future<void> _runWebE2eTest(
  String name, {
1286 1287
  required String buildMode,
  required String renderer,
1288 1289 1290 1291 1292 1293 1294 1295 1296
}) async {
  await _runFlutterDriverWebTest(
    target: path.join('test_driver', '$name.dart'),
    buildMode: buildMode,
    renderer: renderer,
    testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
  );
}

1297
Future<void> _runFlutterDriverWebTest({
1298 1299 1300 1301
  required String target,
  required String buildMode,
  required String renderer,
  required String testAppDirectory,
1302
  String? driver,
1303 1304
  bool expectFailure = false,
  bool silenceBrowserOutput = false,
1305
}) async {
1306
  printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
1307 1308 1309 1310 1311 1312 1313 1314
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1315
      ...flutterTestArgs,
1316
      'drive',
1317
      if (driver != null) '--driver=$driver',
1318 1319 1320 1321 1322
      '--target=$target',
      '--browser-name=chrome',
      '-d',
      'web-server',
      '--$buildMode',
1323
      '--web-renderer=$renderer',
1324
    ],
1325
    expectNonZeroExit: expectFailure,
1326 1327 1328 1329
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
1330 1331 1332 1333 1334 1335 1336 1337 1338
    removeLine: (String line) {
      if (!silenceBrowserOutput) {
        return false;
      }
      if (line.trim().startsWith('[INFO]')) {
        return true;
      }
      return false;
    },
1339 1340 1341
  );
}

1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
// Compiles a sample web app and checks that its JS doesn't contain certain
// debug code that we expect to be tree shaken out.
//
// The app is compiled in `--profile` mode to prevent the compiler from
// minifying the symbols.
Future<void> _runWebTreeshakeTest() async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
  final String target = path.join('lib', 'treeshaking_main.dart');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--target=$target',
      '--profile',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
  final String javaScript = mainDartJs.readAsStringSync();

  // Check that we're not looking at minified JS. Otherwise this test would result in false positive.
1373
  expect(javaScript.contains('RootElement'), true);
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387

  const String word = 'debugFillProperties';
  int count = 0;
  int pos = javaScript.indexOf(word);
  final int contentLength = javaScript.length;
  while (pos != -1) {
    count += 1;
    pos += word.length;
    if (pos >= contentLength || count > 100) {
      break;
    }
    pos = javaScript.indexOf(word, pos);
  }

1388 1389 1390 1391 1392 1393 1394 1395
  // The following are classes from `timeline.dart` that should be treeshaken
  // off unless the app (typically a benchmark) uses methods that need them.
  expect(javaScript.contains('AggregatedTimedBlock'), false);
  expect(javaScript.contains('AggregatedTimings'), false);
  expect(javaScript.contains('_BlockBuffer'), false);
  expect(javaScript.contains('_StringListChain'), false);
  expect(javaScript.contains('_Float64ListChain'), false);

1396 1397 1398 1399 1400 1401 1402 1403 1404
  const int kMaxExpectedDebugFillProperties = 11;
  if (count > kMaxExpectedDebugFillProperties) {
    throw Exception(
      'Too many occurrences of "$word" in compiled JavaScript.\n'
      'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
    );
  }
}

1405
/// Returns the commit hash of the flutter/packages repository that's rolled in.
1406
///
1407
/// The flutter/packages repository is a downstream dependency, it is only used
1408
/// by flutter/flutter for testing purposes, to assure stable tests for a given
1409 1410
/// flutter commit the flutter/packages commit hash to test against is coded in
/// the bin/internal/flutter_packages.version file.
1411
///
1412 1413 1414 1415
/// The `filesystem` parameter specified filesystem to read the packages version file from.
/// The `packagesVersionFile` parameter allows specifying an alternative path for the
/// packages version file, when null [flutterPackagesVersionFile] is used.
Future<String> getFlutterPackagesVersion({
1416
  fs.FileSystem fileSystem = const LocalFileSystem(),
1417
  String? packagesVersionFile,
1418
}) async {
1419
  final File versionFile = fileSystem.file(packagesVersionFile ?? flutterPackagesVersionFile);
1420 1421 1422 1423
  final String versionFileContents = await versionFile.readAsString();
  return versionFileContents.trim();
}

1424 1425
/// Executes the test suite for the flutter/packages repo.
Future<void> _runFlutterPackagesTests() async {
1426
  Future<void> runAnalyze() async {
1427 1428
    printProgress('${green}Running analysis for flutter/packages$reset');
    final Directory checkout = Directory.systemTemp.createTempSync('flutter_packages.');
1429 1430 1431 1432 1433 1434
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'clone',
1435
        'https://github.com/flutter/packages.git',
1436
        '.',
1437 1438 1439
      ],
      workingDirectory: checkout.path,
    );
1440
    final String packagesCommit = await getFlutterPackagesVersion();
1441 1442 1443 1444 1445 1446
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'checkout',
1447
        packagesCommit,
1448 1449 1450
      ],
      workingDirectory: checkout.path,
    );
1451 1452
    // Prep the repository tooling.
    // This test does not use tool_runner.sh because in this context the test
1453 1454
    // should always run on the entire packages repo, while tool_runner.sh
    // is designed for flutter/packages CI and only analyzes changed repository
1455 1456
    // files when run for anything but master.
    final String toolDir = path.join(checkout.path, 'script', 'tool');
1457
    await runCommand(
1458
      'dart',
1459
      <String>[
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
        'pub',
        'get',
      ],
      workingDirectory: toolDir,
    );
    final String toolScript = path.join(toolDir, 'bin', 'flutter_plugin_tools.dart');
    await runCommand(
      'dart',
      <String>[
        'run',
        toolScript,
1471
        'analyze',
1472 1473 1474 1475 1476 1477 1478
        // Fetch the oldest possible dependencies, rather than the newest, to
        // insulate flutter/flutter from out-of-band failures when new versions
        // of dependencies are published. This compensates for the fact that
        // flutter/packages doesn't use pinned dependencies, and for the
        // purposes of this test using old dependencies is fine. See
        // https://github.com/flutter/flutter/issues/129633
        '--downgrade',
1479
        '--custom-analysis=script/configs/custom_analysis.yaml',
1480 1481 1482 1483 1484 1485 1486 1487 1488
      ],
      workingDirectory: checkout.path,
    );
  }
  await selectSubshard(<String, ShardRunner>{
    'analyze': runAnalyze,
  });
}

1489 1490 1491 1492 1493 1494
/// Runs the skp_generator from the flutter/tests repo.
///
/// See also the customer_tests shard.
///
/// Generated SKPs are ditched, this just verifies that it can run without failure.
Future<void> _runSkpGeneratorTests() async {
1495
  printProgress('${green}Running skp_generator from flutter/tests$reset');
1496 1497 1498 1499 1500 1501 1502 1503
  final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.');
  await runCommand(
    'git',
    <String>[
      '-c',
      'core.longPaths=true',
      'clone',
      'https://github.com/flutter/tests.git',
1504
      '.',
1505 1506 1507 1508 1509 1510 1511 1512 1513 1514
    ],
    workingDirectory: checkout.path,
  );
  await runCommand(
    './build.sh',
    <String>[ ],
    workingDirectory: path.join(checkout.path, 'skp_generator'),
  );
}

1515 1516 1517 1518
// The `chromedriver` process created by this test.
//
// If an existing chromedriver is already available on port 4444, the existing
// process is reused and this variable remains null.
1519
Command? _chromeDriver;
1520 1521 1522

Future<bool> _isChromeDriverRunning() async {
  try {
1523 1524 1525
    final RawSocket socket = await RawSocket.connect('localhost', 4444);
    socket.shutdown(SocketDirection.both);
    await socket.close();
1526 1527 1528 1529 1530 1531 1532 1533 1534
    return true;
  } on SocketException {
    return false;
  }
}

Future<void> _ensureChromeDriverIsRunning() async {
  // If we cannot connect to ChromeDriver, assume it is not running. Launch it.
  if (!await _isChromeDriverRunning()) {
1535
    printProgress('Starting chromedriver');
1536 1537
    // Assume chromedriver is in the PATH.
    _chromeDriver = await startCommand(
1538 1539 1540
      // TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
      // and it doesn't use most of startCommand's features; we could simplify this a lot by
      // inlining the relevant parts of startCommand here.
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
      'chromedriver',
      <String>['--port=4444'],
    );
    while (!await _isChromeDriverRunning()) {
      await Future<void>.delayed(const Duration(milliseconds: 100));
      print('Waiting for chromedriver to start up.');
    }
  }

  final HttpClient client = HttpClient();
  final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
  final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
  final HttpClientResponse response = await request.close();
1554
  final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join()) as Map<String, dynamic>;
1555
  client.close();
1556
  final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
1557 1558 1559 1560 1561 1562 1563 1564 1565
  if (!webDriverReady) {
    throw Exception('WebDriver not available.');
  }
}

Future<void> _stopChromeDriver() async {
  if (_chromeDriver == null) {
    return;
  }
1566
  print('Stopping chromedriver');
1567
  _chromeDriver!.process.kill();
1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
}

/// Exercises the old gallery in a browser for a long period of time, looking
/// for memory leaks and dangling pointers.
///
/// This is not a performance test.
///
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
///
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
1580
  printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
1581 1582 1583 1584 1585 1586 1587 1588 1589
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1590
      ...flutterTestArgs,
1591
      'drive',
1592 1593
      if (canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=true',
1594 1595
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1596 1597
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
      '--driver=test_driver/transitions_perf_e2e_test.dart',
      '--target=test_driver/transitions_perf_e2e.dart',
      '--browser-name=chrome',
      '-d',
      'web-server',
      '--$buildMode',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
}

1612
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
1613
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
1614
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--$buildMode',
      '-t',
1629
      entrypoint,
1630 1631 1632 1633 1634 1635 1636 1637
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1638 1639
  final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
  final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
1640
  final String result = await evalTestAppInChrome(
1641
    appUrl: 'http://localhost:$serverPort/index.html',
1642
    appDirectory: appBuildDirectory,
1643 1644
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1645 1646
  );

1647 1648 1649 1650 1651
  if (!result.contains('--- TEST SUCCEEDED ---')) {
    foundError(<String>[
      result,
      '${red}Web stack trace integration test failed.$reset',
    ]);
1652 1653 1654
  }
}

1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
/// Run a web integration test in release mode.
Future<void> _runWebReleaseTest(String target, {
  List<String> additionalArguments = const<String>[],
}) async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1671
      ...flutterTestArgs,
1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
      'build',
      'web',
      '--release',
      ...additionalArguments,
      '-t',
      target,
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1686 1687
  final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
  final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
1688
  final String result = await evalTestAppInChrome(
1689
    appUrl: 'http://localhost:$serverPort/index.html',
1690
    appDirectory: appBuildDirectory,
1691 1692
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1693 1694
  );

1695 1696 1697 1698 1699
  if (!result.contains('--- TEST SUCCEEDED ---')) {
    foundError(<String>[
      result,
      '${red}Web release mode test failed.$reset',
    ]);
1700 1701 1702
  }
}

1703 1704 1705
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
///
/// Instead, we use `flutter run --debug` and sniff out the standard output.
1706 1707 1708
Future<void> _runWebDebugTest(String target, {
  List<String> additionalArguments = const<String>[],
}) async {
1709 1710
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  bool success = false;
1711 1712 1713
  final Map<String, String> environment = <String, String>{
    'FLUTTER_WEB': 'true',
  };
1714
  adjustEnvironmentToEnableFlutterAsserts(environment);
1715
  final CommandResult result = await runCommand(
1716 1717 1718 1719 1720 1721 1722
    flutter,
    <String>[
      'run',
      '--debug',
      '-d',
      'chrome',
      '--web-run-headless',
1723
      '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1724
      '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1725
      ...additionalArguments,
1726 1727
      '-t',
      target,
1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
    ],
    outputMode: OutputMode.capture,
    outputListener: (String line, Process process) {
      if (line.contains('--- TEST SUCCEEDED ---')) {
        success = true;
      }
      if (success || line.contains('--- TEST FAILED ---')) {
        process.stdin.add('q'.codeUnits);
      }
    },
    workingDirectory: testAppDirectory,
1739
    environment: environment,
1740 1741
  );

1742 1743 1744 1745 1746 1747
  if (!success) {
    foundError(<String>[
      result.flattenedStdout!,
      result.flattenedStderr!,
      '${red}Web stack trace integration test failed.$reset',
    ]);
1748 1749 1750
  }
}

1751
Future<void> _runFlutterWebTest(String webRenderer, String workingDirectory, List<String> tests) async {
1752 1753 1754 1755 1756 1757 1758 1759
  await runCommand(
    flutter,
    <String>[
      'test',
      if (ciProvider == CiProviders.cirrus)
        '--concurrency=1',  // do not parallelize on Cirrus, to reduce flakiness
      '-v',
      '--platform=chrome',
1760
      '--web-renderer=$webRenderer',
1761
      '--dart-define=DART_HHH_BOT=$_runningInDartHHHBot',
1762
      ...flutterTestArgs,
1763 1764 1765 1766 1767 1768 1769
      ...tests,
    ],
    workingDirectory: workingDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
1770 1771
}

1772 1773 1774 1775 1776
// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
1777
Future<void> _runDartTest(String workingDirectory, {
1778
  List<String>? testPaths,
1779 1780
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
1781
  String? coverage,
1782
  bool forceSingleCore = false,
1783
  Duration? perTestTimeout,
1784
  bool includeLocalEngineEnv = false,
1785
  bool ensurePrecompiledTool = true,
1786
  bool shuffleTests = true,
1787
  bool collectMetrics = false,
Dan Field's avatar
Dan Field committed
1788
}) async {
1789 1790
  int? cpus;
  final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
1791 1792 1793
  if (cpuVariable != null) {
    cpus = int.tryParse(cpuVariable, radix: 10);
    if (cpus == null) {
1794 1795 1796 1797 1798
      foundError(<String>[
        '${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset',
        'Actual value: "$cpuVariable"',
      ]);
      return;
1799 1800 1801 1802
    }
  } else {
    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
  }
1803 1804 1805 1806 1807
  // Integration tests that depend on external processes like chrome
  // can get stuck if there are multiple instances running at once.
  if (forceSingleCore) {
    cpus = 1;
  }
1808

1809 1810
  const LocalFileSystem fileSystem = LocalFileSystem();
  final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json'));
1811 1812 1813
  final List<String> args = <String>[
    'run',
    'test',
1814
    '--file-reporter=json:${metricFile.path}',
1815
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1816 1817 1818 1819 1820
    '-j$cpus',
    if (!hasColor)
      '--no-color',
    if (coverage != null)
      '--coverage=$coverage',
1821
    if (perTestTimeout != null)
1822
      '--timeout=${perTestTimeout.inMilliseconds}ms',
1823 1824 1825 1826
    if (testPaths != null)
      for (final String testPath in testPaths)
        testPath,
  ];
1827
  final Map<String, String> environment = <String, String>{
1828
    'FLUTTER_ROOT': flutterRoot,
1829 1830 1831 1832
    if (includeLocalEngineEnv)
      ...localEngineEnv,
    if (Directory(pubCache).existsSync())
      'PUB_CACHE': pubCache,
1833 1834
  };
  if (enableFlutterToolAsserts) {
1835
    adjustEnvironmentToEnableFlutterAsserts(environment);
1836
  }
1837 1838 1839 1840
  if (ensurePrecompiledTool) {
    // We rerun the `flutter` tool here just to make sure that it is compiled
    // before tests run, because the tests might time out if they have to rebuild
    // the tool themselves.
1841
    await runCommand(flutter, <String>['--version'], environment: environment);
1842
  }
1843 1844 1845 1846 1847 1848 1849
  await runCommand(
    dart,
    args,
    workingDirectory: workingDirectory,
    environment: environment,
    removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
  );
1850

1851 1852 1853 1854
  final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
  final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
  info.writeAsStringSync(json.encode(test.errors));

1855 1856 1857
  if (collectMetrics) {
    try {
      final List<String> testList = <String>[];
1858
      final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
1859 1860 1861 1862 1863
      for (final TestSpecs testSpecs in allTestSpecs.values) {
        testList.add(testSpecs.toJson());
      }
      if (testList.isNotEmpty) {
        final String testJson = json.encode(testList);
1864 1865
        final File testResults = fileSystem.file(
            path.join(flutterRoot, 'test_results.json'));
1866 1867
        testResults.writeAsStringSync(testJson);
      }
1868
    } on fs.FileSystemException catch (e) {
1869 1870 1871
      print('Failed to generate metrics: $e');
    }
  }
1872 1873
}

1874
Future<void> _runFlutterTest(String workingDirectory, {
1875
  String? script,
1876 1877
  bool expectFailure = false,
  bool printOutput = true,
1878
  OutputChecker? outputChecker,
1879
  List<String> options = const <String>[],
1880
  Map<String, String>? environment,
1881
  List<String> tests = const <String>[],
1882
  bool shuffleTests = true,
1883
  bool fatalWarnings = true,
Dan Field's avatar
Dan Field committed
1884
}) async {
1885
  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
1886

1887
  final List<String> tags = <String>[];
1888
  // Recipe-configured reduced test shards will only execute tests with the
1889
  // appropriate tag.
1890
  if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
1891 1892 1893
    tags.addAll(<String>['-t', 'reduced-test-set']);
  }

1894 1895
  final List<String> args = <String>[
    'test',
1896
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1897
    if (fatalWarnings) '--fatal-warnings',
1898
    ...options,
1899
    ...tags,
1900
    ...flutterTestArgs,
1901
  ];
Dan Field's avatar
Dan Field committed
1902

1903 1904 1905
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
1906 1907 1908 1909 1910 1911 1912 1913
      foundError(<String>[
        '${red}Could not find test$reset: $green$fullScriptPath$reset',
        'Working directory: $cyan$workingDirectory$reset',
        'Script: $green$script$reset',
        if (!printOutput)
          'This is one of the tests that does not normally print output.',
      ]);
      return;
1914
    }
1915
    args.add(script);
1916
  }
1917

1918
  args.addAll(tests);
1919

1920 1921 1922
  final OutputMode outputMode = outputChecker == null && printOutput
    ? OutputMode.print
    : OutputMode.capture;
1923

1924 1925 1926 1927 1928 1929 1930 1931
  final CommandResult result = await runCommand(
    flutter,
    args,
    workingDirectory: workingDirectory,
    expectNonZeroExit: expectFailure,
    outputMode: outputMode,
    environment: environment,
  );
1932

1933 1934 1935 1936
  if (outputChecker != null) {
    final String? message = outputChecker(result);
    if (message != null) {
      foundError(<String>[message]);
1937
    }
1938
  }
1939 1940
}

1941 1942 1943
/// This will force the next run of the Flutter tool (if it uses the provided
/// environment) to have asserts enabled, by setting an environment variable.
void adjustEnvironmentToEnableFlutterAsserts(Map<String, String> environment) {
1944 1945 1946 1947 1948 1949
  // If an existing env variable exists append to it, but only if
  // it doesn't appear to already include enable-asserts.
  String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
  if (!toolsArgs.contains('--enable-asserts')) {
    toolsArgs += ' --enable-asserts';
  }
1950
  environment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
1951 1952
}

1953 1954 1955 1956 1957
enum CiProviders {
  cirrus,
  luci,
}

1958
CiProviders? get ciProvider {
1959 1960 1961 1962 1963 1964 1965 1966
  if (Platform.environment['CIRRUS_CI'] == 'true') {
    return CiProviders.cirrus;
  }
  if (Platform.environment['LUCI_CONTEXT'] != null) {
    return CiProviders.luci;
  }
  return null;
}
1967

1968 1969 1970 1971 1972
/// Checks the given file's contents to determine if they match the allowed
/// pattern for version strings.
///
/// Returns null if the contents are good. Returns a string if they are bad.
/// The string is an error message.
1973
Future<String?> verifyVersion(File file) async {
1974 1975
  final RegExp pattern = RegExp(
    r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
1976
  if (!file.existsSync()) {
1977
    return 'The version logic failed to create the Flutter version file.';
1978
  }
1979
  final String version = await file.readAsString();
1980
  if (version == '0.0.0-unknown') {
1981
    return 'The version logic failed to determine the Flutter version.';
1982 1983
  }
  if (!version.contains(pattern)) {
1984
    return 'The version logic generated an invalid version string: "$version".';
1985
  }
1986 1987 1988
  return null;
}

1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
  // Example: "1_3" means the first (one-indexed) shard of three total shards.
2001
  final String? subshardName = Platform.environment[subshardKey];
2002 2003 2004 2005
  if (subshardName == null) {
    print('$kSubshardKey environment variable is missing, skipping sharding');
    return tests;
  }
2006
  printProgress('$bold$subshardKey=$subshardName$reset');
2007 2008

  final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
2009
  final Match? match = pattern.firstMatch(subshardName);
2010
  if (match == null || match.groupCount != 2) {
2011 2012 2013
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"',
    ]);
2014
    throw Exception('Invalid subshard name: $subshardName');
2015 2016
  }
  // One-indexed.
2017
  final int index = int.parse(match.group(1)!);
2018
  final int total = int.parse(match.group(2)!);
2019
  if (index > total) {
2020 2021 2022 2023
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.',
    ]);
    return <T>[];
2024 2025
  }

2026
  final int testsPerShard = (tests.length / total).ceil();
2027
  final int start = (index - 1) * testsPerShard;
2028
  final int end = math.min(index * testsPerShard, tests.length);
2029

2030
  print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
2031 2032 2033
  return tests.sublist(start, end);
}

2034
Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
2035 2036 2037 2038 2039 2040 2041 2042
  final List<ShardRunner> sublist = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  for (final ShardRunner test in sublist) {
    await test();
  }
}

Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
2043 2044 2045 2046

const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
2047
  String? item = Platform.environment[key];
2048
  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
2049
    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
2050 2051 2052 2053
    assert(positionInTaskName < parts.length);
    item = parts[positionInTaskName];
  }
  if (item == null) {
2054
    for (final String currentItem in items.keys) {
2055
      printProgress('$bold$key=$currentItem$reset');
2056
      await items[currentItem]!();
2057 2058
    }
  } else {
2059
    printProgress('$bold$key=$item$reset');
2060
    if (!items.containsKey(item)) {
2061 2062 2063 2064 2065
      foundError(<String>[
        '${red}Invalid $name: $item$reset',
        'The available ${name}s are: ${items.keys.join(", ")}',
      ]);
      return;
2066
    }
2067
    await items[item]!();
2068 2069
  }
}