test.dart 67 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:convert';
7
import 'dart:io';
8
import 'dart:math' as math;
9

10 11
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
12
import 'package:path/path.dart' as path;
13

14
import 'browser.dart';
Dan Field's avatar
Dan Field committed
15
import 'flutter_compact_formatter.dart';
16
import 'run_command.dart';
17
import 'service_worker_test.dart';
18
import 'utils.dart';
19

20
typedef ShardRunner = Future<void> Function();
21

22 23 24 25 26 27
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
28
typedef OutputChecker = String? Function(CommandResult);
29

30 31
final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
32
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
33 34 35
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'pub$bat');
36
final String pubCache = path.join(flutterRoot, '.pub-cache');
37
final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
38
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
39
final String flutterPluginsVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_plugins.version');
40 41 42 43 44 45 46 47 48 49 50

String get platformFolderName {
  if (Platform.isWindows)
    return 'windows-x64';
  if (Platform.isMacOS)
    return 'darwin-x64';
  if (Platform.isLinux)
    return 'linux-x64';
  throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.');
}
final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe');
51 52 53

/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
54
final List<String> flutterTestArgs = <String>[];
55

56 57 58 59
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};

60
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
61

62 63 64
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';

65 66
/// The number of Cirrus jobs that run Web tests in parallel.
///
67 68 69
/// The default is 8 shards. Typically .cirrus.yml would define the
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
///
70 71 72 73
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
74
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
75
  ? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
76
  : 8;
77
/// Tests that we don't run on Web for compilation reasons.
78
//
79 80
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
const List<String> kWebTestFileKnownFailures = <String>[
81
  'test/services/message_codecs_vm_test.dart',
82
  'test/examples/sector_layout_test.dart',
83
];
84

85
const String kSmokeTestShardName = 'smoke_tests';
86
const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
87

88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
// The seed used to shuffle tests.  If not passed with
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;
String get shuffleSeed {
  if (_shuffleSeed == null) {
    // Change the seed at 7am, UTC.
    final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
    // Generates YYYYMMDD as the seed, so that testing continues to fail for a
    // day after the seed changes, and on other days the seed can be used to
    // replicate failures.
    _shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
  }
  return _shuffleSeed!;
}

104
/// When you call this, you can pass additional arguments to pass custom
105
/// arguments to flutter test. For example, you might want to call this
106
/// script with the parameter --local-engine=host_debug_unopt to
107
/// use your own build of the engine.
108
///
109
/// To run the tool_tests part, run it with SHARD=tool_tests
110
///
111
/// Examples:
112
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
113
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
114
Future<void> main(List<String> args) async {
115 116 117
  print('$clock STARTING ANALYSIS');
  try {
    flutterTestArgs.addAll(args);
118
    final Set<String> removeArgs = <String>{};
119
    for (final String arg in args) {
120
      if (arg.startsWith('--local-engine=')) {
121
        localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
122 123
      }
      if (arg.startsWith('--local-engine-src-path=')) {
124
        localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
125 126 127 128 129
      }
      if (arg.startsWith('--test-randomize-ordering-seed=')) {
        _shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
        removeArgs.add(arg);
      }
130
    }
131
    flutterTestArgs.removeWhere((String arg) => removeArgs.contains(arg));
132 133 134 135 136
    if (Platform.environment.containsKey(CIRRUS_TASK_NAME))
      print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
    print('═' * 80);
    await _runSmokeTests();
    print('═' * 80);
137
    await selectShard(<String, ShardRunner>{
138
      'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
139 140 141 142
      'build_tests': _runBuildTests,
      'framework_coverage': _runFrameworkCoverage,
      'framework_tests': _runFrameworkTests,
      'tool_tests': _runToolTests,
143 144
      // web_tool_tests is also used by HHH: https://dart.googlesource.com/recipes/+/refs/heads/master/recipes/dart/flutter_engine.py
      'web_tool_tests': _runWebToolTests,
145
      'tool_integration_tests': _runIntegrationToolTests,
146
      // All the unit/widget tests run using `flutter test --platform=chrome`
147
      'web_tests': _runWebUnitTests,
148
      // All web integration tests
149
      'web_long_running_tests': _runWebLongRunningTests,
150
      'flutter_plugins': _runFlutterPluginsTests,
151
      'skp_generator': _runSkpGeneratorTests,
152
      kSmokeTestShardName: () async {}, // No-op, the smoke tests already ran. Used for testing this script.
153 154 155 156 157
    });
  } on ExitException catch (error) {
    error.apply();
  }
  print('$clock ${bold}Test successful.$reset');
158 159
}

160 161 162 163 164 165 166 167 168 169 170 171 172 173
/// Verify the Flutter Engine is the revision in
/// bin/cache/internal/engine.version.
Future<void> _validateEngineHash() async {
  final String luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
  if (luciBotId.startsWith('luci-dart-')) {
    // The Dart HHH bots intentionally modify the local artifact cache
    // and then use this script to run Flutter's test suites.
    // Because the artifacts have been changed, this particular test will return
    // a false positive and should be skipped.
    print('${yellow}Skipping Flutter Engine Version Validation for swarming '
          'bot $luciBotId.');
    return;
  }
  final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
174
  final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
175
  final String actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
176 177 178 179 180 181 182 183 184
    return line.startsWith('Flutter Engine Version:');
  });
  if (!actualVersion.contains(expectedVersion)) {
    print('${red}Expected "Flutter Engine Version: $expectedVersion", '
          'but found "$actualVersion".');
    exit(1);
  }
}

185
Future<void> _runSmokeTests() async {
186
  print('${green}Running smoketests...$reset');
187 188 189

  await _validateEngineHash();

190 191
  // Verify that the tests actually return failure on failure and success on
  // success.
192
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
193 194 195 196 197

  // We want to run the smoketests in parallel, because they each take some time
  // to run (e.g. compiling), so we don't want to run them in series, especially
  // on 20-core machines. However, we have a race condition, so for now...
  // Race condition issue: https://github.com/flutter/flutter/issues/90026
198 199
  final List<ShardRunner> tests = <ShardRunner>[
    () => _runFlutterTest(
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
      automatedTests,
      script: path.join('test_smoke_test', 'pass_test.dart'),
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'fail_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
      expectFailure: true,
      printOutput: false,
      outputChecker: (CommandResult result) {
        return result.flattenedStdout!.contains('failingPendingTimerTest')
          ? null
          : 'Failed to find the stack trace for the pending Timer.\n\n'
            'stdout:\n${result.flattenedStdout}\n\n'
            'stderr:\n${result.flattenedStderr}';
    }),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash1_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash2_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
234
    () => _runFlutterTest(
235 236 237 238 239
      automatedTests,
      script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
240
    () => _runFlutterTest(
241 242 243 244 245
      automatedTests,
      script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
246
    () => _runFlutterTest(
247 248 249 250 251
      automatedTests,
      script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
252 253 254 255 256 257 258
  ];

  List<ShardRunner> testsToRun;

  // Smoke tests are special and run first for all test shards.
  // Run all smoke tests for other shards.
  // Only shard smoke tests when explicitly specified.
259
  final String? shardName = Platform.environment[kShardKey];
260 261 262 263 264 265 266 267
  if (shardName == kSmokeTestShardName) {
    testsToRun = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  } else {
    testsToRun = tests;
  }
  for (final ShardRunner test in testsToRun) {
    await test();
  }
268

269
  // Verify that we correctly generated the version file.
270
  final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
271 272
  if (versionError != null)
    exitWithError(<String>[versionError]);
273 274
}

275 276 277 278 279 280
Future<void> _runGeneralToolTests() async {
  await _pubRunTest(
    path.join(flutterRoot, 'packages', 'flutter_tools'),
    testPaths: <String>[path.join('test', 'general.shard')],
    enableFlutterToolAsserts: false,
    // Detect unit test time regressions (poor time delay handling, etc).
281 282
    // This overrides the 15 minute default for tools tests.
    // See the README.md and dart_test.yaml files in the flutter_tools package.
283 284 285 286 287 288 289 290
    perTestTimeout: const Duration(seconds: 2),
  );
}

Future<void> _runCommandsToolTests() async {
  await _pubRunTest(
    path.join(flutterRoot, 'packages', 'flutter_tools'),
    forceSingleCore: true,
291
    testPaths: <String>[path.join('test', 'commands.shard')],
292 293 294
  );
}

295 296 297 298 299
Future<void> _runWebToolTests() async {
  await _pubRunTest(
    path.join(flutterRoot, 'packages', 'flutter_tools'),
    forceSingleCore: true,
    testPaths: <String>[path.join('test', 'web.shard')],
300
    includeLocalEngineEnv: true,
301 302 303
  );
}

304 305 306 307 308 309 310 311 312 313 314 315 316 317
Future<void> _runIntegrationToolTests() async {
  final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
  final List<String> allTests = Directory(path.join(toolsPath, 'test', 'integration.shard'))
      .listSync(recursive: true).whereType<File>()
      .map<String>((FileSystemEntity entry) => path.relative(entry.path, from: toolsPath))
      .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();

  await _pubRunTest(
    toolsPath,
    forceSingleCore: true,
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
  );
}

318
Future<void> _runToolTests() async {
319 320 321 322
  await selectSubshard(<String, ShardRunner>{
    'general': _runGeneralToolTests,
    'commands': _runCommandsToolTests,
  });
323 324
}

325 326
Future<void> runForbiddenFromReleaseTests() async {
  // Build a release APK to get the snapshot json.
327
  final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.');
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
  final List<String> command = <String>[
    'build',
    'apk',
    '--target-platform',
    'android-arm64',
    '--release',
    '--analyze-size',
    '--code-size-directory',
    tempDirectory.path,
    '-v',
  ];

  await runCommand(
    flutter,
    command,
    workingDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
  );

  // First, a smoke test.
  final List<String> smokeTestArgs = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::Widget',
  ];
  await runCommand(
    dart,
    smokeTestArgs,
    workingDirectory: flutterRoot,
    expectNonZeroExit: true,
  );

  // Actual test.
  final List<String> args = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/widget_inspector.dart::WidgetInspectorService',
  ];
  await runCommand(
    dart,
    args,
    workingDirectory: flutterRoot,
  );
}

374 375 376 377 378
/// Verifies that APK, and IPA (if on macOS), and native desktop builds the
/// examples apps without crashing. It does not actually launch the apps. That
/// happens later in the devicelab. This is just a smoke-test. In particular,
/// this will verify we can build when there are spaces in the path name for the
/// Flutter SDK and target app.
379 380
///
/// Also does some checking about types included in hello_world.
381
Future<void> _runBuildTests() async {
382 383 384
  final List<Directory> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync()
    // API example builds will be tested in a separate shard.
    .where((FileSystemEntity entity) => entity is Directory && path.basename(entity.path) != 'api').cast<Directory>().toList()
385
    ..add(Directory(path.join(flutterRoot, 'packages', 'integration_test', 'example')))
386 387 388 389 390 391
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'channels')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'hybrid_android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_platform_view_tests')))
392
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
393
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
394

395 396 397
  // The tests are randomly distributed into subshards so as to get a uniform
  // distribution of costs, but the seed is fixed so that issues are reproducible.
  final List<ShardRunner> tests = <ShardRunner>[
398
    for (final Directory exampleDirectory in exampleDirectories)
399 400 401 402 403 404 405 406 407 408 409 410 411
      () => _runExampleProjectBuildTests(exampleDirectory),
    ...<ShardRunner>[
      // Web compilation tests.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web'),
            path.join('lib', 'main.dart'),
          ),
      // Should not fail to compile with dart:io.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web_compile_tests'),
            path.join('lib', 'dart_io_import.dart'),
          ),
    ],
412
    runForbiddenFromReleaseTests,
413 414
  ]..shuffle(math.Random(0));

415
  await _runShardRunnerIndexOfTotalSubshard(tests);
416 417
}

418
Future<void> _runExampleProjectBuildTests(Directory exampleDirectory, [File? mainFile]) async {
419 420 421 422
  // Only verify caching with flutter gallery.
  final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
  final String examplePath = exampleDirectory.path;
  final bool hasNullSafety = File(path.join(examplePath, 'null_safety')).existsSync();
423 424 425 426
  final List<String> additionalArgs = <String>[
    if (hasNullSafety) '--no-sound-null-safety',
    if (mainFile != null) path.relative(mainFile.path, from: exampleDirectory.absolute.path),
  ];
427 428 429 430 431 432 433 434 435 436 437 438 439
  if (Directory(path.join(examplePath, 'android')).existsSync()) {
    await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
  } else {
    print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
  }
  if (Platform.isMacOS) {
    if (Directory(path.join(examplePath, 'ios')).existsSync()) {
      await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
    }
440
  }
441
  if (Platform.isLinux) {
442 443 444 445 446 447 448
    if (Directory(path.join(examplePath, 'linux')).existsSync()) {
      await _flutterBuildLinux(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildLinux(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no linux directory, skipping Linux');
    }
  }
449
  if (Platform.isMacOS) {
450 451 452 453 454 455 456
    if (Directory(path.join(examplePath, 'macos')).existsSync()) {
      await _flutterBuildMacOS(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildMacOS(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no macos directory, skipping macOS');
    }
  }
457
  if (Platform.isWindows) {
458
    if (Directory(path.join(examplePath, 'windows')).existsSync()) {
459 460
      await _flutterBuildWin32(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildWin32(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
461 462 463 464
    } else {
      print('Example project ${path.basename(examplePath)} has no windows directory, skipping Win32');
    }
  }
465
}
466

467
Future<void> _flutterBuildApk(String relativePathToApplication, {
468
  required bool release,
469
  bool verifyCaching = false,
470 471
  List<String> additionalArgs = const <String>[],
}) async {
472
  print('${green}Testing APK build$reset for $cyan$relativePathToApplication$reset...');
473 474 475 476
  await _flutterBuild(relativePathToApplication, 'APK', 'apk',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
477
  );
478 479
}

480
Future<void> _flutterBuildIpa(String relativePathToApplication, {
481
  required bool release,
482
  List<String> additionalArgs = const <String>[],
483
  bool verifyCaching = false,
484
}) async {
485 486
  assert(Platform.isMacOS);
  print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...');
487 488 489 490 491 492 493
  await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: <String>[...additionalArgs, '--no-codesign'],
  );
}

494
Future<void> _flutterBuildLinux(String relativePathToApplication, {
495
  required bool release,
496 497 498 499 500 501 502 503 504 505 506 507 508
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isLinux);
  await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
  print('${green}Testing Linux build$reset for $cyan$relativePathToApplication$reset...');
  await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

509
Future<void> _flutterBuildMacOS(String relativePathToApplication, {
510
  required bool release,
511 512 513 514 515 516 517 518 519 520 521 522 523
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isMacOS);
  await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
  print('${green}Testing macOS build$reset for $cyan$relativePathToApplication$reset...');
  await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

524
Future<void> _flutterBuildWin32(String relativePathToApplication, {
525
  required bool release,
526 527 528 529 530 531 532 533 534 535 536 537 538
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isWindows);
  await runCommand(flutter, <String>['config', '--enable-windows-desktop']);
  print('${green}Testing Windows build$reset for $cyan$relativePathToApplication$reset...');
  await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

539 540 541 542
Future<void> _flutterBuild(
  String relativePathToApplication,
  String platformLabel,
  String platformBuildName, {
543
  required bool release,
544 545 546
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
547
  await runCommand(flutter,
548 549
    <String>[
      'build',
550
      platformBuildName,
551 552 553 554 555 556 557
      ...additionalArgs,
      if (release)
        '--release'
      else
        '--debug',
      '-v',
    ],
558 559
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
560

561
  if (verifyCaching) {
562
    print('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
563 564 565
    await runCommand(flutter,
      <String>[
        'build',
566
        platformBuildName,
567 568 569 570 571 572 573 574 575 576 577 578 579
        '--performance-measurement-file=perf.json',
        ...additionalArgs,
        if (release)
          '--release'
        else
          '--debug',
        '-v',
      ],
      workingDirectory: path.join(flutterRoot, relativePathToApplication),
    );
    final File file = File(path.join(flutterRoot, relativePathToApplication, 'perf.json'));
    if (!_allTargetsCached(file)) {
      print('${red}Not all build targets cached after second run.$reset');
580
      print('The target performance data was: ${file.readAsStringSync().replaceAll('},', '},\n')}');
581 582 583 584 585 586
      exit(1);
    }
  }
}

bool _allTargetsCached(File performanceFile) {
587 588 589 590 591
  final Map<String, Object?> data = json.decode(performanceFile.readAsStringSync())
    as Map<String, Object?>;
  final List<Map<String, Object?>> targets = (data['targets']! as List<Object?>)
    .cast<Map<String, Object?>>();
  return targets.every((Map<String, Object?> element) => element['skipped'] == true);
592 593
}

594 595 596 597 598 599 600 601 602
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
  print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
  await runCommand(flutter,
    <String>['build', 'web', '-v', '--target=$target'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: expectNonZeroExit,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
603
  );
604 605
}

606 607 608 609 610 611 612 613 614 615 616
Future<void> _runAddToAppLifeCycleTests() async {
  if (Platform.isMacOS) {
    print('${green}Running add-to-app life cycle iOS integration tests$reset...');
    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
    await runCommand('./build_and_test.sh',
      <String>[],
      workingDirectory: addToAppDir,
    );
  }
}

617
Future<void> _runFrameworkTests() async {
618 619
  final List<String> soundNullSafetyOptions     = <String>['--null-assertions', '--sound-null-safety'];
  final List<String> mixedModeNullSafetyOptions = <String>['--null-assertions', '--no-sound-null-safety'];
620
  final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
621

622
  Future<void> runWidgets() async {
623
    print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset');
624 625 626
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
627
        options: <String>[trackWidgetCreationOption, ...soundNullSafetyOptions],
628 629 630
        tests: <String>[ path.join('test', 'widgets') + path.separator ],
      );
    }
631
    // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
632 633 634 635 636 637
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
        options: <String>[trackWidgetCreationOption],
      );
    }
638 639 640 641
    // Run release mode tests (see packages/flutter/test_release/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      options: <String>['--dart-define=dart.vm.product=true', ...soundNullSafetyOptions],
642
      tests: <String>['test_release${path.separator}'],
643
    );
644 645
  }

646
  Future<void> runLibraries() async {
647
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
648
      .listSync(followLinks: false)
649
      .whereType<Directory>()
650
      .where((Directory dir) => dir.path.endsWith('widgets') == false)
651
      .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
652
      .toList();
653
    print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset');
654 655 656
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
657
        options: <String>[trackWidgetCreationOption, ...soundNullSafetyOptions],
658 659 660
        tests: tests,
      );
    }
661 662
  }

663 664 665 666 667 668 669 670 671 672 673 674 675 676
  Future<void> runExampleTests() async {
    // TODO(gspencergoog): Currently Linux LUCI bots can't run desktop Flutter applications, https://github.com/flutter/flutter/issues/90676
    if (!Platform.isLinux || ciProvider != CiProviders.luci) {
      await runCommand(
        flutter,
        <String>['config', '--enable-${Platform.operatingSystem}-desktop'],
        workingDirectory: flutterRoot,
      );
      await runCommand(
        dart,
        <String>[path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')],
        workingDirectory: path.join(flutterRoot, 'examples', 'api'),
      );
    }
677
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'api'), options: soundNullSafetyOptions);
678 679 680 681
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), options: soundNullSafetyOptions);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), options: soundNullSafetyOptions);
  }

682 683 684 685 686 687 688 689 690 691 692 693
  Future<void> runFixTests() async {
    final List<String> args = <String>[
      'fix',
      '--compare-to-golden',
    ];
    await runCommand(
      dart,
      args,
      workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_fixes'),
    );
  }

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
  Future<void> runPrivateTests() async {
    final List<String> args = <String>[
      'run',
      '--sound-null-safety',
      'test_private.dart',
    ];
    final Map<String, String> pubEnvironment = <String, String>{
      'FLUTTER_ROOT': flutterRoot,
    };
    if (Directory(pubCache).existsSync()) {
      pubEnvironment['PUB_CACHE'] = pubCache;
    }

    // If an existing env variable exists append to it, but only if
    // it doesn't appear to already include enable-asserts.
    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
    if (!toolsArgs.contains('--enable-asserts')) {
      toolsArgs += ' --enable-asserts';
    }
    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
    // The flutter_tool will originally have been snapshotted without asserts.
    // We need to force it to be regenerated with them enabled.
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot'));
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp'));

    await runCommand(
      pub,
      args,
      workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'),
      environment: pubEnvironment,
    );
  }

727 728
  Future<void> runMisc() async {
    print('${green}Running package tests$reset for directories other than packages/flutter');
729
    await runExampleTests();
730
    await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
731
    await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
732 733
    await _pubRunTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'conductor', 'ui'));
734 735 736
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
737
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
738
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
739
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')], options: soundNullSafetyOptions);
740
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'));
741
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'), options: soundNullSafetyOptions);
742 743
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), options: soundNullSafetyOptions);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), options: soundNullSafetyOptions);
744
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), options: soundNullSafetyOptions);
745
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'), options: mixedModeNullSafetyOptions);
Dan Field's avatar
Dan Field committed
746 747 748 749
    await _runFlutterTest(
      path.join(flutterRoot, 'dev', 'tracing_tests'),
      options: <String>['--enable-vmservice'],
    );
750
    await runFixTests();
751
    await runPrivateTests();
752 753 754 755
    const String httpClientWarning =
      'Warning: At least one test in this suite creates an HttpClient. When\n'
      'running a test suite that uses TestWidgetsFlutterBinding, all HTTP\n'
      'requests will return status code 400, and no network request will\n'
756
      'actually be made. Any test expecting a real network connection and\n'
757 758 759 760 761 762 763 764 765
      'status code will fail.\n'
      'To test code that needs an HttpClient, provide your own HttpClient\n'
      'implementation to the code under test, so that your test can\n'
      'consistently provide a testable response to the code under test.';
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter_test'),
      script: path.join('test', 'bindings_test_failure.dart'),
      expectFailure: true,
      printOutput: false,
766
      outputChecker: (CommandResult result) {
767
        final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout!);
768
        if (matches == null || matches.isEmpty || matches.length > 1) {
769 770 771
          return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n'
                 'stdout:\n${result.flattenedStdout}\n\n'
                 'stderr:\n${result.flattenedStderr}';
772 773 774 775
        }
        return null;
      },
    );
776
  }
777

778 779 780 781 782
  await selectSubshard(<String, ShardRunner>{
    'widgets': runWidgets,
    'libraries': runLibraries,
    'misc': runMisc,
  });
783 784
}

785
Future<void> _runFrameworkCoverage() async {
786 787 788
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
789 790
    print('Expected to find: $cyan${coverageFile.absolute}$reset');
    print('This file is normally obtained by running `${green}flutter update-packages$reset`.');
791 792 793 794 795 796 797 798
    exit(1);
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
799 800
    print('Expected to find: $cyan${coverageFile.absolute}$reset');
    print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.');
801 802
    exit(1);
  }
803 804
}

805
Future<void> _runWebUnitTests() async {
806 807 808 809 810 811 812 813 814 815 816 817 818 819
  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};

  final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));

  final List<String> allTests = flutterPackageTestDirectory
    .listSync()
    .whereType<Directory>()
    .expand((Directory directory) => directory
      .listSync(recursive: true)
      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
    )
    .whereType<File>()
    .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
820
    .where((String filePath) => !kWebTestFileKnownFailures.contains(path.split(filePath).join('/')))
821 822 823 824 825 826 827
    .toList()
    // Finally we shuffle the list because we want the average cost per file to be uniformly
    // distributed. If the list is not sorted then different shards and batches may have
    // very different characteristics.
    // We use a constant seed for repeatability.
    ..shuffle(math.Random(0));

828 829 830
  assert(webShardCount >= 1);
  final int testsPerShard = (allTests.length / webShardCount).ceil();
  assert(testsPerShard * webShardCount >= allTests.length);
831 832

  // This for loop computes all but the last shard.
833
  for (int index = 0; index < webShardCount - 1; index += 1) {
834 835 836 837 838 839 840 841 842 843 844 845 846
    subshards['$index'] = () => _runFlutterWebTest(
      flutterPackageDirectory.path,
      allTests.sublist(
        index * testsPerShard,
        (index + 1) * testsPerShard,
      ),
    );
  }

  // The last shard also runs the flutter_web_plugins tests.
  //
  // We make sure the last shard ends in _last so it's easier to catch mismatches
  // between `.cirrus.yml` and `test.dart`.
847
  subshards['${webShardCount - 1}_last'] = () async {
848 849 850
    await _runFlutterWebTest(
      flutterPackageDirectory.path,
      allTests.sublist(
851
        (webShardCount - 1) * testsPerShard,
852 853 854 855 856 857 858
        allTests.length,
      ),
    );
    await _runFlutterWebTest(
      path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
      <String>['test'],
    );
859 860 861 862
    await _runFlutterWebTest(
        path.join(flutterRoot, 'packages', 'flutter_driver'),
        <String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
    );
863 864 865 866 867
  };

  await selectSubshard(subshards);
}

868 869 870
/// Coarse-grained integration tests running on the Web.
Future<void> _runWebLongRunningTests() async {
  final List<ShardRunner> tests = <ShardRunner>[
871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911
    for (String buildMode in _kAllBuildModes)
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('test_driver', 'failure.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
        // This test intentionally fails and prints stack traces in the browser
        // logs. To avoid confusion, silence browser output.
        silenceBrowserOutput: true,
      ),

    // This test specifically tests how images are loaded in HTML mode, so we don't run it in CanvasKit mode.
    () => _runWebE2eTest('image_loading_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('image_loading_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('image_loading_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),

    // This test is only known to work in debug mode.
    () => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),

    () => _runWebTreeshakeTest(),

912 913 914 915
    () => _runFlutterDriverWebTest(
      testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
      target: 'test_driver/smoke_web_engine.dart',
      buildMode: 'profile',
916
      renderer: 'auto',
917
    ),
918 919 920 921 922 923
    () => _runGalleryE2eWebTest('debug'),
    () => _runGalleryE2eWebTest('debug', canvasKit: true),
    () => _runGalleryE2eWebTest('profile'),
    () => _runGalleryE2eWebTest('profile', canvasKit: true),
    () => _runGalleryE2eWebTest('release'),
    () => _runGalleryE2eWebTest('release', canvasKit: true),
924
    () => runWebServiceWorkerTest(headless: true),
925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
    () => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/stack_trace.dart'),
    () => _runWebDebugTest('lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/web_directory_loading.dart'),
    () => _runWebDebugTest('test/test.dart'),
    () => _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true),
    () => _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true),
    () => _runWebDebugTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
    () => _runWebReleaseTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
    () => _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
      '--sound-null-safety',
    ]),
    () => _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
      '--sound-null-safety',
    ]),
953
  ];
954 955 956 957 958

  // Shuffling mixes fast tests with slow tests so shards take roughly the same
  // amount of time to run.
  tests.shuffle(math.Random(0));

959
  await _ensureChromeDriverIsRunning();
960
  await _runShardRunnerIndexOfTotalSubshard(tests);
961
  await _stopChromeDriver();
962 963
}

964 965 966
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
Future<void> _runWebE2eTest(
  String name, {
967 968
  required String buildMode,
  required String renderer,
969 970 971 972 973 974 975 976 977
}) async {
  await _runFlutterDriverWebTest(
    target: path.join('test_driver', '$name.dart'),
    buildMode: buildMode,
    renderer: renderer,
    testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
  );
}

978
Future<void> _runFlutterDriverWebTest({
979 980 981 982
  required String target,
  required String buildMode,
  required String renderer,
  required String testAppDirectory,
983 984
  bool expectFailure = false,
  bool silenceBrowserOutput = false,
985
}) async {
986
  print('${green}Running integration tests $target in $buildMode mode.$reset');
987 988 989 990 991 992 993 994
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
995
      ...flutterTestArgs,
996 997 998 999 1000 1001 1002
      'drive',
      '--target=$target',
      '--browser-name=chrome',
      '--no-sound-null-safety',
      '-d',
      'web-server',
      '--$buildMode',
1003
      '--web-renderer=$renderer',
1004
    ],
1005
    expectNonZeroExit: expectFailure,
1006 1007 1008 1009
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
1010 1011 1012 1013 1014 1015 1016 1017 1018
    removeLine: (String line) {
      if (!silenceBrowserOutput) {
        return false;
      }
      if (line.trim().startsWith('[INFO]')) {
        return true;
      }
      return false;
    },
1019 1020 1021 1022
  );
  print('${green}Integration test passed.$reset');
}

1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
// Compiles a sample web app and checks that its JS doesn't contain certain
// debug code that we expect to be tree shaken out.
//
// The app is compiled in `--profile` mode to prevent the compiler from
// minifying the symbols.
Future<void> _runWebTreeshakeTest() async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
  final String target = path.join('lib', 'treeshaking_main.dart');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--target=$target',
      '--no-sound-null-safety',
      '--profile',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
  final String javaScript = mainDartJs.readAsStringSync();

  // Check that we're not looking at minified JS. Otherwise this test would result in false positive.
  expect(javaScript.contains('RenderObjectToWidgetElement'), true);

  const String word = 'debugFillProperties';
  int count = 0;
  int pos = javaScript.indexOf(word);
  final int contentLength = javaScript.length;
  while (pos != -1) {
    count += 1;
    pos += word.length;
    if (pos >= contentLength || count > 100) {
      break;
    }
    pos = javaScript.indexOf(word, pos);
  }

  const int kMaxExpectedDebugFillProperties = 11;
  if (count > kMaxExpectedDebugFillProperties) {
    throw Exception(
      'Too many occurrences of "$word" in compiled JavaScript.\n'
      'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
    );
  }
}

1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
/// Returns the commit hash of the flutter/plugins repository that's rolled in.
///
/// The flutter/plugins repository is a downstream dependency, it is only used
/// by flutter/flutter for testing purposes, to assure stable tests for a given
/// flutter commit the flutter/plugins commit hash to test against is coded in
/// the bin/internal/flutter_plugins.version file.
///
/// The `filesystem` parameter specified filesystem to read the plugins version file from.
/// The `pluginsVersionFile` parameter allows specifying an alternative path for the
/// plugins version file, when null [flutterPluginsVersionFile] is used.
Future<String> getFlutterPluginsVersion({
  fs.FileSystem fileSystem = const LocalFileSystem(),
1091
  String? pluginsVersionFile,
1092 1093 1094 1095 1096 1097
}) async {
  final File versionFile = fileSystem.file(pluginsVersionFile ?? flutterPluginsVersionFile);
  final String versionFileContents = await versionFile.readAsString();
  return versionFileContents.trim();
}

1098 1099 1100 1101
/// Executes the test suite for the flutter/plugins repo.
Future<void> _runFlutterPluginsTests() async {
  Future<void> runAnalyze() async {
    print('${green}Running analysis for flutter/plugins$reset');
1102
    final Directory checkout = Directory.systemTemp.createTempSync('flutter_plugins.');
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'clone',
        'https://github.com/flutter/plugins.git',
        '.'
      ],
      workingDirectory: checkout.path,
    );
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
    final String pluginsCommit = await getFlutterPluginsVersion();
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'checkout',
        pluginsCommit,
      ],
      workingDirectory: checkout.path,
    );
1125 1126 1127 1128 1129 1130
    // Prep the repository tooling.
    // This test does not use tool_runner.sh because in this context the test
    // should always run on the entire plugins repo, while tool_runner.sh
    // is designed for flutter/plugins CI and only analyzes changed repository
    // files when run for anything but master.
    final String toolDir = path.join(checkout.path, 'script', 'tool');
1131
    await runCommand(
1132
      'dart',
1133
      <String>[
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
        'pub',
        'get',
      ],
      workingDirectory: toolDir,
    );
    final String toolScript = path.join(toolDir, 'bin', 'flutter_plugin_tools.dart');
    await runCommand(
      'dart',
      <String>[
        'run',
        toolScript,
1145
        'analyze',
1146
        '--custom-analysis=script/configs/custom_analysis.yaml',
1147 1148 1149 1150 1151 1152 1153 1154 1155
      ],
      workingDirectory: checkout.path,
    );
  }
  await selectSubshard(<String, ShardRunner>{
    'analyze': runAnalyze,
  });
}

1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
/// Runs the skp_generator from the flutter/tests repo.
///
/// See also the customer_tests shard.
///
/// Generated SKPs are ditched, this just verifies that it can run without failure.
Future<void> _runSkpGeneratorTests() async {
  print('${green}Running skp_generator from flutter/tests$reset');
  final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.');
  await runCommand(
    'git',
    <String>[
      '-c',
      'core.longPaths=true',
      'clone',
      'https://github.com/flutter/tests.git',
      '.'
    ],
    workingDirectory: checkout.path,
  );
  await runCommand(
    './build.sh',
    <String>[ ],
    workingDirectory: path.join(checkout.path, 'skp_generator'),
  );
}

1182 1183 1184 1185
// The `chromedriver` process created by this test.
//
// If an existing chromedriver is already available on port 4444, the existing
// process is reused and this variable remains null.
1186
Command? _chromeDriver;
1187 1188 1189

Future<bool> _isChromeDriverRunning() async {
  try {
1190 1191 1192
    final RawSocket socket = await RawSocket.connect('localhost', 4444);
    socket.shutdown(SocketDirection.both);
    await socket.close();
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
    return true;
  } on SocketException {
    return false;
  }
}

Future<void> _ensureChromeDriverIsRunning() async {
  // If we cannot connect to ChromeDriver, assume it is not running. Launch it.
  if (!await _isChromeDriverRunning()) {
    print('Starting chromedriver');
    // Assume chromedriver is in the PATH.
    _chromeDriver = await startCommand(
      'chromedriver',
      <String>['--port=4444'],
    );
    while (!await _isChromeDriverRunning()) {
      await Future<void>.delayed(const Duration(milliseconds: 100));
      print('Waiting for chromedriver to start up.');
    }
  }

  final HttpClient client = HttpClient();
  final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
  final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
  final HttpClientResponse response = await request.close();
1218
  final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join()) as Map<String, dynamic>;
1219
  client.close();
1220
  final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
1221 1222 1223 1224 1225 1226 1227 1228 1229
  if (!webDriverReady) {
    throw Exception('WebDriver not available.');
  }
}

Future<void> _stopChromeDriver() async {
  if (_chromeDriver == null) {
    return;
  }
1230
  print('Stopping chromedriver');
1231
  _chromeDriver!.process.kill();
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
}

/// Exercises the old gallery in a browser for a long period of time, looking
/// for memory leaks and dangling pointers.
///
/// This is not a performance test.
///
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
///
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
  print('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1254
      ...flutterTestArgs,
1255 1256 1257
      'drive',
      if (canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=true',
1258 1259
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1260 1261
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1262 1263 1264
      '--driver=test_driver/transitions_perf_e2e_test.dart',
      '--target=test_driver/transitions_perf_e2e.dart',
      '--browser-name=chrome',
1265
      '--no-sound-null-safety',
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
      '-d',
      'web-server',
      '--$buildMode',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
  print('${green}Integration test passed.$reset');
}

1278
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
1279
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
1280
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--$buildMode',
      '-t',
1295
      entrypoint,
1296 1297 1298 1299 1300 1301 1302 1303
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1304 1305
  final int serverPort = await findAvailablePort();
  final int browserDebugPort = await findAvailablePort();
1306
  final String result = await evalTestAppInChrome(
1307
    appUrl: 'http://localhost:$serverPort/index.html',
1308
    appDirectory: appBuildDirectory,
1309 1310
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
  );

  if (result.contains('--- TEST SUCCEEDED ---')) {
    print('${green}Web stack trace integration test passed.$reset');
  } else {
    print(result);
    print('${red}Web stack trace integration test failed.$reset');
    exit(1);
  }
}

1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
/// Run a web integration test in release mode.
Future<void> _runWebReleaseTest(String target, {
  List<String> additionalArguments = const<String>[],
}) async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1338
      ...flutterTestArgs,
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
      'build',
      'web',
      '--release',
      ...additionalArguments,
      '-t',
      target,
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1353 1354
  final int serverPort = await findAvailablePort();
  final int browserDebugPort = await findAvailablePort();
1355
  final String result = await evalTestAppInChrome(
1356
    appUrl: 'http://localhost:$serverPort/index.html',
1357
    appDirectory: appBuildDirectory,
1358 1359
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
  );

  if (result.contains('--- TEST SUCCEEDED ---')) {
    print('${green}Web release mode test passed.$reset');
  } else {
    print(result);
    print('${red}Web release mode test failed.$reset');
    exit(1);
  }
}

1371 1372 1373
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
///
/// Instead, we use `flutter run --debug` and sniff out the standard output.
1374
Future<void> _runWebDebugTest(String target, {
1375
  bool enableNullSafety = false,
1376 1377
  List<String> additionalArguments = const<String>[],
}) async {
1378 1379
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  bool success = false;
1380
  final CommandResult result = await runCommand(
1381 1382 1383 1384
    flutter,
    <String>[
      'run',
      '--debug',
1385 1386
      if (enableNullSafety)
        ...<String>[
1387 1388
          '--no-sound-null-safety',
          '--null-assertions',
1389
        ],
1390 1391 1392
      '-d',
      'chrome',
      '--web-run-headless',
1393
      '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1394
      '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1395
      ...additionalArguments,
1396 1397
      '-t',
      target,
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
    ],
    outputMode: OutputMode.capture,
    outputListener: (String line, Process process) {
      if (line.contains('--- TEST SUCCEEDED ---')) {
        success = true;
      }
      if (success || line.contains('--- TEST FAILED ---')) {
        process.stdin.add('q'.codeUnits);
      }
    },
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  if (success) {
    print('${green}Web stack trace integration test passed.$reset');
  } else {
1417 1418
    print(result.flattenedStdout!);
    print(result.flattenedStderr!);
1419 1420 1421 1422 1423
    print('${red}Web stack trace integration test failed.$reset');
    exit(1);
  }
}

1424
Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async {
1425 1426 1427 1428 1429 1430 1431 1432
  await runCommand(
    flutter,
    <String>[
      'test',
      if (ciProvider == CiProviders.cirrus)
        '--concurrency=1',  // do not parallelize on Cirrus, to reduce flakiness
      '-v',
      '--platform=chrome',
1433 1434
      // TODO(ferhatb): Run web tests with both rendering backends.
      '--web-renderer=html', // use html backend for web tests.
1435
      '--sound-null-safety', // web tests do not autodetect yet.
1436
      ...flutterTestArgs,
1437 1438 1439 1440 1441 1442 1443
      ...tests,
    ],
    workingDirectory: workingDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
1444 1445
}

1446 1447 1448 1449 1450
// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
1451
Future<void> _pubRunTest(String workingDirectory, {
1452
  List<String>? testPaths,
1453 1454
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
1455
  String? coverage,
1456
  bool forceSingleCore = false,
1457
  Duration? perTestTimeout,
1458
  bool includeLocalEngineEnv = false,
1459
  bool ensurePrecompiledTool = true,
1460
  bool shuffleTests = true,
Dan Field's avatar
Dan Field committed
1461
}) async {
1462 1463
  int? cpus;
  final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
  if (cpuVariable != null) {
    cpus = int.tryParse(cpuVariable, radix: 10);
    if (cpus == null) {
      print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset');
      print('Actual value: "$cpuVariable"');
      exit(1);
    }
  } else {
    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
  }
1474 1475 1476 1477 1478
  // Integration tests that depend on external processes like chrome
  // can get stuck if there are multiple instances running at once.
  if (forceSingleCore) {
    cpus = 1;
  }
1479 1480 1481 1482

  final List<String> args = <String>[
    'run',
    'test',
1483
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1484 1485 1486 1487 1488 1489 1490 1491 1492
    if (useFlutterTestFormatter)
      '-rjson'
    else
      '-rcompact',
    '-j$cpus',
    if (!hasColor)
      '--no-color',
    if (coverage != null)
      '--coverage=$coverage',
1493 1494
    if (perTestTimeout != null)
      '--timeout=${perTestTimeout.inMilliseconds.toString()}ms',
1495 1496 1497 1498
    if (testPaths != null)
      for (final String testPath in testPaths)
        testPath,
  ];
1499 1500
  final Map<String, String> pubEnvironment = <String, String>{
    'FLUTTER_ROOT': flutterRoot,
1501
    if (includeLocalEngineEnv) ...localEngineEnv,
1502
  };
1503 1504 1505
  if (Directory(pubCache).existsSync()) {
    pubEnvironment['PUB_CACHE'] = pubCache;
  }
1506 1507 1508 1509 1510
  if (enableFlutterToolAsserts) {
    // If an existing env variable exists append to it, but only if
    // it doesn't appear to already include enable-asserts.
    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
    if (!toolsArgs.contains('--enable-asserts'))
1511
      toolsArgs += ' --enable-asserts';
1512
    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
1513 1514 1515 1516
    // The flutter_tool will originally have been snapshotted without asserts.
    // We need to force it to be regenerated with them enabled.
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot'));
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp'));
1517
  }
1518 1519 1520 1521 1522 1523
  if (ensurePrecompiledTool) {
    // We rerun the `flutter` tool here just to make sure that it is compiled
    // before tests run, because the tests might time out if they have to rebuild
    // the tool themselves.
    await runCommand(flutter, <String>['--version'], environment: pubEnvironment);
  }
1524 1525
  if (useFlutterTestFormatter) {
    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
    Stream<String> testOutput;
    try {
      testOutput = runAndGetStdout(
        pub,
        args,
        workingDirectory: workingDirectory,
        environment: pubEnvironment,
      );
    } finally {
      formatter.finish();
    }
1537
    await _processTestOutput(formatter, testOutput);
1538 1539 1540 1541
  } else {
    await runCommand(
      pub,
      args,
1542 1543 1544
      workingDirectory: workingDirectory,
      environment: pubEnvironment,
      removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
1545 1546
    );
  }
1547 1548
}

1549
Future<void> _runFlutterTest(String workingDirectory, {
1550
  String? script,
1551 1552
  bool expectFailure = false,
  bool printOutput = true,
1553
  OutputChecker? outputChecker,
1554
  List<String> options = const <String>[],
1555
  Map<String, String>? environment,
1556
  List<String> tests = const <String>[],
1557
  bool shuffleTests = true,
Dan Field's avatar
Dan Field committed
1558
}) async {
1559
  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
1560

1561 1562 1563 1564 1565 1566 1567
  final List<String> tags = <String>[];
  // Recipe configured reduced test shards will only execute tests with the
  // appropriate tag.
  if ((Platform.environment['REDUCED_TEST_SET'] ?? 'False') == 'True') {
    tags.addAll(<String>['-t', 'reduced-test-set']);
  }

1568 1569
  final List<String> args = <String>[
    'test',
1570
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1571
    ...options,
1572
    ...tags,
1573
    ...flutterTestArgs,
1574
  ];
Dan Field's avatar
Dan Field committed
1575

1576
  final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
1577
  if (shouldProcessOutput)
1578
    args.add('--machine');
Dan Field's avatar
Dan Field committed
1579

1580 1581 1582
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
1583 1584 1585
      print('${red}Could not find test$reset: $green$fullScriptPath$reset');
      print('Working directory: $cyan$workingDirectory$reset');
      print('Script: $green$script$reset');
1586 1587 1588 1589
      if (!printOutput)
        print('This is one of the tests that does not normally print output.');
      exit(1);
    }
1590
    args.add(script);
1591
  }
1592

1593
  args.addAll(tests);
1594

1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
  if (!shouldProcessOutput) {
    final OutputMode outputMode = outputChecker == null && printOutput
      ? OutputMode.print
      : OutputMode.capture;

    final CommandResult result = await runCommand(
      flutter,
      args,
      workingDirectory: workingDirectory,
      expectNonZeroExit: expectFailure,
      outputMode: outputMode,
      environment: environment,
    );

    if (outputChecker != null) {
1610
      final String? message = outputChecker(result);
1611 1612
      if (message != null)
        exitWithError(<String>[message]);
1613
    }
1614 1615
    return;
  }
1616

1617 1618 1619 1620 1621
  if (useFlutterTestFormatter) {
    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
    Stream<String> testOutput;
    try {
      testOutput = runAndGetStdout(
1622 1623 1624 1625
        flutter,
        args,
        workingDirectory: workingDirectory,
        expectNonZeroExit: expectFailure,
1626
        environment: environment,
1627
      );
1628 1629
    } finally {
      formatter.finish();
1630
    }
1631
    await _processTestOutput(formatter, testOutput);
1632
  } else {
1633 1634 1635 1636 1637 1638
    await runCommand(
      flutter,
      args,
      workingDirectory: workingDirectory,
      expectNonZeroExit: expectFailure,
    );
1639
  }
1640 1641
}

1642
Map<String, String> _initGradleEnvironment() {
1643
  final String? androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
1644 1645 1646
      ? Platform.environment['ANDROID_SDK_ROOT']
      : Platform.environment['ANDROID_HOME'];
  if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
1647
    print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT.$reset');
1648 1649 1650
    exit(1);
  }
  return <String, String>{
1651
    'ANDROID_HOME': androidSdkRoot!,
1652 1653
    'ANDROID_SDK_ROOT': androidSdkRoot,
  };
1654
}
1655

1656 1657 1658 1659 1660 1661 1662 1663 1664
final Map<String, String> gradleEnvironment = _initGradleEnvironment();

void deleteFile(String path) {
  // This is technically a race condition but nobody else should be running
  // while this script runs, so we should be ok. (Sadly recursive:true does not
  // obviate the need for existsSync, at least on Windows.)
  final File file = File(path);
  if (file.existsSync())
    file.deleteSync();
1665 1666
}

1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
enum CiProviders {
  cirrus,
  luci,
}

Future<void> _processTestOutput(
  FlutterCompactFormatter formatter,
  Stream<String> testOutput,
) async {
  final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
    print('Processing...');
  });

  await testOutput.forEach(formatter.processRawOutput);
  heartbeat.cancel();
  formatter.finish();
1683 1684
}

1685
CiProviders? get ciProvider {
1686 1687 1688 1689 1690 1691 1692 1693
  if (Platform.environment['CIRRUS_CI'] == 'true') {
    return CiProviders.cirrus;
  }
  if (Platform.environment['LUCI_CONTEXT'] != null) {
    return CiProviders.luci;
  }
  return null;
}
1694

1695 1696 1697 1698
/// Returns the name of the branch being tested.
String get branchName {
  switch(ciProvider) {
    case CiProviders.cirrus:
1699
      return Platform.environment['CIRRUS_BRANCH']!;
1700
    case CiProviders.luci:
1701 1702 1703
      return Platform.environment['LUCI_BRANCH']!;
    case null:
      return '';
1704 1705 1706
  }
}

1707 1708 1709 1710 1711
/// Checks the given file's contents to determine if they match the allowed
/// pattern for version strings.
///
/// Returns null if the contents are good. Returns a string if they are bad.
/// The string is an error message.
1712
Future<String?> verifyVersion(File file) async {
1713 1714
  final RegExp pattern = RegExp(
    r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724
  final String version = await file.readAsString();
  if (!file.existsSync())
    return 'The version logic failed to create the Flutter version file.';
  if (version == '0.0.0-unknown')
    return 'The version logic failed to determine the Flutter version.';
  if (!version.contains(pattern))
    return 'The version logic generated an invalid version string: "$version".';
  return null;
}

1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
  // Example: "1_3" means the first (one-indexed) shard of three total shards.
1737
  final String? subshardName = Platform.environment[subshardKey];
1738 1739 1740 1741 1742 1743 1744
  if (subshardName == null) {
    print('$kSubshardKey environment variable is missing, skipping sharding');
    return tests;
  }
  print('$bold$subshardKey=$subshardName$reset');

  final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
1745
  final Match? match = pattern.firstMatch(subshardName);
1746 1747
  if (match == null || match.groupCount != 2) {
    print('${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"');
1748
    exit(1);
1749 1750
  }
  // One-indexed.
1751 1752
  final int index = int.parse(match!.group(1)!);
  final int total = int.parse(match.group(2)!);
1753 1754 1755 1756 1757
  if (index > total) {
    print('${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.');
    exit(1);
  }

1758
  final int testsPerShard = (tests.length / total).ceil();
1759
  final int start = (index - 1) * testsPerShard;
1760
  final int end = math.min(index * testsPerShard, tests.length);
1761 1762 1763 1764 1765

  print('Selecting subshard $index of $total (range ${start + 1}-$end of ${tests.length})');
  return tests.sublist(start, end);
}

1766
Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
1767 1768 1769 1770 1771 1772
  final List<ShardRunner> sublist = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  for (final ShardRunner test in sublist) {
    await test();
  }
}

1773
/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
1774
/// the shard and sub-shard (parsing it in the form shard-subshard-platform, ignoring
1775 1776
/// the platform).
///
1777
/// For local testing you can just set the SHARD and SUBSHARD
1778
/// environment variables. For example, to run all the framework tests you can
1779 1780 1781 1782 1783 1784
/// just set SHARD=framework_tests. Some shards support named subshards, like
/// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
/// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of two"
/// as in run the first half of the tests).
///
/// To run specifically the third subshard of
1785
/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
1786 1787
Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
1788 1789 1790 1791

const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
1792
  String? item = Platform.environment[key];
1793
  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
1794
    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
1795 1796 1797 1798
    assert(positionInTaskName < parts.length);
    item = parts[positionInTaskName];
  }
  if (item == null) {
1799
    for (final String currentItem in items.keys) {
1800
      print('$bold$key=$currentItem$reset');
1801
      await items[currentItem]!();
1802 1803 1804
      print('');
    }
  } else {
1805 1806 1807 1808 1809 1810
    if (!items.containsKey(item)) {
      print('${red}Invalid $name: $item$reset');
      print('The available ${name}s are: ${items.keys.join(", ")}');
      exit(1);
    }
    print('$bold$key=$item$reset');
1811
    await items[item]!();
1812 1813
  }
}