test.dart 78.9 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
// Runs the tests for the flutter/flutter repository.
//
//
// By default, test output is filtered and only errors are shown. (If a
// particular test takes longer than _quietTimeout in utils.dart, the output is
// shown then also, in case something has hung.)
//
//  --verbose stops the output cleanup and just outputs everything verbatim.
//
//
// By default, errors are non-fatal; all tests are executed and the output
// ends with a summary of the errors that were detected.
//
// Exit code is 1 if there was an error.
//
//  --abort-on-error causes the script to exit immediately when hitting an error.
//
//
// By default, all tests are run. However, the tests support being split by
// shard and subshard. (Inspect the code to see what shards and subshards are
// supported.)
//
// If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
// the shard and sub-shard, by parsing it in the form shard-subshard-platform,
// ignoring the platform.
//
// For local testing you can just set the SHARD and SUBSHARD environment
// variables. For example, to run all the framework tests you can just set
// SHARD=framework_tests. Some shards support named subshards, like
// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of
// two" as in run the first half of the tests).
//
// So for example to run specifically the third subshard of the Web tests you
// would set SHARD=web_tests SUBSHARD=2 (it's zero-based).
//
// By default, where supported, tests within a shard are executed in a random
// order to (eventually) catch inter-test dependencies.
//
//  --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
//
//
// All other arguments are treated as arguments to pass to the flutter tool when
// running tests.

50
import 'dart:convert';
51
import 'dart:core' as system show print;
52
import 'dart:core' hide print;
53
import 'dart:io' as system show exit;
54
import 'dart:io' hide exit;
55
import 'dart:math' as math;
Ian Hickson's avatar
Ian Hickson committed
56
import 'dart:typed_data';
57

Ian Hickson's avatar
Ian Hickson committed
58
import 'package:archive/archive.dart';
59 60
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
61
import 'package:path/path.dart' as path;
62

63
import 'browser.dart';
64
import 'run_command.dart';
65
import 'service_worker_test.dart';
66
import 'tool_subsharding.dart';
67
import 'utils.dart';
68

69
typedef ShardRunner = Future<void> Function();
70

71 72 73 74 75 76
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
77
typedef OutputChecker = String? Function(CommandResult);
78

79 80
final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
81
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
82 83
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
84
final String pubCache = path.join(flutterRoot, '.pub-cache');
85
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
86
final String flutterPluginsVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_plugins.version');
87 88

String get platformFolderName {
89
  if (Platform.isWindows) {
90
    return 'windows-x64';
91 92
  }
  if (Platform.isMacOS) {
93
    return 'darwin-x64';
94 95
  }
  if (Platform.isLinux) {
96
    return 'linux-x64';
97
  }
98 99 100
  throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.');
}
final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe');
101 102 103

/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
104
final List<String> flutterTestArgs = <String>[];
105

106 107 108 109
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};

110 111 112
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';

113 114
/// The number of Cirrus jobs that run Web tests in parallel.
///
115 116 117
/// The default is 8 shards. Typically .cirrus.yml would define the
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
///
118 119 120 121
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
122
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
123
  ? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
124
  : 8;
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

/// Tests that we don't run on Web.
///
/// In general avoid adding new tests here. If a test cannot run on the web
/// because it fails at runtime, such as when a piece of functionality is not
/// implemented or not implementable on the web, prefer using `skip` in the
/// test code. Only add tests here that cannot be skipped using `skip`. For
/// example:
///
///  * Test code cannot be compiled because it uses Dart VM-specific
///    functionality. In this case `skip` doesn't help because the code cannot
///    reach the point where it can even run the skipping logic.
///  * Migrations. It is OK to put tests here that need to be temporarily
///    disabled in certain modes because of some migration or initial bringup.
///
/// The key in the map is the renderer type that the list applies to. The value
/// is the list of tests known to fail for that renderer.
142
//
143
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String>>{
  'html': <String>[
    // These tests are not compilable on the web due to dependencies on
    // VM-specific functionality.
    'test/services/message_codecs_vm_test.dart',
    'test/examples/sector_layout_test.dart',
  ],
  'canvaskit': <String>[
    // These tests are not compilable on the web due to dependencies on
    // VM-specific functionality.
    'test/services/message_codecs_vm_test.dart',
    'test/examples/sector_layout_test.dart',

    // These tests are broken and need to be fixed.
    // TODO(yjbanov): https://github.com/flutter/flutter/issues/71604
    'test/painting/decoration_test.dart',
    'test/material/text_selection_theme_test.dart',
    'test/material/date_picker_test.dart',
    'test/rendering/layers_test.dart',
    'test/painting/text_style_test.dart',
    'test/widgets/image_test.dart',
    'test/cupertino/colors_test.dart',
    'test/cupertino/slider_test.dart',
    'test/material/text_field_test.dart',
    'test/rendering/proxy_box_test.dart',
    'test/widgets/app_overrides_test.dart',
    'test/material/calendar_date_picker_test.dart',
    'test/material/ink_paint_test.dart',
    'test/rendering/editable_test.dart',
    'test/cupertino/dialog_test.dart',
    'test/widgets/shape_decoration_test.dart',
    'test/material/time_picker_theme_test.dart',
    'test/cupertino/picker_test.dart',
    'test/material/chip_theme_test.dart',
    'test/cupertino/nav_bar_test.dart',
    'test/widgets/performance_overlay_test.dart',
    'test/widgets/html_element_view_test.dart',
    'test/cupertino/scaffold_test.dart',
    'test/rendering/platform_view_test.dart',
183
    'test/cupertino/context_menu_action_test.dart',
184 185
  ],
};
186

187
const String kTestHarnessShardName = 'test_harness_tests';
188
const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
189

190
// The seed used to shuffle tests. If not passed with
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;
String get shuffleSeed {
  if (_shuffleSeed == null) {
    // Change the seed at 7am, UTC.
    final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
    // Generates YYYYMMDD as the seed, so that testing continues to fail for a
    // day after the seed changes, and on other days the seed can be used to
    // replicate failures.
    _shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
  }
  return _shuffleSeed!;
}

206
/// When you call this, you can pass additional arguments to pass custom
207
/// arguments to flutter test. For example, you might want to call this
208
/// script with the parameter --local-engine=host_debug_unopt to
209
/// use your own build of the engine.
210
///
211
/// To run the tool_tests part, run it with SHARD=tool_tests
212
///
213
/// Examples:
214
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
215
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
216
Future<void> main(List<String> args) async {
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
  try {
    printProgress('STARTING ANALYSIS');
    for (final String arg in args) {
      if (arg.startsWith('--local-engine=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
        flutterTestArgs.add(arg);
      } else if (arg.startsWith('--local-engine-src-path=')) {
        localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
        flutterTestArgs.add(arg);
      } else if (arg.startsWith('--test-randomize-ordering-seed=')) {
        _shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
      } else if (arg.startsWith('--verbose')) {
        print = (Object? message) {
          system.print(message);
        };
      } else if (arg.startsWith('--abort-on-error')) {
        onError = () {
          system.exit(1);
        };
      } else {
        flutterTestArgs.add(arg);
      }
239
    }
240 241
    if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
      printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
242
    }
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
    await selectShard(<String, ShardRunner>{
      'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
      'build_tests': _runBuildTests,
      'framework_coverage': _runFrameworkCoverage,
      'framework_tests': _runFrameworkTests,
      'tool_tests': _runToolTests,
      // web_tool_tests is also used by HHH: https://dart.googlesource.com/recipes/+/refs/heads/master/recipes/dart/flutter_engine.py
      'web_tool_tests': _runWebToolTests,
      'tool_integration_tests': _runIntegrationToolTests,
      'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
      // All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
      'web_tests': _runWebHtmlUnitTests,
      // All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
      'web_canvaskit_tests': _runWebCanvasKitUnitTests,
      // All web integration tests
      'web_long_running_tests': _runWebLongRunningTests,
      'flutter_plugins': _runFlutterPluginsTests,
      'skp_generator': _runSkpGeneratorTests,
      kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
    });
  } catch (error, stackTrace) {
    foundError(<String>[
      'UNEXPECTED ERROR!',
      error.toString(),
      ...stackTrace.toString().split('\n'),
      'The test.dart script should be corrected to catch this error and call foundError().',
      '${yellow}Some tests are likely to have been skipped.$reset',
    ]);
    system.exit(255);
272 273
  }
  if (hasError) {
274
    printProgress('${bold}Test failed.$reset');
275
    reportErrorsAndExit();
276
  }
277 278
  printProgress('${bold}Test successful.$reset');
  system.exit(0);
279 280
}

281 282 283
final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
final bool _runningInDartHHHBot = _luciBotId.startsWith('luci-dart-');

284 285 286
/// Verify the Flutter Engine is the revision in
/// bin/cache/internal/engine.version.
Future<void> _validateEngineHash() async {
287
  if (_runningInDartHHHBot) {
288 289 290 291
    // The Dart HHH bots intentionally modify the local artifact cache
    // and then use this script to run Flutter's test suites.
    // Because the artifacts have been changed, this particular test will return
    // a false positive and should be skipped.
292
    print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $_luciBotId.');
293 294 295
    return;
  }
  final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
296
  final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
  if (result.flattenedStdout!.isNotEmpty) {
    foundError(<String>[
      '${red}The stdout of `$flutterTester --help` was not empty:$reset',
      ...result.flattenedStdout!.split('\n').map((String line) => ' $gray$reset $line'),
    ]);
  }
  final String actualVersion;
  try {
    actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
      return line.startsWith('Flutter Engine Version:');
    });
  } on StateError {
    foundError(<String>[
      '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset',
      ...result.flattenedStderr!.split('\n').map((String line) => ' $gray$reset $line'),
    ]);
    return;
  }
315
  if (!actualVersion.contains(expectedVersion)) {
316
    foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']);
317 318 319
  }
}

320
Future<void> _runTestHarnessTests() async {
321
  printProgress('${green}Running test harness tests...$reset');
322 323 324

  await _validateEngineHash();

325 326
  // Verify that the tests actually return failure on failure and success on
  // success.
327
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
328

329
  // We want to run these tests in parallel, because they each take some time
330 331 332
  // to run (e.g. compiling), so we don't want to run them in series, especially
  // on 20-core machines. However, we have a race condition, so for now...
  // Race condition issue: https://github.com/flutter/flutter/issues/90026
333 334
  final List<ShardRunner> tests = <ShardRunner>[
    () => _runFlutterTest(
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
      automatedTests,
      script: path.join('test_smoke_test', 'pass_test.dart'),
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'fail_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
      expectFailure: true,
      printOutput: false,
      outputChecker: (CommandResult result) {
        return result.flattenedStdout!.contains('failingPendingTimerTest')
          ? null
          : 'Failed to find the stack trace for the pending Timer.\n\n'
            'stdout:\n${result.flattenedStdout}\n\n'
            'stderr:\n${result.flattenedStderr}';
    }),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash1_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
    () => _runFlutterTest(
      automatedTests,
      script: path.join('test_smoke_test', 'crash2_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
369
    () => _runFlutterTest(
370 371 372 373 374
      automatedTests,
      script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
375
    () => _runFlutterTest(
376 377 378 379 380
      automatedTests,
      script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
      expectFailure: true,
      printOutput: false,
    ),
381
    () => _runFlutterTest(
382 383 384 385 386
      automatedTests,
      script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
      expectFailure: true,
      printOutput: false,
    ),
387 388 389 390
  ];

  List<ShardRunner> testsToRun;

391
  // Run all tests unless sharding is explicitly specified.
392
  final String? shardName = Platform.environment[kShardKey];
393
  if (shardName == kTestHarnessShardName) {
394 395 396 397 398 399 400
    testsToRun = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  } else {
    testsToRun = tests;
  }
  for (final ShardRunner test in testsToRun) {
    await test();
  }
401

402
  // Verify that we correctly generated the version file.
403
  final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
404
  if (versionError != null) {
405
    foundError(<String>[versionError]);
406
  }
407 408
}

409 410
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');

411
Future<void> _runGeneralToolTests() async {
412
  await _runDartTest(
413
    _toolsPath,
414 415
    testPaths: <String>[path.join('test', 'general.shard')],
    enableFlutterToolAsserts: false,
416

417
    // Detect unit test time regressions (poor time delay handling, etc).
418 419
    // This overrides the 15 minute default for tools tests.
    // See the README.md and dart_test.yaml files in the flutter_tools package.
420 421 422 423 424
    perTestTimeout: const Duration(seconds: 2),
  );
}

Future<void> _runCommandsToolTests() async {
425
  await _runDartTest(
426
    _toolsPath,
427
    forceSingleCore: true,
428
    testPaths: <String>[path.join('test', 'commands.shard')],
429 430 431
  );
}

432
Future<void> _runWebToolTests() async {
433 434 435 436 437 438 439 440
  final List<File> allFiles = Directory(path.join(_toolsPath, 'test', 'web.shard'))
      .listSync(recursive: true).whereType<File>().toList();
  final List<String> allTests = <String>[];
  for (final File file in allFiles) {
    if (file.path.endsWith('_test.dart')) {
      allTests.add(file.path);
    }
  }
441
  await _runDartTest(
442
    _toolsPath,
443
    forceSingleCore: true,
444
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
445
    includeLocalEngineEnv: true,
446 447 448
  );
}

449
Future<void> _runToolHostCrossArchTests() {
450
  return _runDartTest(
451 452 453 454 455 456 457
    _toolsPath,
    // These are integration tests
    forceSingleCore: true,
    testPaths: <String>[path.join('test', 'host_cross_arch.shard')],
  );
}

458
Future<void> _runIntegrationToolTests() async {
459
  final List<String> allTests = Directory(path.join(_toolsPath, 'test', 'integration.shard'))
460
      .listSync(recursive: true).whereType<File>()
461
      .map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
462 463
      .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();

464
  await _runDartTest(
465
    _toolsPath,
466 467
    forceSingleCore: true,
    testPaths: _selectIndexOfTotalSubshard<String>(allTests),
468
    collectMetrics: true,
469 470 471
  );
}

472
Future<void> _runToolTests() async {
473 474 475 476
  await selectSubshard(<String, ShardRunner>{
    'general': _runGeneralToolTests,
    'commands': _runCommandsToolTests,
  });
477 478
}

479 480
Future<void> runForbiddenFromReleaseTests() async {
  // Build a release APK to get the snapshot json.
481
  final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.');
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
  final List<String> command = <String>[
    'build',
    'apk',
    '--target-platform',
    'android-arm64',
    '--release',
    '--analyze-size',
    '--code-size-directory',
    tempDirectory.path,
    '-v',
  ];

  await runCommand(
    flutter,
    command,
    workingDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
  );

  // First, a smoke test.
  final List<String> smokeTestArgs = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::Widget',
  ];
  await runCommand(
    dart,
    smokeTestArgs,
    workingDirectory: flutterRoot,
    expectNonZeroExit: true,
  );

  // Actual test.
  final List<String> args = <String>[
    path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
    '--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
    '--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
    '--forbidden-type', 'package:flutter/src/widgets/widget_inspector.dart::WidgetInspectorService',
520 521
    '--forbidden-type', 'package:flutter/src/widgets/framework.dart::DebugCreator',
    '--forbidden-type', 'package:flutter/src/foundation/print.dart::debugPrint',
522 523 524 525 526 527 528 529
  ];
  await runCommand(
    dart,
    args,
    workingDirectory: flutterRoot,
  );
}

530 531 532 533 534
/// Verifies that APK, and IPA (if on macOS), and native desktop builds the
/// examples apps without crashing. It does not actually launch the apps. That
/// happens later in the devicelab. This is just a smoke-test. In particular,
/// this will verify we can build when there are spaces in the path name for the
/// Flutter SDK and target app.
535 536
///
/// Also does some checking about types included in hello_world.
537
Future<void> _runBuildTests() async {
538 539 540
  final List<Directory> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync()
    // API example builds will be tested in a separate shard.
    .where((FileSystemEntity entity) => entity is Directory && path.basename(entity.path) != 'api').cast<Directory>().toList()
541
    ..add(Directory(path.join(flutterRoot, 'packages', 'integration_test', 'example')))
542 543 544 545 546 547
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'channels')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'hybrid_android_views')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery')))
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_platform_view_tests')))
548
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
549
    ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
550

551 552 553
  // The tests are randomly distributed into subshards so as to get a uniform
  // distribution of costs, but the seed is fixed so that issues are reproducible.
  final List<ShardRunner> tests = <ShardRunner>[
554
    for (final Directory exampleDirectory in exampleDirectories)
555 556 557 558 559 560 561 562 563 564 565 566 567
      () => _runExampleProjectBuildTests(exampleDirectory),
    ...<ShardRunner>[
      // Web compilation tests.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web'),
            path.join('lib', 'main.dart'),
          ),
      // Should not fail to compile with dart:io.
      () => _flutterBuildDart2js(
            path.join('dev', 'integration_tests', 'web_compile_tests'),
            path.join('lib', 'dart_io_import.dart'),
          ),
    ],
568
    runForbiddenFromReleaseTests,
569 570
  ]..shuffle(math.Random(0));

571
  await _runShardRunnerIndexOfTotalSubshard(tests);
572 573
}

574
Future<void> _runExampleProjectBuildTests(Directory exampleDirectory, [File? mainFile]) async {
575 576
  // Only verify caching with flutter gallery.
  final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
577
  final String examplePath = path.relative(exampleDirectory.path, from: Directory.current.path);
578
  final bool hasNullSafety = File(path.join(examplePath, 'null_safety')).existsSync();
579 580 581 582
  final List<String> additionalArgs = <String>[
    if (hasNullSafety) '--no-sound-null-safety',
    if (mainFile != null) path.relative(mainFile.path, from: exampleDirectory.absolute.path),
  ];
583 584 585 586 587 588 589 590 591 592 593 594 595
  if (Directory(path.join(examplePath, 'android')).existsSync()) {
    await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
  } else {
    print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
  }
  if (Platform.isMacOS) {
    if (Directory(path.join(examplePath, 'ios')).existsSync()) {
      await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
    }
596
  }
597
  if (Platform.isLinux) {
598 599 600 601 602 603 604
    if (Directory(path.join(examplePath, 'linux')).existsSync()) {
      await _flutterBuildLinux(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildLinux(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no linux directory, skipping Linux');
    }
  }
605
  if (Platform.isMacOS) {
606 607 608 609 610 611 612
    if (Directory(path.join(examplePath, 'macos')).existsSync()) {
      await _flutterBuildMacOS(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildMacOS(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
    } else {
      print('Example project ${path.basename(examplePath)} has no macos directory, skipping macOS');
    }
  }
613
  if (Platform.isWindows) {
614
    if (Directory(path.join(examplePath, 'windows')).existsSync()) {
615 616
      await _flutterBuildWin32(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
      await _flutterBuildWin32(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
617 618 619 620
    } else {
      print('Example project ${path.basename(examplePath)} has no windows directory, skipping Win32');
    }
  }
621
}
622

623
Future<void> _flutterBuildApk(String relativePathToApplication, {
624
  required bool release,
625
  bool verifyCaching = false,
626 627
  List<String> additionalArgs = const <String>[],
}) async {
628
  printProgress('${green}Testing APK ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
629 630 631 632
  await _flutterBuild(relativePathToApplication, 'APK', 'apk',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
633
  );
634 635
}

636
Future<void> _flutterBuildIpa(String relativePathToApplication, {
637
  required bool release,
638
  List<String> additionalArgs = const <String>[],
639
  bool verifyCaching = false,
640
}) async {
641
  assert(Platform.isMacOS);
642
  printProgress('${green}Testing IPA ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
643 644 645 646 647 648 649
  await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: <String>[...additionalArgs, '--no-codesign'],
  );
}

650
Future<void> _flutterBuildLinux(String relativePathToApplication, {
651
  required bool release,
652 653 654 655 656
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isLinux);
  await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
657
  printProgress('${green}Testing Linux ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
658 659 660 661 662 663 664
  await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

665
Future<void> _flutterBuildMacOS(String relativePathToApplication, {
666
  required bool release,
667 668 669 670 671
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isMacOS);
  await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
672
  printProgress('${green}Testing macOS ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
673 674 675 676 677 678 679
  await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

680
Future<void> _flutterBuildWin32(String relativePathToApplication, {
681
  required bool release,
682 683 684 685
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
  assert(Platform.isWindows);
686
  printProgress('${green}Testing ${release ? 'release' : 'debug'} Windows build$reset for $cyan$relativePathToApplication$reset...');
687 688 689 690 691 692 693
  await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
    release: release,
    verifyCaching: verifyCaching,
    additionalArgs: additionalArgs
  );
}

694 695 696 697
Future<void> _flutterBuild(
  String relativePathToApplication,
  String platformLabel,
  String platformBuildName, {
698
  required bool release,
699 700 701
  bool verifyCaching = false,
  List<String> additionalArgs = const <String>[],
}) async {
702
  await runCommand(flutter,
703 704
    <String>[
      'build',
705
      platformBuildName,
706 707 708 709 710 711 712
      ...additionalArgs,
      if (release)
        '--release'
      else
        '--debug',
      '-v',
    ],
713 714
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
715

716
  if (verifyCaching) {
717
    printProgress('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
718 719 720
    await runCommand(flutter,
      <String>[
        'build',
721
        platformBuildName,
722 723 724 725 726 727 728 729 730 731 732 733
        '--performance-measurement-file=perf.json',
        ...additionalArgs,
        if (release)
          '--release'
        else
          '--debug',
        '-v',
      ],
      workingDirectory: path.join(flutterRoot, relativePathToApplication),
    );
    final File file = File(path.join(flutterRoot, relativePathToApplication, 'perf.json'));
    if (!_allTargetsCached(file)) {
734 735 736 737
      foundError(<String>[
        '${red}Not all build targets cached after second run.$reset',
        'The target performance data was: ${file.readAsStringSync().replaceAll('},', '},\n')}',
      ]);
738 739 740 741 742
    }
  }
}

bool _allTargetsCached(File performanceFile) {
743 744 745 746 747
  final Map<String, Object?> data = json.decode(performanceFile.readAsStringSync())
    as Map<String, Object?>;
  final List<Map<String, Object?>> targets = (data['targets']! as List<Object?>)
    .cast<Map<String, Object?>>();
  return targets.every((Map<String, Object?> element) => element['skipped'] == true);
748 749
}

750
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
751
  printProgress('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
752 753 754 755 756 757 758
  await runCommand(flutter,
    <String>['build', 'web', '-v', '--target=$target'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: expectNonZeroExit,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
759
  );
760 761
}

762 763
Future<void> _runAddToAppLifeCycleTests() async {
  if (Platform.isMacOS) {
764
    printProgress('${green}Running add-to-app life cycle iOS integration tests$reset...');
765 766 767 768 769
    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
    await runCommand('./build_and_test.sh',
      <String>[],
      workingDirectory: addToAppDir,
    );
770 771
  } else {
    printProgress('${yellow}Skipped on this platform (only iOS has add-to-add lifecycle tests at this time).$reset');
772 773 774
  }
}

775
Future<void> _runFrameworkTests() async {
776 777
  final List<String> soundNullSafetyOptions     = <String>['--null-assertions', '--sound-null-safety'];
  final List<String> mixedModeNullSafetyOptions = <String>['--null-assertions', '--no-sound-null-safety'];
778
  final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
779

780
  Future<void> runWidgets() async {
781
    printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
782 783 784
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
785
        options: <String>[trackWidgetCreationOption, ...soundNullSafetyOptions],
786 787 788
        tests: <String>[ path.join('test', 'widgets') + path.separator ],
      );
    }
789
    // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
790 791 792 793
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
        options: <String>[trackWidgetCreationOption],
794
        fatalWarnings: false, // until we've migrated video_player
795 796
      );
    }
797 798 799 800
    // Run release mode tests (see packages/flutter/test_release/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      options: <String>['--dart-define=dart.vm.product=true', ...soundNullSafetyOptions],
801
      tests: <String>['test_release${path.separator}'],
802
    );
803 804 805 806 807 808
    // Run profile mode tests (see packages/flutter/test_profile/README.md)
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      options: <String>['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true', ...soundNullSafetyOptions],
      tests: <String>['test_profile${path.separator}'],
    );
809 810
  }

811
  Future<void> runLibraries() async {
812
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
813
      .listSync(followLinks: false)
814
      .whereType<Directory>()
815
      .where((Directory dir) => dir.path.endsWith('widgets') == false)
816
      .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
817
      .toList();
818
    printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
819 820 821
    for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
      await _runFlutterTest(
        path.join(flutterRoot, 'packages', 'flutter'),
822
        options: <String>[trackWidgetCreationOption, ...soundNullSafetyOptions],
823 824 825
        tests: tests,
      );
    }
826 827
  }

828 829 830 831 832 833 834 835 836 837 838 839 840 841
  Future<void> runExampleTests() async {
    // TODO(gspencergoog): Currently Linux LUCI bots can't run desktop Flutter applications, https://github.com/flutter/flutter/issues/90676
    if (!Platform.isLinux || ciProvider != CiProviders.luci) {
      await runCommand(
        flutter,
        <String>['config', '--enable-${Platform.operatingSystem}-desktop'],
        workingDirectory: flutterRoot,
      );
      await runCommand(
        dart,
        <String>[path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')],
        workingDirectory: path.join(flutterRoot, 'examples', 'api'),
      );
    }
842
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'api'), options: soundNullSafetyOptions);
843 844 845 846
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), options: soundNullSafetyOptions);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), options: soundNullSafetyOptions);
  }

Ian Hickson's avatar
Ian Hickson committed
847 848 849 850 851 852 853 854 855 856 857 858
  Future<void> runTracingTests() async {
    final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests');

    // run the tests for debug mode
    await _runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);

    Future<List<String>> verifyTracingAppBuild({
      required String modeArgument,
      required String sourceFile,
      required Set<String> allowed,
      required Set<String> disallowed,
    }) async {
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
      try {
        await runCommand(
          flutter,
          <String>[
            'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile),
          ],
          workingDirectory: tracingDirectory,
        );
        final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync());
        final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
        final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
        final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
        await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
        final List<String> results = <String>[];
        for (final String pattern in allowed) {
          if (!libappStrings.contains(pattern)) {
            results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.');
          }
Ian Hickson's avatar
Ian Hickson committed
877
        }
878 879 880 881
        for (final String pattern in disallowed) {
          if (libappStrings.contains(pattern)) {
            results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.');
          }
Ian Hickson's avatar
Ian Hickson committed
882
        }
883 884 885 886 887 888
        return results;
      } catch (error, stackTrace) {
        return <String>[
          error.toString(),
          ...stackTrace.toString().trimRight().split('\n'),
        ];
Ian Hickson's avatar
Ian Hickson committed
889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
      }
    }

    final List<String> results = <String>[];
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'profile',
      sourceFile: 'control.dart', // this is the control, the other two below are the actual test
      allowed: <String>{
        'TIMELINE ARGUMENTS TEST CONTROL FILE',
        'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
      },
    ));
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'profile',
      sourceFile: 'test.dart',
      allowed: <String>{
        'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls
        'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds
        // (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort)
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
        'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
        'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker
      },
    ));
    results.addAll(await verifyTracingAppBuild(
      modeArgument: 'release',
      sourceFile: 'test.dart',
      allowed: <String>{
        'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls
      },
      disallowed: <String>{
        'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE',
        'BUILD', 'LAYOUT', 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds
        'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
        'toTimelineArguments used in non-debug build', // not included in release builds
      },
    ));
    if (results.isNotEmpty) {
932
      foundError(results);
Ian Hickson's avatar
Ian Hickson committed
933 934 935
    }
  }

936 937 938 939 940 941 942 943 944 945 946 947
  Future<void> runFixTests() async {
    final List<String> args = <String>[
      'fix',
      '--compare-to-golden',
    ];
    await runCommand(
      dart,
      args,
      workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_fixes'),
    );
  }

948 949 950
  Future<void> runPrivateTests() async {
    final List<String> args = <String>[
      '--sound-null-safety',
951 952
      'run',
      'bin/test_private.dart',
953
    ];
954
    final Map<String, String> environment = <String, String>{
955
      'FLUTTER_ROOT': flutterRoot,
956 957
      if (Directory(pubCache).existsSync())
        'PUB_CACHE': pubCache,
958
    };
959
    adjustEnvironmentToEnableFlutterAsserts(environment);
960
    await runCommand(
961
      dart,
962 963
      args,
      workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'),
964
      environment: environment,
965 966 967
    );
  }

968 969 970 971 972 973 974 975 976 977 978
  // Tests that take longer than average to run. This is usually because they
  // need to compile something large or make use of the analyzer for the test.
  // These tests need to be platform agnostic as they are only run on a linux
  // machine to save on execution time and cost.
  Future<void> runSlow() async {
    printProgress('${green}Running slow package tests$reset for directories other than packages/flutter');
    await runTracingTests();
    await runFixTests();
    await runPrivateTests();
  }

979
  Future<void> runMisc() async {
980
    printProgress('${green}Running package tests$reset for directories other than packages/flutter');
981
    await _runTestHarnessTests();
982
    await runExampleTests();
983 984 985
    await _runDartTest(path.join(flutterRoot, 'dev', 'bots'));
    await _runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
    await _runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
986
    // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed.
987
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
988
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
989 990
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
991
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
992
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
993
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
994
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')], options: soundNullSafetyOptions);
995 996 997 998 999
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: <String>[
      '--enable-vmservice',
      // Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard.
      '--exclude-tags=web',
    ]);
1000
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'), options: soundNullSafetyOptions);
1001 1002
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), options: soundNullSafetyOptions);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), options: soundNullSafetyOptions);
1003
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), options: soundNullSafetyOptions);
1004
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'), options: mixedModeNullSafetyOptions);
1005 1006 1007 1008
    const String httpClientWarning =
      'Warning: At least one test in this suite creates an HttpClient. When\n'
      'running a test suite that uses TestWidgetsFlutterBinding, all HTTP\n'
      'requests will return status code 400, and no network request will\n'
1009
      'actually be made. Any test expecting a real network connection and\n'
1010 1011 1012 1013 1014 1015 1016 1017 1018
      'status code will fail.\n'
      'To test code that needs an HttpClient, provide your own HttpClient\n'
      'implementation to the code under test, so that your test can\n'
      'consistently provide a testable response to the code under test.';
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter_test'),
      script: path.join('test', 'bindings_test_failure.dart'),
      expectFailure: true,
      printOutput: false,
1019
      outputChecker: (CommandResult result) {
1020
        final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout!);
1021
        if (matches == null || matches.isEmpty || matches.length > 1) {
1022 1023 1024
          return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n'
                 'stdout:\n${result.flattenedStdout}\n\n'
                 'stderr:\n${result.flattenedStderr}';
1025 1026 1027 1028
        }
        return null;
      },
    );
1029
  }
1030

1031 1032 1033
  await selectSubshard(<String, ShardRunner>{
    'widgets': runWidgets,
    'libraries': runLibraries,
1034
    'slow': runSlow,
1035 1036
    'misc': runMisc,
  });
1037 1038
}

1039
Future<void> _runFrameworkCoverage() async {
1040 1041
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
1042 1043
    foundError(<String>[
      '${red}Coverage file not found.$reset',
1044
      'Expected to find: $cyan${coverageFile.absolute.path}$reset',
1045 1046 1047
      'This file is normally obtained by running `${green}flutter update-packages$reset`.',
    ]);
    return;
1048 1049 1050 1051 1052 1053
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
1054 1055
    foundError(<String>[
      '${red}Coverage file not found.$reset',
1056
      'Expected to find: $cyan${coverageFile.absolute.path}$reset',
1057 1058 1059
      'This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.',
    ]);
    return;
1060
  }
1061 1062
}

1063 1064 1065 1066 1067 1068 1069 1070 1071
Future<void> _runWebHtmlUnitTests() {
  return _runWebUnitTests('html');
}

Future<void> _runWebCanvasKitUnitTests() {
  return _runWebUnitTests('canvaskit');
}

Future<void> _runWebUnitTests(String webRenderer) async {
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};

  final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));

  final List<String> allTests = flutterPackageTestDirectory
    .listSync()
    .whereType<Directory>()
    .expand((Directory directory) => directory
      .listSync(recursive: true)
      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
    )
    .whereType<File>()
    .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
1086
    .where((String filePath) => !kWebTestFileKnownFailures[webRenderer]!.contains(path.split(filePath).join('/')))
1087 1088 1089 1090 1091 1092 1093
    .toList()
    // Finally we shuffle the list because we want the average cost per file to be uniformly
    // distributed. If the list is not sorted then different shards and batches may have
    // very different characteristics.
    // We use a constant seed for repeatability.
    ..shuffle(math.Random(0));

1094 1095 1096
  assert(webShardCount >= 1);
  final int testsPerShard = (allTests.length / webShardCount).ceil();
  assert(testsPerShard * webShardCount >= allTests.length);
1097 1098

  // This for loop computes all but the last shard.
1099
  for (int index = 0; index < webShardCount - 1; index += 1) {
1100
    subshards['$index'] = () => _runFlutterWebTest(
1101
      webRenderer,
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
      flutterPackageDirectory.path,
      allTests.sublist(
        index * testsPerShard,
        (index + 1) * testsPerShard,
      ),
    );
  }

  // The last shard also runs the flutter_web_plugins tests.
  //
  // We make sure the last shard ends in _last so it's easier to catch mismatches
  // between `.cirrus.yml` and `test.dart`.
1114
  subshards['${webShardCount - 1}_last'] = () async {
1115
    await _runFlutterWebTest(
1116
      webRenderer,
1117 1118
      flutterPackageDirectory.path,
      allTests.sublist(
1119
        (webShardCount - 1) * testsPerShard,
1120 1121 1122 1123
        allTests.length,
      ),
    );
    await _runFlutterWebTest(
1124
      webRenderer,
1125 1126 1127
      path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
      <String>['test'],
    );
1128
    await _runFlutterWebTest(
1129 1130 1131
      webRenderer,
      path.join(flutterRoot, 'packages', 'flutter_driver'),
      <String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
1132
    );
1133 1134 1135 1136 1137
  };

  await selectSubshard(subshards);
}

1138 1139 1140
/// Coarse-grained integration tests running on the Web.
Future<void> _runWebLongRunningTests() async {
  final List<ShardRunner> tests = <ShardRunner>[
1141
    for (String buildMode in _kAllBuildModes) ...<ShardRunner>[
1142 1143 1144 1145 1146 1147 1148 1149 1150
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('test_driver', 'failure.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
        // This test intentionally fails and prints stack traces in the browser
        // logs. To avoid confusion, silence browser output.
        silenceBrowserOutput: true,
      ),
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('integration_test', 'example_test.dart'),
        driver: path.join('test_driver', 'integration_test.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
      ),
      () => _runFlutterDriverWebTest(
        testAppDirectory: path.join('packages', 'integration_test', 'example'),
        target: path.join('integration_test', 'extended_test.dart'),
        driver: path.join('test_driver', 'extended_integration_test.dart'),
        buildMode: buildMode,
        renderer: 'canvaskit',
      ),
    ],
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194

    // This test specifically tests how images are loaded in HTML mode, so we don't run it in CanvasKit mode.
    () => _runWebE2eTest('image_loading_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('image_loading_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('image_loading_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('platform_messages_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),

    // This test is only known to work in debug mode.
    () => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
    () => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),

    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),

1195 1196 1197 1198 1199
    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    () => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'debug', renderer: 'auto'),
    () => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'profile', renderer: 'canvaskit'),
    () => _runWebE2eTest('capabilities_integration_html', buildMode: 'release', renderer: 'html'),

1200 1201 1202 1203 1204
    // This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
    // CacheWidth and CacheHeight are only currently supported in CanvasKit mode, so we don't run the test in HTML mode.
    () => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'debug', renderer: 'auto'),
    () => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'profile', renderer: 'canvaskit'),

1205 1206
    () => _runWebTreeshakeTest(),

1207 1208 1209 1210
    () => _runFlutterDriverWebTest(
      testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
      target: 'test_driver/smoke_web_engine.dart',
      buildMode: 'profile',
1211
      renderer: 'auto',
1212
    ),
1213 1214 1215 1216 1217 1218
    () => _runGalleryE2eWebTest('debug'),
    () => _runGalleryE2eWebTest('debug', canvasKit: true),
    () => _runGalleryE2eWebTest('profile'),
    () => _runGalleryE2eWebTest('profile', canvasKit: true),
    () => _runGalleryE2eWebTest('release'),
    () => _runGalleryE2eWebTest('release', canvasKit: true),
1219 1220 1221
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
1222
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
1223
    () => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
1224 1225 1226
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
1227
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
1228
    () => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
1229
    () => runWebServiceWorkerTestWithGeneratedEntrypoint(headless: true),
1230
    () => runWebServiceWorkerTestWithBlockedServiceWorkers(headless: true),
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
    () => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
    () => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
    () => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/stack_trace.dart'),
    () => _runWebDebugTest('lib/framework_stack_trace.dart'),
    () => _runWebDebugTest('lib/web_directory_loading.dart'),
    () => _runWebDebugTest('test/test.dart'),
    () => _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true),
    () => _runWebDebugTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
    () => _runWebReleaseTest('lib/web_define_loading.dart',
      additionalArguments: <String>[
        '--dart-define=test.valueA=Example,A',
        '--dart-define=test.valueB=Value',
      ]
    ),
    () => _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
      '--sound-null-safety',
    ]),
    () => _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
      '--sound-null-safety',
    ]),
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
    () => _runFlutterWebTest(
      'html',
      path.join(flutterRoot, 'packages', 'integration_test'),
      <String>['test/web_extension_test.dart'],
    ),
    () => _runFlutterWebTest(
      'canvaskit',
      path.join(flutterRoot, 'packages', 'integration_test'),
      <String>['test/web_extension_test.dart'],
    ),
1268
  ];
1269 1270 1271 1272 1273

  // Shuffling mixes fast tests with slow tests so shards take roughly the same
  // amount of time to run.
  tests.shuffle(math.Random(0));

1274
  await _ensureChromeDriverIsRunning();
1275
  await _runShardRunnerIndexOfTotalSubshard(tests);
1276
  await _stopChromeDriver();
1277 1278
}

1279 1280 1281
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
Future<void> _runWebE2eTest(
  String name, {
1282 1283
  required String buildMode,
  required String renderer,
1284 1285 1286 1287 1288 1289 1290 1291 1292
}) async {
  await _runFlutterDriverWebTest(
    target: path.join('test_driver', '$name.dart'),
    buildMode: buildMode,
    renderer: renderer,
    testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
  );
}

1293
Future<void> _runFlutterDriverWebTest({
1294 1295 1296 1297
  required String target,
  required String buildMode,
  required String renderer,
  required String testAppDirectory,
1298
  String? driver,
1299 1300
  bool expectFailure = false,
  bool silenceBrowserOutput = false,
1301
}) async {
1302
  printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
1303 1304 1305 1306 1307 1308 1309 1310
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1311
      ...flutterTestArgs,
1312
      'drive',
1313
      if (driver != null) '--driver=$driver',
1314 1315 1316 1317 1318 1319
      '--target=$target',
      '--browser-name=chrome',
      '--no-sound-null-safety',
      '-d',
      'web-server',
      '--$buildMode',
1320
      '--web-renderer=$renderer',
1321
    ],
1322
    expectNonZeroExit: expectFailure,
1323 1324 1325 1326
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
1327 1328 1329 1330 1331 1332 1333 1334 1335
    removeLine: (String line) {
      if (!silenceBrowserOutput) {
        return false;
      }
      if (line.trim().startsWith('[INFO]')) {
        return true;
      }
      return false;
    },
1336 1337 1338
  );
}

1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
// Compiles a sample web app and checks that its JS doesn't contain certain
// debug code that we expect to be tree shaken out.
//
// The app is compiled in `--profile` mode to prevent the compiler from
// minifying the symbols.
Future<void> _runWebTreeshakeTest() async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
  final String target = path.join('lib', 'treeshaking_main.dart');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--target=$target',
      '--no-sound-null-safety',
      '--profile',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
  final String javaScript = mainDartJs.readAsStringSync();

  // Check that we're not looking at minified JS. Otherwise this test would result in false positive.
  expect(javaScript.contains('RenderObjectToWidgetElement'), true);

  const String word = 'debugFillProperties';
  int count = 0;
  int pos = javaScript.indexOf(word);
  final int contentLength = javaScript.length;
  while (pos != -1) {
    count += 1;
    pos += word.length;
    if (pos >= contentLength || count > 100) {
      break;
    }
    pos = javaScript.indexOf(word, pos);
  }

  const int kMaxExpectedDebugFillProperties = 11;
  if (count > kMaxExpectedDebugFillProperties) {
    throw Exception(
      'Too many occurrences of "$word" in compiled JavaScript.\n'
      'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
    );
  }
}

1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
/// Returns the commit hash of the flutter/plugins repository that's rolled in.
///
/// The flutter/plugins repository is a downstream dependency, it is only used
/// by flutter/flutter for testing purposes, to assure stable tests for a given
/// flutter commit the flutter/plugins commit hash to test against is coded in
/// the bin/internal/flutter_plugins.version file.
///
/// The `filesystem` parameter specified filesystem to read the plugins version file from.
/// The `pluginsVersionFile` parameter allows specifying an alternative path for the
/// plugins version file, when null [flutterPluginsVersionFile] is used.
Future<String> getFlutterPluginsVersion({
  fs.FileSystem fileSystem = const LocalFileSystem(),
1407
  String? pluginsVersionFile,
1408 1409 1410 1411 1412 1413
}) async {
  final File versionFile = fileSystem.file(pluginsVersionFile ?? flutterPluginsVersionFile);
  final String versionFileContents = await versionFile.readAsString();
  return versionFileContents.trim();
}

1414 1415 1416
/// Executes the test suite for the flutter/plugins repo.
Future<void> _runFlutterPluginsTests() async {
  Future<void> runAnalyze() async {
1417
    printProgress('${green}Running analysis for flutter/plugins$reset');
1418
    final Directory checkout = Directory.systemTemp.createTempSync('flutter_plugins.');
1419 1420 1421 1422 1423 1424 1425
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'clone',
        'https://github.com/flutter/plugins.git',
1426
        '.',
1427 1428 1429
      ],
      workingDirectory: checkout.path,
    );
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
    final String pluginsCommit = await getFlutterPluginsVersion();
    await runCommand(
      'git',
      <String>[
        '-c',
        'core.longPaths=true',
        'checkout',
        pluginsCommit,
      ],
      workingDirectory: checkout.path,
    );
1441 1442 1443 1444 1445 1446
    // Prep the repository tooling.
    // This test does not use tool_runner.sh because in this context the test
    // should always run on the entire plugins repo, while tool_runner.sh
    // is designed for flutter/plugins CI and only analyzes changed repository
    // files when run for anything but master.
    final String toolDir = path.join(checkout.path, 'script', 'tool');
1447
    await runCommand(
1448
      'dart',
1449
      <String>[
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
        'pub',
        'get',
      ],
      workingDirectory: toolDir,
    );
    final String toolScript = path.join(toolDir, 'bin', 'flutter_plugin_tools.dart');
    await runCommand(
      'dart',
      <String>[
        'run',
        toolScript,
1461
        'analyze',
1462
        '--custom-analysis=script/configs/custom_analysis.yaml',
1463 1464 1465 1466 1467 1468 1469 1470 1471
      ],
      workingDirectory: checkout.path,
    );
  }
  await selectSubshard(<String, ShardRunner>{
    'analyze': runAnalyze,
  });
}

1472 1473 1474 1475 1476 1477
/// Runs the skp_generator from the flutter/tests repo.
///
/// See also the customer_tests shard.
///
/// Generated SKPs are ditched, this just verifies that it can run without failure.
Future<void> _runSkpGeneratorTests() async {
1478
  printProgress('${green}Running skp_generator from flutter/tests$reset');
1479 1480 1481 1482 1483 1484 1485 1486
  final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.');
  await runCommand(
    'git',
    <String>[
      '-c',
      'core.longPaths=true',
      'clone',
      'https://github.com/flutter/tests.git',
1487
      '.',
1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
    ],
    workingDirectory: checkout.path,
  );
  await runCommand(
    './build.sh',
    <String>[ ],
    workingDirectory: path.join(checkout.path, 'skp_generator'),
  );
}

1498 1499 1500 1501
// The `chromedriver` process created by this test.
//
// If an existing chromedriver is already available on port 4444, the existing
// process is reused and this variable remains null.
1502
Command? _chromeDriver;
1503 1504 1505

Future<bool> _isChromeDriverRunning() async {
  try {
1506 1507 1508
    final RawSocket socket = await RawSocket.connect('localhost', 4444);
    socket.shutdown(SocketDirection.both);
    await socket.close();
1509 1510 1511 1512 1513 1514 1515 1516 1517
    return true;
  } on SocketException {
    return false;
  }
}

Future<void> _ensureChromeDriverIsRunning() async {
  // If we cannot connect to ChromeDriver, assume it is not running. Launch it.
  if (!await _isChromeDriverRunning()) {
1518
    printProgress('Starting chromedriver');
1519 1520
    // Assume chromedriver is in the PATH.
    _chromeDriver = await startCommand(
1521 1522 1523
      // TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
      // and it doesn't use most of startCommand's features; we could simplify this a lot by
      // inlining the relevant parts of startCommand here.
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
      'chromedriver',
      <String>['--port=4444'],
    );
    while (!await _isChromeDriverRunning()) {
      await Future<void>.delayed(const Duration(milliseconds: 100));
      print('Waiting for chromedriver to start up.');
    }
  }

  final HttpClient client = HttpClient();
  final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
  final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
  final HttpClientResponse response = await request.close();
1537
  final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join()) as Map<String, dynamic>;
1538
  client.close();
1539
  final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
1540 1541 1542 1543 1544 1545 1546 1547 1548
  if (!webDriverReady) {
    throw Exception('WebDriver not available.');
  }
}

Future<void> _stopChromeDriver() async {
  if (_chromeDriver == null) {
    return;
  }
1549
  print('Stopping chromedriver');
1550
  _chromeDriver!.process.kill();
1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
}

/// Exercises the old gallery in a browser for a long period of time, looking
/// for memory leaks and dangling pointers.
///
/// This is not a performance test.
///
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
///
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
1563
  printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
1564 1565 1566 1567 1568 1569 1570 1571 1572
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1573
      ...flutterTestArgs,
1574 1575 1576
      'drive',
      if (canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=true',
1577 1578
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1579 1580
      if (!canvasKit)
        '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1581 1582 1583
      '--driver=test_driver/transitions_perf_e2e_test.dart',
      '--target=test_driver/transitions_perf_e2e.dart',
      '--browser-name=chrome',
1584
      '--no-sound-null-safety',
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
      '-d',
      'web-server',
      '--$buildMode',
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
}

1596
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
1597
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
1598
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
      'build',
      'web',
      '--$buildMode',
      '-t',
1613
      entrypoint,
1614 1615 1616 1617 1618 1619 1620 1621
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1622 1623
  final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
  final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
1624
  final String result = await evalTestAppInChrome(
1625
    appUrl: 'http://localhost:$serverPort/index.html',
1626
    appDirectory: appBuildDirectory,
1627 1628
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1629 1630
  );

1631 1632 1633 1634 1635
  if (!result.contains('--- TEST SUCCEEDED ---')) {
    foundError(<String>[
      result,
      '${red}Web stack trace integration test failed.$reset',
    ]);
1636 1637 1638
  }
}

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
/// Run a web integration test in release mode.
Future<void> _runWebReleaseTest(String target, {
  List<String> additionalArguments = const<String>[],
}) async {
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');

  // Build the app.
  await runCommand(
    flutter,
    <String>[ 'clean' ],
    workingDirectory: testAppDirectory,
  );
  await runCommand(
    flutter,
    <String>[
1655
      ...flutterTestArgs,
1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
      'build',
      'web',
      '--release',
      ...additionalArguments,
      '-t',
      target,
    ],
    workingDirectory: testAppDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );

  // Run the app.
1670 1671
  final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
  final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
1672
  final String result = await evalTestAppInChrome(
1673
    appUrl: 'http://localhost:$serverPort/index.html',
1674
    appDirectory: appBuildDirectory,
1675 1676
    serverPort: serverPort,
    browserDebugPort: browserDebugPort,
1677 1678
  );

1679 1680 1681 1682 1683
  if (!result.contains('--- TEST SUCCEEDED ---')) {
    foundError(<String>[
      result,
      '${red}Web release mode test failed.$reset',
    ]);
1684 1685 1686
  }
}

1687 1688 1689
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
///
/// Instead, we use `flutter run --debug` and sniff out the standard output.
1690
Future<void> _runWebDebugTest(String target, {
1691
  bool enableNullSafety = false,
1692 1693
  List<String> additionalArguments = const<String>[],
}) async {
1694 1695
  final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
  bool success = false;
1696 1697 1698
  final Map<String, String> environment = <String, String>{
    'FLUTTER_WEB': 'true',
  };
1699
  adjustEnvironmentToEnableFlutterAsserts(environment);
1700
  final CommandResult result = await runCommand(
1701 1702 1703 1704
    flutter,
    <String>[
      'run',
      '--debug',
1705 1706
      if (enableNullSafety)
        ...<String>[
1707 1708
          '--no-sound-null-safety',
          '--null-assertions',
1709
        ],
1710 1711 1712
      '-d',
      'chrome',
      '--web-run-headless',
1713
      '--dart-define=FLUTTER_WEB_USE_SKIA=false',
1714
      '--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
1715
      ...additionalArguments,
1716 1717
      '-t',
      target,
1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728
    ],
    outputMode: OutputMode.capture,
    outputListener: (String line, Process process) {
      if (line.contains('--- TEST SUCCEEDED ---')) {
        success = true;
      }
      if (success || line.contains('--- TEST FAILED ---')) {
        process.stdin.add('q'.codeUnits);
      }
    },
    workingDirectory: testAppDirectory,
1729
    environment: environment,
1730 1731
  );

1732 1733 1734 1735 1736 1737
  if (!success) {
    foundError(<String>[
      result.flattenedStdout!,
      result.flattenedStderr!,
      '${red}Web stack trace integration test failed.$reset',
    ]);
1738 1739 1740
  }
}

1741
Future<void> _runFlutterWebTest(String webRenderer, String workingDirectory, List<String> tests) async {
1742 1743 1744 1745 1746 1747 1748 1749
  await runCommand(
    flutter,
    <String>[
      'test',
      if (ciProvider == CiProviders.cirrus)
        '--concurrency=1',  // do not parallelize on Cirrus, to reduce flakiness
      '-v',
      '--platform=chrome',
1750
      '--web-renderer=$webRenderer',
1751
      '--dart-define=DART_HHH_BOT=$_runningInDartHHHBot',
1752
      '--sound-null-safety',
1753
      ...flutterTestArgs,
1754 1755 1756 1757 1758 1759 1760
      ...tests,
    ],
    workingDirectory: workingDirectory,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
  );
1761 1762
}

1763 1764 1765 1766 1767
// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
1768
Future<void> _runDartTest(String workingDirectory, {
1769
  List<String>? testPaths,
1770 1771
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
1772
  String? coverage,
1773
  bool forceSingleCore = false,
1774
  Duration? perTestTimeout,
1775
  bool includeLocalEngineEnv = false,
1776
  bool ensurePrecompiledTool = true,
1777
  bool shuffleTests = true,
1778
  bool collectMetrics = false,
Dan Field's avatar
Dan Field committed
1779
}) async {
1780 1781
  int? cpus;
  final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
1782 1783 1784
  if (cpuVariable != null) {
    cpus = int.tryParse(cpuVariable, radix: 10);
    if (cpus == null) {
1785 1786 1787 1788 1789
      foundError(<String>[
        '${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset',
        'Actual value: "$cpuVariable"',
      ]);
      return;
1790 1791 1792 1793
    }
  } else {
    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
  }
1794 1795 1796 1797 1798
  // Integration tests that depend on external processes like chrome
  // can get stuck if there are multiple instances running at once.
  if (forceSingleCore) {
    cpus = 1;
  }
1799

1800 1801
  const LocalFileSystem fileSystem = LocalFileSystem();
  final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json'));
1802 1803 1804
  final List<String> args = <String>[
    'run',
    'test',
1805
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1806 1807 1808 1809 1810
    '-j$cpus',
    if (!hasColor)
      '--no-color',
    if (coverage != null)
      '--coverage=$coverage',
1811
    if (perTestTimeout != null)
1812
      '--timeout=${perTestTimeout.inMilliseconds}ms',
1813 1814 1815
    if (testPaths != null)
      for (final String testPath in testPaths)
        testPath,
1816 1817
    if (collectMetrics)
      '--file-reporter=json:${metricFile.path}',
1818
  ];
1819
  final Map<String, String> environment = <String, String>{
1820
    'FLUTTER_ROOT': flutterRoot,
1821 1822 1823 1824
    if (includeLocalEngineEnv)
      ...localEngineEnv,
    if (Directory(pubCache).existsSync())
      'PUB_CACHE': pubCache,
1825 1826
  };
  if (enableFlutterToolAsserts) {
1827
    adjustEnvironmentToEnableFlutterAsserts(environment);
1828
  }
1829 1830 1831 1832
  if (ensurePrecompiledTool) {
    // We rerun the `flutter` tool here just to make sure that it is compiled
    // before tests run, because the tests might time out if they have to rebuild
    // the tool themselves.
1833
    await runCommand(flutter, <String>['--version'], environment: environment);
1834
  }
1835 1836 1837 1838 1839 1840 1841
  await runCommand(
    dart,
    args,
    workingDirectory: workingDirectory,
    environment: environment,
    removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
  );
1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858

  if (collectMetrics) {
    try {
      final List<String> testList = <String>[];
      final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
      for (final TestSpecs testSpecs in allTestSpecs.values) {
        testList.add(testSpecs.toJson());
      }
      if (testList.isNotEmpty) {
        final String testJson = json.encode(testList);
        final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
        testResults.writeAsStringSync(testJson);
      }
    } on fs.FileSystemException catch (e){
      print('Failed to generate metrics: $e');
    }
  }
1859 1860
}

1861
Future<void> _runFlutterTest(String workingDirectory, {
1862
  String? script,
1863 1864
  bool expectFailure = false,
  bool printOutput = true,
1865
  OutputChecker? outputChecker,
1866
  List<String> options = const <String>[],
1867
  Map<String, String>? environment,
1868
  List<String> tests = const <String>[],
1869
  bool shuffleTests = true,
1870
  bool fatalWarnings = true,
Dan Field's avatar
Dan Field committed
1871
}) async {
1872
  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
1873

1874
  final List<String> tags = <String>[];
1875
  // Recipe-configured reduced test shards will only execute tests with the
1876
  // appropriate tag.
1877
  if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
1878 1879 1880
    tags.addAll(<String>['-t', 'reduced-test-set']);
  }

1881 1882
  final List<String> args = <String>[
    'test',
1883
    if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
1884
    if (fatalWarnings) '--fatal-warnings',
1885
    ...options,
1886
    ...tags,
1887
    ...flutterTestArgs,
1888
  ];
Dan Field's avatar
Dan Field committed
1889

1890 1891 1892
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
1893 1894 1895 1896 1897 1898 1899 1900
      foundError(<String>[
        '${red}Could not find test$reset: $green$fullScriptPath$reset',
        'Working directory: $cyan$workingDirectory$reset',
        'Script: $green$script$reset',
        if (!printOutput)
          'This is one of the tests that does not normally print output.',
      ]);
      return;
1901
    }
1902
    args.add(script);
1903
  }
1904

1905
  args.addAll(tests);
1906

1907 1908 1909
  final OutputMode outputMode = outputChecker == null && printOutput
    ? OutputMode.print
    : OutputMode.capture;
1910

1911 1912 1913 1914 1915 1916 1917 1918
  final CommandResult result = await runCommand(
    flutter,
    args,
    workingDirectory: workingDirectory,
    expectNonZeroExit: expectFailure,
    outputMode: outputMode,
    environment: environment,
  );
1919

1920 1921 1922 1923
  if (outputChecker != null) {
    final String? message = outputChecker(result);
    if (message != null) {
      foundError(<String>[message]);
1924
    }
1925
  }
1926 1927
}

1928 1929 1930
/// This will force the next run of the Flutter tool (if it uses the provided
/// environment) to have asserts enabled, by setting an environment variable.
void adjustEnvironmentToEnableFlutterAsserts(Map<String, String> environment) {
1931 1932 1933 1934 1935 1936
  // If an existing env variable exists append to it, but only if
  // it doesn't appear to already include enable-asserts.
  String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
  if (!toolsArgs.contains('--enable-asserts')) {
    toolsArgs += ' --enable-asserts';
  }
1937
  environment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
1938 1939
}

1940 1941 1942 1943 1944
enum CiProviders {
  cirrus,
  luci,
}

1945
CiProviders? get ciProvider {
1946 1947 1948 1949 1950 1951 1952 1953
  if (Platform.environment['CIRRUS_CI'] == 'true') {
    return CiProviders.cirrus;
  }
  if (Platform.environment['LUCI_CONTEXT'] != null) {
    return CiProviders.luci;
  }
  return null;
}
1954

1955 1956 1957 1958 1959
/// Checks the given file's contents to determine if they match the allowed
/// pattern for version strings.
///
/// Returns null if the contents are good. Returns a string if they are bad.
/// The string is an error message.
1960
Future<String?> verifyVersion(File file) async {
1961 1962
  final RegExp pattern = RegExp(
    r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
1963
  if (!file.existsSync()) {
1964
    return 'The version logic failed to create the Flutter version file.';
1965
  }
1966
  final String version = await file.readAsString();
1967
  if (version == '0.0.0-unknown') {
1968
    return 'The version logic failed to determine the Flutter version.';
1969 1970
  }
  if (!version.contains(pattern)) {
1971
    return 'The version logic generated an invalid version string: "$version".';
1972
  }
1973 1974 1975
  return null;
}

1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
  // Example: "1_3" means the first (one-indexed) shard of three total shards.
1988
  final String? subshardName = Platform.environment[subshardKey];
1989 1990 1991 1992
  if (subshardName == null) {
    print('$kSubshardKey environment variable is missing, skipping sharding');
    return tests;
  }
1993
  printProgress('$bold$subshardKey=$subshardName$reset');
1994 1995

  final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
1996
  final Match? match = pattern.firstMatch(subshardName);
1997
  if (match == null || match.groupCount != 2) {
1998 1999 2000
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"',
    ]);
2001
    throw Exception('Invalid subshard name: $subshardName');
2002 2003
  }
  // One-indexed.
2004
  final int index = int.parse(match.group(1)!);
2005
  final int total = int.parse(match.group(2)!);
2006
  if (index > total) {
2007 2008 2009 2010
    foundError(<String>[
      '${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.',
    ]);
    return <T>[];
2011 2012
  }

2013
  final int testsPerShard = (tests.length / total).ceil();
2014
  final int start = (index - 1) * testsPerShard;
2015
  final int end = math.min(index * testsPerShard, tests.length);
2016

2017
  print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
2018 2019 2020
  return tests.sublist(start, end);
}

2021
Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
2022 2023 2024 2025 2026 2027 2028 2029
  final List<ShardRunner> sublist = _selectIndexOfTotalSubshard<ShardRunner>(tests);
  for (final ShardRunner test in sublist) {
    await test();
  }
}

Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
2030 2031 2032 2033

const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
2034
  String? item = Platform.environment[key];
2035
  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
2036
    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
2037 2038 2039 2040
    assert(positionInTaskName < parts.length);
    item = parts[positionInTaskName];
  }
  if (item == null) {
2041
    for (final String currentItem in items.keys) {
2042
      printProgress('$bold$key=$currentItem$reset');
2043
      await items[currentItem]!();
2044 2045
    }
  } else {
2046
    printProgress('$bold$key=$item$reset');
2047
    if (!items.containsKey(item)) {
2048 2049 2050 2051 2052
      foundError(<String>[
        '${red}Invalid $name: $item$reset',
        'The available ${name}s are: ${items.keys.join(", ")}',
      ]);
      return;
2053
    }
2054
    await items[item]!();
2055 2056
  }
}