test.dart 39.4 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:io';
7
import 'dart:math' as math;
8

Dan Field's avatar
Dan Field committed
9 10 11
import 'package:googleapis/bigquery/v2.dart' as bq;
import 'package:googleapis_auth/auth_io.dart' as auth;
import 'package:http/http.dart' as http;
12
import 'package:path/path.dart' as path;
13

Dan Field's avatar
Dan Field committed
14
import 'flutter_compact_formatter.dart';
15
import 'run_command.dart';
16
import 'utils.dart';
17

18
typedef ShardRunner = Future<void> Function();
19

20 21 22 23 24 25 26 27
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
typedef OutputChecker = String Function(CapturedOutput);

28 29 30 31
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
32
final String pubCache = path.join(flutterRoot, '.pub-cache');
33
final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
34 35 36

/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
37
final List<String> flutterTestArgs = <String>[];
38

39
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
40

41
final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true';
42

43 44 45 46
/// The number of Cirrus jobs that run host-only devicelab tests in parallel.
///
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
47
const int kDeviceLabShardCount = 4;
48 49 50 51 52 53 54

/// The number of Cirrus jobs that run Web tests in parallel.
///
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
55
const int kWebShardCount = 8;
56 57 58 59 60 61 62 63

/// Maximum number of Web tests to run in a single `flutter test`. We found that
/// large batches can get flaky, possibly because we reuse a single instance of
/// the browser, and after many tests the browser's state gets corrupted.
const int kWebBatchSize = 20;

/// Tests that we don't run on Web for various reasons.
//
Yegor's avatar
Yegor committed
64
// TODO(yjbanov): we're getting rid of this blacklist as part of https://github.com/flutter/flutter/projects/60
65
const List<String> kWebTestFileBlacklist = <String>[
66
  // This test doesn't compile because it depends on code outside the flutter package.
Yegor's avatar
Yegor committed
67
  'test/examples/sector_layout_test.dart',
68 69 70 71 72 73 74
  'test/widgets/selectable_text_test.dart',
  'test/widgets/color_filter_test.dart',
  'test/widgets/editable_text_cursor_test.dart',
  'test/widgets/raw_keyboard_listener_test.dart',
  'test/widgets/editable_text_test.dart',
  'test/widgets/widget_inspector_test.dart',
  'test/widgets/shortcuts_test.dart',
75 76
  'test/material/text_form_field_test.dart',
  'test/material/data_table_test.dart',
Yegor's avatar
Yegor committed
77 78 79 80 81 82 83 84 85 86 87 88
  'test/cupertino/dialog_test.dart',
  'test/cupertino/nav_bar_test.dart',
  'test/cupertino/nav_bar_transition_test.dart',
  'test/cupertino/refresh_test.dart',
  'test/cupertino/switch_test.dart',
  'test/cupertino/text_field_test.dart',
  'test/cupertino/date_picker_test.dart',
  'test/cupertino/slider_test.dart',
  'test/cupertino/text_field_test.dart',
  'test/cupertino/segmented_control_test.dart',
  'test/cupertino/route_test.dart',
  'test/cupertino/activity_indicator_test.dart',
89
];
90

91
/// When you call this, you can pass additional arguments to pass custom
92
/// arguments to flutter test. For example, you might want to call this
93
/// script with the parameter --local-engine=host_debug_unopt to
94
/// use your own build of the engine.
95
///
96
/// To run the tool_tests part, run it with SHARD=tool_tests
97
///
98
/// Examples:
99
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
100
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
101
Future<void> main(List<String> args) async {
102 103 104 105 106 107 108 109 110 111
  print('$clock STARTING ANALYSIS');
  try {
    flutterTestArgs.addAll(args);
    if (Platform.environment.containsKey(CIRRUS_TASK_NAME))
      print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
    print('═' * 80);
    await _runSmokeTests();
    print('═' * 80);
    await selectShard(const <String, ShardRunner>{
      'add_to_app_tests': _runAddToAppTests,
112
      'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
113 114 115 116 117 118 119 120 121 122 123 124
      'build_tests': _runBuildTests,
      'framework_coverage': _runFrameworkCoverage,
      'framework_tests': _runFrameworkTests,
      'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
      'tool_coverage': _runToolCoverage,
      'tool_tests': _runToolTests,
      'web_tests': _runWebTests,
    });
  } on ExitException catch (error) {
    error.apply();
  }
  print('$clock ${bold}Test successful.$reset');
125 126
}

127
Future<void> _runSmokeTests() async {
128
  print('${green}Running smoketests...$reset');
129 130
  // Verify that the tests actually return failure on failure and success on
  // success.
131
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
132 133 134
  // We run the "pass" and "fail" smoke tests first, and alone, because those
  // are particularly critical and sensitive. If one of these fails, there's no
  // point even trying the others.
135 136 137 138 139
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pass_test.dart'),
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
140
    script: path.join('test_smoke_test', 'fail_test.dart'),
141 142 143
    expectFailure: true,
    printOutput: false,
  );
144
  // We run the timeout tests individually because they are timing-sensitive.
145
  await _runFlutterTest(automatedTests,
146 147
    script: path.join('test_smoke_test', 'timeout_pass_test.dart'),
    expectFailure: false,
148 149
    printOutput: false,
  );
150
  await _runFlutterTest(automatedTests,
151
    script: path.join('test_smoke_test', 'timeout_fail_test.dart'),
152 153 154
    expectFailure: true,
    printOutput: false,
  );
155 156 157 158
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
    expectFailure: true,
    printOutput: false,
159 160 161 162 163
    outputChecker: (CapturedOutput output) {
      return output.stdout.contains('failingPendingTimerTest')
        ? null
        : 'Failed to find the stack trace for the pending Timer.';
    }
164
  );
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
  // We run the remaining smoketests in parallel, because they each take some
  // time to run (e.g. compiling), so we don't want to run them in series,
  // especially on 20-core machines...
  await Future.wait<void>(
    <Future<void>>[
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash1_test.dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash2_test.dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
        expectFailure: true,
        printOutput: false,
      ),
195
      runCommand(flutter,
196 197
        <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
        workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
198
        expectNonZeroExit: true,
199
        outputMode: OutputMode.discard,
200 201
      ),
    ],
202 203
  );

204
  // Verify that we correctly generated the version file.
205
  final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
206 207
  if (versionError != null)
    exitWithError(<String>[versionError]);
208 209
}

Dan Field's avatar
Dan Field committed
210
Future<bq.BigqueryApi> _getBigqueryApi() async {
211 212 213
  if (!useFlutterTestFormatter) {
    return null;
  }
Dan Field's avatar
Dan Field committed
214 215
  // TODO(dnfield): How will we do this on LUCI?
  final String privateKey = Platform.environment['GCLOUD_SERVICE_ACCOUNT_KEY'];
216 217 218 219 220
  // If we're on Cirrus and a non-collaborator is doing this, we can't get the key.
  if (privateKey == null || privateKey.isEmpty || privateKey.startsWith('ENCRYPTED[')) {
    return null;
  }
  try {
221
    final auth.ServiceAccountCredentials accountCredentials = auth.ServiceAccountCredentials(
222 223 224 225 226 227 228 229
      'flutter-ci-test-reporter@flutter-infra.iam.gserviceaccount.com',
      auth.ClientId.serviceAccount('114390419920880060881.apps.googleusercontent.com'),
      '-----BEGIN PRIVATE KEY-----\n$privateKey\n-----END PRIVATE KEY-----\n',
    );
    final List<String> scopes = <String>[bq.BigqueryApi.BigqueryInsertdataScope];
    final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes);
    return bq.BigqueryApi(client);
  } catch (e) {
230
    print('${red}Failed to get BigQuery API client.$reset');
231
    print(e);
Dan Field's avatar
Dan Field committed
232 233 234 235
    return null;
  }
}

236
Future<void> _runToolCoverage() async {
237
  await runCommand( // Precompile tests to speed up subsequent runs.
238 239 240 241
    pub,
    <String>['run', 'build_runner', 'build'],
    workingDirectory: toolRoot,
  );
242 243 244 245 246 247 248 249
  await runCommand(
    dart,
    <String>[path.join('tool', 'tool_coverage.dart')],
    workingDirectory: toolRoot,
    environment: <String, String>{
      'FLUTTER_ROOT': flutterRoot,
    }
  );
250 251
}

252
Future<void> _runToolTests() async {
Dan Field's avatar
Dan Field committed
253
  final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
254

255 256 257 258 259 260 261 262 263 264 265 266
  const String kDotShard = '.shard';
  const String kTest = 'test';
  final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');

  final Map<String, ShardRunner> subshards = Map<String, ShardRunner>.fromIterable(
    Directory(path.join(toolsPath, kTest))
      .listSync()
      .map<String>((FileSystemEntity entry) => entry.path)
      .where((String name) => name.endsWith(kDotShard))
      .map<String>((String name) => path.basenameWithoutExtension(name)),
    // The `dynamic` on the next line is because Map.fromIterable isn't generic.
    value: (dynamic subshard) => () async {
267 268 269 270 271
      // Due to https://github.com/flutter/flutter/issues/46180, skip the hermetic directory
      // on Windows.
      final String suffix = Platform.isWindows && subshard == 'commands'
        ? 'permeable'
        : '';
272 273
      await _pubRunTest(
        toolsPath,
274
        testPath: path.join(kTest, '$subshard$kDotShard', suffix),
275 276
        useBuildRunner: canUseBuildRunner,
        tableData: bigqueryApi?.tabledata,
277
        enableFlutterToolAsserts: true,
278 279 280
      );
    },
  );
281

282
  await selectSubshard(subshards);
283 284
}

285 286 287 288 289 290 291
// Example apps that should not be built by _runBuildTests`
const List<String> _excludedExampleApplications = <String>[
  // This application contains no platform code and cannot be built, except for
  // as a part of a '--fast-start' Android application.
  'splash',
];

292 293 294 295
/// Verifies that AOT, APK, and IPA (if on macOS) builds the examples apps
/// without crashing. It does not actually launch the apps. That happens later
/// in the devicelab. This is just a smoke-test. In particular, this will verify
/// we can build when there are spaces in the path name for the Flutter SDK and
296 297
/// target app.
Future<void> _runBuildTests() async {
298
  final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list();
299
  await for (final FileSystemEntity fileEntity in exampleDirectories) {
300 301 302
    if (fileEntity is! Directory) {
      continue;
    }
303 304 305
    if (_excludedExampleApplications.any(fileEntity.path.endsWith)) {
      continue;
    }
306 307 308
    final String examplePath = fileEntity.path;
    await _flutterBuildAot(examplePath);
    await _flutterBuildApk(examplePath);
309 310 311
    if (Platform.isMacOS) {
      await _flutterBuildIpa(examplePath);
    }
312
  }
313 314 315 316 317 318 319 320 321 322 323 324 325 326

  final String branch = Platform.environment['CIRRUS_BRANCH'];
  if (branch != 'beta' && branch != 'stable') {
    // Web compilation tests.
    await _flutterBuildDart2js(
      path.join('dev', 'integration_tests', 'web'),
      path.join('lib', 'main.dart'),
    );
    // Should not fail to compile with dart:io.
    await _flutterBuildDart2js(
      path.join('dev', 'integration_tests', 'web_compile_tests'),
      path.join('lib', 'dart_io_import.dart'),
    );
  }
327
}
328

329
Future<void> _flutterBuildAot(String relativePathToApplication) async {
330
  print('${green}Testing AOT build$reset for $cyan$relativePathToApplication$reset...');
331 332 333 334 335
  await runCommand(flutter,
    <String>['build', 'aot', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
}
336

337
Future<void> _flutterBuildApk(String relativePathToApplication) async {
338
  print('${green}Testing APK --debug build$reset for $cyan$relativePathToApplication$reset...');
339 340 341 342
  await runCommand(flutter,
    <String>['build', 'apk', '--debug', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
343 344
}

345
Future<void> _flutterBuildIpa(String relativePathToApplication) async {
346 347
  assert(Platform.isMacOS);
  print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...');
348 349 350 351 352 353 354
  // Install Cocoapods.  We don't have these checked in for the examples,
  // and build ios doesn't take care of it automatically.
  final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
  if (podfile.existsSync()) {
    await runCommand('pod',
      <String>['install'],
      workingDirectory: podfile.parent.path,
355 356 357
      environment: <String, String>{
        'LANG': 'en_US.UTF-8',
      },
358 359 360 361
    );
  }
  await runCommand(flutter,
    <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
362 363 364 365
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
  );
}

366 367 368 369 370 371 372 373 374
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
  print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
  await runCommand(flutter,
    <String>['build', 'web', '-v', '--target=$target'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: expectNonZeroExit,
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
    },
375
  );
376 377
}

378 379 380 381 382 383 384 385 386 387 388
Future<void> _runAddToAppTests() async {
  if (Platform.isMacOS) {
    print('${green}Running add-to-app iOS integration tests$reset...');
    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
    await runCommand('./build_and_test.sh',
      <String>[],
      workingDirectory: addToAppDir,
    );
  }
}

389 390 391 392 393 394 395 396 397 398 399
Future<void> _runAddToAppLifeCycleTests() async {
  if (Platform.isMacOS) {
    print('${green}Running add-to-app life cycle iOS integration tests$reset...');
    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
    await runCommand('./build_and_test.sh',
      <String>[],
      workingDirectory: addToAppDir,
    );
  }
}

400
Future<void> _runFrameworkTests() async {
Dan Field's avatar
Dan Field committed
401
  final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
402

403
  Future<void> runWidgets() async {
404
    print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset');
405 406
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
407
      options: <String>['--track-widget-creation'],
408
      tableData: bigqueryApi?.tabledata,
409
      tests: <String>[ path.join('test', 'widgets') + path.separator ],
410 411 412
    );
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
413
      options: <String>['--no-track-widget-creation'],
414
      tableData: bigqueryApi?.tabledata,
415
      tests: <String>[ path.join('test', 'widgets') + path.separator ],
416
    );
417 418 419
    // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--no-track-widget-creation'], tableData: bigqueryApi?.tabledata);
420 421
  }

422
  Future<void> runLibraries() async {
423 424 425
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
      .listSync(followLinks: false, recursive: false)
      .whereType<Directory>()
426
      .where((Directory dir) => dir.path.endsWith('widgets') == false)
427
      .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
428
      .toList();
429
    print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset');
430 431
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
432
      options: <String>['--track-widget-creation'],
433 434 435 436 437
      tableData: bigqueryApi?.tabledata,
      tests: tests,
    );
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
438
      options: <String>['--no-track-widget-creation'],
439 440 441 442 443
      tableData: bigqueryApi?.tabledata,
      tests: tests,
    );
  }

444 445
  Future<void> runMisc() async {
    print('${green}Running package tests$reset for directories other than packages/flutter');
446 447 448
    await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata);
449
    await _pubRunTest(path.join(flutterRoot, 'dev', 'tools'), tableData: bigqueryApi?.tabledata);
450 451 452
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata);
453
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
454 455 456
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata);
457
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata, tests: <String>[path.join('test', 'src', 'real_tests')]);
458
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'), tableData: bigqueryApi?.tabledata);
459 460 461 462 463 464 465 466 467 468 469
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(
      path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'),
      tableData: bigqueryApi?.tabledata,
      environment: <String, String>{
        'FLUTTER_EXPERIMENTAL_BUILD': 'true',
      },
    );
  }
470

471 472 473 474 475
  await selectSubshard(<String, ShardRunner>{
    'widgets': runWidgets,
    'libraries': runLibraries,
    'misc': runMisc,
  });
476 477
}

478
Future<void> _runFrameworkCoverage() async {
479 480 481
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
482 483
    print('Expected to find: $cyan${coverageFile.absolute}$reset');
    print('This file is normally obtained by running `${green}flutter update-packages$reset`.');
484 485 486 487 488 489 490 491
    exit(1);
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
492 493
    print('Expected to find: $cyan${coverageFile.absolute}$reset');
    print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.');
494 495
    exit(1);
  }
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
}

Future<void> _runWebTests() async {
  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};

  final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));

  final List<String> allTests = flutterPackageTestDirectory
    .listSync()
    .whereType<Directory>()
    .expand((Directory directory) => directory
      .listSync(recursive: true)
      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
    )
    .whereType<File>()
    .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
    .where((String filePath) => !kWebTestFileBlacklist.contains(filePath))
    .toList()
    // Finally we shuffle the list because we want the average cost per file to be uniformly
    // distributed. If the list is not sorted then different shards and batches may have
    // very different characteristics.
    // We use a constant seed for repeatability.
    ..shuffle(math.Random(0));

  assert(kWebShardCount >= 1);
  final int testsPerShard = (allTests.length / kWebShardCount).ceil();
  assert(testsPerShard * kWebShardCount >= allTests.length);

  // This for loop computes all but the last shard.
  for (int index = 0; index < kWebShardCount - 1; index += 1) {
    subshards['$index'] = () => _runFlutterWebTest(
      flutterPackageDirectory.path,
      allTests.sublist(
        index * testsPerShard,
        (index + 1) * testsPerShard,
      ),
    );
  }

  // The last shard also runs the flutter_web_plugins tests.
  //
  // We make sure the last shard ends in _last so it's easier to catch mismatches
  // between `.cirrus.yml` and `test.dart`.
  subshards['${kWebShardCount - 1}_last'] = () async {
    await _runFlutterWebTest(
      flutterPackageDirectory.path,
      allTests.sublist(
        (kWebShardCount - 1) * testsPerShard,
        allTests.length,
      ),
    );
    await _runFlutterWebTest(
      path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
      <String>['test'],
    );
552 553 554 555
    await _runFlutterWebTest(
        path.join(flutterRoot, 'packages', 'flutter_driver'),
        <String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
    );
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
  };

  await selectSubshard(subshards);
}

Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async {
  final List<String> batch = <String>[];
  for (int i = 0; i < tests.length; i += 1) {
    final String testFilePath = tests[i];
    batch.add(testFilePath);
    if (batch.length == kWebBatchSize || i == tests.length - 1) {
      await runCommand(
        flutter,
        <String>[
          'test',
          if (ciProvider == CiProviders.cirrus)
            '--concurrency=1',  // do not parallelize on Cirrus, to reduce flakiness
          '-v',
          '--platform=chrome',
          ...?flutterTestArgs,
          ...batch,
        ],
        workingDirectory: workingDirectory,
        environment: <String, String>{
          'FLUTTER_WEB': 'true',
          'FLUTTER_LOW_RESOURCE_MODE': 'true',
        },
      );
      batch.clear();
    }
  }
587 588
}

589
Future<void> _pubRunTest(String workingDirectory, {
Dan Field's avatar
Dan Field committed
590
  String testPath,
591 592
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
Dan Field's avatar
Dan Field committed
593 594
  bq.TabledataResourceApi tableData,
}) async {
595
  final List<String> args = <String>['run'];
596
  if (useBuildRunner) {
597 598 599 600 601 602 603 604
    final String posixTestPath = path.posix.joinAll(path.split(testPath));
    args.addAll(<String>[
      'build_runner',
      'test',
      '--build-filter=$posixTestPath/*.dill',
      '--build-filter=$posixTestPath/**/*.dill',
      '--',
    ]);
605 606 607 608
  } else {
    args.add('test');
  }
  args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact');
609 610 611 612 613 614 615 616 617 618 619 620 621
  int cpus;
  final String cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
  if (cpuVariable != null) {
    cpus = int.tryParse(cpuVariable, radix: 10);
    if (cpus == null) {
      print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset');
      print('Actual value: "$cpuVariable"');
      exit(1);
    }
  } else {
    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
  }
  args.add('-j$cpus');
622 623 624 625
  if (!hasColor)
    args.add('--no-color');
  if (testPath != null)
    args.add(testPath);
626 627 628
  final Map<String, String> pubEnvironment = <String, String>{
    'FLUTTER_ROOT': flutterRoot,
  };
629 630 631
  if (Directory(pubCache).existsSync()) {
    pubEnvironment['PUB_CACHE'] = pubCache;
  }
632 633 634 635 636
  if (enableFlutterToolAsserts) {
    // If an existing env variable exists append to it, but only if
    // it doesn't appear to already include enable-asserts.
    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
    if (!toolsArgs.contains('--enable-asserts'))
637
      toolsArgs += ' --enable-asserts';
638
    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
639 640 641 642
    // The flutter_tool will originally have been snapshotted without asserts.
    // We need to force it to be regenerated with them enabled.
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot'));
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp'));
643
  }
644 645
  if (useFlutterTestFormatter) {
    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
646 647 648 649 650 651 652 653 654 655 656
    Stream<String> testOutput;
    try {
      testOutput = runAndGetStdout(
        pub,
        args,
        workingDirectory: workingDirectory,
        environment: pubEnvironment,
      );
    } finally {
      formatter.finish();
    }
657 658 659 660 661
    await _processTestOutput(formatter, testOutput, tableData);
  } else {
    await runCommand(
      pub,
      args,
662 663 664
      workingDirectory: workingDirectory,
      environment: pubEnvironment,
      removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
665 666
    );
  }
667 668
}

669
Future<void> _runFlutterTest(String workingDirectory, {
670
  String script,
671 672
  bool expectFailure = false,
  bool printOutput = true,
673
  OutputChecker outputChecker,
674 675
  List<String> options = const <String>[],
  bool skip = false,
Dan Field's avatar
Dan Field committed
676
  bq.TabledataResourceApi tableData,
677
  Map<String, String> environment,
678
  List<String> tests = const <String>[],
Dan Field's avatar
Dan Field committed
679
}) async {
680
  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
681

682 683 684 685 686
  final List<String> args = <String>[
    'test',
    ...options,
    ...?flutterTestArgs,
  ];
Dan Field's avatar
Dan Field committed
687

688
  final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
689
  if (shouldProcessOutput)
Dan Field's avatar
Dan Field committed
690 691
    args.add('--machine');

692 693 694
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
695 696 697
      print('${red}Could not find test$reset: $green$fullScriptPath$reset');
      print('Working directory: $cyan$workingDirectory$reset');
      print('Script: $green$script$reset');
698 699 700 701 702 703
      if (!printOutput)
        print('This is one of the tests that does not normally print output.');
      if (skip)
        print('This is one of the tests that is normally skipped in this configuration.');
      exit(1);
    }
704
    args.add(script);
705
  }
706 707 708

  args.addAll(tests);

709
  if (!shouldProcessOutput) {
710 711 712 713 714 715 716 717 718 719 720 721 722
    OutputMode outputMode = OutputMode.discard;
    CapturedOutput output;

    if (outputChecker != null) {
      outputMode = OutputMode.capture;
      output = CapturedOutput();
    } else if (printOutput) {
      outputMode = OutputMode.print;
    }

    await runCommand(
      flutter,
      args,
Dan Field's avatar
Dan Field committed
723
      workingDirectory: workingDirectory,
724
      expectNonZeroExit: expectFailure,
725 726
      outputMode: outputMode,
      output: output,
Dan Field's avatar
Dan Field committed
727
      skip: skip,
728
      environment: environment,
Dan Field's avatar
Dan Field committed
729
    );
730 731 732

    if (outputChecker != null) {
      final String message = outputChecker(output);
733 734
      if (message != null)
        exitWithError(<String>[message]);
735 736
    }
    return;
Dan Field's avatar
Dan Field committed
737
  }
738 739

  if (useFlutterTestFormatter) {
740
    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
741 742 743 744 745 746 747 748 749 750 751 752
    Stream<String> testOutput;
    try {
      testOutput = runAndGetStdout(
        flutter,
        args,
        workingDirectory: workingDirectory,
        expectNonZeroExit: expectFailure,
        environment: environment,
      );
    } finally {
      formatter.finish();
    }
753
    await _processTestOutput(formatter, testOutput, tableData);
754 755 756 757 758 759 760 761
  } else {
    await runCommand(
      flutter,
      args,
      workingDirectory: workingDirectory,
      expectNonZeroExit: expectFailure,
    );
  }
762 763
}

764 765 766 767 768 769 770 771 772 773 774 775
Map<String, String> _initGradleEnvironment() {
  final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
      ? Platform.environment['ANDROID_SDK_ROOT']
      : Platform.environment['ANDROID_HOME'];
  if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
    print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT (or ANDROID_HOME).$reset');
    exit(1);
  }
  return <String, String>{
    'ANDROID_HOME': androidSdkRoot,
    'ANDROID_SDK_ROOT': androidSdkRoot,
  };
776
}
777

778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
final Map<String, String> gradleEnvironment = _initGradleEnvironment();

Future<void> _runHostOnlyDeviceLabTests() async {
  // Please don't add more tests here. We should not be using the devicelab
  // logic to run tests outside devicelab, that's just confusing.
  // Instead, create tests that are not devicelab tests, and run those.

  // TODO(ianh): Move the tests that are not running on devicelab any more out
  // of the device lab directory.

  // List the tests to run.
  // We split these into subshards. The tests are randomly distributed into
  // those subshards so as to get a uniform distribution of costs, but the
  // seed is fixed so that issues are reproducible.
  final List<ShardRunner> tests = <ShardRunner>[
    // Keep this in alphabetical order.
794
    () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment),
795 796 797
    if (Platform.isMacOS) () => _runDevicelabTest('flutter_create_offline_test_mac'),
    if (Platform.isLinux) () => _runDevicelabTest('flutter_create_offline_test_linux'),
    if (Platform.isWindows) () => _runDevicelabTest('flutter_create_offline_test_windows'),
798
    () => _runDevicelabTest('gradle_fast_start_test', environment: gradleEnvironment),
799
    // TODO(ianh): Fails on macOS looking for "dexdump", https://github.com/flutter/flutter/issues/42494
800 801 802 803 804 805 806
    if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment),
    () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment),
    () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment),
    () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment),
    () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment),
    () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment),

807
    () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: true),
808
    () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true),
809
    () => _runDevicelabTest('plugin_dependencies_test', environment: gradleEnvironment),
810

811
    if (Platform.isMacOS) () => _runDevicelabTest('module_test_ios'),
xster's avatar
xster committed
812
    if (Platform.isMacOS) () => _runDevicelabTest('build_ios_framework_module_test'),
813
    if (Platform.isMacOS) () => _runDevicelabTest('plugin_lint_mac'),
814
    () => _runDevicelabTest('plugin_test', environment: gradleEnvironment),
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
  ]..shuffle(math.Random(0));

  final int testsPerShard = tests.length ~/ kDeviceLabShardCount;
  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};

  for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) {
    String last = '';
    List<ShardRunner> sublist;
    if (subshard < kDeviceLabShardCount - 1) {
      sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard);
    } else {
      sublist = tests.sublist(subshard * testsPerShard, tests.length);
      // We make sure the last shard ends in _last so it's easier to catch mismatches
      // between `.cirrus.yml` and `test.dart`.
      last = '_last';
    }
    subshards['$subshard$last'] = () async {
832
      for (final ShardRunner test in sublist)
833 834
        await test();
    };
835
  }
836 837

  await selectSubshard(subshards);
838 839
}

840 841
Future<void> _runDevicelabTest(String testName, {
  Map<String, String> environment,
842 843 844 845
  // testEmbeddingV2 is only supported by certain specific devicelab tests.
  // Don't use it unless you're sure the test actually supports it.
  // You can check by looking to see if the test examines the environment
  // for the ENABLE_ANDROID_EMBEDDING_V2 variable.
846 847
  bool testEmbeddingV2 = false,
}) async {
848 849 850 851
  await runCommand(
    dart,
    <String>['bin/run.dart', '-t', testName],
    workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
852 853 854 855 856
    environment: <String, String>{
      ...?environment,
      if (testEmbeddingV2)
        'ENABLE_ANDROID_EMBEDDING_V2': 'true',
    },
857 858 859
  );
}

860 861 862 863 864 865 866
void deleteFile(String path) {
  // This is technically a race condition but nobody else should be running
  // while this script runs, so we should be ok. (Sadly recursive:true does not
  // obviate the need for existsSync, at least on Windows.)
  final File file = File(path);
  if (file.existsSync())
    file.deleteSync();
867 868
}

869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
enum CiProviders {
  cirrus,
  luci,
}

Future<void> _processTestOutput(
  FlutterCompactFormatter formatter,
  Stream<String> testOutput,
  bq.TabledataResourceApi tableData,
) async {
  final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
    print('Processing...');
  });

  await testOutput.forEach(formatter.processRawOutput);
  heartbeat.cancel();
  formatter.finish();
  if (tableData == null || formatter.tests.isEmpty) {
887
    return;
888
  }
889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
  final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
  final String authors = await _getAuthors();
  request.rows = List<bq.TableDataInsertAllRequestRows>.from(
    formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
      bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
        'json': <String, dynamic>{
          'source': <String, dynamic>{
            'provider': ciProviderName,
            'url': ciUrl,
            'platform': <String, dynamic>{
              'os': Platform.operatingSystem,
              'version': Platform.operatingSystemVersion,
            },
          },
          'test': <String, dynamic>{
            'name': result.name,
            'result': result.status.toString(),
            'file': result.path,
            'line': result.line,
            'column': result.column,
            'time': result.totalTime,
          },
          'git': <String, dynamic>{
            'author': authors,
            'pull_request': prNumber,
            'commit': gitHash,
            'organization': 'flutter',
            'repository': 'flutter',
          },
          'error': result.status != TestStatus.failed ? null : <String, dynamic>{
            'message': result.errorMessage,
            'stack_trace': result.stackTrace,
          },
          'information': result.messages,
        },
      }),
    ),
    growable: false,
  );
  final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
  if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
    print('${red}BigQuery insert errors:');
    print(response.toJson());
    print(reset);
933 934 935
  }
}

936 937 938 939 940 941 942 943 944
CiProviders get ciProvider {
  if (Platform.environment['CIRRUS_CI'] == 'true') {
    return CiProviders.cirrus;
  }
  if (Platform.environment['LUCI_CONTEXT'] != null) {
    return CiProviders.luci;
  }
  return null;
}
945

946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
String get ciProviderName {
  switch (ciProvider) {
    case CiProviders.cirrus:
      return 'cirrusci';
    case CiProviders.luci:
      return 'luci';
  }
  return 'unknown';
}

int get prNumber {
  switch (ciProvider) {
    case CiProviders.cirrus:
      return Platform.environment['CIRRUS_PR'] == null
          ? -1
          : int.tryParse(Platform.environment['CIRRUS_PR']);
    case CiProviders.luci:
      return -1; // LUCI doesn't know about this.
  }
  return -1;
}

Future<String> _getAuthors() async {
  final String exe = Platform.isWindows ? '.exe' : '';
  final String author = await runAndGetStdout(
    'git$exe', <String>['-c', 'log.showSignature=false', 'log', gitHash, '--pretty="%an <%ae>"'],
    workingDirectory: flutterRoot,
  ).first;
  return author;
}

String get ciUrl {
  switch (ciProvider) {
    case CiProviders.cirrus:
      return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
    case CiProviders.luci:
      return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
  }
  return '';
}

String get gitHash {
  switch(ciProvider) {
    case CiProviders.cirrus:
      return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
    case CiProviders.luci:
      return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
  }
  return '';
}

/// Checks the given file's contents to determine if they match the allowed
/// pattern for version strings.
///
/// Returns null if the contents are good. Returns a string if they are bad.
/// The string is an error message.
Future<String> verifyVersion(File file) async {
  final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
  final String version = await file.readAsString();
  if (!file.existsSync())
    return 'The version logic failed to create the Flutter version file.';
  if (version == '0.0.0-unknown')
    return 'The version logic failed to determine the Flutter version.';
  if (!version.contains(pattern))
    return 'The version logic generated an invalid version string: "$version".';
  return null;
}

/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
1015
/// the shard and sub-shard (parsing it in the form shard-subshard-platform, ignoring
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
/// the platform).
///
/// However, for local testing you can just set the SHARD and SUBSHARD
/// environment variables. For example, to run all the framework tests you can
/// just set SHARD=framework_tests. To run specifically the third subshard of
/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard', 1);

const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
  String item = Platform.environment[key];
  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME].split('-');
    assert(positionInTaskName < parts.length);
    item = parts[positionInTaskName];
  }
  if (item == null) {
1035
    for (final String currentItem in items.keys) {
1036 1037 1038 1039 1040
      print('$bold$key=$currentItem$reset');
      await items[currentItem]();
      print('');
    }
  } else {
1041 1042 1043 1044 1045 1046 1047 1048 1049
    if (!items.containsKey(item)) {
      print('${red}Invalid $name: $item$reset');
      print('The available ${name}s are: ${items.keys.join(", ")}');
      exit(1);
    }
    print('$bold$key=$item$reset');
    await items[item]();
  }
}