test.dart 33.8 KB
Newer Older
1 2 3 4
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:io';
7

Dan Field's avatar
Dan Field committed
8 9 10
import 'package:googleapis/bigquery/v2.dart' as bq;
import 'package:googleapis_auth/auth_io.dart' as auth;
import 'package:http/http.dart' as http;
11
import 'package:meta/meta.dart';
12
import 'package:path/path.dart' as path;
13

Dan Field's avatar
Dan Field committed
14
import 'flutter_compact_formatter.dart';
15
import 'run_command.dart';
16

17
typedef ShardRunner = Future<void> Function();
18

19 20 21 22 23 24 25 26
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
typedef OutputChecker = String Function(CapturedOutput);

27 28 29 30
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
31
final String pubCache = path.join(flutterRoot, '.pub-cache');
32
final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
33
final List<String> flutterTestArgs = <String>[];
34

35
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
36
final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true';
37

38
const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
39
  'tests': _runTests,
40
  'web_tests': _runWebTests,
41
  'tool_tests': _runToolTests,
42
  'tool_coverage': _runToolCoverage,
43
  'build_tests': _runBuildTests,
44
  'coverage': _runCoverage,
45
  'integration_tests': _runIntegrationTests,
Dan Field's avatar
Dan Field committed
46
  'add2app_test': _runAdd2AppTest,
47 48
};

49
/// When you call this, you can pass additional arguments to pass custom
50
/// arguments to flutter test. For example, you might want to call this
51
/// script with the parameter --local-engine=host_debug_unopt to
52
/// use your own build of the engine.
53
///
54
/// To run the tool_tests part, run it with SHARD=tool_tests
55 56
///
/// For example:
57
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
58
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
59
Future<void> main(List<String> args) async {
60 61
  flutterTestArgs.addAll(args);

62 63
  final String shard = Platform.environment['SHARD'];
  if (shard != null) {
64 65 66 67 68
    if (!_kShards.containsKey(shard)) {
      print('Invalid shard: $shard');
      print('The available shards are: ${_kShards.keys.join(", ")}');
      exit(1);
    }
69
    print('${bold}SHARD=$shard$reset');
70 71 72 73 74
    await _kShards[shard]();
  } else {
    for (String currentShard in _kShards.keys) {
      print('${bold}SHARD=$currentShard$reset');
      await _kShards[currentShard]();
75
      print('');
76 77
    }
  }
78 79
}

80
Future<void> _runSmokeTests() async {
81 82
  // Verify that the tests actually return failure on failure and success on
  // success.
83
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
84 85 86
  // We run the "pass" and "fail" smoke tests first, and alone, because those
  // are particularly critical and sensitive. If one of these fails, there's no
  // point even trying the others.
87 88 89 90 91
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pass_test.dart'),
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
92
    script: path.join('test_smoke_test', 'fail_test.dart'),
93 94 95
    expectFailure: true,
    printOutput: false,
  );
96
  // We run the timeout tests individually because they are timing-sensitive.
97
  await _runFlutterTest(automatedTests,
98 99
    script: path.join('test_smoke_test', 'timeout_pass_test.dart'),
    expectFailure: false,
100 101
    printOutput: false,
  );
102
  await _runFlutterTest(automatedTests,
103
    script: path.join('test_smoke_test', 'timeout_fail_test.dart'),
104 105 106
    expectFailure: true,
    printOutput: false,
  );
107 108 109 110 111 112 113 114 115
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
    expectFailure: true,
    printOutput: false,
    outputChecker: (CapturedOutput output) =>
      output.stdout.contains('failingPendingTimerTest')
      ? null
      : 'Failed to find the stack trace for the pending Timer.',
  );
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
  // We run the remaining smoketests in parallel, because they each take some
  // time to run (e.g. compiling), so we don't want to run them in series,
  // especially on 20-core machines...
  await Future.wait<void>(
    <Future<void>>[
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash1_test.dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash2_test.dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
        expectFailure: true,
        printOutput: false,
      ),
146
      runCommand(flutter,
147 148
        <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
        workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
149
        expectNonZeroExit: true,
150
        outputMode: OutputMode.discard,
151 152
      ),
    ],
153 154
  );

155
  // Verify that we correctly generated the version file.
156 157 158 159
  final bool validVersion = await verifyVersion(path.join(flutterRoot, 'version'));
  if (!validVersion) {
    exit(1);
  }
160 161
}

Dan Field's avatar
Dan Field committed
162
Future<bq.BigqueryApi> _getBigqueryApi() async {
163 164 165
  if (!useFlutterTestFormatter) {
    return null;
  }
Dan Field's avatar
Dan Field committed
166 167
  // TODO(dnfield): How will we do this on LUCI?
  final String privateKey = Platform.environment['GCLOUD_SERVICE_ACCOUNT_KEY'];
168 169 170 171 172
  // If we're on Cirrus and a non-collaborator is doing this, we can't get the key.
  if (privateKey == null || privateKey.isEmpty || privateKey.startsWith('ENCRYPTED[')) {
    return null;
  }
  try {
173
    final auth.ServiceAccountCredentials accountCredentials = auth.ServiceAccountCredentials(
174 175 176 177 178 179 180 181 182 183
      'flutter-ci-test-reporter@flutter-infra.iam.gserviceaccount.com',
      auth.ClientId.serviceAccount('114390419920880060881.apps.googleusercontent.com'),
      '-----BEGIN PRIVATE KEY-----\n$privateKey\n-----END PRIVATE KEY-----\n',
    );
    final List<String> scopes = <String>[bq.BigqueryApi.BigqueryInsertdataScope];
    final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes);
    return bq.BigqueryApi(client);
  } catch (e) {
    print('Failed to get BigQuery API client.');
    print(e);
Dan Field's avatar
Dan Field committed
184 185 186 187
    return null;
  }
}

188
Future<void> _runToolCoverage() async {
189
  await runCommand( // Precompile tests to speed up subsequent runs.
190 191 192 193
    pub,
    <String>['run', 'build_runner', 'build'],
    workingDirectory: toolRoot,
  );
194 195 196 197 198 199 200 201
  await runCommand(
    dart,
    <String>[path.join('tool', 'tool_coverage.dart')],
    workingDirectory: toolRoot,
    environment: <String, String>{
      'FLUTTER_ROOT': flutterRoot,
    }
  );
202 203
}

204
Future<void> _runToolTests() async {
Dan Field's avatar
Dan Field committed
205
  final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
206 207
  await _runSmokeTests();

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
  const String kDotShard = '.shard';
  const String kTest = 'test';
  final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');

  final Map<String, ShardRunner> subshards = Map<String, ShardRunner>.fromIterable(
    Directory(path.join(toolsPath, kTest))
      .listSync()
      .map<String>((FileSystemEntity entry) => entry.path)
      .where((String name) => name.endsWith(kDotShard))
      .map<String>((String name) => path.basenameWithoutExtension(name)),
    // The `dynamic` on the next line is because Map.fromIterable isn't generic.
    value: (dynamic subshard) => () async {
      await _pubRunTest(
        toolsPath,
        testPath: path.join(kTest, '$subshard$kDotShard'),
        useBuildRunner: canUseBuildRunner,
        tableData: bigqueryApi?.tabledata,
        // TODO(ianh): The integration tests fail to start on Windows if asserts are enabled.
        // See https://github.com/flutter/flutter/issues/36476
        enableFlutterToolAsserts: !(subshard == 'integration' && Platform.isWindows),
      );
    },
  );
231

232
  await selectSubshard(subshards);
233 234
}

235 236
/// Verifies that AOT, APK, and IPA (if on macOS) builds the
/// examples apps without crashing. It does not actually
237 238 239 240 241
/// launch the apps. That happens later in the devicelab. This is
/// just a smoke-test. In particular, this will verify we can build
/// when there are spaces in the path name for the Flutter SDK and
/// target app.
Future<void> _runBuildTests() async {
242 243 244 245 246 247 248 249 250
  final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list();
  await for (FileSystemEntity fileEntity in exampleDirectories) {
    if (fileEntity is! Directory) {
      continue;
    }
    final String examplePath = fileEntity.path;

    await _flutterBuildAot(examplePath);
    await _flutterBuildApk(examplePath);
251
    await _flutterBuildIpa(examplePath);
252
  }
253 254
  // Web compilation tests.
  await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web'), path.join('lib', 'main.dart'));
255
  // Should not fail to compile with dart:io.
256 257 258
  await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web_compile_tests'),
    path.join('lib', 'dart_io_import.dart'),
  );
259 260 261 262

  print('${bold}DONE: All build tests successful.$reset');
}

263
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
264 265
  print('Running Dart2JS build tests...');
  await runCommand(flutter,
266
    <String>['build', 'web', '-v', '--target=$target'],
267
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
268
    expectNonZeroExit: expectNonZeroExit,
269 270
    environment: <String, String>{
      'FLUTTER_WEB': 'true',
271
    },
272 273 274
  );
  print('Done.');
}
275

276 277 278 279 280 281 282 283 284
Future<void> _flutterBuildAot(String relativePathToApplication) async {
  print('Running AOT build tests...');
  await runCommand(flutter,
    <String>['build', 'aot', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: false,
  );
  print('Done.');
}
285

286
Future<void> _flutterBuildApk(String relativePathToApplication) async {
287 288 289
  if (
        (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) &&
        (Platform.environment['ANDROID_SDK_ROOT']?.isEmpty ?? true)) {
290 291 292 293 294 295 296 297 298
    return;
  }
  print('Running APK build tests...');
  await runCommand(flutter,
    <String>['build', 'apk', '--debug', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: false,
  );
  print('Done.');
299 300
}

301
Future<void> _flutterBuildIpa(String relativePathToApplication) async {
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
  if (!Platform.isMacOS) {
    return;
  }
  print('Running IPA build tests...');
  // Install Cocoapods.  We don't have these checked in for the examples,
  // and build ios doesn't take care of it automatically.
  final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
  if (podfile.existsSync()) {
    await runCommand('pod',
      <String>['install'],
      workingDirectory: podfile.parent.path,
      expectNonZeroExit: false,
    );
  }
  await runCommand(flutter,
    <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
318
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
319
    expectNonZeroExit: false,
320
  );
321
  print('Done.');
322 323
}

Dan Field's avatar
Dan Field committed
324 325 326 327 328 329 330 331 332 333 334 335 336 337
Future<void> _runAdd2AppTest() async {
  if (!Platform.isMacOS) {
    return;
  }
  print('Running Add2App iOS integration tests...');
  final String add2AppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
  await runCommand('./build_and_test.sh',
    <String>[],
    workingDirectory: add2AppDir,
    expectNonZeroExit: false,
  );
  print('Done.');
}

338
Future<void> _runTests() async {
Dan Field's avatar
Dan Field committed
339
  final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
340
  await _runSmokeTests();
341
  final String subShard = Platform.environment['SUBSHARD'];
342

343 344 345 346 347
  Future<void> runWidgets() async {
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      tableData: bigqueryApi?.tabledata,
      tests: <String>[
348
        path.join('test', 'widgets') + path.separator,
349 350 351 352 353 354 355 356 357 358 359
      ],
    );
    // Only packages/flutter/test/widgets/widget_inspector_test.dart really
    // needs to be run with --track-widget-creation but it is nice to run
    // all of the tests in package:flutter with the flag to ensure that
    // the Dart kernel transformer triggered by the flag does not break anything.
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      options: <String>['--track-widget-creation'],
      tableData: bigqueryApi?.tabledata,
      tests: <String>[
360
        path.join('test', 'widgets') + path.separator,
361 362 363 364 365 366 367 368
      ],
    );
  }

  Future<void> runFrameworkOthers() async {
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
      .listSync(followLinks: false, recursive: false)
      .whereType<Directory>()
369 370
      .where((Directory dir) => dir.path.endsWith('widgets') == false)
      .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
      .toList();

    print('Running tests for: ${tests.join(';')}');

    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      tableData: bigqueryApi?.tabledata,
      tests: tests,
    );
    // Only packages/flutter/test/widgets/widget_inspector_test.dart really
    // needs to be run with --track-widget-creation but it is nice to run
    // all of the tests in package:flutter with the flag to ensure that
    // the Dart kernel transformer triggered by the flag does not break anything.
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
      options: <String>['--track-widget-creation'],
      tableData: bigqueryApi?.tabledata,
      tests: tests,
    );
  }

  Future<void> runExtras() async {
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata);
    // Regression test to ensure that code outside of package:flutter can run
    // with --track-widget-creation.
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
    // Smoke test for code generation.
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: <String, String>{
      'FLUTTER_EXPERIMENTAL_BUILD': 'true',
    });
  }
  switch (subShard) {
    case 'widgets':
      await runWidgets();
      break;
    case 'framework_other':
      await runFrameworkOthers();
      break;
    case 'extras':
      runExtras();
      break;
    default:
      print('Unknown sub-shard $subShard, running all tests!');
      await runWidgets();
      await runFrameworkOthers();
      await runExtras();

  }
433 434 435 436

  print('${bold}DONE: All tests successful.$reset');
}

437
Future<void> _runWebTests() async {
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
  final Directory flutterPackageDir = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  final Directory testDir = Directory(path.join(flutterPackageDir.path, 'test'));

  // TODO(yjbanov): we're getting rid of this blacklist as part of https://github.com/flutter/flutter/projects/60
  const List<String> kBlacklist = <String>[
    'test/cupertino',
    'test/examples',
    'test/material',
    'test/painting',
    'test/rendering',
    'test/semantics',
    'test/widgets',
  ];

  final List<String> directories = testDir
    .listSync()
    .whereType<Directory>()
    .map<String>((Directory dir) => path.relative(dir.path, from: flutterPackageDir.path))
    .where((String relativePath) => !kBlacklist.contains(relativePath))
    .toList();

  await _runFlutterWebTest(flutterPackageDir.path, tests: directories);
460
  await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter_web_plugins'), tests: <String>['test']);
461 462
}

463
Future<void> _runCoverage() async {
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file is normally obtained by running `flutter update-packages`.');
    exit(1);
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file should have been generated by the `flutter test --coverage` script, but was not.');
    exit(1);
  }
  print('${bold}DONE: Coverage collection successful.$reset');
482 483
}

484
Future<void> _pubRunTest(String workingDirectory, {
Dan Field's avatar
Dan Field committed
485
  String testPath,
486 487
  bool enableFlutterToolAsserts = true,
  bool useBuildRunner = false,
Dan Field's avatar
Dan Field committed
488 489
  bq.TabledataResourceApi tableData,
}) async {
490 491 492 493 494 495 496 497 498 499 500 501
  final List<String> args = <String>['run', '--verbose'];
  if (useBuildRunner) {
    args.addAll(<String>['build_runner', 'test', '--']);
  } else {
    args.add('test');
  }
  args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact');
  args.add('-j1'); // TODO(ianh): Scale based on CPUs.
  if (!hasColor)
    args.add('--no-color');
  if (testPath != null)
    args.add(testPath);
502 503 504
  final Map<String, String> pubEnvironment = <String, String>{
    'FLUTTER_ROOT': flutterRoot,
  };
505 506 507
  if (Directory(pubCache).existsSync()) {
    pubEnvironment['PUB_CACHE'] = pubCache;
  }
508 509 510 511 512
  if (enableFlutterToolAsserts) {
    // If an existing env variable exists append to it, but only if
    // it doesn't appear to already include enable-asserts.
    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
    if (!toolsArgs.contains('--enable-asserts'))
513
      toolsArgs += ' --enable-asserts';
514
    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
515 516 517 518
    // The flutter_tool will originally have been snapshotted without asserts.
    // We need to force it to be regenerated with them enabled.
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot'));
    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp'));
519
  }
520 521 522 523 524 525 526
  if (useFlutterTestFormatter) {
    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
    final Stream<String> testOutput = runAndGetStdout(
      pub,
      args,
      workingDirectory: workingDirectory,
      environment: pubEnvironment,
527
      beforeExit: formatter.finish,
528 529 530 531 532 533
    );
    await _processTestOutput(formatter, testOutput, tableData);
  } else {
    await runCommand(
      pub,
      args,
534 535 536
      workingDirectory: workingDirectory,
      environment: pubEnvironment,
      removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
537 538
    );
  }
539 540
}

541 542 543 544 545 546
void deleteFile(String path) {
  // There's a race condition here but in theory we're not racing anyone
  // while this script runs, so should be ok.
  final File file = File(path);
  if (file.existsSync())
    file.deleteSync();
Dan Field's avatar
Dan Field committed
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
}

enum CiProviders {
  cirrus,
  luci,
}

CiProviders _getCiProvider() {
  if (Platform.environment['CIRRUS_CI'] == 'true') {
    return CiProviders.cirrus;
  }
  if (Platform.environment['LUCI_CONTEXT'] != null) {
    return CiProviders.luci;
  }
  return null;
}

String _getCiProviderName() {
  switch(_getCiProvider()) {
    case CiProviders.cirrus:
      return 'cirrusci';
    case CiProviders.luci:
      return 'luci';
  }
  return 'unknown';
}

int _getPrNumber() {
  switch(_getCiProvider()) {
    case CiProviders.cirrus:
577 578 579
      return Platform.environment['CIRRUS_PR'] == null
          ? -1
          : int.tryParse(Platform.environment['CIRRUS_PR']);
Dan Field's avatar
Dan Field committed
580 581 582 583 584 585 586 587 588
    case CiProviders.luci:
      return -1; // LUCI doesn't know about this.
  }
  return -1;
}

Future<String> _getAuthors() async {
  final String exe = Platform.isWindows ? '.exe' : '';
  final String author = await runAndGetStdout(
589
    'git$exe', <String>['-c', 'log.showSignature=false', 'log', _getGitHash(), '--pretty="%an <%ae>"'],
Dan Field's avatar
Dan Field committed
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614
    workingDirectory: flutterRoot,
  ).first;
  return author;
}

String _getCiUrl() {
  switch(_getCiProvider()) {
    case CiProviders.cirrus:
      return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
    case CiProviders.luci:
      return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
  }
  return '';
}

String _getGitHash() {
  switch(_getCiProvider()) {
    case CiProviders.cirrus:
      return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
    case CiProviders.luci:
      return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
  }
  return '';
}

615 616 617 618 619
Future<void> _processTestOutput(
  FlutterCompactFormatter formatter,
  Stream<String> testOutput,
  bq.TabledataResourceApi tableData,
) async {
Dan Field's avatar
Dan Field committed
620 621 622 623
  final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
    print('Processing...');
  });

Dan Field's avatar
Dan Field committed
624
  await testOutput.forEach(formatter.processRawOutput);
Dan Field's avatar
Dan Field committed
625
  heartbeat.cancel();
626
  formatter.finish();
Dan Field's avatar
Dan Field committed
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
  if (tableData == null || formatter.tests.isEmpty) {
    return;
  }
  final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
  final String authors = await _getAuthors();
  request.rows = List<bq.TableDataInsertAllRequestRows>.from(
    formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
      bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
        'json': <String, dynamic>{
          'source': <String, dynamic>{
            'provider': _getCiProviderName(),
            'url': _getCiUrl(),
            'platform': <String, dynamic>{
              'os': Platform.operatingSystem,
              'version': Platform.operatingSystemVersion,
            },
          },
          'test': <String, dynamic>{
            'name': result.name,
            'result': result.status.toString(),
            'file': result.path,
            'line': result.line,
            'column': result.column,
            'time': result.totalTime,
          },
          'git': <String, dynamic>{
            'author': authors,
            'pull_request': _getPrNumber(),
            'commit': _getGitHash(),
            'organization': 'flutter',
            'repository': 'flutter',
          },
          'error': result.status != TestStatus.failed ? null : <String, dynamic>{
            'message': result.errorMessage,
            'stack_trace': result.stackTrace,
          },
          'information': result.messages,
        },
      }),
    ),
    growable: false,
  );
  final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
  if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
    print('${red}BigQuery insert errors:');
    print(response.toJson());
    print(reset);
  }
675 676
}

677 678 679 680
class EvalResult {
  EvalResult({
    this.stdout,
    this.stderr,
681
    this.exitCode = 0,
682 683 684 685
  });

  final String stdout;
  final String stderr;
686
  final int exitCode;
687 688
}

689 690
Future<void> _runFlutterWebTest(String workingDirectory, {
  List<String> tests,
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
}) async {
  final List<String> allTests = <String>[];
  for (String testDirPath in tests) {
    final Directory testDir = Directory(path.join(workingDirectory, testDirPath));
    allTests.addAll(
      testDir.listSync(recursive: true)
        .whereType<File>()
        .where((File file) => file.path.endsWith('_test.dart'))
        .map((File file) => path.relative(file.path, from: workingDirectory))
    );
  }
  print(allTests.join('\n'));
  print('${allTests.length} tests total');

  // Maximum number of tests to run in a single `flutter test`. We found that
  // large batches can get flaky, possibly because we reuse a single instance
  // of the browser, and after many tests the browser's state gets corrupted.
  const int kBatchSize = 20;
  List<String> batch = <String>[];
  for (int i = 0; i < allTests.length; i += 1) {
    final String testFilePath = allTests[i];
    batch.add(testFilePath);
    if (batch.length == kBatchSize || i == allTests.length - 1) {
      await _runFlutterWebTestBatch(workingDirectory, batch: batch);
      batch = <String>[];
    }
  }
}

Future<void> _runFlutterWebTestBatch(String workingDirectory, {
  List<String> batch,
722
}) async {
723 724
  final List<String> args = <String>[
    'test',
725 726
    if (_getCiProvider() == CiProviders.cirrus)
      '--concurrency=1',  // do not parallelize on Cirrus to reduce flakiness
727 728 729
    '-v',
    '--platform=chrome',
    ...?flutterTestArgs,
730
    ...batch,
731
  ];
732

733 734 735 736 737 738 739 740
  // TODO(jonahwilliams): fix relative path issues to make this unecessary.
  final Directory oldCurrent = Directory.current;
  Directory.current = Directory(path.join(flutterRoot, 'packages', 'flutter'));
  try {
    await runCommand(
      flutter,
      args,
      workingDirectory: workingDirectory,
741
      expectFlaky: false,
742 743 744 745 746 747 748 749
      environment: <String, String>{
        'FLUTTER_WEB': 'true',
        'FLUTTER_LOW_RESOURCE_MODE': 'true',
      },
    );
  } finally {
    Directory.current = oldCurrent;
  }
750 751
}

752
Future<void> _runFlutterTest(String workingDirectory, {
753
  String script,
754 755
  bool expectFailure = false,
  bool printOutput = true,
756
  OutputChecker outputChecker,
757 758
  List<String> options = const <String>[],
  bool skip = false,
Dan Field's avatar
Dan Field committed
759
  bq.TabledataResourceApi tableData,
760
  Map<String, String> environment,
761
  List<String> tests = const <String>[],
Dan Field's avatar
Dan Field committed
762
}) async {
James Lin's avatar
James Lin committed
763 764
  assert(!printOutput || outputChecker == null,
      'Output either can be printed or checked but not both');
765

766 767 768 769 770
  final List<String> args = <String>[
    'test',
    ...options,
    ...?flutterTestArgs,
  ];
Dan Field's avatar
Dan Field committed
771

772
  final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
773
  if (shouldProcessOutput) {
Dan Field's avatar
Dan Field committed
774 775 776
    args.add('--machine');
  }

777 778 779 780 781 782 783 784 785 786 787 788
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
      print('Could not find test: $fullScriptPath');
      print('Working directory: $workingDirectory');
      print('Script: $script');
      if (!printOutput)
        print('This is one of the tests that does not normally print output.');
      if (skip)
        print('This is one of the tests that is normally skipped in this configuration.');
      exit(1);
    }
789
    args.add(script);
790
  }
791 792 793

  args.addAll(tests);

794
  if (!shouldProcessOutput) {
795 796 797 798 799 800 801 802 803 804 805 806 807
    OutputMode outputMode = OutputMode.discard;
    CapturedOutput output;

    if (outputChecker != null) {
      outputMode = OutputMode.capture;
      output = CapturedOutput();
    } else if (printOutput) {
      outputMode = OutputMode.print;
    }

    await runCommand(
      flutter,
      args,
Dan Field's avatar
Dan Field committed
808
      workingDirectory: workingDirectory,
809
      expectNonZeroExit: expectFailure,
810 811
      outputMode: outputMode,
      output: output,
Dan Field's avatar
Dan Field committed
812
      skip: skip,
813
      environment: environment,
Dan Field's avatar
Dan Field committed
814
    );
815 816 817 818 819 820 821 822 823 824 825

    if (outputChecker != null) {
      final String message = outputChecker(output);
      if (message != null) {
        print('$redLine');
        print(message);
        print('$redLine');
        exit(1);
      }
    }
    return;
Dan Field's avatar
Dan Field committed
826
  }
827 828

  if (useFlutterTestFormatter) {
829 830 831 832 833 834 835 836 837 838
    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
    final Stream<String> testOutput = runAndGetStdout(
      flutter,
      args,
      workingDirectory: workingDirectory,
      expectNonZeroExit: expectFailure,
      beforeExit: formatter.finish,
      environment: environment,
    );
    await _processTestOutput(formatter, testOutput, tableData);
839 840 841 842 843 844 845 846
  } else {
    await runCommand(
      flutter,
      args,
      workingDirectory: workingDirectory,
      expectNonZeroExit: expectFailure,
    );
  }
847 848
}

849 850 851 852 853 854 855
// the optional `file` argument is an override for testing
@visibleForTesting
Future<bool> verifyVersion(String filename, [File file]) async {
  final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
  file ??= File(filename);
  final String version = await file.readAsString();
  if (!file.existsSync()) {
856
    print('$redLine');
857
    print('The version logic failed to create the Flutter version file.');
858
    print('$redLine');
859
    return false;
860 861
  }
  if (version == '0.0.0-unknown') {
862
    print('$redLine');
863
    print('The version logic failed to determine the Flutter version.');
864
    print('$redLine');
865
    return false;
866 867
  }
  if (!version.contains(pattern)) {
868
    print('$redLine');
869
    print('The version logic generated an invalid version string: "$version".');
870
    print('$redLine');
871
    return false;
872
  }
873
  return true;
874
}
875 876

Future<void> _runIntegrationTests() async {
877
  final String subShard = Platform.environment['SUBSHARD'];
878

879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
  switch (subShard) {
    case 'gradle1':
    case 'gradle2':
      // This runs some gradle integration tests if the subshard is Android.
      await _androidGradleTests(subShard);
      break;
    default:
      await _runDevicelabTest('dartdocs');

      if (Platform.isLinux) {
        await _runDevicelabTest('flutter_create_offline_test_linux');
      } else if (Platform.isWindows) {
        await _runDevicelabTest('flutter_create_offline_test_windows');
      } else if (Platform.isMacOS) {
        await _runDevicelabTest('flutter_create_offline_test_mac');
894
        await _runDevicelabTest('plugin_lint_mac');
895 896
// TODO(jmagman): Re-enable once flakiness is resolved.
//        await _runDevicelabTest('module_test_ios');
897 898 899
      }
      // This does less work if the subshard isn't Android.
      await _androidPluginTest();
900 901 902 903 904 905 906 907 908 909 910 911
  }
}

Future<void> _runDevicelabTest(String testName, {Map<String, String> env}) async {
  await runCommand(
    dart,
    <String>['bin/run.dart', '-t', testName],
    workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
    environment: env,
  );
}

912
String get androidSdkRoot {
913 914 915 916
  final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
      ? Platform.environment['ANDROID_SDK_ROOT']
      : Platform.environment['ANDROID_HOME'];
  if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
917 918 919 920 921 922 923 924
    return null;
  }
  return androidSdkRoot;
}

Future<void> _androidPluginTest() async {
  if (androidSdkRoot == null) {
    print('No Android SDK detected, skipping Android Plugin test.');
925 926 927 928 929 930 931 932
    return;
  }

  final Map<String, String> env = <String, String> {
    'ANDROID_HOME': androidSdkRoot,
    'ANDROID_SDK_ROOT': androidSdkRoot,
  };

933 934 935 936
  await _runDevicelabTest('plugin_test', env: env);
}

Future<void> _androidGradleTests(String subShard) async {
937
  // TODO(dnfield): gradlew is crashing on the cirrus image and it's not clear why.
938 939 940 941 942 943 944 945 946 947 948
  if (androidSdkRoot == null || Platform.isWindows) {
    print('No Android SDK detected or on Windows, skipping Android gradle test.');
    return;
  }

  final Map<String, String> env = <String, String> {
    'ANDROID_HOME': androidSdkRoot,
    'ANDROID_SDK_ROOT': androidSdkRoot,
  };

  if (subShard == 'gradle1') {
949 950
    await _runDevicelabTest('gradle_plugin_light_apk_test', env: env);
    await _runDevicelabTest('gradle_plugin_fat_apk_test', env: env);
951
    await _runDevicelabTest('gradle_r8_test', env: env);
952
    await _runDevicelabTest('gradle_non_android_plugin_test', env: env);
953
    await _runDevicelabTest('gradle_jetifier_test', env: env);
954 955
  }
  if (subShard == 'gradle2') {
956
    await _runDevicelabTest('gradle_plugin_bundle_test', env: env);
957
    await _runDevicelabTest('module_test', env: env);
958
    await _runDevicelabTest('module_host_with_custom_build_test', env: env);
959
    await _runDevicelabTest('build_aar_module_test', env: env);
960
  }
961
}
962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983

Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard');
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard');

Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name) async {
  final String item = Platform.environment[key];
  if (item != null) {
    if (!items.containsKey(item)) {
      print('${red}Invalid $name: $item$reset');
      print('The available ${name}s are: ${items.keys.join(", ")}');
      exit(1);
    }
    print('$bold$key=$item$reset');
    await items[item]();
  } else {
    for (String currentItem in items.keys) {
      print('$bold$key=$currentItem$reset');
      await items[currentItem]();
      print('');
    }
  }
}