test.dart 13.4 KB
Newer Older
1 2 3 4
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:io';
7

8
import 'package:path/path.dart' as path;
9 10

import 'run_command.dart';
11

12
typedef ShardRunner = Future<void> Function();
13

14 15 16 17
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
18
final String pubCache = path.join(flutterRoot, '.pub-cache');
19
final List<String> flutterTestArgs = <String>[];
20

21
const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
22
  'tests': _runTests,
23
  'tool_tests': _runToolTests,
24
  'build_tests': _runBuildTests,
25 26 27
  'coverage': _runCoverage,
};

28 29
const Duration _kLongTimeout = Duration(minutes: 45);
const Duration _kShortTimeout = Duration(minutes: 5);
30

31
/// When you call this, you can pass additional arguments to pass custom
32
/// arguments to flutter test. For example, you might want to call this
33
/// script with the parameter --local-engine=host_debug_unopt to
34
/// use your own build of the engine.
35
///
36
/// To run the tool_tests part, run it with SHARD=tool_tests
37 38
///
/// For example:
39
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
40
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
41
Future<void> main(List<String> args) async {
42 43
  flutterTestArgs.addAll(args);

44 45
  final String shard = Platform.environment['SHARD'];
  if (shard != null) {
46 47 48 49 50
    if (!_kShards.containsKey(shard)) {
      print('Invalid shard: $shard');
      print('The available shards are: ${_kShards.keys.join(", ")}');
      exit(1);
    }
51
    print('${bold}SHARD=$shard$reset');
52 53 54 55 56
    await _kShards[shard]();
  } else {
    for (String currentShard in _kShards.keys) {
      print('${bold}SHARD=$currentShard$reset');
      await _kShards[currentShard]();
57
      print('');
58 59
    }
  }
60 61
}

62
Future<void> _runSmokeTests() async {
63 64
  // Verify that the tests actually return failure on failure and success on
  // success.
65
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
66 67 68
  // We run the "pass" and "fail" smoke tests first, and alone, because those
  // are particularly critical and sensitive. If one of these fails, there's no
  // point even trying the others.
69 70 71
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pass_test.dart'),
    printOutput: false,
72
    timeout: _kShortTimeout,
73 74
  );
  await _runFlutterTest(automatedTests,
75
    script: path.join('test_smoke_test', 'fail_test.dart'),
76 77
    expectFailure: true,
    printOutput: false,
78
    timeout: _kShortTimeout,
79
  );
80
  // We run the timeout tests individually because they are timing-sensitive.
81
  await _runFlutterTest(automatedTests,
82 83
    script: path.join('test_smoke_test', 'timeout_pass_test.dart'),
    expectFailure: false,
84
    printOutput: false,
85
    timeout: _kShortTimeout,
86
  );
87
  await _runFlutterTest(automatedTests,
88
    script: path.join('test_smoke_test', 'timeout_fail_test.dart'),
89 90 91 92
    expectFailure: true,
    printOutput: false,
    timeout: _kShortTimeout,
  );
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
  // We run the remaining smoketests in parallel, because they each take some
  // time to run (e.g. compiling), so we don't want to run them in series,
  // especially on 20-core machines...
  await Future.wait<void>(
    <Future<void>>[
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash1_test.dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash2_test.dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
128
      runCommand(flutter,
129 130
        <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
        workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
131
        expectNonZeroExit: true,
132 133 134 135
        printOutput: false,
        timeout: _kShortTimeout,
      ),
    ],
136 137
  );

138 139
  // Verify that we correctly generated the version file.
  await _verifyVersion(path.join(flutterRoot, 'version'));
140 141
}

142
Future<void> _runToolTests() async {
143 144
  await _runSmokeTests();

145 146 147 148
  await _pubRunTest(
    path.join(flutterRoot, 'packages', 'flutter_tools'),
    enableFlutterToolAsserts: true,
  );
149 150 151 152

  print('${bold}DONE: All tests successful.$reset');
}

153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
/// Verifies that AOT, APK, and IPA (if on macOS) builds of some
/// examples apps finish without crashing. It does not actually
/// launch the apps. That happens later in the devicelab. This is
/// just a smoke-test. In particular, this will verify we can build
/// when there are spaces in the path name for the Flutter SDK and
/// target app.
Future<void> _runBuildTests() async {
  final List<String> paths = <String>[
    path.join('examples', 'hello_world'),
    path.join('examples', 'flutter_gallery'),
    path.join('examples', 'flutter_view'),
  ];
  for (String path in paths) {
    await _flutterBuildAot(path);
    await _flutterBuildApk(path);
    await _flutterBuildIpa(path);
  }

  print('${bold}DONE: All build tests successful.$reset');
}

Future<void> _flutterBuildAot(String relativePathToApplication) async {
  print('Running AOT build tests...');
  await runCommand(flutter,
    <String>['build', 'aot', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: false,
    timeout: _kShortTimeout,
  );
  print('Done.');
}
184

185 186
Future<void> _flutterBuildApk(String relativePathToApplication) async {
  // TODO(dnfield): See if we can get Android SDK on all Cirrus platforms.
187 188 189
  if (
        (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) &&
        (Platform.environment['ANDROID_SDK_ROOT']?.isEmpty ?? true)) {
190 191 192 193 194 195 196 197 198 199
    return;
  }
  print('Running APK build tests...');
  await runCommand(flutter,
    <String>['build', 'apk', '--debug', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: false,
    timeout: _kShortTimeout,
  );
  print('Done.');
200 201
}

202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
Future<void> _flutterBuildIpa(String relativePathToApplication) async {
  if (!Platform.isMacOS) {
    return;
  }
  print('Running IPA build tests...');
  // Install Cocoapods.  We don't have these checked in for the examples,
  // and build ios doesn't take care of it automatically.
  final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
  if (podfile.existsSync()) {
    await runCommand('pod',
      <String>['install'],
      workingDirectory: podfile.parent.path,
      expectNonZeroExit: false,
      timeout: _kShortTimeout,
    );
  }
  await runCommand(flutter,
    <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
220 221 222 223
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: false,
    timeout: _kShortTimeout,
  );
224
  print('Done.');
225 226
}

227
Future<void> _runTests() async {
228
  await _runSmokeTests();
229

230
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'));
231 232 233 234 235
  // Only packages/flutter/test/widgets/widget_inspector_test.dart really
  // needs to be run with --track-widget-creation but it is nice to run
  // all of the tests in package:flutter with the flag to ensure that
  // the Dart kernel transformer triggered by the flag does not break anything.
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), options: <String>['--track-widget-creation']);
236 237 238 239
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
240
  await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
241
  await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'));
242
  await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'));
243
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'));
244 245 246 247 248 249
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'));
250 251 252
  // Regression test to ensure that code outside of package:flutter can run
  // with --track-widget-creation.
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation']);
253
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'));
254 255 256 257

  print('${bold}DONE: All tests successful.$reset');
}

258
Future<void> _runCoverage() async {
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file is normally obtained by running `flutter update-packages`.');
    exit(1);
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file should have been generated by the `flutter test --coverage` script, but was not.');
    exit(1);
  }

  print('${bold}DONE: Coverage collection successful.$reset');
278 279
}

280
Future<void> _pubRunTest(
281 282
  String workingDirectory, {
  String testPath,
283
  bool enableFlutterToolAsserts = false
284
}) {
285
  final List<String> args = <String>['run', 'test', '-rcompact', '-j1'];
286 287
  if (!hasColor)
    args.add('--no-color');
288 289
  if (testPath != null)
    args.add(testPath);
290
  final Map<String, String> pubEnvironment = <String, String>{};
291
  if (Directory(pubCache).existsSync()) {
292 293
    pubEnvironment['PUB_CACHE'] = pubCache;
  }
294 295 296 297 298 299 300 301
  if (enableFlutterToolAsserts) {
    // If an existing env variable exists append to it, but only if
    // it doesn't appear to already include enable-asserts.
    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
    if (!toolsArgs.contains('--enable-asserts'))
        toolsArgs += ' --enable-asserts';
    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
  }
302
  return runCommand(
303 304 305 306
    pub, args,
    workingDirectory: workingDirectory,
    environment: pubEnvironment,
  );
307 308
}

309 310 311 312
class EvalResult {
  EvalResult({
    this.stdout,
    this.stderr,
313
    this.exitCode = 0,
314 315 316 317
  });

  final String stdout;
  final String stderr;
318
  final int exitCode;
319 320
}

321
Future<void> _runFlutterTest(String workingDirectory, {
322
  String script,
323 324 325 326 327
  bool expectFailure = false,
  bool printOutput = true,
  List<String> options = const <String>[],
  bool skip = false,
  Duration timeout = _kLongTimeout,
328
}) {
329
  final List<String> args = <String>['test']..addAll(options);
330
  if (flutterTestArgs != null && flutterTestArgs.isNotEmpty)
331
    args.addAll(flutterTestArgs);
332 333 334 335 336 337 338 339 340 341 342 343
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
      print('Could not find test: $fullScriptPath');
      print('Working directory: $workingDirectory');
      print('Script: $script');
      if (!printOutput)
        print('This is one of the tests that does not normally print output.');
      if (skip)
        print('This is one of the tests that is normally skipped in this configuration.');
      exit(1);
    }
344
    args.add(script);
345
  }
346
  return runCommand(flutter, args,
347
    workingDirectory: workingDirectory,
348
    expectNonZeroExit: expectFailure,
349
    printOutput: printOutput,
350
    skip: skip,
351
    timeout: timeout,
352 353 354
  );
}

355
Future<void> _verifyVersion(String filename) async {
356
  if (!File(filename).existsSync()) {
357
    print('$redLine');
358
    print('The version logic failed to create the Flutter version file.');
359
    print('$redLine');
360 361
    exit(1);
  }
362
  final String version = await File(filename).readAsString();
363
  if (version == '0.0.0-unknown') {
364
    print('$redLine');
365
    print('The version logic failed to determine the Flutter version.');
366
    print('$redLine');
367 368
    exit(1);
  }
369
  final RegExp pattern = RegExp(r'^[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$');
370
  if (!version.contains(pattern)) {
371
    print('$redLine');
372
    print('The version logic generated an invalid version string.');
373
    print('$redLine');
374 375
    exit(1);
  }
376
}