test.dart 11.8 KB
Newer Older
1 2 3 4
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:io';
7
import 'dart:math' as math;
8

9
import 'package:path/path.dart' as path;
10 11

import 'run_command.dart';
12

13
typedef ShardRunner = Future<void> Function();
14

15 16 17 18
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
19
final String pubCache = path.join(flutterRoot, '.pub-cache');
20
final List<String> flutterTestArgs = <String>[];
21

22
const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
23
  'tests': _runTests,
24
  'tool_tests': _runToolTests,
25
  'aot_build_tests': _runAotBuildTests,
26 27 28
  'coverage': _runCoverage,
};

29 30
const Duration _kLongTimeout = Duration(minutes: 45);
const Duration _kShortTimeout = Duration(minutes: 5);
31

32
/// When you call this, you can pass additional arguments to pass custom
33
/// arguments to flutter test. For example, you might want to call this
34
/// script with the parameter --local-engine=host_debug_unopt to
35
/// use your own build of the engine.
36
///
37
/// To run the tool_tests part, run it with SHARD=tool_tests
38 39
///
/// For example:
40
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
41
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
42
Future<void> main(List<String> args) async {
43 44
  flutterTestArgs.addAll(args);

45 46
  final String shard = Platform.environment['SHARD'];
  if (shard != null) {
47 48 49 50 51
    if (!_kShards.containsKey(shard)) {
      print('Invalid shard: $shard');
      print('The available shards are: ${_kShards.keys.join(", ")}');
      exit(1);
    }
52
    print('${bold}SHARD=$shard$reset');
53 54 55 56 57
    await _kShards[shard]();
  } else {
    for (String currentShard in _kShards.keys) {
      print('${bold}SHARD=$currentShard$reset');
      await _kShards[currentShard]();
58
      print('');
59 60
    }
  }
61 62
}

63
Future<void> _runSmokeTests() async {
64 65
  // Verify that the tests actually return failure on failure and success on
  // success.
66
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
67 68 69
  // We run the "pass" and "fail" smoke tests first, and alone, because those
  // are particularly critical and sensitive. If one of these fails, there's no
  // point even trying the others.
70 71 72
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pass_test.dart'),
    printOutput: false,
73
    timeout: _kShortTimeout,
74 75
  );
  await _runFlutterTest(automatedTests,
76
    script: path.join('test_smoke_test', 'fail_test.dart'),
77 78
    expectFailure: true,
    printOutput: false,
79
    timeout: _kShortTimeout,
80
  );
81
  // We run the timeout tests individually because they are timing-sensitive.
82
  await _runFlutterTest(automatedTests,
83 84
    script: path.join('test_smoke_test', 'timeout_pass_test.dart'),
    expectFailure: false,
85
    printOutput: false,
86
    timeout: _kShortTimeout,
87
  );
88
  await _runFlutterTest(automatedTests,
89
    script: path.join('test_smoke_test', 'timeout_fail_test.dart'),
90 91 92 93
    expectFailure: true,
    printOutput: false,
    timeout: _kShortTimeout,
  );
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
  // We run the remaining smoketests in parallel, because they each take some
  // time to run (e.g. compiling), so we don't want to run them in series,
  // especially on 20-core machines...
  await Future.wait<void>(
    <Future<void>>[
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash1_test.dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'crash2_test.dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
      _runFlutterTest(automatedTests,
        script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
        expectFailure: true,
        printOutput: false,
        timeout: _kShortTimeout,
      ),
129
      runCommand(flutter,
130 131
        <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
        workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
132
        expectNonZeroExit: true,
133 134 135 136
        printOutput: false,
        timeout: _kShortTimeout,
      ),
    ],
137 138
  );

139 140
  // Verify that we correctly generated the version file.
  await _verifyVersion(path.join(flutterRoot, 'version'));
141 142
}

143
Future<void> _runToolTests() async {
144 145
  await _runSmokeTests();

146 147 148 149
  await _pubRunTest(
    path.join(flutterRoot, 'packages', 'flutter_tools'),
    enableFlutterToolAsserts: true,
  );
150 151 152 153

  print('${bold}DONE: All tests successful.$reset');
}

154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
/// Verifies that AOT builds of some examples apps finish
/// without crashing. It does not actually launch the AOT-built
/// apps. That happens later in the devicelab. This is just
/// a smoke-test.
Future<void> _runAotBuildTests() async {
  await _flutterBuildAot(path.join('examples', 'hello_world'));
  await _flutterBuildAot(path.join('examples', 'flutter_gallery'));
  await _flutterBuildAot(path.join('examples', 'flutter_view'));

  print('${bold}DONE: All AOT build tests successful.$reset');
}

Future<void> _flutterBuildAot(String relativePathToApplication) {
  return runCommand(flutter,
    <String>['build', 'aot'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
    expectNonZeroExit: false,
    timeout: _kShortTimeout,
  );
}

175
Future<void> _runTests() async {
176
  await _runSmokeTests();
177

178
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'));
179 180 181 182 183
  // Only packages/flutter/test/widgets/widget_inspector_test.dart really
  // needs to be run with --track-widget-creation but it is nice to run
  // all of the tests in package:flutter with the flag to ensure that
  // the Dart kernel transformer triggered by the flag does not break anything.
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), options: <String>['--track-widget-creation']);
184 185 186 187
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
188
  await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
189
  await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'));
190
  await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'));
191
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'));
192 193 194 195 196 197
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'));
198 199 200
  // Regression test to ensure that code outside of package:flutter can run
  // with --track-widget-creation.
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation']);
201
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'));
202 203 204 205

  print('${bold}DONE: All tests successful.$reset');
}

206
Future<void> _runCoverage() async {
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file is normally obtained by running `flutter update-packages`.');
    exit(1);
  }
  coverageFile.deleteSync();
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file should have been generated by the `flutter test --coverage` script, but was not.');
    exit(1);
  }

  print('${bold}DONE: Coverage collection successful.$reset');
226 227
}

228
Future<void> _pubRunTest(
229 230
  String workingDirectory, {
  String testPath,
231
  bool enableFlutterToolAsserts = false
232
}) {
233
  final List<String> args = <String>['run', 'test', '-rcompact'];
234 235
  final int concurrency = math.max(1, Platform.numberOfProcessors - 1);
  args.add('-j$concurrency');
236 237
  if (!hasColor)
    args.add('--no-color');
238 239
  if (testPath != null)
    args.add(testPath);
240
  final Map<String, String> pubEnvironment = <String, String>{};
241
  if (Directory(pubCache).existsSync()) {
242 243
    pubEnvironment['PUB_CACHE'] = pubCache;
  }
244 245 246 247 248 249 250 251
  if (enableFlutterToolAsserts) {
    // If an existing env variable exists append to it, but only if
    // it doesn't appear to already include enable-asserts.
    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
    if (!toolsArgs.contains('--enable-asserts'))
        toolsArgs += ' --enable-asserts';
    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
  }
252
  return runCommand(
253 254 255 256
    pub, args,
    workingDirectory: workingDirectory,
    environment: pubEnvironment,
  );
257 258
}

259 260 261 262
class EvalResult {
  EvalResult({
    this.stdout,
    this.stderr,
263
    this.exitCode = 0,
264 265 266 267
  });

  final String stdout;
  final String stderr;
268
  final int exitCode;
269 270
}

271
Future<void> _runFlutterTest(String workingDirectory, {
272
  String script,
273 274 275 276 277
  bool expectFailure = false,
  bool printOutput = true,
  List<String> options = const <String>[],
  bool skip = false,
  Duration timeout = _kLongTimeout,
278
}) {
279
  final List<String> args = <String>['test']..addAll(options);
280
  if (flutterTestArgs != null && flutterTestArgs.isNotEmpty)
281
    args.addAll(flutterTestArgs);
282 283 284 285 286 287 288 289 290 291 292 293
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
      print('Could not find test: $fullScriptPath');
      print('Working directory: $workingDirectory');
      print('Script: $script');
      if (!printOutput)
        print('This is one of the tests that does not normally print output.');
      if (skip)
        print('This is one of the tests that is normally skipped in this configuration.');
      exit(1);
    }
294
    args.add(script);
295
  }
296
  return runCommand(flutter, args,
297
    workingDirectory: workingDirectory,
298
    expectNonZeroExit: expectFailure,
299
    printOutput: printOutput,
300
    skip: skip,
301
    timeout: timeout,
302 303 304
  );
}

305
Future<void> _verifyVersion(String filename) async {
306
  if (!File(filename).existsSync()) {
307
    print('$redLine');
308
    print('The version logic failed to create the Flutter version file.');
309
    print('$redLine');
310 311
    exit(1);
  }
312
  final String version = await File(filename).readAsString();
313
  if (version == '0.0.0-unknown') {
314
    print('$redLine');
315
    print('The version logic failed to determine the Flutter version.');
316
    print('$redLine');
317 318
    exit(1);
  }
319
  final RegExp pattern = RegExp(r'^[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$');
320
  if (!version.contains(pattern)) {
321
    print('$redLine');
322
    print('The version logic generated an invalid version string.');
323
    print('$redLine');
324 325
    exit(1);
  }
326
}