test.dart 14.7 KB
Newer Older
1 2 3 4
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:convert';
7
import 'dart:io';
8

9
import 'package:path/path.dart' as path;
10

11 12
typedef Future<Null> ShardRunner();

13 14 15 16 17 18 19 20 21 22 23 24 25
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
final String flutterTestArgs = Platform.environment['FLUTTER_TEST_ARGS'];
final bool hasColor = stdout.supportsAnsiEscapes;

final String bold = hasColor ? '\x1B[1m' : '';
final String red = hasColor ? '\x1B[31m' : '';
final String green = hasColor ? '\x1B[32m' : '';
final String yellow = hasColor ? '\x1B[33m' : '';
final String cyan = hasColor ? '\x1B[36m' : '';
final String reset = hasColor ? '\x1B[0m' : '';
26

27 28 29 30 31 32 33
const Map<String, ShardRunner> _kShards = const <String, ShardRunner>{
  'docs': _generateDocs,
  'analyze': _analyzeRepo,
  'tests': _runTests,
  'coverage': _runCoverage,
};

34 35 36 37
/// When you call this, you can set FLUTTER_TEST_ARGS to pass custom
/// arguments to flutter test. For example, you might want to call this
/// script using FLUTTER_TEST_ARGS=--local-engine=host_debug_unopt to
/// use your own build of the engine.
38 39 40 41 42 43
///
/// To run the analysis part, run it with SHARD=analyze
///
/// For example:
/// SHARD=analyze bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// FLUTTER_TEST_ARGS=--local-engine=host_debug_unopt bin/cache/dart-sdk/bin/dart dev/bots/test.dart
44
Future<Null> main() async {
45 46 47 48 49 50 51 52 53 54 55
  final String shard = Platform.environment['SHARD'] ?? 'tests';
  if (!_kShards.containsKey(shard))
    throw new ArgumentError('Invalid shard: $shard');
  await _kShards[shard]();
}

Future<Null> _generateDocs() async {
  print('${bold}DONE: test.dart does nothing in the docs shard.$reset');
}

Future<Null> _analyzeRepo() async {
56 57
  await _verifyNoBadImports(flutterRoot);

58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
  // Analyze all the Dart code in the repo.
  await _runFlutterAnalyze(flutterRoot,
    options: <String>['--flutter-repo'],
  );

  // Analyze all the sample code in the repo
  await _runCommand(dart, <String>[path.join(flutterRoot, 'dev', 'bots', 'analyze-sample-code.dart')],
    workingDirectory: flutterRoot,
  );

  // Try with the --watch analyzer, to make sure it returns success also.
  // The --benchmark argument exits after one run.
  await _runFlutterAnalyze(flutterRoot,
    options: <String>['--flutter-repo', '--watch', '--benchmark'],
  );

  // Try an analysis against a big version of the gallery.
  await _runCommand(dart, <String>[path.join(flutterRoot, 'dev', 'tools', 'mega_gallery.dart')],
    workingDirectory: flutterRoot,
  );
  await _runFlutterAnalyze(path.join(flutterRoot, 'dev', 'benchmarks', 'mega_gallery'),
    options: <String>['--watch', '--benchmark'],
  );

  print('${bold}DONE: Analysis successful.$reset');
}

Future<Null> _runTests() async {
  // Verify that the tests actually return failure on failure and success on success.
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'fail_test.dart'),
    expectFailure: true,
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'pass_test.dart'),
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'crash1_test.dart'),
    expectFailure: true,
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'crash2_test.dart'),
    expectFailure: true,
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
    expectFailure: true,
    printOutput: false,
  );
  await _runFlutterTest(automatedTests,
    script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
    expectFailure: true,
    printOutput: false,
  );
  await _runCommand(flutter, <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
    workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
    expectFailure: true,
    printOutput: false,
  );

  // Run tests.
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
  await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools'));

  await _runAllDartTests(path.join(flutterRoot, 'dev', 'devicelab'));
  await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'));
  await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'));

  print('${bold}DONE: All tests successful.$reset');
}

Future<Null> _runCoverage() async {
  if (Platform.environment['TRAVIS'] == null ||
142 143 144
      Platform.environment['TRAVIS_PULL_REQUEST'] != 'false' ||
      Platform.environment['TRAVIS_OS_NAME'] != 'linux') {
    print('${bold}DONE: test.dart does not run coverage for Travis pull requests or not non-Linux environments');
145
    return;
146
  }
147

148 149 150 151 152 153 154 155
  final File coverageFile = new File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file is normally obtained by running `flutter update-packages`.');
    exit(1);
  }
  coverageFile.deleteSync();
156 157 158
  await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
    options: const <String>['--coverage'],
  );
159 160 161 162 163 164
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
    print('Expected to find: ${coverageFile.absolute}');
    print('This file should have been generated by the `flutter test --coverage` script, but was not.');
    exit(1);
  }
165 166

  print('${bold}DONE: Coverage collection successful.$reset');
167 168
}

169 170 171 172
Future<Null> _pubRunTest(
  String workingDirectory, {
  String testPath,
}) {
173
  final List<String> args = <String>['run', 'test', '-j1', '-rexpanded'];
174 175
  if (testPath != null)
    args.add(testPath);
176
  return _runCommand(pub, args, workingDirectory: workingDirectory);
177 178
}

179
Future<Null> _runCommand(String executable, List<String> arguments, {
180 181 182 183 184
  String workingDirectory,
  Map<String, String> environment,
  bool expectFailure: false,
  bool printOutput: true,
  bool skip: false,
185
}) async {
186 187
  final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
  final String relativeWorkingDir = path.relative(workingDirectory);
188
  if (skip) {
189
    _printProgress('SKIPPING', relativeWorkingDir, commandDescription);
190 191
    return null;
  }
192
  _printProgress('RUNNING', relativeWorkingDir, commandDescription);
193

194
  final Process process = await Process.start(executable, arguments,
195 196
    workingDirectory: workingDirectory,
    environment: environment,
197 198
  );

199
  Future<List<List<int>>> savedStdout, savedStderr;
200 201 202
  if (printOutput) {
    stdout.addStream(process.stdout);
    stderr.addStream(process.stderr);
203 204 205
  } else {
    savedStdout = process.stdout.toList();
    savedStderr = process.stderr.toList();
206 207
  }

208
  final int exitCode = await process.exitCode;
209
  if ((exitCode == 0) == expectFailure) {
210 211 212 213
    if (!printOutput) {
      print(UTF8.decode((await savedStdout).expand((List<int> ints) => ints).toList()));
      print(UTF8.decode((await savedStderr).expand((List<int> ints) => ints).toList()));
    }
214
    print(
215 216 217
      '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset\n'
      '${bold}ERROR:$red Last command exited with $exitCode (expected: ${expectFailure ? 'non-zero' : 'zero'}).$reset\n'
      '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset'
218 219 220 221 222 223 224 225 226 227 228 229
    );
    exit(1);
  }
}

Future<Null> _runFlutterTest(String workingDirectory, {
    String script,
    bool expectFailure: false,
    bool printOutput: true,
    List<String> options: const <String>[],
    bool skip: false,
}) {
230
  final List<String> args = <String>['test']..addAll(options);
231
  if (flutterTestArgs != null && flutterTestArgs.isNotEmpty)
232 233 234
    args.add(flutterTestArgs);
  if (script != null)
    args.add(script);
235
  return _runCommand(flutter, args,
236 237 238 239
    workingDirectory: workingDirectory,
    expectFailure: expectFailure,
    printOutput: printOutput,
    skip: skip || Platform.isWindows, // TODO(goderbauer): run on Windows when sky_shell is available
240 241 242 243
  );
}

Future<Null> _runAllDartTests(String workingDirectory, {
244
  Map<String, String> environment,
245
}) {
246 247
  final List<String> args = <String>['--checked', path.join('test', 'all.dart')];
  return _runCommand(dart, args,
248 249
    workingDirectory: workingDirectory,
    environment: environment,
250 251 252 253
  );
}

Future<Null> _runFlutterAnalyze(String workingDirectory, {
254
  List<String> options: const <String>[]
255
}) {
256
  return _runCommand(flutter, <String>['analyze']..addAll(options),
257
    workingDirectory: workingDirectory,
258 259 260
  );
}

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
Future<Null> _verifyNoBadImports(String workingDirectory) async {
  final List<String> errors = <String>[];
  final String libPath = path.join(workingDirectory, 'packages', 'flutter', 'lib');
  final String srcPath = path.join(workingDirectory, 'packages', 'flutter', 'lib', 'src');
  // Verify there's one libPath/*.dart for each srcPath/*/.
  <String>[];
  final List<String> packages = new Directory(libPath).listSync()
    .where((FileSystemEntity entity) => entity is File && path.extension(entity.path) == '.dart')
    .map<String>((FileSystemEntity entity) => path.basenameWithoutExtension(entity.path))
    .toList()..sort();
  final List<String> directories = new Directory(srcPath).listSync()
    .where((FileSystemEntity entity) => entity is Directory)
    .map<String>((FileSystemEntity entity) => path.basename(entity.path))
    .toList()..sort();
  if (!_matches(packages, directories)) {
    errors.add(
      'flutter/lib/*.dart does not match flutter/lib/src/*/:\n'
      'These are the exported packages:\n' +
      packages.map((String path) => '  lib/$path.dart').join('\n') +
      'These are the directories:\n' +
      directories.map((String path) => '  lib/src/$path/').join('\n')
    );
  }
  // Verify that the imports are well-ordered.
  final Map<String, Set<String>> dependencyMap = new Map<String, Set<String>>.fromIterable(
    directories,
    key: (String directory) => directory,
    value: (String directory) => _findDependencies(path.join(srcPath, directory), errors, checkForMeta: directory != 'foundation'),
  );
  for (String package in dependencyMap.keys) {
    if (dependencyMap[package].contains(package)) {
      errors.add(
        'One of the files in the $yellow$package$reset package imports that package recursively.'
      );
    }
  }
  for (String package in dependencyMap.keys) {
    final List<String> loop = _deepSearch(dependencyMap, package);
    if (loop != null) {
      errors.add(
        '${yellow}Dependency loop:$reset ' +
        loop.join(' depends on ')
      );
    }
  }
  // Fail if any errors
  if (errors.isNotEmpty) {
    print('$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset');
    if (errors.length == 1) {
      print('${bold}An error was detected when looking at import dependencies within the Flutter package:$reset\n');
    } else {
      print('${bold}Multiple errors were detected when looking at import dependencies within the Flutter package:$reset\n');
    }
    print(errors.join('\n\n'));
    print('$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset\n');
    exit(1);
  }
}

bool _matches<T>(List<T> a, List<T> b) {
  assert(a != null);
  assert(b != null);
  if (a.length != b.length)
    return false;
  for (int index = 0; index < a.length; index += 1) {
    if (a[index] != b[index])
      return false;
  }
  return true;
}

final RegExp _importPattern = new RegExp(r"import 'package:flutter/([^.]+)\.dart'");
final RegExp _importMetaPattern = new RegExp(r"import 'package:meta/meta.dart'");

Set<String> _findDependencies(String srcPath, List<String> errors, { bool checkForMeta: false }) {
  return new Directory(srcPath).listSync().where((FileSystemEntity entity) {
    return entity is File && path.extension(entity.path) == '.dart';
  }).map<Set<String>>((FileSystemEntity entity) {
    final Set<String> result = new Set<String>();
    final File file = entity;
    for (String line in file.readAsLinesSync()) {
      Match match = _importPattern.firstMatch(line);
      if (match != null)
        result.add(match.group(1));
      if (checkForMeta) {
        match = _importMetaPattern.firstMatch(line);
        if (match != null) {
          errors.add(
            '${file.path}\nThis package imports the ${yellow}meta$reset package.\n'
            'You should instead import the "foundation.dart" library.'
          );
        }
      }
    }
    return result;
  }).reduce((Set<String> value, Set<String> element) {
    value ??= new Set<String>();
    value.addAll(element);
    return value;
  });
}

List<T> _deepSearch<T>(Map<T, Set<T>> map, T start, [ Set<T> seen ]) {
  for (T key in map[start]) {
    if (key == start)
      continue; // we catch these separately
    if (seen != null && seen.contains(key))
      return <T>[start, key];
    final List<T> result = _deepSearch(
      map,
      key,
      (seen == null ? new Set<T>.from(<T>[start]) : new Set<T>.from(seen))..add(key),
    );
    if (result != null) {
      result.insert(0, start);
      // Only report the shortest chains.
      // For example a->b->a, rather than c->a->b->a.
      // Since we visit every node, we know the shortest chains are those
      // that start and end on the loop.
      if (result.first == result.last)
        return result;
    }
  }
  return null;
}

387 388 389
void _printProgress(String action, String workingDir, String command) {
  const String arrow = '⏩';
  print('$arrow $action: cd $cyan$workingDir$reset; $yellow$command$reset');
390
}