Commit bd67926f authored by Todd Volkert's avatar Todd Volkert Committed by GitHub

Run coverage in a dedicated shard on Travis (#10755)

parent 615410d2
...@@ -17,6 +17,7 @@ install: ...@@ -17,6 +17,7 @@ install:
env: env:
- SHARD=analyze - SHARD=analyze
- SHARD=tests - SHARD=tests
- SHARD=coverage
- SHARD=docs - SHARD=docs
before_script: before_script:
- ./dev/bots/travis_setup.sh - ./dev/bots/travis_setup.sh
......
...@@ -8,6 +8,8 @@ import 'dart:io'; ...@@ -8,6 +8,8 @@ import 'dart:io';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
typedef Future<Null> ShardRunner();
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script)))); final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter'); final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart'); final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
...@@ -22,6 +24,13 @@ final String yellow = hasColor ? '\x1B[33m' : ''; ...@@ -22,6 +24,13 @@ final String yellow = hasColor ? '\x1B[33m' : '';
final String cyan = hasColor ? '\x1B[36m' : ''; final String cyan = hasColor ? '\x1B[36m' : '';
final String reset = hasColor ? '\x1B[0m' : ''; final String reset = hasColor ? '\x1B[0m' : '';
const Map<String, ShardRunner> _kShards = const <String, ShardRunner>{
'docs': _generateDocs,
'analyze': _analyzeRepo,
'tests': _runTests,
'coverage': _runCoverage,
};
/// When you call this, you can set FLUTTER_TEST_ARGS to pass custom /// When you call this, you can set FLUTTER_TEST_ARGS to pass custom
/// arguments to flutter test. For example, you might want to call this /// arguments to flutter test. For example, you might want to call this
/// script using FLUTTER_TEST_ARGS=--local-engine=host_debug_unopt to /// script using FLUTTER_TEST_ARGS=--local-engine=host_debug_unopt to
...@@ -33,94 +42,111 @@ final String reset = hasColor ? '\x1B[0m' : ''; ...@@ -33,94 +42,111 @@ final String reset = hasColor ? '\x1B[0m' : '';
/// SHARD=analyze bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// SHARD=analyze bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// FLUTTER_TEST_ARGS=--local-engine=host_debug_unopt bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// FLUTTER_TEST_ARGS=--local-engine=host_debug_unopt bin/cache/dart-sdk/bin/dart dev/bots/test.dart
Future<Null> main() async { Future<Null> main() async {
if (Platform.environment['SHARD'] == 'docs') { final String shard = Platform.environment['SHARD'] ?? 'tests';
print('${bold}DONE: test.dart does nothing in the docs shard.$reset'); if (!_kShards.containsKey(shard))
} else if (Platform.environment['SHARD'] == 'analyze') { throw new ArgumentError('Invalid shard: $shard');
// Analyze all the Dart code in the repo. await _kShards[shard]();
await _runFlutterAnalyze(flutterRoot, }
options: <String>['--flutter-repo'],
);
// Analyze all the sample code in the repo Future<Null> _generateDocs() async {
await _runCommand(dart, <String>[path.join(flutterRoot, 'dev', 'bots', 'analyze-sample-code.dart')], print('${bold}DONE: test.dart does nothing in the docs shard.$reset');
workingDirectory: flutterRoot, }
);
// Try with the --watch analyzer, to make sure it returns success also. Future<Null> _analyzeRepo() async {
// The --benchmark argument exits after one run. // Analyze all the Dart code in the repo.
await _runFlutterAnalyze(flutterRoot, await _runFlutterAnalyze(flutterRoot,
options: <String>['--flutter-repo', '--watch', '--benchmark'], options: <String>['--flutter-repo'],
); );
// Try an analysis against a big version of the gallery. // Analyze all the sample code in the repo
await _runCommand(dart, <String>[path.join(flutterRoot, 'dev', 'tools', 'mega_gallery.dart')], await _runCommand(dart, <String>[path.join(flutterRoot, 'dev', 'bots', 'analyze-sample-code.dart')],
workingDirectory: flutterRoot, workingDirectory: flutterRoot,
); );
await _runFlutterAnalyze(path.join(flutterRoot, 'dev', 'benchmarks', 'mega_gallery'),
options: <String>['--watch', '--benchmark'],
);
print('${bold}DONE: Analysis successful.$reset'); // Try with the --watch analyzer, to make sure it returns success also.
} else { // The --benchmark argument exits after one run.
// Verify that the tests actually return failure on failure and success on success. await _runFlutterAnalyze(flutterRoot,
final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests'); options: <String>['--flutter-repo', '--watch', '--benchmark'],
await _runFlutterTest(automatedTests, );
script: path.join('test_smoke_test', 'fail_test.dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'pass_test.dart'),
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'crash1_test.dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'crash2_test.dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
expectFailure: true,
printOutput: false,
);
await _runCommand(flutter, <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
expectFailure: true,
printOutput: false,
);
final List<String> coverageFlags = <String>[]; // Try an analysis against a big version of the gallery.
if (Platform.environment['TRAVIS'] != null && Platform.environment['TRAVIS_PULL_REQUEST'] == 'false') await _runCommand(dart, <String>[path.join(flutterRoot, 'dev', 'tools', 'mega_gallery.dart')],
coverageFlags.add('--coverage'); workingDirectory: flutterRoot,
);
await _runFlutterAnalyze(path.join(flutterRoot, 'dev', 'benchmarks', 'mega_gallery'),
options: <String>['--watch', '--benchmark'],
);
// Run tests. print('${bold}DONE: Analysis successful.$reset');
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), }
options: coverageFlags,
); Future<Null> _runTests() async {
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver')); // Verify that the tests actually return failure on failure and success on success.
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test')); final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools')); await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'fail_test.dart'),
await _runAllDartTests(path.join(flutterRoot, 'dev', 'devicelab')); expectFailure: true,
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests')); printOutput: false,
await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world')); );
await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers')); await _runFlutterTest(automatedTests,
await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks')); script: path.join('test_smoke_test', 'pass_test.dart'),
await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery')); printOutput: false,
await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog')); );
await _runFlutterTest(automatedTests,
print('${bold}DONE: All tests successful.$reset'); script: path.join('test_smoke_test', 'crash1_test.dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'crash2_test.dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
expectFailure: true,
printOutput: false,
);
await _runFlutterTest(automatedTests,
script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
expectFailure: true,
printOutput: false,
);
await _runCommand(flutter, <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
expectFailure: true,
printOutput: false,
);
// Run tests.
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'));
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools'));
await _runAllDartTests(path.join(flutterRoot, 'dev', 'devicelab'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'));
await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'));
await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'));
await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'));
await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'));
print('${bold}DONE: All tests successful.$reset');
}
Future<Null> _runCoverage() async {
if (Platform.environment['TRAVIS'] == null ||
Platform.environment['TRAVIS_PULL_REQUEST'] != 'false') {
print('${bold}DONE: test.dart does not run coverage for Travis pull requests');
return;
} }
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
options: const <String>['--coverage'],
);
print('${bold}DONE: Coverage collection successful.$reset');
} }
Future<Null> _pubRunTest( Future<Null> _pubRunTest(
......
...@@ -6,7 +6,7 @@ export PATH="$PWD/bin:$PWD/bin/cache/dart-sdk/bin:$PATH" ...@@ -6,7 +6,7 @@ export PATH="$PWD/bin:$PWD/bin/cache/dart-sdk/bin:$PATH"
LCOV_FILE=./packages/flutter/coverage/lcov.info LCOV_FILE=./packages/flutter/coverage/lcov.info
if [ "$SHARD" = "tests" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ] && [ -f "$LCOV_FILE" ]; then if [ "$SHARD" = "coverage" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ] && [ -f "$LCOV_FILE" ]; then
GSUTIL=$HOME/google-cloud-sdk/bin/gsutil GSUTIL=$HOME/google-cloud-sdk/bin/gsutil
GCLOUD=$HOME/google-cloud-sdk/bin/gcloud GCLOUD=$HOME/google-cloud-sdk/bin/gcloud
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment