Separate tools tests from other tests and put them in a different shard. (#19593)

.cirrus.yml

@@ -38,6 +38,15 @@ task:
       container:
         cpu: 4
         memory: 8G
+    - name: tool_tests-linux
+      env:
+        SHARD: tool_tests
+      test_script: |
+        export TEST_COMMIT_RANGE="$(git merge-base --fork-point FETCH_HEAD HEAD)..HEAD"
+        dart ./dev/bots/test.dart
+      container:
+        cpu: 4
+        memory: 8G
 
 task:
   name: tests-windows
@@ -58,6 +67,25 @@ task:
     export TEST_COMMIT_RANGE="$(git merge-base --fork-point FETCH_HEAD HEAD)..HEAD"
     bin\cache\dart-sdk\bin\dart.exe -c dev\bots\test.dart
 
+task:
+  name: tool_tests-windows
+  env:
+    SHARD: tool_tests
+  windows_container:
+    image: cirrusci/windowsservercore:2016
+    os_version: 2016
+    cpu: 4
+  env:
+    CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk"
+  git_fetch_script: git fetch origin
+  setup_script:
+    - bin\flutter.bat config --no-analytics
+    - bin\flutter.bat update-packages
+    - git fetch origin master
+  test_all_script: |
+    export TEST_COMMIT_RANGE="$(git merge-base --fork-point FETCH_HEAD HEAD)..HEAD"
+    bin\cache\dart-sdk\bin\dart.exe -c dev\bots\test.dart
+
 task:
   name: tests-macos
   env:
@@ -72,3 +100,18 @@ task:
     ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
     export TEST_COMMIT_RANGE="$(git merge-base --fork-point FETCH_HEAD HEAD)..HEAD"
     bin/cache/dart-sdk/bin/dart -c dev/bots/test.dart
+
+task:
+  name: tool_tests-macos
+  env:
+    SHARD: tool_tests
+  osx_instance:
+    image: high-sierra-xcode-9.4.1
+  git_fetch_script: git fetch origin
+  setup_script:
+    - bin/flutter config --no-analytics
+    - bin/flutter update-packages
+  test_all_script: |
+    ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+    export TEST_COMMIT_RANGE="$(git merge-base --fork-point FETCH_HEAD HEAD)..HEAD"
+    bin/cache/dart-sdk/bin/dart -c dev/bots/test.dart

.travis.yml

@@ -6,6 +6,7 @@ osx_image: xcode9.2
 env:
   - SHARD=analyze
   - SHARD=tests
+  - SHARD=tool_tests
   - SHARD=docs
   - SHARD=build_and_deploy_gallery
 matrix:

appveyor.yml

@@ -11,5 +11,7 @@ cache:
 build: off
 
 test_script:
+- cmd: set SHARD=tool_tests
+- cmd: bin\cache\dart-sdk\bin\dart.exe -c dev\bots\test.dart
 - cmd: set SHARD=tests
 - cmd: bin\cache\dart-sdk\bin\dart.exe -c dev\bots\test.dart

dev/bots/test.dart

@@ -24,10 +24,13 @@ final String green = hasColor ? '\x1B[32m' : '';
 final String yellow = hasColor ? '\x1B[33m' : '';
 final String cyan = hasColor ? '\x1B[36m' : '';
 final String reset = hasColor ? '\x1B[0m' : '';
+const String arrow = '⏩';
+const String clock = '🕐';
 
 const Map<String, ShardRunner> _kShards = const <String, ShardRunner>{
   'analyze': _analyzeRepo,
   'tests': _runTests,
+  'tool_tests': _runToolTests,
   'coverage': _runCoverage,
   // 'docs': handled by travis_script.sh and docs.sh
   // 'build_and_deploy_gallery': handled by travis_script.sh
@@ -172,7 +175,7 @@ Future<Null> _analyzeRepo() async {
   print('${bold}DONE: Analysis successful.$reset');
 }
 
-Future<Null> _runTests() async {
+Future<Null> _runSmokeTests() async {
   // Verify that the tests actually return failure on failure and success on
   // success.
   final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
@@ -250,14 +253,24 @@ Future<Null> _runTests() async {
 
   // Verify that we correctly generated the version file.
   await _verifyVersion(path.join(flutterRoot, 'version'));
+}
+
+Future<Null> _runToolTests() async {
+  await _runSmokeTests();
+
+  await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools'));
+
+  print('${bold}DONE: All tests successful.$reset');
+}
+
+Future<Null> _runTests() async {
+  await _runSmokeTests();
 
-  // Run tests.
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'));
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'));
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
   await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
-  await _pubRunTest(path.join(flutterRoot, 'packages', 'flutter_tools'));
   await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
   await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'));
   await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
@@ -336,6 +349,7 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
   }
   _printProgress('RUNNING', relativeWorkingDir, commandDescription);
 
+  final DateTime start = new DateTime.now();
   final Process process = await Process.start(executable, arguments,
     workingDirectory: workingDirectory,
     environment: environment,
@@ -349,6 +363,8 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
     stderr: utf8.decode((await savedStderr).expand((List<int> ints) => ints).toList()),
   );
 
+  print('$clock ELAPSED TIME: $bold${elapsedTime(start)}$reset for $commandDescription in $relativeWorkingDir: ');
+
   if (exitCode != 0) {
     stderr.write(result.stderr);
     print(
@@ -364,6 +380,10 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
   return result;
 }
 
+String elapsedTime(DateTime start) {
+  return new DateTime.now().difference(start).toString();
+}
+
 Future<Null> _runCommand(String executable, List<String> arguments, {
   String workingDirectory,
   Map<String, String> environment,
@@ -381,6 +401,7 @@ Future<Null> _runCommand(String executable, List<String> arguments, {
   }
   _printProgress('RUNNING', relativeWorkingDir, commandDescription);
 
+  final DateTime start = new DateTime.now();
   final Process process = await Process.start(executable, arguments,
     workingDirectory: workingDirectory,
     environment: environment,
@@ -401,6 +422,7 @@ Future<Null> _runCommand(String executable, List<String> arguments, {
     stderr.writeln('Process timed out after $timeout');
     return expectFailure ? 0 : 1;
   });
+  print('$clock ELAPSED TIME: $bold${elapsedTime(start)}$reset for $commandDescription in $relativeWorkingDir: ');
   if ((exitCode == 0) == expectFailure || (expectedExitCode != null && exitCode != expectedExitCode)) {
     if (!printOutput) {
       stdout.writeln(utf8.decode((await savedStdout).expand((List<int> ints) => ints).toList()));
@@ -409,6 +431,8 @@ Future<Null> _runCommand(String executable, List<String> arguments, {
     print(
       '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset\n'
       '${bold}ERROR:$red Last command exited with $exitCode (expected: ${expectFailure ? 'non-zero' : 'zero'}).$reset\n'
+      '${bold}Command:$cyan $commandDescription$reset\n'
+      '${bold}Relative working directory:$red $relativeWorkingDir$reset\n'
       '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset'
     );
     exit(1);
@@ -606,7 +630,6 @@ Future<Null> _verifyNoBadImportsInFlutterTools(String workingDirectory) async {
 }
 
 void _printProgress(String action, String workingDir, String command) {
-  const String arrow = '⏩';
   print('$arrow $action: cd $cyan$workingDir$reset; $yellow$command$reset');
 }
 
@@ -666,9 +689,10 @@ String _getPackageFor(File entity, Directory flutterRootDir) {
 
 bool _isGeneratedPluginRegistrant(File file) {
   final String filename = path.basename(file.path);
-  return filename == 'GeneratedPluginRegistrant.java' ||
-      filename == 'GeneratedPluginRegistrant.h' ||
-      filename == 'GeneratedPluginRegistrant.m';
+  return !file.path.contains('.pub-cache')
+      && (filename == 'GeneratedPluginRegistrant.java' ||
+          filename == 'GeneratedPluginRegistrant.h' ||
+          filename == 'GeneratedPluginRegistrant.m');
 }
 
 Future<Null> _verifyVersion(String filename) async {