Move commands into their own shard (#39140)

.cirrus.yml

@@ -25,7 +25,7 @@ task:
   setup_script: ./dev/bots/cirrus_setup.sh
   matrix:
     - name: docs
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master'"
       env:
         SHARD: docs
         # For uploading master docs to Firebase master branch staging site
@@ -67,7 +67,7 @@ task:
         cpu: 4
         memory: 12G
     - name: tests_widgets-linux
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tests
@@ -81,7 +81,7 @@ task:
         cpu: 4
         memory: 12G
     - name: tests_framework_other-linux
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tests
@@ -95,7 +95,7 @@ task:
         cpu: 4
         memory: 12G
     - name: tests_extras-linux
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tests
@@ -106,23 +106,23 @@ task:
         cpu: 4
         memory: 12G
     # all of the tests except the ones in test/integration and test/commands/create_test for packages/flutter_tools
-    - name: tool_tests-linux
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: tool_tests_general-linux
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
-        SUBSHARD: tool
+        SUBSHARD: general
       test_script:
         - dart --enable-asserts ./dev/bots/test.dart
       container:
         cpu: 4
         memory: 12G
-    - name: tool_tests_create-linux
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: tool_tests_commands-linux
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
-        SUBSHARD: create
+        SUBSHARD: commands
       test_script:
         - dart --enable-asserts ./dev/bots/test.dart
       container:
@@ -130,7 +130,7 @@ task:
         memory: 12G
     # all of the tests in test/integration for packages/flutter_tools
     - name: tool_tests_integration-linux
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
@@ -141,7 +141,7 @@ task:
         cpu: 4
         memory: 12G
     - name: tool_coverage-linux
-      skip: "!changesInclude('packages/flutter_tools/**/*.dart') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**/*.dart') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         CODECOV_TOKEN: ENCRYPTED[7c76a7f8c9264f3b7f3fd63fcf186f93c62c4dfe43ec288861c2f506d456681032b89efe7b7a139c82156350ca2c752c]
@@ -276,23 +276,22 @@ task:
     - dart --enable-asserts dev\bots\test.dart
   matrix:
     # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools
-    - name: tool_tests-windows
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: tool_tests_general-windows
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
-        SUBSHARD: tool
-        SHARD_INDEX: 1
+        SUBSHARD: general
     # all of the tests in test/commands/create_test
-    - name: tool_tests_create-windows
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: tool_tests_commands-windows
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
-        SUBSHARD: create
+        SUBSHARD: commands
     # all of the tests in test/integration for packages/flutter_tools
     - name: tool_tests_integration-windows
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
@@ -333,7 +332,7 @@ task:
         - cd dev\bots
         - pub run test
     - name: tests_widgets-windows
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tests
@@ -344,7 +343,7 @@ task:
       test_all_script:
         - dart --enable-asserts dev\bots\test.dart
     - name: tests_framework_other-windows
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tests
@@ -355,7 +354,7 @@ task:
       test_all_script:
         - dart --enable-asserts dev\bots\test.dart
     - name: tests_extras-windows
-      skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tests
@@ -461,20 +460,19 @@ task:
     - bin/cache/dart-sdk/bin/dart --enable-asserts dev/bots/test.dart
   matrix:
     # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools
-    - name: tool_tests-macos
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: tool_tests_general-macos
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
-        SUBSHARD: tool
-        SHARD_INDEX: 1
+        SUBSHARD: general
     # all of the tests in test/commands/create_test
-    - name: tool_tests_create-macos
-      skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: tool_tests_commands-macos
+      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
       env:
         GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
         SHARD: tool_tests
-        SUBSHARD: create
+        SUBSHARD: commands
     # all of the tests in test/integration for packages/flutter_tools
     - name: tool_tests_integration-macos
       only_if: $CIRRUS_BRANCH == 'master'
@@ -573,8 +571,8 @@ docker_builder:
     - tests_widgets-linux
     - tests_framework_other-linux
     - tests_extras-linux
-    - tool_tests-linux
-    - tool_tests_create-linux
+    - tool_tests_general-linux
+    - tool_tests_commands-linux
     - tool_tests_integration-linux
     - build_tests-linux
     - integration_tests-linux

bin/flutter.bat

@@ -60,7 +60,7 @@ IF NOT EXIST "%cache_dir%" (
 
 
 REM To debug the tool, you can uncomment the following lines to enable checked mode and set an observatory port:
-REM SET FLUTTER_TOOL_ARGS="--checked %FLUTTER_TOOL_ARGS%"
+REM SET FLUTTER_TOOL_ARGS="--enable-asserts %FLUTTER_TOOL_ARGS%"
 REM SET FLUTTER_TOOL_ARGS="%FLUTTER_TOOL_ARGS% --observe=65432"
 
 :acquire_lock

dev/bots/test.dart

@@ -33,8 +33,7 @@ final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
 final List<String> flutterTestArgs = <String>[];
 
 final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
-
-final bool noUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] == 'true';
+final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true';
 
 const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
   'tests': _runTests,
@@ -186,84 +185,51 @@ Future<bq.BigqueryApi> _getBigqueryApi() async {
   }
 }
 
-// Partition tool tests into two groups, see explanation on `_runToolCoverage`.
-List<List<String>> _partitionToolTests() {
-  final String toolTestDir = path.join(toolRoot, 'test');
-  final List<String> pending = <String>[
-    for (FileSystemEntity entity in Directory(toolTestDir).listSync(recursive: true))
-      if (entity is File && entity.path.endsWith('_test.dart'))
-        path.relative(entity.path, from: toolRoot),
-  ];
-
-  // Shuffle the tests to avoid giving an expensive test directory like
-  // integration to a single run of tests.
-  pending.shuffle();
-  final int aboutHalf = pending.length ~/ 2;
-  final List<String> groupA = pending.take(aboutHalf).toList();
-  final List<String> groupB = pending.skip(aboutHalf).toList();
-  return <List<String>>[groupA, groupB];
-}
-
-// Tools tests run with coverage enabled have much higher memory usage than
-// our current CI infrastructure can support. We partition the tests into
-// two sets and run them in separate invocations of dart to reduce peak memory
-// usage. codecov.io automatically handles merging different coverage files
-// together, so producing separate files is OK.
-//
-// See: https://github.com/flutter/flutter/issues/35025
 Future<void> _runToolCoverage() async {
-  final List<List<String>> tests = _partitionToolTests();
-  // Precompile tests to speed up subsequent runs.
-  await runCommand(
+  await runCommand( // Precompile tests to speed up subsequent runs.
     pub,
     <String>['run', 'build_runner', 'build'],
     workingDirectory: toolRoot,
   );
-
-  // The name of this subshard has to match the --file path provided at
-  // the end of this test script in `.cirrus.yml`.
-  const List<String> subshards = <String>['A', 'B'];
-  for (int i = 0; i < tests.length; i++) {
-    final List<String> testGroup = tests[i];
   await runCommand(
     dart,
-      <String>[path.join('tool', 'tool_coverage.dart'), '--', ...testGroup],
+    <String>[path.join('tool', 'tool_coverage.dart')],
     workingDirectory: toolRoot,
     environment: <String, String>{
       'FLUTTER_ROOT': flutterRoot,
-        'SUBSHARD': subshards[i],
-      },
-    );
     }
+  );
 }
 
 Future<void> _runToolTests() async {
   final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
   await _runSmokeTests();
 
-  // The flutter_tool will currently be snapshotted without asserts. We need
-  // to force it to be regenerated with them enabled.
-  if (!Platform.isWindows) {
-    File(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot')).deleteSync();
-    File(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp')).deleteSync();
-  }
-  // reduce overhead of build_runner in the create case.
-  if (noUseBuildRunner || Platform.environment['SUBSHARD'] == 'create') {
+  const String kDotShard = '.shard';
+  const String kTest = 'test';
+  final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
+
+  final Map<String, ShardRunner> subshards = Map<String, ShardRunner>.fromIterable(
+    Directory(path.join(toolsPath, kTest))
+      .listSync()
+      .map<String>((FileSystemEntity entry) => entry.path)
+      .where((String name) => name.endsWith(kDotShard))
+      .map<String>((String name) => path.basenameWithoutExtension(name)),
+    // The `dynamic` on the next line is because Map.fromIterable isn't generic.
+    value: (dynamic subshard) => () async {
       await _pubRunTest(
-      path.join(flutterRoot, 'packages', 'flutter_tools'),
+        toolsPath,
+        testPath: path.join(kTest, '$subshard$kDotShard'),
+        useBuildRunner: canUseBuildRunner,
         tableData: bigqueryApi?.tabledata,
-      enableFlutterToolAsserts: !Platform.isWindows,
+        // TODO(ianh): The integration tests fail to start on Windows if asserts are enabled.
+        // See https://github.com/flutter/flutter/issues/36476
+        enableFlutterToolAsserts: !(subshard == 'integration' && Platform.isWindows),
       );
-  } else {
-    await _buildRunnerTest(
-      path.join(flutterRoot, 'packages', 'flutter_tools'),
-      flutterRoot,
-      tableData: bigqueryApi?.tabledata,
-      enableFlutterToolAsserts: !Platform.isWindows,
+    },
   );
-  }
 
-  print('${bold}DONE: All tests successful.$reset');
+  await selectSubshard(subshards);
 }
 
 /// Verifies that AOT, APK, and IPA (if on macOS) builds the
@@ -503,31 +469,33 @@ Future<void> _runCoverage() async {
     print('This file should have been generated by the `flutter test --coverage` script, but was not.');
     exit(1);
   }
-
   print('${bold}DONE: Coverage collection successful.$reset');
 }
 
-Future<void> _buildRunnerTest(
-  String workingDirectory,
-  String flutterRoot, {
+Future<void> _pubRunTest(String workingDirectory, {
   String testPath,
-  bool enableFlutterToolAsserts = false,
+  bool enableFlutterToolAsserts = true,
+  bool useBuildRunner = false,
   bq.TabledataResourceApi tableData,
 }) async {
-  final List<String> args = <String>[
-    'run',
-    'build_runner',
-    'test',
-    '--',
-    if (useFlutterTestFormatter) '-rjson' else '-rcompact',
-    '-j1',
-    if (!hasColor) '--no-color',
-    if (testPath != null) testPath,
-  ];
+  final List<String> args = <String>['run', '--verbose'];
+  if (useBuildRunner) {
+    args.addAll(<String>['build_runner', 'test', '--']);
+  } else {
+    args.add('test');
+  }
+  args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact');
+  args.add('-j1'); // TODO(ianh): Scale based on CPUs.
+  if (!hasColor)
+    args.add('--no-color');
+  if (testPath != null)
+    args.add(testPath);
   final Map<String, String> pubEnvironment = <String, String>{
     'FLUTTER_ROOT': flutterRoot,
-    if (Directory(pubCache).existsSync()) 'PUB_CACHE': pubCache,
   };
+  if (Directory(pubCache).existsSync()) {
+    pubEnvironment['PUB_CACHE'] = pubCache;
+  }
   if (enableFlutterToolAsserts) {
     // If an existing env variable exists append to it, but only if
     // it doesn't appear to already include enable-asserts.
@@ -535,21 +503,11 @@ Future<void> _buildRunnerTest(
     if (!toolsArgs.contains('--enable-asserts'))
       toolsArgs += ' --enable-asserts';
     pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
+    // The flutter_tool will originally have been snapshotted without asserts.
+    // We need to force it to be regenerated with them enabled.
+    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot'));
+    deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp'));
   }
-
-  final String subShard = Platform.environment['SUBSHARD'];
-  switch (subShard) {
-    case 'integration':
-      args.addAll(<String>['--tags', 'integration']);
-      break;
-    case 'create':
-      args.addAll(<String>['--tags', 'create']);
-      break;
-    case 'tool':
-      args.addAll(<String>['-x', 'integration', '-x', 'create']);
-      break;
-  }
-
   if (useFlutterTestFormatter) {
     final FlutterCompactFormatter formatter = FlutterCompactFormatter();
     final Stream<String> testOutput = runAndGetStdout(
@@ -564,68 +522,19 @@ Future<void> _buildRunnerTest(
     await runCommand(
       pub,
       args,
-      workingDirectory:workingDirectory,
-      environment:pubEnvironment,
-      removeLine: (String line) => line.contains('[INFO]'),
+      workingDirectory: workingDirectory,
+      environment: pubEnvironment,
+      removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
     );
   }
 }
 
-Future<void> _pubRunTest(
-  String workingDirectory, {
-  String testPath,
-  bool enableFlutterToolAsserts = false,
-  bq.TabledataResourceApi tableData,
-}) async {
-  final List<String> args = <String>[
-    'run',
-    'test',
-    if (useFlutterTestFormatter) '-rjson' else '-rcompact',
-    '-j1',
-    if (!hasColor) '--no-color',
-    if (testPath != null) testPath,
-  ];
-  final Map<String, String> pubEnvironment = <String, String>{
-    if (Directory(pubCache).existsSync()) 'PUB_CACHE': pubCache,
-  };
-  if (enableFlutterToolAsserts) {
-    // If an existing env variable exists append to it, but only if
-    // it doesn't appear to already include enable-asserts.
-    String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
-    if (!toolsArgs.contains('--enable-asserts'))
-        toolsArgs += ' --enable-asserts';
-    pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
-  }
-
-  final String subShard = Platform.environment['SUBSHARD'];
-  switch (subShard) {
-    case 'integration':
-      args.addAll(<String>['--tags', 'integration']);
-      break;
-    case 'tool':
-      args.addAll(<String>['--exclude-tags', 'integration']);
-      break;
-    case 'create':
-      args.addAll(<String>[path.join('test', 'general.shard', 'commands', 'create_test.dart')]);
-      break;
-  }
-
-  if (useFlutterTestFormatter) {
-    final FlutterCompactFormatter formatter = FlutterCompactFormatter();
-    final Stream<String> testOutput = runAndGetStdout(
-      pub,
-      args,
-      workingDirectory: workingDirectory,
-      beforeExit: formatter.finish,
-    );
-    await _processTestOutput(formatter, testOutput, tableData);
-  } else {
-    await runCommand(
-      pub,
-      args,
-      workingDirectory: workingDirectory,
-    );
-  }
+void deleteFile(String path) {
+  // There's a race condition here but in theory we're not racing anyone
+  // while this script runs, so should be ok.
+  final File file = File(path);
+  if (file.existsSync())
+    file.deleteSync();
 }
 
 enum CiProviders {
@@ -1039,3 +948,25 @@ Future<void> _androidGradleTests(String subShard) async {
     await _runDevicelabTest('module_host_with_custom_build_test', env: env);
   }
 }
+
+Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard');
+Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard');
+
+Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name) async {
+  final String item = Platform.environment[key];
+  if (item != null) {
+    if (!items.containsKey(item)) {
+      print('${red}Invalid $name: $item$reset');
+      print('The available ${name}s are: ${items.keys.join(", ")}');
+      exit(1);
+    }
+    print('$bold$key=$item$reset');
+    await items[item]();
+  } else {
+    for (String currentItem in items.keys) {
+      print('$bold$key=$currentItem$reset');
+      await items[currentItem]();
+      print('');
+    }
+  }
+}

packages/flutter_tools/test/commands.shard/README.md

+This directory contains tests for specific `flutter` commands.
+
+Tests that are self-contained unit tests should go in `hermetic/`.
+
+Tests that are more end-to-end, e.g. that involve actually running
+subprocesses, should go in `permeable/`.
+
+The `../../tool/coverage_tool.dart` script (which is used to collect
+coverage for the tool) runs only the tests in the `hermetic` directory
+when collecting coverage.

packages/flutter_tools/test/general.shard/commands/create_usage_test.dart → packages/flutter_tools/test/commands.shard/hermetic/create_usage_test.dart

@@ -29,7 +29,7 @@ void main() {
           fs.path.join('flutter', 'packages', 'flutter_test', 'pubspec.yaml'),
           fs.path.join('flutter', 'bin', 'cache', 'artifacts', 'gradle_wrapper', 'wrapper'),
           fs.path.join('usr', 'local', 'bin', 'adb'),
-          fs.path.join('Android', 'platform-tools', 'foo'),
+          fs.path.join('Android', 'platform-tools', 'adb.exe'),
         ];
         for (String path in paths) {
           fs.file(path).createSync(recursive: true);
@@ -43,25 +43,17 @@ void main() {
       final CreateCommand command = CreateCommand();
       final CommandRunner<void> runner = createTestCommandRunner(command);
 
-      await runner.run(<String>[
-        'create', '--flutter-root=flutter', '--no-pub', '--template=module', 'testy']);
-      expect(await command.usageValues,
-             containsPair(CustomDimensions.commandCreateProjectType, 'module'));
+      await runner.run(<String>['create', '--flutter-root=flutter', '--no-pub', '--template=module', 'testy']);
+      expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'module'));
 
-      await runner.run(<String>[
-        'create',  '--flutter-root=flutter', '--no-pub', '--template=app', 'testy']);
-      expect(await command.usageValues,
-             containsPair(CustomDimensions.commandCreateProjectType, 'app'));
+      await runner.run(<String>['create', '--flutter-root=flutter', '--no-pub', '--template=app', 'testy']);
+      expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'app'));
 
-      await runner.run(<String>[
-        'create',  '--flutter-root=flutter', '--no-pub', '--template=package', 'testy']);
-      expect(await command.usageValues,
-             containsPair(CustomDimensions.commandCreateProjectType, 'package'));
+      await runner.run(<String>['create', '--flutter-root=flutter', '--no-pub', '--template=package', 'testy']);
+      expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'package'));
 
-      await runner.run(<String>[
-        'create', '--flutter-root=flutter', '--no-pub', '--template=plugin', 'testy']);
-      expect(await command.usageValues,
-             containsPair(CustomDimensions.commandCreateProjectType, 'plugin'));
+      await runner.run(<String>['create', '--flutter-root=flutter', '--no-pub', '--template=plugin', 'testy']);
+      expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'plugin'));
     }));
 
     test('set iOS host language type as usage value', () => testbed.run(() async {

packages/flutter_tools/test/general.shard/commands/ide_config_test.dart → packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart

@@ -46,11 +46,9 @@ void main() {
         fs.path.join(basePath, '.idea'): 'dir',
         fs.path.join(basePath, '.idea', 'modules.xml$suffix'): 'modules $marker',
         fs.path.join(basePath, '.idea', 'vcs.xml$suffix'): 'vcs $marker',
-        fs.path.join(basePath, '.idea', '.name$suffix'):
-            'codeStyleSettings $marker',
+        fs.path.join(basePath, '.idea', '.name$suffix'): 'codeStyleSettings $marker',
         fs.path.join(basePath, '.idea', 'runConfigurations'): 'dir',
-        fs.path.join(basePath, '.idea', 'runConfigurations', 'hello_world.xml$suffix'):
-            'hello_world $marker',
+        fs.path.join(basePath, '.idea', 'runConfigurations', 'hello_world.xml$suffix'): 'hello_world $marker',
         fs.path.join(basePath, 'flutter.iml$suffix'): 'flutter $marker',
         fs.path.join(basePath, 'packages', 'new', 'deep.iml$suffix'): 'deep $marker',
       };

packages/flutter_tools/test/general.shard/commands/create_test.dart → packages/flutter_tools/test/commands.shard/permeable/create_test.dart

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// This test performs too poorly to run with coverage enabled.
-@Tags(<String>['create', 'no_coverage'])
 import 'dart:async';
 import 'dart:convert';
 import 'dart:typed_data';
@@ -25,7 +23,6 @@ import 'package:process/process.dart';
 import '../../src/common.dart';
 import '../../src/context.dart';
 
-
 const String frameworkRevision = '12345678';
 const String frameworkChannel = 'omega';
 final Generator _kNoColorTerminalPlatform = () => FakePlatform.fromPlatform(const LocalPlatform())..stdoutSupportsAnsi = false;
@@ -116,7 +113,9 @@ void main() {
       'ios/',
     ]);
     return _runFlutterTest(projectDir);
-  }, timeout: allowForRemotePubInvocation);
+  },
+    timeout: allowForRemotePubInvocation,
+  );
 
   testUsingContext('cannot create a project if non-empty non-project directory exists with .metadata', () async {
     await projectDir.absolute.childDirectory('blag').create(recursive: true);

packages/flutter_tools/tool/tool_coverage.dart

@@ -39,7 +39,18 @@ Future<void> main(List<String> arguments) async {
       <Runtime>[Runtime.vm],
       () => vmPlatform,
     );
-    await test.main(<String>['-x', 'no_coverage', '--no-color', '-r', 'compact', '-j', '1', ...arguments]);
+    if (arguments.isEmpty) {
+      arguments = <String>[
+        path.join('test', 'general.shard'),
+        path.join('test', 'commands.shard', 'hermetic'),
+      ];
+    }
+    await test.main(<String>[
+      '--no-color',
+      '-r', 'compact',
+      '-j', '1',
+      ...arguments
+    ]);
     exit(exitCode);
   });
 }
@@ -141,8 +152,7 @@ class VMPlatform extends PlatformPlugin {
     final String result = await coverageCollector.finalizeCoverage(
       formatter: formatter,
     );
-    final String prefix = Platform.environment['SUBSHARD'] ?? '';
-    final String outputLcovPath = path.join('coverage', '$prefix.lcov.info');
+    final String outputLcovPath = path.join('coverage', 'lcov.info');
     File(outputLcovPath)
       ..createSync(recursive: true)
       ..writeAsStringSync(result);