Clean up test infrastructure (#43030)

See #41880 for history.

dev/README.md

@@ -2,3 +2,6 @@ This directory contains tools and resources that the Flutter team uses
 during development of the framework. The tools in this directory
 should not be necessary for developing Flutter applications, though of
 course they may be interesting if you are curious.
+
+The tests in this directory are run in the `framework_tests_misc-*`
+shards.

dev/bots/cirrus_setup.sh → dev/bots/accept_android_sdk_licenses.sh

 #!/bin/bash
 set -e
 
+# This script is only meant to be run by the Cirrus CI system, not locally.
+# It must be run from the root of the Flutter repo.
+
 function error() {
   echo "$@" 1>&2
 }
 
-# This script is only meant to be run by the Cirrus CI system, not locally.
-# It must be run from the root of the Flutter repo.
-
 function accept_android_licenses() {
   yes "y" | flutter doctor --android-licenses > /dev/null 2>&1
 }
 
 echo "Flutter SDK directory is: $PWD"
 
-# Run flutter to download dependencies and precompile things, and to disable
-# analytics on the bots.
-echo "Downloading build dependencies and pre-compiling Flutter snapshot"
-./bin/flutter config --no-analytics
-
-# Run doctor, to print it to the log for debugging purposes.
-./bin/flutter doctor -v
-
 # Accept licenses.
 echo "Accepting Android licenses."
 accept_android_licenses || (error "Accepting Android licenses failed." && false)
-
-# Run pub get in all the repo packages.
-echo "Updating packages for Flutter."
-./bin/flutter update-packages

dev/bots/deploy_gallery.sh

@@ -24,11 +24,9 @@ set -x
 
 cd "$FLUTTER_ROOT"
 
-if [[ "$SHARD" = "deploy_gallery" ]]; then
-  version="$(<version)"
-  if [[ "$OS" == "linux" ]]; then
+version="$(<version)"
+if [[ "$OS" == "linux" ]]; then
   echo "Building Flutter Gallery $version for Android..."
-
   # ANDROID_SDK_ROOT must be set in the env.
   (
     cd examples/flutter_gallery
@@ -47,9 +45,9 @@ if [[ "$SHARD" = "deploy_gallery" ]]; then
       fastlane deploy_play_store
     )
   else
-      echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits"
+    echo "(Not deploying; Flutter Gallery is only deployed to Play store for tagged dev branch commits.)"
   fi
-  elif [[ "$OS" == "darwin" ]]; then
+elif [[ "$OS" == "darwin" ]]; then
   echo "Building Flutter Gallery $version for iOS..."
   (
     cd examples/flutter_gallery
@@ -95,17 +93,21 @@ if [[ "$SHARD" = "deploy_gallery" ]]; then
         fastlane build_and_deploy_testflight upload:true
       )
     else
-        echo "Archiving with distribution profile..."
+      # On iOS the signing can break as well, so we verify this regularly (not just
+      # on tagged dev branch commits). We can only do this post-merge, though, because
+      # the secrets aren't available on PRs.
+      echo "Testing archiving with distribution profile..."
       (
         cd examples/flutter_gallery/ios
         fastlane build_and_deploy_testflight
       )
-        echo "Archive is only deployed to TestFlight on tagged dev branch commits"
+      echo "(Not deploying; Flutter Gallery is only deployed to TestFlight for tagged dev branch commits.)"
     fi
   else
-      echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits"
-    fi
+    echo "(Not archiving or deploying; Flutter Gallery archiving is only tested post-commit.)"
   fi
 else
-  echo "Doing nothing: not on the 'deploy_gallery' SHARD."
+  echo "Unknown OS: $OS"
+  echo "Aborted."
+  exit 1
 fi

dev/bots/run_command.dart

@@ -95,7 +95,6 @@ Future<void> runCommand(String executable, List<String> arguments, {
   OutputMode outputMode = OutputMode.print,
   CapturedOutput output,
   bool skip = false,
-  bool expectFlaky = false,
   bool Function(String) removeLine,
 }) async {
   assert((outputMode == OutputMode.capture) == (output != null),
@@ -145,10 +144,6 @@ Future<void> runCommand(String executable, List<String> arguments, {
     output.stderr = _flattenToString(await savedStderr);
   }
 
-  // If the test is flaky we don't care about the actual exit.
-  if (expectFlaky)
-    return;
-
   if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) {
     if (failureMessage != null) {
       print(failureMessage);

dev/bots/test/bot_test.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'common.dart';
+
+void main() {
+  test('BOT variable is set on bots', () {
+    expect(Platform.environment['BOT'], 'true');
+  });
+}

dev/bots/test/sdk_directory_has_space_test.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import 'common.dart';
+
+void main() {
+  test('We are in a directory with a space in it', () async {
+    // The Flutter SDK should be in a directory with a space in it, to make sure
+    // our tools support that.
+    final String expectedName = Platform.environment['FLUTTER_SDK_PATH_WITH_SPACE'];
+    expect(expectedName, 'flutter sdk');
+    expect(expectedName, contains(' '));
+    final List<String> parts = path.split(Directory.current.absolute.path);
+    expect(parts.reversed.take(3), <String>['bots', 'dev', expectedName]);
+  });
+}

dev/bots/test/test_test.dart

@@ -29,7 +29,11 @@ void main() {
       ];
       for (String version in valid_versions) {
         when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version));
-        expect(await verifyVersion(version, file), isTrue, reason: '$version is invalid');
+        expect(
+          await verifyVersion(file),
+          isNull,
+          reason: '$version is valid but verifyVersionFile said it was bad',
+        );
       }
     });
 
@@ -41,10 +45,15 @@ void main() {
         '1.2.3-pre',
         '1.2.3-pre.1+hotfix.1',
         '  1.2.3',
+        '1.2.3-hotfix.1',
       ];
       for (String version in invalid_versions) {
         when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version));
-        expect(await verifyVersion(version, file), isFalse);
+        expect(
+          await verifyVersion(file),
+          'The version logic generated an invalid version string: "$version".',
+          reason: '$version is invalid but verifyVersionFile said it was fine',
+        );
       }
     });
   });

packages/flutter_tools/dart_test.yaml

-tags:
-  "no_coverage":
-  "create":
-  "integration":
+# package:test configuration
+# https://github.com/dart-lang/test/blob/master/pkgs/test/doc/configuration.md
 
+# Some of our tests take an absurdly long time to run, and on some
+# hosts they can take even longer due to the host suddenly being
+# overloaded. For this reason, we set the test timeout to
+# significantly more than it would be by default, and we never set the
+# timeouts in the tests themselves.
+timeout: 15m

packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart

@@ -135,7 +135,7 @@ void main() {
       return _updateIdeConfig(
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
     testUsingContext('creates non-existent files', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -155,7 +155,7 @@ void main() {
       return _updateIdeConfig(
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
     testUsingContext('overwrites existing files with --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -181,7 +181,7 @@ void main() {
         args: <String>['--overwrite'],
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
     testUsingContext('only adds new templates without --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -212,7 +212,7 @@ void main() {
         args: <String>['--update-templates'],
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
     testUsingContext('update all templates with --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -239,7 +239,7 @@ void main() {
         args: <String>['--update-templates', '--overwrite'],
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
     testUsingContext('removes deleted imls with --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -275,7 +275,7 @@ void main() {
         args: <String>['--update-templates', '--overwrite'],
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
     testUsingContext('removes deleted imls with --overwrite, including empty parent dirs', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -316,7 +316,7 @@ void main() {
         args: <String>['--update-templates', '--overwrite'],
         expectedContents: expectedContents,
       );
-    }, timeout: const Timeout.factor(2.0));
+    });
 
   });
 }

packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart

@@ -16,9 +16,6 @@ import 'package:flutter_tools/src/runner/flutter_command.dart';
 import '../../src/common.dart';
 import '../../src/context.dart';
 
-/// Test case timeout for tests involving project analysis.
-const Timeout allowForSlowAnalyzeTests = Timeout.factor(5.0);
-
 final Generator _kNoColorTerminalPlatform = () => FakePlatform.fromPlatform(const LocalPlatform())..stdoutSupportsAnsi = false;
 final Map<Type, Generator> noColorTerminalOverride = <Type, Generator>{
   Platform: _kNoColorTerminalPlatform,
@@ -54,7 +51,7 @@ void main() {
         ],
       );
       expect(libMain.existsSync(), isTrue);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -65,7 +62,7 @@ void main() {
         arguments: <String>['analyze'],
         statusTextContains: <String>['No issues found!'],
       );
-    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -80,6 +77,7 @@ void main() {
     }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
+
     // Analyze in the current directory - no arguments
     testUsingContext('working directory with errors', () async {
       // Break the code to produce the "The parameter 'onPressed' is required" hint
@@ -110,7 +108,7 @@ void main() {
         exitMessageContains: '2 issues found.',
         toolExit: true,
       );
-    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
       ...noColorTerminalOverride,
     });
@@ -140,7 +138,7 @@ void main() {
         exitMessageContains: '3 issues found.',
         toolExit: true,
       );
-    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
       ...noColorTerminalOverride
     });

packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart

@@ -75,7 +75,7 @@ void main() {
 
     expect(await command.usageValues,
         containsPair(CustomDimensions.commandBuildBundleIsModule, 'true'));
-  }, timeout: allowForCreateFlutterProject);
+  });
 
   testUsingContext('bundle getUsage indicate that project is not a module', () async {
     final String projectPath = await createProject(tempDir,
@@ -85,7 +85,7 @@ void main() {
 
     expect(await command.usageValues,
         containsPair(CustomDimensions.commandBuildBundleIsModule, 'false'));
-  }, timeout: allowForCreateFlutterProject);
+  });
 
   testUsingContext('bundle getUsage indicate the target platform', () async {
     final String projectPath = await createProject(tempDir,
@@ -95,7 +95,7 @@ void main() {
 
     expect(await command.usageValues,
         containsPair(CustomDimensions.commandBuildBundleTargetPlatform, 'android-arm'));
-  }, timeout: allowForCreateFlutterProject);
+  });
 
   testUsingContext('bundle fails to build for Windows if feature is disabled', () async {
     fs.file('lib/main.dart').createSync(recursive: true);

packages/flutter_tools/test/commands.shard/permeable/packages_test.dart

@@ -207,7 +207,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectZeroPluginsInjected(projectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -220,7 +220,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectZeroPluginsInjected(projectPath);
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -234,7 +234,7 @@ void main() {
 
       expect(await getCommand.usageValues,
              containsPair(CustomDimensions.commandPackagesNumberPlugins, '0'));
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -248,7 +248,7 @@ void main() {
 
       expect(await getCommand.usageValues,
              containsPair(CustomDimensions.commandPackagesProjectModule, 'false'));
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -262,7 +262,7 @@ void main() {
 
       expect(await getCommand.usageValues,
              containsPair(CustomDimensions.commandPackagesProjectModule, 'true'));
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -275,7 +275,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectZeroPluginsInjected(projectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -288,7 +288,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectModulePluginInjected(projectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -309,7 +309,7 @@ void main() {
 
       expectDependenciesResolved(exampleProjectPath);
       expectPluginInjected(exampleProjectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
   });

packages/flutter_tools/test/general.shard/commands/build_aar_test.dart

@@ -57,7 +57,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('indicate that project is a plugin', () async {
       final String projectPath = await createProject(tempDir,
@@ -69,7 +69,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('indicate the target platform', () async {
       final String projectPath = await createProject(tempDir,
@@ -82,7 +82,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
   });
 
   group('Gradle', () {

packages/flutter_tools/test/general.shard/commands/build_apk_test.dart

@@ -48,7 +48,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('split per abi', () async {
       final String projectPath = await createProject(tempDir,
@@ -65,7 +65,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('build type', () async {
       final String projectPath = await createProject(tempDir,
@@ -92,7 +92,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
   });
 
   group('Gradle', () {
@@ -223,8 +223,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('shrinking is disabled when --no-shrink is passed', () async {
       final String projectPath = await createProject(tempDir,
@@ -255,8 +254,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('guides the user when the shrinker fails', () async {
       final String projectPath = await createProject(tempDir,
@@ -312,8 +310,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('reports when the app isn\'t using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -366,8 +363,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('reports when the app is using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -422,8 +418,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
   });
 }
 

packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart

@@ -48,7 +48,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('build type', () async {
       final String projectPath = await createProject(tempDir,
@@ -75,7 +75,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
   });
 
   group('Gradle', () {
@@ -212,8 +212,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('shrinking is disabled when --no-shrink is passed', () async {
       final String projectPath = await createProject(
@@ -246,8 +245,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('guides the user when the shrinker fails', () async {
       final String projectPath = await createProject(tempDir,
@@ -303,8 +301,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('reports when the app isn\'t using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -357,8 +354,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
 
     testUsingContext('reports when the app is using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -413,8 +409,7 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    },
-    timeout: allowForCreateFlutterProject);
+    });
   });
 }
 

packages/flutter_tools/test/integration.shard/daemon_mode_test.dart

@@ -2,13 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
 import 'dart:convert';
+import 'dart:io';
 
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -21,49 +17,26 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  group('daemon_mode', () {
-    Directory tempDir;
-    final BasicProject _project = BasicProject();
-    Process process;
+  test('device.getDevices', () async {
+    final Directory tempDir = createResolvedTempDirectorySync('daemon_mode_test.');
 
-    setUp(() async {
-      tempDir = createResolvedTempDirectorySync('daemon_mode_test.');
+    final BasicProject _project = BasicProject();
     await _project.setUpIn(tempDir);
-    });
-
-    tearDown(() async {
-      tryToDelete(tempDir);
-      process?.kill();
-    });
 
-    test('device.getDevices', () async {
-      final String flutterBin =
-          fs.path.join(getFlutterRoot(), 'bin', 'flutter');
+    final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter');
 
     const ProcessManager processManager = LocalProcessManager();
-      process = await processManager.start(
+    final Process process = await processManager.start(
       <String>[flutterBin, '--show-test-device', 'daemon'],
-          workingDirectory: tempDir.path);
-
-      final StreamController<String> stdout =
-          StreamController<String>.broadcast();
+      workingDirectory: tempDir.path,
+    );
 
-      transformToLines(process.stdout)
-          .listen((String line) => stdout.add(line));
-
-      final Stream<Map<String, dynamic>> stream =
-        stdout.stream.where((String line) {
-          final Map<String, dynamic> response = parseFlutterResponse(line);
-          // ignore 'Starting device daemon...'
-          if (response == null) {
-            return false;
-          }
-          // TODO(devoncarew): Remove this after #25440 lands.
-          if (response['event'] == 'daemon.showMessage') {
-            return false;
-          }
-          return true;
-        }).map(parseFlutterResponse);
+    final StreamController<String> stdout = StreamController<String>.broadcast();
+    transformToLines(process.stdout).listen((String line) => stdout.add(line));
+    final Stream<Map<String, dynamic>> stream = stdout
+      .stream
+      .map<Map<String, dynamic>>(parseFlutterResponse)
+      .where((Map<String, dynamic> value) => value != null);
 
     Map<String, dynamic> response = await stream.first;
     expect(response['event'], 'daemon.connected');
@@ -94,6 +67,8 @@ void main() {
     final dynamic result = response['result'];
     expect(result, isList);
     expect(result, isNotEmpty);
+
+    tryToDelete(tempDir);
+    process.kill();
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // This test uses the `flutter` tool, which could be blocked behind the startup lock for a long time.
 }

packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart

@@ -2,11 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
+import 'dart:io';
+
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 
@@ -16,23 +13,14 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  group('debugger', () {
-    Directory tempDir;
-    final SteppingProject _project = SteppingProject();
-    FlutterRunTestDriver _flutter;
+  test('can step over statements', () async {
+    final Directory tempDir = createResolvedTempDirectorySync('debugger_stepping_test.');
 
-    setUp(() async {
-      tempDir = createResolvedTempDirectorySync('debugger_stepping_test.');
+    final SteppingProject _project = SteppingProject();
     await _project.setUpIn(tempDir);
-      _flutter = FlutterRunTestDriver(tempDir);
-    });
 
-    tearDown(() async {
-      await _flutter.stop();
-      tryToDelete(tempDir);
-    });
+    final FlutterRunTestDriver _flutter = FlutterRunTestDriver(tempDir);
 
-    test('can step over statements', () async {
     await _flutter.run(withDebugger: true, startPaused: true);
     await _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine);
     await _flutter.resume();
@@ -51,8 +39,11 @@ void main() {
       final int expectedLine = _project.lineForStep(i);
 
       expect(actualLine, equals(expectedLine),
-          reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine');
+        reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine'
+      );
     }
+
+    await _flutter.stop();
+    tryToDelete(tempDir);
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart

@@ -2,12 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
+import 'dart:io';
 
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -20,22 +16,21 @@ import 'test_data/tests_project.dart';
 import 'test_driver.dart';
 import 'test_utils.dart';
 
-void main() {
-  group('flutter run expression evaluation', () {
-    Directory tempDir;
+void batch1() {
   final BasicProject _project = BasicProject();
+  Directory tempDir;
   FlutterRunTestDriver _flutter;
 
-    setUp(() async {
+  Future<void> initProject() async {
     tempDir = createResolvedTempDirectorySync('run_expression_eval_test.');
     await _project.setUpIn(tempDir);
     _flutter = FlutterRunTestDriver(tempDir);
-    });
+  }
 
-    tearDown(() async {
+  Future<void> cleanProject() async {
     await _flutter.stop();
     tryToDelete(tempDir);
-    });
+  }
 
   Future<void> breakInBuildMethod(FlutterTestDriver flutter) async {
     await _flutter.breakAt(
@@ -51,89 +46,103 @@ void main() {
     );
   }
 
-    test('can evaluate trivial expressions in top level function', () async {
+  test('flutter run expression evaluation - can evaluate trivial expressions in top level function', () async {
+    await initProject();
     await _flutter.run(withDebugger: true);
     await breakInTopLevelFunction(_flutter);
     await evaluateTrivialExpressions(_flutter);
+    await cleanProject();
   });
 
-    test('can evaluate trivial expressions in build method', () async {
+  test('flutter run expression evaluation - can evaluate trivial expressions in build method', () async {
+    await initProject();
     await _flutter.run(withDebugger: true);
     await breakInBuildMethod(_flutter);
     await evaluateTrivialExpressions(_flutter);
+    await cleanProject();
   });
 
-    test('can evaluate complex expressions in top level function', () async {
+  test('flutter run expression evaluation - can evaluate complex expressions in top level function', () async {
+    await initProject();
     await _flutter.run(withDebugger: true);
     await breakInTopLevelFunction(_flutter);
     await evaluateComplexExpressions(_flutter);
+    await cleanProject();
   });
 
-    test('can evaluate complex expressions in build method', () async {
+  test('flutter run expression evaluation - can evaluate complex expressions in build method', () async {
+    await initProject();
     await _flutter.run(withDebugger: true);
     await breakInBuildMethod(_flutter);
     await evaluateComplexExpressions(_flutter);
+    await cleanProject();
   });
 
-    test('can evaluate expressions returning complex objects in top level function', () async {
+  test('flutter run expression evaluation - can evaluate expressions returning complex objects in top level function', () async {
+    await initProject();
     await _flutter.run(withDebugger: true);
     await breakInTopLevelFunction(_flutter);
     await evaluateComplexReturningExpressions(_flutter);
+    await cleanProject();
   });
 
-    test('can evaluate expressions returning complex objects in build method', () async {
+  test('flutter run expression evaluation - can evaluate expressions returning complex objects in build method', () async {
+    await initProject();
     await _flutter.run(withDebugger: true);
     await breakInBuildMethod(_flutter);
     await evaluateComplexReturningExpressions(_flutter);
+    await cleanProject();
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
+}
 
-  group('flutter test expression evaluation', () {
-    Directory tempDir;
+void batch2() {
   final TestsProject _project = TestsProject();
+  Directory tempDir;
   FlutterTestTestDriver _flutter;
 
-    setUp(() async {
+  Future<void> initProject() async {
     tempDir = createResolvedTempDirectorySync('test_expression_eval_test.');
     await _project.setUpIn(tempDir);
     _flutter = FlutterTestTestDriver(tempDir);
-    });
+  }
 
-    tearDown(() async {
-      await _flutter.quit();
+  Future<void> cleanProject() async {
+    await _flutter?.quit();
     tryToDelete(tempDir);
-    });
+  }
 
-    test('can evaluate trivial expressions in a test', () async {
+  test('flutter test expression evaluation - can evaluate trivial expressions in a test', () async {
+    await initProject();
     await _flutter.test(
       withDebugger: true,
       beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
     );
     await _flutter.waitForPause();
     await evaluateTrivialExpressions(_flutter);
-      await _flutter.resume();
+    await cleanProject();
   });
 
-    test('can evaluate complex expressions in a test', () async {
+  test('flutter test expression evaluation - can evaluate complex expressions in a test', () async {
+    await initProject();
     await _flutter.test(
       withDebugger: true,
       beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
     );
     await _flutter.waitForPause();
     await evaluateComplexExpressions(_flutter);
-      await _flutter.resume();
+    await cleanProject();
   });
 
-    test('can evaluate expressions returning complex objects in a test', () async {
+  test('flutter test expression evaluation - can evaluate expressions returning complex objects in a test', () async {
+    await initProject();
     await _flutter.test(
       withDebugger: true,
       beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
     );
     await _flutter.waitForPause();
     await evaluateComplexReturningExpressions(_flutter);
-      await _flutter.resume();
+    await cleanProject();
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
 
 Future<void> evaluateTrivialExpressions(FlutterTestDriver flutter) async {
@@ -164,3 +173,8 @@ Future<void> evaluateComplexReturningExpressions(FlutterTestDriver flutter) asyn
   final InstanceRef res = await flutter.evaluate(resp.id, r'"$year-$month-$day"');
   expect(res.valueAsString, equals('${now.year}-${now.month}-${now.day}'));
 }
+
+void main() {
+  batch1();
+  batch2();
+}

packages/flutter_tools/test/integration.shard/flutter_attach_test.dart

@@ -2,11 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
+import 'dart:io';
+
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 
@@ -33,7 +30,6 @@ void main() {
     tryToDelete(tempDir);
   });
 
-  group('attached process', () {
   test('writes pid-file', () async {
     final File pidFile = tempDir.childFile('test.pid');
     await _flutterRun.run(withDebugger: true);
@@ -43,11 +39,13 @@ void main() {
     );
     expect(pidFile.existsSync(), isTrue);
   });
+
   test('can hot reload', () async {
     await _flutterRun.run(withDebugger: true);
     await _flutterAttach.attach(_flutterRun.vmServicePort);
     await _flutterAttach.hotReload();
   });
+
   test('can detach, reattach, hot reload', () async {
     await _flutterRun.run(withDebugger: true);
     await _flutterAttach.attach(_flutterRun.vmServicePort);
@@ -55,6 +53,7 @@ void main() {
     await _flutterAttach.attach(_flutterRun.vmServicePort);
     await _flutterAttach.hotReload();
   });
+
   test('killing process behaves the same as detach ', () async {
     await _flutterRun.run(withDebugger: true);
     await _flutterAttach.attach(_flutterRun.vmServicePort);
@@ -63,5 +62,4 @@ void main() {
     await _flutterAttach.attach(_flutterRun.vmServicePort);
     await _flutterAttach.hotReload();
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/integration.shard/flutter_run_test.dart

@@ -2,11 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/io.dart';
@@ -18,7 +13,6 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  group('flutter_run', () {
   Directory tempDir;
   final BasicProject _project = BasicProject();
   FlutterRunTestDriver _flutter;
@@ -34,7 +28,7 @@ void main() {
     tryToDelete(tempDir);
   });
 
-    test('reports an error if an invalid device is supplied', () async {
+  test('flutter run reports an error if an invalid device is supplied', () async {
     // This test forces flutter to check for all possible devices to catch issues
     // like https://github.com/flutter/flutter/issues/21418 which were skipped
     // over because other integration tests run using flutter-tester which short-cuts
@@ -55,10 +49,9 @@ void main() {
     }
   });
 
-    test('writes pid-file', () async {
+  test('flutter run writes pid-file', () async {
     final File pidFile = tempDir.childFile('test.pid');
     await _flutter.run(pidFile: pidFile);
     expect(pidFile.existsSync(), isTrue);
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/integration.shard/hot_reload_test.dart

@@ -2,11 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
 
 import 'package:file/file.dart';
@@ -20,7 +15,6 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  group('hot reload tests', () {
   Directory tempDir;
   final HotReloadProject _project = HotReloadProject();
   FlutterRunTestDriver _flutter;
@@ -154,5 +148,4 @@ void main() {
     await _flutter.resume();
     await subscription.cancel();
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/integration.shard/lifetime_test.dart

@@ -2,11 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
 
 import 'package:file/file.dart';
@@ -23,7 +18,6 @@ import 'test_utils.dart';
 const Duration requiredLifespan = Duration(seconds: 5);
 
 void main() {
-  group('flutter run', () {
   final BasicProject _project = BasicProject();
   FlutterRunTestDriver _flutter;
   Directory tempDir;
@@ -39,16 +33,15 @@ void main() {
     tryToDelete(tempDir);
   });
 
-    test('does not terminate when a debugger is attached', () async {
+  test('flutter run does not terminate when a debugger is attached', () async {
     await _flutter.run(withDebugger: true);
     await Future<void>.delayed(requiredLifespan);
     expect(_flutter.hasExited, equals(false));
   });
 
-    test('does not terminate when a debugger is attached and pause-on-exceptions', () async {
+  test('fluter run does not terminate when a debugger is attached and pause-on-exceptions', () async {
     await _flutter.run(withDebugger: true, pauseOnExceptions: true);
     await Future<void>.delayed(requiredLifespan);
     expect(_flutter.hasExited, equals(false));
   });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/src/common.dart

@@ -127,13 +127,6 @@ Future<String> createProject(Directory temp, { List<String> arguments }) async {
   return projectPath;
 }
 
-/// Test case timeout for tests involving remote calls to `pub get` or similar.
-const Timeout allowForRemotePubInvocation = Timeout.factor(10.0);
-
-/// Test case timeout for tests involving creating a Flutter project with
-/// `--no-pub`. Use [allowForRemotePubInvocation] when creation involves `pub`.
-const Timeout allowForCreateFlutterProject = Timeout.factor(3.0);
-
 Future<void> expectToolExitLater(Future<dynamic> future, Matcher messageMatcher) async {
   try {
     await future;

packages/flutter_tools/test/src/context.dart

@@ -48,7 +48,6 @@ typedef ContextInitializer = void Function(AppContext testContext);
 void testUsingContext(
   String description,
   dynamic testMethod(), {
-  Timeout timeout,
   Map<Type, Generator> overrides = const <Type, Generator>{},
   bool initializeFlutterRoot = true,
   String testOn,
@@ -137,8 +136,7 @@ void testUsingContext(
         },
       );
     });
-  }, timeout: timeout ?? const Timeout(Duration(seconds: 60)),
-      testOn: testOn, skip: skip);
+  }, testOn: testOn, skip: skip);
 }
 
 void _printBufferedErrors(AppContext testContext) {