Clean up test infrastructure (#43030)

See #41880 for history.

Clean up test infrastructure (#43030)
See #41880 for history.
124dc661 · Ian Hickson · GitHub · 40b1335a · 124dc661 · 124dc661
Unverified Commit 124dc661 authored Oct 18, 2019 by Ian Hickson Committed by GitHub Oct 18, 2019
27 changed files
--- a/.cirrus.yml
+++ b/.cirrus.yml
-container:
+# CIRRUS CONFIGURATION FILE
-  image: gcr.io/flutter-cirrus/build-flutter-image:latest
+use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+environment:
+  # For details about environment variables used in Cirrus, including how encrypted variables work,
+  # see https://cirrus-ci.org/guide/writing-tasks/#environment-variables
+  GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+  # We change Flutter's directory to include a space in its name (see $CIRRUS_WORKING_DIR) so that
+  # we constantly test path names with spaces in them. The FLUTTER_SDK_PATH_WITH_SPACE variable must
+  # therefore have a space in it.
+  FLUTTER_SDK_PATH_WITH_SPACE: "flutter sdk"
+  # We force BOT to true so that all our tools know we're in a CI environment. This avoids any
+  # dependency on precisely how Cirrus is detected by our tools.
+  BOT: "true"
+  # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they might include non-ASCII
+  # characters which makes Gradle crash. See: https://github.com/flutter/flutter/issues/24935
+  # TODO(amirha): remove once we've migrated to newer Gradle
+  CIRRUS_CHANGE_MESSAGE: ""
+  CIRRUS_COMMIT_MESSAGE: ""
+# LINUX SHARDS
 task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+  container:
-  env:
+    image: gcr.io/flutter-cirrus/build-flutter-image:latest
-    # Name the SDK directory to include a space so that we constantly
+    cpu: $CPU
-    # test path names with spaces in them.
+    memory: $MEMORY
-    CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
+  environment:
+    # We shrink our default resource requirement as much as possible because that way we are more
+    # likely to get scheduled. We require 4G of RAM because most of the shards (all but one as of
+    # October 2019) just get OOM-killed with less. Some shards may need more. When increasing the
+    # requirements for select shards, please leave a comment on those shards saying when you
+    # increased the requirements, what numbers you tried, and what the results were.
+    CPU: 1 # 0.1-8 without compute credits, 0.1-30 with (yes, you can go fractional)
+    MEMORY: 4G # 256M-24G without compute credits, 256M-90G with
+    CIRRUS_WORKING_DIR: "/tmp/$FLUTTER_SDK_PATH_WITH_SPACE"
    PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH"
    ANDROID_SDK_ROOT: "/opt/android_sdk"
-  git_fetch_script:
-    - git clean -xfd
-    - git fetch origin
-    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
  pub_cache:
    folder: $HOME/.pub-cache
    fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
@@ -22,345 +45,302 @@ task:
  artifacts_cache:
    folder: bin/cache/artifacts
    fingerprint_script: echo $OS; cat bin/internal/*.version
-  setup_script: ./dev/bots/cirrus_setup.sh
+  setup_script:
+    - date
+    - git clean -xffd
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD, so that "git merge-base" works.
+    - flutter config --no-analytics
+    - flutter doctor -v
+    - flutter update-packages
+    - ./dev/bots/accept_android_sdk_licenses.sh
+    - date
+  on_failure:
+    failure_script:
+      - date
+      - which flutter
  matrix:
-    - name: docs
+    - name: analyze-linux # linux-only
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master'"
+      # environment:
-      env:
+        # Empirically, the analyze-linux shard runs surprisingly fast (under 15 minutes) with just 1
-        SHARD: docs
+        # CPU and 4G RAM as of October 2019. It fails with less than 4G though.
-        # For uploading master docs to Firebase master branch staging site
+      script:
-        FIREBASE_MASTER_TOKEN: ENCRYPTED[eb768d18798fdc5abfe09b224e1724c4d82831d715ccf90df2c79d618c317216cbd99493278361f6fe7948b409b603f0]
-        # For uploading beta docs to Firebase public live site
-        FIREBASE_PUBLIC_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92]
-      docs_script: ./dev/bots/docs.sh
-    - name: deploy_gallery
-      depends_on:
-        - docs
-        - analyze
-        - build_tests-linux
-      env:
-        SHARD: deploy_gallery
-        GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346]
-        ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0f2aca35f05b26add5d9edea2a7449341269a2b7e22d5c667f876996e2e8bc44ff1369431ebf73b7c5581fd95d0e5902]
-      test_script:
-        # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they
-        # might include non-ASCII characters which makes Gradle crash.
-        # See: https://github.com/flutter/flutter/issues/24935
-        # This is a temporary workaround until we figure how to properly configure
-        # a UTF8 locale on Cirrus (or until the Gradle bug is fixed).
-        # TODO(amirh): Set the locale to UTF8.
-        - echo "$CIRRUS_CHANGE_MESSAGE" > /tmp/cirrus_change_message.txt
-        - echo "$CIRRUS_COMMIT_MESSAGE" > /tmp/cirrus_commit_message.txt
-        - export CIRRUS_CHANGE_MESSAGE=""
-        - export CIRRUS_COMMIT_MESSAGE=""
-        - ./dev/bots/deploy_gallery.sh
-        - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt`
-        - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt`
-    - name: analyze
-      test_script:
        - dart --enable-asserts ./dev/bots/analyze.dart
-    - name: bots_tests-linux
-      skip: "!changesInclude('dev/bots/**')"
+    - name: framework_tests-widgets-linux
-      test_script:
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-        - (cd ./dev/bots && pub run test)
+      environment:
-      container:
+        # We use 3 CPUs because that's the minimum required to get framework_tests-widgets-linux
-        cpu: 4
+        # running fast enough that it is not the long pole, as of OCtober 2019.
-        memory: 12G
+        CPU: 3
-    - name: tests_widgets-linux
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tests
-        SUBSHARD: widgets
        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
-      goldctl_script: ./dev/bots/download_goldctl.sh
+      script:
-      test_script:
+        - ./dev/bots/download_goldctl.sh
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: framework_tests-libraries-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: tests_framework_other-linux
+      environment:
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+        # We use 3 CPUs because that's the minimum required to get the
-      env:
+        # framework_tests-libraries-linux shard running fast enough that it is not the long pole, as
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+        # of October 2019.
-        SHARD: tests
+        CPU: 3
-        SUBSHARD: framework_other
        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
-      goldctl_script: ./dev/bots/download_goldctl.sh
+      script:
-      test_script:
+        - ./dev/bots/download_goldctl.sh
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: framework_tests-misc-linux
-        memory: 12G
+      # this includes the tests for directories in dev/
-    - name: tests_extras-linux
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+      environment:
-      env:
+        # We use 3 CPUs because that's the minimum required to get framework_tests-misc-linux
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+        # running fast enough that it is not the long pole, as of October 2019.
-        SHARD: tests
+        CPU: 3
-        SUBSHARD: extras
+      script:
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: tool_tests-general-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    # all of the tests except the ones in test/integration and test/commands/create_test for packages/flutter_tools
+      environment:
-    - name: tool_tests_general-linux
+        # As of October 2019, the tool_tests-general-linux shard got faster with more CPUs up to 4
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+        # CPUs, and needed at least 8G of RAM to not run out of memory.
-      env:
+        CPU: 4
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+        MEMORY: 8G
-        SHARD: tool_tests
+      script:
-        SUBSHARD: general
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: tool_tests-commands-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: tool_tests_commands-linux
+      environment:
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+        # As of October 2019, the tool_tests-commands-linux shard got faster with more CPUs up to 6
-      env:
+        # CPUs, and needed at least 8G of RAM to not run out of memory.
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+        CPU: 6
-        SHARD: tool_tests
+        MEMORY: 8G
-        SUBSHARD: commands
+      script:
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: tool_tests-integration-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    # all of the tests in test/integration for packages/flutter_tools
+      environment:
-    - name: tool_tests_integration-linux
+        # As of October 2019, the tool_tests-integration-linux shard got faster with more CPUs up to
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+        # 6 CPUs, and needed at least 8G of RAM to not run out of memory.
-      env:
+        CPU: 6
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+        MEMORY: 8G
-        SHARD: tool_tests
+      script:
-        SUBSHARD: integration
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: tool_coverage-linux # linux-only
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**/*.dart', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: tool_coverage-linux
+      environment:
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**/*.dart') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+        # As of October 2019, the tool_coverage-linux shard needed at least 12G of RAM to run without
-      env:
+        # getting OOM-killed, and even 8 CPUs took 25 minutes.
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+        CPU: 8
+        MEMORY: 12G
        CODECOV_TOKEN: ENCRYPTED[7c76a7f8c9264f3b7f3fd63fcf186f93c62c4dfe43ec288861c2f506d456681032b89efe7b7a139c82156350ca2c752c]
-        SHARD: tool_coverage
+      script:
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-        - bash <(curl -s https://codecov.io/bash) -c -s ./packages/flutter_tools/coverage/ -f '*.lcov.info' -F flutter_tool
+        - bash <(curl -s https://codecov.io/bash) -c -f packages/flutter_tools/coverage/lcov.info -F flutter_tool
-      container:
-        cpu: 8
+    - name: web_tests-0-linux
-        memory: 24G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: web_tests-linux-shard-0
+      environment:
-      use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+        # As of October 2019, the Web shards needed more than 6G of RAM.
-      env:
+        CPU: 2
-        SHARD: web_tests
+        MEMORY: 8G
-        WEB_SHARD: 0
+      allow_failures: true
-      test_script:
+      script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: web_tests-1-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: web_tests-linux-shard-1
+      environment:
-      use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+        # As of October 2019, the Web shards needed more than 6G of RAM.
-      env:
+        CPU: 2
-        SHARD: web_tests
+        MEMORY: 8G
-        WEB_SHARD: 1
+      allow_failures: true
-      test_script:
+      script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: web_tests-2-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: web_tests-linux-shard-2
+      environment:
-      use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+        # As of October 2019, the Web shards needed more than 6G of RAM.
-      env:
+        CPU: 2
-        SHARD: web_tests
+        MEMORY: 8G
-        WEB_SHARD: 2
+      allow_failures: true
-      test_script:
+      script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: web_tests-3-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: web_tests-linux-shard-3
+      environment:
-      use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+        # As of October 2019, the Web shards needed more than 6G of RAM.
-      env:
+        CPU: 2
-        SHARD: web_tests
+        MEMORY: 8G
-        WEB_SHARD: 3
+      allow_failures: true
-      test_script:
+      script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: web_tests-4-linux
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: web_tests-linux-shard-4
+      environment:
-      use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+        # As of October 2019, the Web shards needed more than 6G of RAM.
-      env:
+        CPU: 2
-        SHARD: web_tests
+        MEMORY: 8G
-        WEB_SHARD: 4
+      allow_failures: true
-      test_script:
+      script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: web_tests-5_last-linux # last Web shard must end with _last
-        memory: 12G
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-    - name: web_tests-linux-shard-5
+      environment:
-      use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+        # As of October 2019, the Web shards needed more than 6G of RAM.
-      env:
+        CPU: 2
-        SHARD: web_tests
+        MEMORY: 8G
-        WEB_SHARD: 5
+      allow_failures: true
-      test_script:
+      script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
-        memory: 12G
    - name: build_tests-linux
-      env:
+      environment:
-        SHARD: build_tests
+        # With 1 CPU and 4G of RAM, as of October 2019, build_tests-linux would get OOM-killed.
-    - name: integration_tests-linux
+        # Increasing the RAM to 12G allowed it to finish in about 30 minutes, any extra CPU (tried 2
-      env:
+        # and 4) reduced that to just over 20 minutes. 6G was enough not to get OOM-killed.
-        SHARD: integration_tests
+        CPU: 2
-      test_script:
+        MEMORY: 6G
-        # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they
+      script:
-        # might include non-ASCII characters which makes Gradle crash.
-        # See: https://github.com/flutter/flutter/issues/24935
-        # This is a temporary workaround until we figure how to properly configure
-        # a UTF8 locale on Cirrus (or until the Gradle bug is fixed).
-        # TODO(amirh): Set the locale to UTF8.
-        - echo "$CIRRUS_CHANGE_MESSAGE" > /tmp/cirrus_change_message.txt
-        - echo "$CIRRUS_COMMIT_MESSAGE" > /tmp/cirrus_commit_message.txt
-        - export CIRRUS_CHANGE_MESSAGE=""
-        - export CIRRUS_COMMIT_MESSAGE=""
-        - dart --enable-asserts ./dev/bots/test.dart
-        - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt`
-        - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt`
-      container:
-        cpu: 4
-        memory: 12G
-    - name: gradle_tests-linux-shard-1
-      env:
-        SHARD: integration_tests
-        SUBSHARD: gradle1
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: hostonly_devicelab_tests-0-linux
-        memory: 12G
+      environment:
-    - name: gradle_tests-linux-shard-2
+        # Some of the host-only devicelab tests are pretty involved and need a lot of RAM.
-      env:
+        CPU: 2
-        SHARD: integration_tests
+        MEMORY: 8G
-        SUBSHARD: gradle2
+      script:
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: hostonly_devicelab_tests-1-linux
-        memory: 12G
+      environment:
-    - name: gradle_embedding_v2_tests-linux-shard-1
+        # Some of the host-only devicelab tests are pretty involved and need a lot of RAM.
-      env:
+        CPU: 2
-        SHARD: integration_tests
+        MEMORY: 8G
-        SUBSHARD: gradle1
+      script:
-        ENABLE_ANDROID_EMBEDDING_V2: 'true'
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: hostonly_devicelab_tests-2-linux
-        memory: 12G
+      environment:
-    - name: gradle_embedding_v2_tests-linux-shard-2
+        # Some of the host-only devicelab tests are pretty involved and need a lot of RAM.
-      env:
+        CPU: 2
-        SHARD: integration_tests
+        MEMORY: 8G
-        SUBSHARD: gradle2
+      script:
-        ENABLE_ANDROID_EMBEDDING_V2: 'true'
-      test_script:
        - dart --enable-asserts ./dev/bots/test.dart
-      container:
-        cpu: 4
+    - name: hostonly_devicelab_tests-3-linux
-        memory: 12G
+      environment:
-    - name: release_smoke_tests
+        # Some of the host-only devicelab tests are pretty involved and need a lot of RAM.
-      env:
+        CPU: 2
-        CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+        MEMORY: 8G
-        GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec]
+      script:
-      test_script:
+        - dart --enable-asserts ./dev/bots/test.dart
-        - echo "$CIRRUS_CHANGE_MESSAGE" > /tmp/cirrus_change_message.txt
-        - echo "$CIRRUS_COMMIT_MESSAGE" > /tmp/cirrus_commit_message.txt
+    - name: hostonly_devicelab_tests-4-linux
-        - export CIRRUS_CHANGE_MESSAGE=""
+      environment:
-        - export CIRRUS_COMMIT_MESSAGE=""
+        # Some of the host-only devicelab tests are pretty involved and need a lot of RAM.
-        - ./dev/bots/firebase_testlab.sh
+        CPU: 2
-        - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt`
+        MEMORY: 8G
-        - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt`
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-5_last-linux
+      environment:
+        # Some of the host-only devicelab tests are pretty involved and need a lot of RAM.
+        CPU: 2
+        MEMORY: 8G
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    # TODO(ianh): name: add_to_app_tests-linux
+    - name: docs-linux # linux-only
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') || $CIRRUS_PR == ''"
+      environment:
+        # Empirically, as of October 2019, the docs-linux shard took about 30 minutes when run with
+        # 1 CPU and 4G of RAM. 2 CPUs reduced that to 20 minutes, more CPUs did not improve matters.
+        CPU: 2
+        # For uploading master docs to Firebase master branch staging site
+        FIREBASE_MASTER_TOKEN: ENCRYPTED[eb768d18798fdc5abfe09b224e1724c4d82831d715ccf90df2c79d618c317216cbd99493278361f6fe7948b409b603f0]
+        # For uploading beta docs to Firebase public live site
+        FIREBASE_PUBLIC_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92]
+      script:
+        - ./dev/bots/docs.sh
    - name: customer_testing-linux
+      # environment:
+        # Empirically, this shard runs fine at 1 CPU and 4G RAM as of October 2019. We will probably
+        # want to grow this container when we invite people to add their tests in large numbers.
      script:
        - rm -rf bin/cache/pkg/tests
        - git clone https://github.com/flutter/tests.git bin/cache/pkg/tests
        - dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test
-task:
+    - name: firebase_test_lab_tests-linux # linux-only
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
+      environment:
-  windows_container:
+        # Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of
-    image: cirrusci/android-sdk:28-windowsservercore-2019
+        # October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM;
-    os_version: 2019
+        # doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it
-    cpu: 4
+        # on the high end of the 20-25 minute range. (This makes sense, as it's just driving the
-  env:
+        # Firebase test lab remotely.) Less than 4G of RAM made it go OOM.
-    CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk"
+        CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    PATH: "$CIRRUS_WORKING_DIR/bin;$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin;$PATH"
+        GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec]
-  git_fetch_script:
+      script:
-    - git clean -xfd
+        - ./dev/bots/firebase_testlab.sh
-    - git fetch origin
-    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
-  pub_cache:
-    folder: $APPDATA\Pub\Cache
-    fingerprint_script:
-      - ps:  $Env:OS; Get-ChildItem -Path "$Env:CIRRUS_WORKING_DIR" pubspec.yaml -Recurse | Select-String -Pattern "PUBSPEC CHECKSUM" -SimpleMatch
-  flutter_pkg_cache:
-    folder: bin\cache\pkg
-    fingerprint_script: echo %OS% & type bin\internal\*.version
-  artifacts_cache:
-    folder: bin\cache\artifacts
-    fingerprint_script: echo %OS% & type bin\internal\engine.version
-  setup_script:
-    - flutter config --no-analytics
-    - flutter doctor -v
-    - flutter update-packages
-    - git fetch origin master
-  test_all_script:
-    - dart --enable-asserts dev\bots\test.dart
-  matrix:
-    # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools
-    - name: tool_tests_general-windows
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tool_tests
-        SUBSHARD: general
-    # all of the tests in test/commands/create_test
-    - name: tool_tests_commands-windows
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tool_tests
-        SUBSHARD: commands
-    # all of the tests in test/integration for packages/flutter_tools
-    - name: tool_tests_integration-windows
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tool_tests
-        SUBSHARD: integration
+    - name: deploy_gallery-linux # linux- and macos- only
+      depends_on:
+        - analyze-linux
+        - framework_tests-widgets-linux
+        - framework_tests-libraries-linux
+        - framework_tests-misc-linux
+        - tool_tests-general-linux
+        - tool_tests-commands-linux
+        - tool_tests-integration-linux
+        - build_tests-linux
+        - hostonly_devicelab_tests-0-linux
+        - hostonly_devicelab_tests-1-linux
+        - hostonly_devicelab_tests-2-linux
+        - hostonly_devicelab_tests-3-linux
+        - hostonly_devicelab_tests-4-linux
+        - hostonly_devicelab_tests-5_last-linux
+        - firebase_test_lab_tests-linux
+      environment:
+        # As of October 2019, 1 CPU and 4G of RAM let deploy_gallery-linux finish in about 15
+        # minutes, once it got started.
+        GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346]
+        ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0f2aca35f05b26add5d9edea2a7449341269a2b7e22d5c667f876996e2e8bc44ff1369431ebf73b7c5581fd95d0e5902]
+      script:
+        - ./dev/bots/deploy_gallery.sh
+# WINDOWS SHARDS
 task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
  windows_container:
    image: cirrusci/android-sdk:28-windowsservercore-2019
    os_version: 2019
-    cpu: 4
+    cpu: $CPU
-  env:
+    memory: $MEMORY
-    CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk"
+  environment:
+    # It's not clear that changing the cpu/memory fields here has any impact, so we use
+    # small numbers...
+    CPU: 2 # 1-8 without compute credits, 1-30 with
+    MEMORY: 2G # 256M-24G without compute credits, 256M-90G with
+    CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\$FLUTTER_SDK_PATH_WITH_SPACE"
    PATH: "$CIRRUS_WORKING_DIR/bin;$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin;$PATH"
-  git_fetch_script:
-    - git clean -xfd
-    - git fetch origin
-    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
  pub_cache:
    folder: $APPDATA\Pub\Cache
    fingerprint_script:
-      - ps:  $Env:OS; Get-ChildItem -Path "$Env:CIRRUS_WORKING_DIR" pubspec.yaml -Recurse | Select-String -Pattern "PUBSPEC CHECKSUM" -SimpleMatch
+      - ps: $Environment:OS; Get-ChildItem -Path "$Environment:CIRRUS_WORKING_DIR" pubspec.yaml -Recurse | Select-String -Pattern "PUBSPEC CHECKSUM" -SimpleMatch
  flutter_pkg_cache:
    folder: bin\cache\pkg
    fingerprint_script: echo %OS% & type bin\internal\*.version
@@ -368,290 +348,285 @@ task:
    folder: bin\cache\artifacts
    fingerprint_script: echo %OS% & type bin\internal\*.version
  setup_script:
+    - git clean -xffd
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD, so that "git merge-base" works.
    - flutter config --no-analytics
    - flutter doctor -v
    - flutter update-packages
    - git fetch origin master
  matrix:
-    - name: bots_tests-windows
+    - name: framework_tests-widgets-windows
-      skip: "!changesInclude('dev/bots/**')"
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-      test_script:
+      environment:
-        - cd dev\bots
-        - pub run test
-    - name: tests_widgets-windows
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tests
-        SUBSHARD: widgets
        GOLDCTL: "C:\\Windows\\Temp\\depot_tools\\goldctl.exe"
        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
-      goldctl_script: powershell dev\bots\download_goldctl.ps1
+      script:
-      test_all_script:
+        - powershell dev\bots\download_goldctl.ps1
        - dart --enable-asserts dev\bots\test.dart
-    - name: tests_framework_other-windows
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: framework_tests-libraries-windows
-      env:
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+      environment:
-        SHARD: tests
-        SUBSHARD: framework_other
        GOLDCTL: "C:\\Windows\\Temp\\depot_tools\\goldctl.exe"
        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
-      goldctl_script: powershell dev\bots\download_goldctl.ps1
+      script:
-      test_all_script:
+        - powershell dev\bots\download_goldctl.ps1
        - dart --enable-asserts dev\bots\test.dart
-    - name: tests_extras-windows
-      skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
+    - name: framework_tests-misc-windows
-      env:
+      # this includes the tests for directories in dev/
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-        SHARD: tests
+      script:
-        SUBSHARD: extras
-      test_all_script:
        - dart --enable-asserts dev\bots\test.dart
+    - name: tool_tests-general-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: tool_tests-commands-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: tool_tests-integration-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    # TODO(ianh): Enable Web tests on Windows
    - name: build_tests-windows
-      env:
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
-        SHARD: build_tests
+      script:
-      container:
-        cpu: 4
-        memory: 12G
-      test_all_script:
-        - dart --enable-asserts dev\bots\test.dart
-    - name: integration_tests-windows
-      env:
-        SHARD: integration_tests
-      container:
-        cpu: 4
-        memory: 12G
-      test_all_script:
        - dart --enable-asserts dev\bots\test.dart
+    - name: hostonly_devicelab_tests-0-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-1-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-2-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-3-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-4-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-5_last-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    # TODO(ianh): name: add_to_app_tests-windows
    - name: customer_testing-windows
-      test_script:
+      script:
        - CMD /S /C "IF EXIST "bin\cache\pkg\tests\" RMDIR /S /Q bin\cache\pkg\tests"
        - git clone https://github.com/flutter/tests.git bin\cache\pkg\tests
        - dart --enable-asserts dev\customer_testing\run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test
-    - name: gradle_tests-windows-shard-1
-      env:
-        SHARD: integration_tests
-        SUBSHARD: gradle1
-      test_script:
-        - dart --enable-asserts dev\bots\test.dart
-      container:
-        cpu: 4
-        memory: 12G
-    - name: gradle_tests-windows-shard-2
-      env:
-        SHARD: integration_tests
-        SUBSHARD: gradle2
-      test_script:
-        - dart --enable-asserts dev\bots\test.dart
-      container:
-        cpu: 4
-        memory: 12G
-    - name: gradle_embedding_v2_tests-windows-shard-1
-      env:
-        SHARD: integration_tests
-        SUBSHARD: gradle1
-        ENABLE_ANDROID_EMBEDDING_V2: 'true'
-      test_script:
-        - dart --enable-asserts dev\bots\test.dart
-      container:
-        cpu: 4
-        memory: 12G
-    - name: gradle_embedding_v2_tests-windows-shard-2
-      env:
-        SHARD: integration_tests
-        SUBSHARD: gradle2
-        ENABLE_ANDROID_EMBEDDING_V2: 'true'
-      test_script:
-        - dart --enable-asserts dev\bots\test.dart
-      container:
-        cpu: 4
-        memory: 12G
+# MACOS SHARDS
+# Mac doesn't use caches because they apparently take longer to populate and save
+# than just fetching the data in the first place.
 task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
-  name: deploy_gallery-macos
-  depends_on:
-    - analyze
-  env:
-    # Name the SDK directory to include a space so that we constantly
-    # test path names with spaces in them.
-    CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
-    SHARD: deploy_gallery
-    # Apple Fastlane password.
-    FASTLANE_PASSWORD: ENCRYPTED[4b1f0b8d52874e9de965acd46c79743f3b81f3a513614179b9be7cf53dc8258753e257bdadb11a298ee455259df21865]
-    # Private repo for publishing certificates.
-    PUBLISHING_MATCH_CERTIFICATE_REPO: ENCRYPTED[3c0e78877d933fc80107aa6f3790fd1cf927250b852d6cb53202be696b4903ed8ca839b809626aaf18050bf7e436fab7]
-    PUBLISHING_MATCH_REPO_TOKEN: ENCRYPTED[3d1230b744c6ed6c788a91bec741b769401dbcd426b18f9af8080bfeefdfc21913ca4047980c5b5b7ce823f12e7b6b19]
-    # Apple Certificates Match Passphrase
-    MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73]
  osx_instance:
-    image: mojave-xcode-10.2
+    image: mojave-xcode-10.2-flutter # see https://cirrus-ci.org/guide/macOS/ for list of images (we should update regularly)
-  # occasionally the clock on these machines is out of sync
+    # cpu is always 2
-  # with the actual time - this should help to verify
+    # memory is always 8G
-  print_date_script:
+  environment:
-    - date
+    CIRRUS_WORKING_DIR: "/tmp/$FLUTTER_SDK_PATH_WITH_SPACE"
-  install_cocoapods_script:
+    FLUTTER_FRAMEWORK_DIR: "$CIRRUS_WORKING_DIR/bin/cache/artifacts/engine/ios/"
-    - sudo gem install cocoapods --no-document
+    PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH"
-  git_fetch_script:
-    - git clean -xfd
-    - git fetch origin
-    - git fetch origin master # To set FETCH_HEAD
-  setup_script:
-    - bin/flutter config --no-analytics
-    - bin/flutter update-packages
-  test_all_script:
-    - ./dev/bots/deploy_gallery.sh
-task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
-  osx_instance:
-    image: mojave-xcode-10.2
-  depends_on:
-    - analyze
-  env:
-    CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
    COCOAPODS_DISABLE_STATS: true
-  print_date_script:
+    CPU: 2
-    - date
+    MEMORY: 8G
-  git_fetch_script:
-    - git clean -xfd
-    - git fetch origin
-    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
  setup_script:
-    - bin/flutter config --no-analytics
-    - bin/flutter doctor -v
-    - bin/flutter update-packages
-  test_all_script:
-    - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
-    - bin/cache/dart-sdk/bin/dart --enable-asserts dev/bots/test.dart
-  matrix:
-    # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools
-    - name: tool_tests_general-macos
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tool_tests
-        SUBSHARD: general
-    # all of the tests in test/commands/create_test
-    - name: tool_tests_commands-macos
-      skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'"
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tool_tests
-        SUBSHARD: commands
-    # all of the tests in test/integration for packages/flutter_tools
-    - name: tool_tests_integration-macos
-      only_if: $CIRRUS_BRANCH == 'master'
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tool_tests
-        SUBSHARD: integration
-task:
-  use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
-  osx_instance:
-    image: mojave-xcode-10.2
-  depends_on:
-    - analyze
-  env:
-    CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
-    COCOAPODS_DISABLE_STATS: true
-    PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH"
-  # occasionally the clock on these machines is out of sync
-  # with the actual time - this should help to verify
-  print_date_script:
    - date
-  install_cocoapods_script:
+    - which flutter
-    - sudo gem install cocoapods --no-document
+    - sudo gem install cocoapods
-  git_fetch_script:
+    - sudo gem install xcpretty
-    - git clean -xfd
+    - git clean -xffd
    - git fetch origin
-    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
+    - git fetch origin master # To set FETCH_HEAD, so that "git merge-base" works.
-  setup_script:
+    - flutter config --no-analytics
-    - bin/flutter config --no-analytics
+    - flutter doctor -v
-    - bin/flutter doctor -v
+    - flutter update-packages
-    - bin/flutter update-packages
+    - date
+    - which flutter
+  on_failure:
+    failure_script:
+      - date
+      - which flutter
  matrix:
-    - name: bots_tests-macos
+    - name: framework_tests-widgets-macos
-      skip: "!changesInclude('dev/bots/**')"
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-      test_script:
+      environment:
-        - (cd ./dev/bots && pub run test)
-    - name: tests_widgets-macos
-      only_if: $CIRRUS_BRANCH == 'master'
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tests
-        SUBSHARD: widgets
        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
-      goldctl_script: ./dev/bots/download_goldctl.sh
+      script:
-      test_all_script:
        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - ./dev/bots/download_goldctl.sh
        - dart --enable-asserts dev/bots/test.dart
-      on_failure:
-        print_failure_time_script: date
+    - name: framework_tests-libraries-macos
-    - name: tests_framework_other-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-      only_if: $CIRRUS_BRANCH == 'master'
+      environment:
-      env:
-        GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb]
-        SHARD: tests
-        SUBSHARD: framework_other
        GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl"
        GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095]
-      goldctl_script: ./dev/bots/download_goldctl.sh
+      script:
-      test_all_script:
        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - ./dev/bots/download_goldctl.sh
        - dart --enable-asserts dev/bots/test.dart
-      on_failure:
-        print_failure_time_script: date
+    - name: framework_tests-misc-macos
-    - name: integration_tests-macos
+      # this includes the tests for directories in dev/
-      only_if: $CIRRUS_BRANCH == 'master'
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-      env:
+      script:
-        SHARD: integration_tests
-      test_all_script:
        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
        - dart --enable-asserts dev/bots/test.dart
-    - name: add2app-macos
-      skip: true # https://github.com/flutter/flutter/issues/39507
+    - name: tool_tests-general-macos
-      env:
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
-        SHARD: add2app_test
+      script:
-      setup_xcpretty_script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
-        - sudo gem install xcpretty
+        - dart --enable-asserts ./dev/bots/test.dart
-      test_all_script:
+    - name: tool_tests-commands-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: tool_tests-integration-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''"
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    # TODO(ianh): Enable Web tests on macOS.
+    - name: build_tests-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-0-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-1-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-2-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-3-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-4-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: hostonly_devicelab_tests-5_last-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - dart --enable-asserts ./dev/bots/test.dart
+    - name: add_to_app_tests-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      skip: true # https://github.com/flutter/flutter/pull/42444
+      script:
        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
        - dart --enable-asserts dev/bots/test.dart
    - name: customer_testing-macos
-      test_script:
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
        - rm -rf bin/cache/pkg/tests
        - git clone https://github.com/flutter/tests.git bin/cache/pkg/tests
        - dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test
+    - name: deploy_gallery-macos # linux- and macos- only
+      depends_on:
+        - analyze-linux
+        - framework_tests-widgets-macos
+        - framework_tests-libraries-macos
+        - framework_tests-misc-macos
+        - tool_tests-general-macos
+        - tool_tests-commands-macos
+        - tool_tests-integration-macos
+        - build_tests-macos
+        - hostonly_devicelab_tests-0-macos
+        - hostonly_devicelab_tests-1-macos
+        - hostonly_devicelab_tests-2-macos
+        - hostonly_devicelab_tests-3-macos
+        - hostonly_devicelab_tests-4-macos
+        - hostonly_devicelab_tests-5_last-macos
+        - firebase_test_lab_tests-linux
+      environment:
+        # Apple Fastlane password.
+        FASTLANE_PASSWORD: ENCRYPTED[4b1f0b8d52874e9de965acd46c79743f3b81f3a513614179b9be7cf53dc8258753e257bdadb11a298ee455259df21865]
+        # Private repo for publishing certificates.
+        PUBLISHING_MATCH_CERTIFICATE_REPO: ENCRYPTED[3c0e78877d933fc80107aa6f3790fd1cf927250b852d6cb53202be696b4903ed8ca839b809626aaf18050bf7e436fab7]
+        PUBLISHING_MATCH_REPO_TOKEN: ENCRYPTED[3d1230b744c6ed6c788a91bec741b769401dbcd426b18f9af8080bfeefdfc21913ca4047980c5b5b7ce823f12e7b6b19]
+        # Apple Certificates Match Passphrase
+        MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73]
+      script:
+        - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
+        - ./dev/bots/deploy_gallery.sh
 docker_builder:
  # Only build a new docker image when we tag a release (for dev, beta, or release.)
  only_if: $CIRRUS_TAG != ''
-  env:
+  environment:
    GCLOUD_CREDENTIALS: ENCRYPTED[f7c098d4dd7f5ee1bfee0bb7e944cce72efbe10e97ad6440ae72de4de6a1c24d23f421a2619c668e94377fb64b0bb3e6]
  depends_on:
-    - docs
+    - docs-linux
-    - analyze
+    - analyze-linux
-    - tests_widgets-linux
+    - framework_tests-widgets-linux
-    - tests_framework_other-linux
+    - framework_tests-libraries-linux
-    - tests_extras-linux
+    - framework_tests-misc-linux
-    - tool_tests_general-linux
+    - tool_tests-general-linux
-    - tool_tests_commands-linux
+    - tool_tests-commands-linux
-    - tool_tests_integration-linux
+    - tool_tests-integration-linux
    - build_tests-linux
-    - integration_tests-linux
+    - hostonly_devicelab_tests-0-linux
-    - gradle_tests-linux-shard-1
+    - hostonly_devicelab_tests-1-linux
-    - gradle_tests-linux-shard-2
+    - hostonly_devicelab_tests-2-linux
-    - gradle_embedding_v2_tests-linux-shard-1
+    - hostonly_devicelab_tests-3-linux
-    - gradle_embedding_v2_tests-linux-shard-2
+    - hostonly_devicelab_tests-4-linux
-    - gradle_tests-windows-shard-1
+    - hostonly_devicelab_tests-5_last-linux
-    - gradle_tests-windows-shard-2
+    - firebase_test_lab_tests-linux
-    - gradle_embedding_v2_tests-windows-shard-1
+  script:
-    - gradle_embedding_v2_tests-windows-shard-2
+    - "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh"
-    - release_smoke_tests
+    - "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh"
-  build_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh"
+    - "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh"
-  login_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh"
-  push_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh"
--- a/dev/README.md
+++ b/dev/README.md
@@ -2,3 +2,6 @@ This directory contains tools and resources that the Flutter team uses
 during development of the framework. The tools in this directory
 should not be necessary for developing Flutter applications, though of
 course they may be interesting if you are curious.
+The tests in this directory are run in the `framework_tests_misc-*`
+shards.
--- a/dev/bots/cirrus_setup.sh
+++ b/dev/bots/cirrus_setup.sh
 #!/bin/bash
 set -e
+# This script is only meant to be run by the Cirrus CI system, not locally.
+# It must be run from the root of the Flutter repo.
 function error() {
  echo "$@" 1>&2
 }
-# This script is only meant to be run by the Cirrus CI system, not locally.
-# It must be run from the root of the Flutter repo.
 function accept_android_licenses() {
  yes "y" | flutter doctor --android-licenses > /dev/null 2>&1
 }
 echo "Flutter SDK directory is: $PWD"
-# Run flutter to download dependencies and precompile things, and to disable
-# analytics on the bots.
-echo "Downloading build dependencies and pre-compiling Flutter snapshot"
-./bin/flutter config --no-analytics
-# Run doctor, to print it to the log for debugging purposes.
-./bin/flutter doctor -v
 # Accept licenses.
 echo "Accepting Android licenses."
 accept_android_licenses || (error "Accepting Android licenses failed." && false)
-# Run pub get in all the repo packages.
-echo "Updating packages for Flutter."
-./bin/flutter update-packages
--- a/dev/bots/deploy_gallery.sh
+++ b/dev/bots/deploy_gallery.sh
@@ -24,11 +24,9 @@ set -x
 cd "$FLUTTER_ROOT"
-if [[ "$SHARD" = "deploy_gallery" ]]; then
+version="$(<version)"
-  version="$(<version)"
+if [[ "$OS" == "linux" ]]; then
-  if [[ "$OS" == "linux" ]]; then
  echo "Building Flutter Gallery $version for Android..."
  # ANDROID_SDK_ROOT must be set in the env.
  (
    cd examples/flutter_gallery
@@ -47,9 +45,9 @@ if [[ "$SHARD" = "deploy_gallery" ]]; then
      fastlane deploy_play_store
    )
  else
-      echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits"
+    echo "(Not deploying; Flutter Gallery is only deployed to Play store for tagged dev branch commits.)"
  fi
-  elif [[ "$OS" == "darwin" ]]; then
+elif [[ "$OS" == "darwin" ]]; then
  echo "Building Flutter Gallery $version for iOS..."
  (
    cd examples/flutter_gallery
@@ -95,17 +93,21 @@ if [[ "$SHARD" = "deploy_gallery" ]]; then
        fastlane build_and_deploy_testflight upload:true
      )
    else
-        echo "Archiving with distribution profile..."
+      # On iOS the signing can break as well, so we verify this regularly (not just
+      # on tagged dev branch commits). We can only do this post-merge, though, because
+      # the secrets aren't available on PRs.
+      echo "Testing archiving with distribution profile..."
      (
        cd examples/flutter_gallery/ios
        fastlane build_and_deploy_testflight
      )
-        echo "Archive is only deployed to TestFlight on tagged dev branch commits"
+      echo "(Not deploying; Flutter Gallery is only deployed to TestFlight for tagged dev branch commits.)"
    fi
  else
-      echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits"
+    echo "(Not archiving or deploying; Flutter Gallery archiving is only tested post-commit.)"
-    fi
  fi
 else
-  echo "Doing nothing: not on the 'deploy_gallery' SHARD."
+  echo "Unknown OS: $OS"
+  echo "Aborted."
+  exit 1
 fi
--- a/dev/bots/run_command.dart
+++ b/dev/bots/run_command.dart
@@ -95,7 +95,6 @@ Future<void> runCommand(String executable, List<String> arguments, {
  OutputMode outputMode = OutputMode.print,
  CapturedOutput output,
  bool skip = false,
-  bool expectFlaky = false,
  bool Function(String) removeLine,
 }) async {
  assert((outputMode == OutputMode.capture) == (output != null),
@@ -145,10 +144,6 @@ Future<void> runCommand(String executable, List<String> arguments, {
    output.stderr = _flattenToString(await savedStderr);
  }
-  // If the test is flaky we don't care about the actual exit.
-  if (expectFlaky)
-    return;
  if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) {
    if (failureMessage != null) {
      print(failureMessage);

--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -4,11 +4,11 @@
 import 'dart:async';
 import 'dart:io';
+import 'dart:math' as math;
 import 'package:googleapis/bigquery/v2.dart' as bq;
 import 'package:googleapis_auth/auth_io.dart' as auth;
 import 'package:http/http.dart' as http;
-import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
 import 'flutter_compact_formatter.dart';
@@ -30,21 +30,54 @@ final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Pl
 final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
 final String pubCache = path.join(flutterRoot, '.pub-cache');
 final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
+/// The arguments to pass to `flutter test` (typically the local engine
+/// configuration) -- prefilled with the arguments passed to test.dart.
 final List<String> flutterTestArgs = <String>[];
 final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
 final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true';
-const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
+/// The number of Cirrus jobs that run host-only devicelab tests in parallel.
-  'tests': _runTests,
+///
-  'web_tests': _runWebTests,
+/// WARNING: if you change this number, also change .cirrus.yml
-  'tool_tests': _runToolTests,
+/// and make sure it runs _all_ shards.
-  'tool_coverage': _runToolCoverage,
+const int kDeviceLabShardCount = 6;
-  'build_tests': _runBuildTests,
-  'coverage': _runCoverage,
+/// The number of Cirrus jobs that run Web tests in parallel.
-  'integration_tests': _runIntegrationTests,
+///
-  'add2app_test': _runAdd2AppTest,
+/// WARNING: if you change this number, also change .cirrus.yml
-};
+/// and make sure it runs _all_ shards.
+///
+/// The last shard also runs the Web plugin tests.
+const int kWebShardCount = 6;
+/// Maximum number of Web tests to run in a single `flutter test`. We found that
+/// large batches can get flaky, possibly because we reuse a single instance of
+/// the browser, and after many tests the browser's state gets corrupted.
+const int kWebBatchSize = 20;
+/// Tests that we don't run on Web for various reasons.
+//
+// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60
+const List<String> kWebTestDirectoryBlacklist = <String>[
+  'cupertino',
+  'examples',
+  'material',
+];
+const List<String> kWebTestFileBlacklist = <String>[
+  'test/widgets/heroes_test.dart',
+  'test/widgets/text_test.dart',
+  'test/widgets/selectable_text_test.dart',
+  'test/widgets/color_filter_test.dart',
+  'test/widgets/editable_text_cursor_test.dart',
+  'test/widgets/shadow_test.dart',
+  'test/widgets/raw_keyboard_listener_test.dart',
+  'test/widgets/editable_text_test.dart',
+  'test/widgets/widget_inspector_test.dart',
+  'test/widgets/draggable_test.dart',
+  'test/widgets/shortcuts_test.dart',
+];
 /// When you call this, you can pass additional arguments to pass custom
 /// arguments to flutter test. For example, you might want to call this
@@ -53,31 +86,30 @@ const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
 ///
 /// To run the tool_tests part, run it with SHARD=tool_tests
 ///
-/// For example:
+/// Examples:
 /// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
 /// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
 Future<void> main(List<String> args) async {
  flutterTestArgs.addAll(args);
+  if (Platform.environment.containsKey(CIRRUS_TASK_NAME))
-  final String shard = Platform.environment['SHARD'];
+    print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
-  if (shard != null) {
+  print('═' * 80);
-    if (!_kShards.containsKey(shard)) {
+  await _runSmokeTests();
-      print('Invalid shard: $shard');
+  print('═' * 80);
-      print('The available shards are: ${_kShards.keys.join(", ")}');
+  await selectShard(const <String, ShardRunner>{
-      exit(1);
+    'add_to_app_tests': _runAddToAppTests,
-    }
+    'build_tests': _runBuildTests,
-    print('${bold}SHARD=$shard$reset');
+    'framework_coverage': _runFrameworkCoverage,
-    await _kShards[shard]();
+    'framework_tests': _runFrameworkTests,
-  } else {
+    'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
-    for (String currentShard in _kShards.keys) {
+    'tool_coverage': _runToolCoverage,
-      print('${bold}SHARD=$currentShard$reset');
+    'tool_tests': _runToolTests,
-      await _kShards[currentShard]();
+    'web_tests': _runWebTests,
-      print('');
+  });
-    }
-  }
 }
 Future<void> _runSmokeTests() async {
+  print('${green}Running smoketests...$reset');
  // Verify that the tests actually return failure on failure and success on
  // success.
  final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
@@ -108,10 +140,11 @@ Future<void> _runSmokeTests() async {
    script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
    expectFailure: true,
    printOutput: false,
-    outputChecker: (CapturedOutput output) =>
+    outputChecker: (CapturedOutput output) {
-      output.stdout.contains('failingPendingTimerTest')
+      return output.stdout.contains('failingPendingTimerTest')
        ? null
-      : 'Failed to find the stack trace for the pending Timer.',
+        : 'Failed to find the stack trace for the pending Timer.';
+    }
  );
  // We run the remaining smoketests in parallel, because they each take some
  // time to run (e.g. compiling), so we don't want to run them in series,
@@ -153,8 +186,11 @@ Future<void> _runSmokeTests() async {
  );
  // Verify that we correctly generated the version file.
-  final bool validVersion = await verifyVersion(path.join(flutterRoot, 'version'));
+  final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
-  if (!validVersion) {
+  if (versionError != null) {
+    print(redLine);
+    print(versionError);
+    print(redLine);
    exit(1);
  }
 }
@@ -179,7 +215,7 @@ Future<bq.BigqueryApi> _getBigqueryApi() async {
    final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes);
    return bq.BigqueryApi(client);
  } catch (e) {
-    print('Failed to get BigQuery API client.');
+    print('${red}Failed to get BigQuery API client.$reset');
    print(e);
    return null;
  }
@@ -203,7 +239,6 @@ Future<void> _runToolCoverage() async {
 Future<void> _runToolTests() async {
  final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
-  await _runSmokeTests();
  const String kDotShard = '.shard';
  const String kTest = 'test';
@@ -232,11 +267,10 @@ Future<void> _runToolTests() async {
  await selectSubshard(subshards);
 }
-/// Verifies that AOT, APK, and IPA (if on macOS) builds the
+/// Verifies that AOT, APK, and IPA (if on macOS) builds the examples apps
-/// examples apps without crashing. It does not actually
+/// without crashing. It does not actually launch the apps. That happens later
-/// launch the apps. That happens later in the devicelab. This is
+/// in the devicelab. This is just a smoke-test. In particular, this will verify
-/// just a smoke-test. In particular, this will verify we can build
+/// we can build when there are spaces in the path name for the Flutter SDK and
-/// when there are spaces in the path name for the Flutter SDK and
 /// target app.
 Future<void> _runBuildTests() async {
  final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list();
@@ -245,64 +279,39 @@ Future<void> _runBuildTests() async {
      continue;
    }
    final String examplePath = fileEntity.path;
    await _flutterBuildAot(examplePath);
    await _flutterBuildApk(examplePath);
+    if (Platform.isMacOS) {
      await _flutterBuildIpa(examplePath);
    }
+  }
  // Web compilation tests.
  await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web'), path.join('lib', 'main.dart'));
  // Should not fail to compile with dart:io.
  await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web_compile_tests'),
    path.join('lib', 'dart_io_import.dart'),
  );
-  print('${bold}DONE: All build tests successful.$reset');
-}
-Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
-  print('Running Dart2JS build tests...');
-  await runCommand(flutter,
-    <String>['build', 'web', '-v', '--target=$target'],
-    workingDirectory: path.join(flutterRoot, relativePathToApplication),
-    expectNonZeroExit: expectNonZeroExit,
-    environment: <String, String>{
-      'FLUTTER_WEB': 'true',
-    },
-  );
-  print('Done.');
 }
 Future<void> _flutterBuildAot(String relativePathToApplication) async {
-  print('Running AOT build tests...');
+  print('${green}Testing AOT build$reset for $cyan$relativePathToApplication$reset...');
  await runCommand(flutter,
    <String>['build', 'aot', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
-    expectNonZeroExit: false,
  );
-  print('Done.');
 }
 Future<void> _flutterBuildApk(String relativePathToApplication) async {
-  if (
+  print('${green}Testing APK --debug build$reset for $cyan$relativePathToApplication$reset...');
-        (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) &&
-        (Platform.environment['ANDROID_SDK_ROOT']?.isEmpty ?? true)) {
-    return;
-  }
-  print('Running APK build tests...');
  await runCommand(flutter,
    <String>['build', 'apk', '--debug', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
-    expectNonZeroExit: false,
  );
-  print('Done.');
 }
 Future<void> _flutterBuildIpa(String relativePathToApplication) async {
-  if (!Platform.isMacOS) {
+  assert(Platform.isMacOS);
-    return;
+  print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...');
-  }
-  print('Running IPA build tests...');
  // Install Cocoapods.  We don't have these checked in for the examples,
  // and build ios doesn't take care of it automatically.
  final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
@@ -310,171 +319,119 @@ Future<void> _flutterBuildIpa(String relativePathToApplication) async {
    await runCommand('pod',
      <String>['install'],
      workingDirectory: podfile.parent.path,
-      expectNonZeroExit: false,
    );
  }
  await runCommand(flutter,
    <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
    workingDirectory: path.join(flutterRoot, relativePathToApplication),
-    expectNonZeroExit: false,
  );
-  print('Done.');
 }
-Future<void> _runAdd2AppTest() async {
+Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
-  if (!Platform.isMacOS) {
+  print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
-    return;
+  await runCommand(flutter,
-  }
+    <String>['build', 'web', '-v', '--target=$target'],
-  print('Running Add2App iOS integration tests...');
+    workingDirectory: path.join(flutterRoot, relativePathToApplication),
-  final String add2AppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
+    expectNonZeroExit: expectNonZeroExit,
+    environment: <String, String>{
+      'FLUTTER_WEB': 'true',
+    },
+  );
+}
+Future<void> _runAddToAppTests() async {
+  if (Platform.isMacOS) {
+    print('${green}Running add-to-app iOS integration tests$reset...');
+    final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
    await runCommand('./build_and_test.sh',
      <String>[],
-    workingDirectory: add2AppDir,
+      workingDirectory: addToAppDir,
-    expectNonZeroExit: false,
    );
-  print('Done.');
+  }
 }
-Future<void> _runTests() async {
+Future<void> _runFrameworkTests() async {
  final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
-  await _runSmokeTests();
-  final String subShard = Platform.environment['SUBSHARD'];
  Future<void> runWidgets() async {
+    print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset');
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
+      options: <String>['--track-widget-creation'],
      tableData: bigqueryApi?.tabledata,
-      tests: <String>[
+      tests: <String>[ path.join('test', 'widgets') + path.separator ],
-        path.join('test', 'widgets') + path.separator,
-      ],
    );
-    // Only packages/flutter/test/widgets/widget_inspector_test.dart really
-    // needs to be run with --track-widget-creation but it is nice to run
-    // all of the tests in package:flutter with the flag to ensure that
-    // the Dart kernel transformer triggered by the flag does not break anything.
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
-      options: <String>['--track-widget-creation'],
+      options: <String>['--no-track-widget-creation'],
      tableData: bigqueryApi?.tabledata,
-      tests: <String>[
+      tests: <String>[ path.join('test', 'widgets') + path.separator ],
-        path.join('test', 'widgets') + path.separator,
-      ],
    );
+    // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
+    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
+    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--no-track-widget-creation'], tableData: bigqueryApi?.tabledata);
  }
-  Future<void> runFrameworkOthers() async {
+  Future<void> runLibraries() async {
    final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
      .listSync(followLinks: false, recursive: false)
      .whereType<Directory>()
      .where((Directory dir) => dir.path.endsWith('widgets') == false)
-      .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
+      .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
      .toList();
+    print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset');
-    print('Running tests for: ${tests.join(';')}');
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
+      options: <String>['--track-widget-creation'],
      tableData: bigqueryApi?.tabledata,
      tests: tests,
    );
-    // Only packages/flutter/test/widgets/widget_inspector_test.dart really
-    // needs to be run with --track-widget-creation but it is nice to run
-    // all of the tests in package:flutter with the flag to ensure that
-    // the Dart kernel transformer triggered by the flag does not break anything.
    await _runFlutterTest(
      path.join(flutterRoot, 'packages', 'flutter'),
-      options: <String>['--track-widget-creation'],
+      options: <String>['--no-track-widget-creation'],
      tableData: bigqueryApi?.tabledata,
      tests: tests,
    );
  }
-  Future<void> runExtras() async {
+  Future<void> runMisc() async {
-    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
+    print('${green}Running package tests$reset for directories other than packages/flutter');
-    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
-    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
-    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata);
    await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata);
+    await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata);
    await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata);
-    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata);
+    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
-    // Regression test to ensure that code outside of package:flutter can run
+    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
-    // with --track-widget-creation.
+    await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
-    await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
+    await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
-    await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
+    await _runFlutterTest(
-    // Smoke test for code generation.
+      path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'),
-    await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: <String, String>{
+      tableData: bigqueryApi?.tabledata,
+      environment: <String, String>{
        'FLUTTER_EXPERIMENTAL_BUILD': 'true',
-    });
+      },
+    );
  }
-  switch (subShard) {
-    case 'widgets':
-      await runWidgets();
-      break;
-    case 'framework_other':
-      await runFrameworkOthers();
-      break;
-    case 'extras':
-      runExtras();
-      break;
-    default:
-      print('Unknown sub-shard $subShard, running all tests!');
-      await runWidgets();
-      await runFrameworkOthers();
-      await runExtras();
-  }
-  print('${bold}DONE: All tests successful.$reset');
-}
-// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60
+  await selectSubshard(<String, ShardRunner>{
-const List<String> kWebTestDirectoryBlacklist = <String>[
+    'widgets': runWidgets,
-  'test/cupertino',
+    'libraries': runLibraries,
-  'test/examples',
+    'misc': runMisc,
-  'test/material',
+  });
-];
-const List<String> kWebTestFileBlacklist = <String>[
-  'test/widgets/heroes_test.dart',
-  'test/widgets/text_test.dart',
-  'test/widgets/selectable_text_test.dart',
-  'test/widgets/color_filter_test.dart',
-  'test/widgets/editable_text_cursor_test.dart',
-  'test/widgets/shadow_test.dart',
-  'test/widgets/raw_keyboard_listener_test.dart',
-  'test/widgets/editable_text_test.dart',
-  'test/widgets/widget_inspector_test.dart',
-  'test/widgets/draggable_test.dart',
-  'test/widgets/shortcuts_test.dart',
-];
-Future<void> _runWebTests() async {
-  final Directory flutterPackageDir = Directory(path.join(flutterRoot, 'packages', 'flutter'));
-  final Directory testDir = Directory(path.join(flutterPackageDir.path, 'test'));
-  final List<String> directories = testDir
-    .listSync()
-    .whereType<Directory>()
-    .map<String>((Directory dir) => path.relative(dir.path, from: flutterPackageDir.path))
-    .where((String relativePath) => !kWebTestDirectoryBlacklist.contains(relativePath))
-    .toList();
-  await _runFlutterWebTest(flutterPackageDir.path, tests: directories);
-  await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter_web_plugins'), tests: <String>['test']);
 }
-Future<void> _runCoverage() async {
+Future<void> _runFrameworkCoverage() async {
  final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
-    print('Expected to find: ${coverageFile.absolute}');
+    print('Expected to find: $cyan${coverageFile.absolute}$reset');
-    print('This file is normally obtained by running `flutter update-packages`.');
+    print('This file is normally obtained by running `${green}flutter update-packages$reset`.');
    exit(1);
  }
  coverageFile.deleteSync();
@@ -483,11 +440,98 @@ Future<void> _runCoverage() async {
  );
  if (!coverageFile.existsSync()) {
    print('${red}Coverage file not found.$reset');
-    print('Expected to find: ${coverageFile.absolute}');
+    print('Expected to find: $cyan${coverageFile.absolute}$reset');
-    print('This file should have been generated by the `flutter test --coverage` script, but was not.');
+    print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.');
    exit(1);
  }
-  print('${bold}DONE: Coverage collection successful.$reset');
+}
+Future<void> _runWebTests() async {
+  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
+  final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
+  final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));
+  final List<String> allTests = flutterPackageTestDirectory
+    .listSync()
+    .whereType<Directory>()
+    .where((Directory directory) => !kWebTestDirectoryBlacklist.contains(path.basename(directory.path)))
+    .expand((Directory directory) => directory
+      .listSync(recursive: true)
+      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
+    )
+    .whereType<File>()
+    .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
+    .where((String filePath) => !kWebTestFileBlacklist.contains(filePath))
+    .toList()
+    // Finally we shuffle the list because we want the average cost per file to be uniformly
+    // distributed. If the list is not sorted then different shards and batches may have
+    // very different characteristics.
+    // We use a constant seed for repeatability.
+    ..shuffle(math.Random(0));
+  assert(kWebShardCount >= 1);
+  final int testsPerShard = (allTests.length / kWebShardCount).ceil();
+  assert(testsPerShard * kWebShardCount >= allTests.length);
+  // This for loop computes all but the last shard.
+  for (int index = 0; index < kWebShardCount - 1; index += 1) {
+    subshards['$index'] = () => _runFlutterWebTest(
+      flutterPackageDirectory.path,
+      allTests.sublist(
+        index * testsPerShard,
+        (index + 1) * testsPerShard,
+      ),
+    );
+  }
+  // The last shard also runs the flutter_web_plugins tests.
+  //
+  // We make sure the last shard ends in _last so it's easier to catch mismatches
+  // between `.cirrus.yml` and `test.dart`.
+  subshards['${kWebShardCount - 1}_last'] = () async {
+    await _runFlutterWebTest(
+      flutterPackageDirectory.path,
+      allTests.sublist(
+        (kWebShardCount - 1) * testsPerShard,
+        allTests.length,
+      ),
+    );
+    await _runFlutterWebTest(
+      path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
+      <String>['test'],
+    );
+  };
+  await selectSubshard(subshards);
+}
+Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async {
+  final List<String> batch = <String>[];
+  for (int i = 0; i < tests.length; i += 1) {
+    final String testFilePath = tests[i];
+    batch.add(testFilePath);
+    if (batch.length == kWebBatchSize || i == tests.length - 1) {
+      await runCommand(
+        flutter,
+        <String>[
+          'test',
+          if (ciProvider == CiProviders.cirrus)
+            '--concurrency=1',  // do not parallelize on Cirrus, to reduce flakiness
+          '-v',
+          '--platform=chrome',
+          ...?flutterTestArgs,
+          ...batch,
+        ],
+        workingDirectory: workingDirectory,
+        environment: <String, String>{
+          'FLUTTER_WEB': 'true',
+          'FLUTTER_LOW_RESOURCE_MODE': 'true',
+        },
+      );
+      batch.clear();
+    }
+  }
 }
 Future<void> _pubRunTest(String workingDirectory, {
@@ -496,14 +540,33 @@ Future<void> _pubRunTest(String workingDirectory, {
  bool useBuildRunner = false,
  bq.TabledataResourceApi tableData,
 }) async {
-  final List<String> args = <String>['run', '--verbose'];
+  final List<String> args = <String>['run'];
  if (useBuildRunner) {
-    args.addAll(<String>['build_runner', 'test', '--']);
+    final String posixTestPath = path.posix.joinAll(path.split(testPath));
+    args.addAll(<String>[
+      'build_runner',
+      'test',
+      '--build-filter=$posixTestPath/*.dill',
+      '--build-filter=$posixTestPath/**/*.dill',
+      '--',
+    ]);
  } else {
    args.add('test');
  }
  args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact');
-  args.add('-j1'); // TODO(ianh): Scale based on CPUs.
+  int cpus;
+  final String cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
+  if (cpuVariable != null) {
+    cpus = int.tryParse(cpuVariable, radix: 10);
+    if (cpus == null) {
+      print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset');
+      print('Actual value: "$cpuVariable"');
+      exit(1);
+    }
+  } else {
+    cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
+  }
+  args.add('-j$cpus');
  if (!hasColor)
    args.add('--no-color');
  if (testPath != null)
@@ -547,238 +610,6 @@ Future<void> _pubRunTest(String workingDirectory, {
  }
 }
-void deleteFile(String path) {
-  // There's a race condition here but in theory we're not racing anyone
-  // while this script runs, so should be ok.
-  final File file = File(path);
-  if (file.existsSync())
-    file.deleteSync();
-}
-enum CiProviders {
-  cirrus,
-  luci,
-}
-CiProviders _getCiProvider() {
-  if (Platform.environment['CIRRUS_CI'] == 'true') {
-    return CiProviders.cirrus;
-  }
-  if (Platform.environment['LUCI_CONTEXT'] != null) {
-    return CiProviders.luci;
-  }
-  return null;
-}
-String _getCiProviderName() {
-  switch(_getCiProvider()) {
-    case CiProviders.cirrus:
-      return 'cirrusci';
-    case CiProviders.luci:
-      return 'luci';
-  }
-  return 'unknown';
-}
-int _getPrNumber() {
-  switch(_getCiProvider()) {
-    case CiProviders.cirrus:
-      return Platform.environment['CIRRUS_PR'] == null
-          ? -1
-          : int.tryParse(Platform.environment['CIRRUS_PR']);
-    case CiProviders.luci:
-      return -1; // LUCI doesn't know about this.
-  }
-  return -1;
-}
-Future<String> _getAuthors() async {
-  final String exe = Platform.isWindows ? '.exe' : '';
-  final String author = await runAndGetStdout(
-    'git$exe', <String>['-c', 'log.showSignature=false', 'log', _getGitHash(), '--pretty="%an <%ae>"'],
-    workingDirectory: flutterRoot,
-  ).first;
-  return author;
-}
-String _getCiUrl() {
-  switch(_getCiProvider()) {
-    case CiProviders.cirrus:
-      return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
-    case CiProviders.luci:
-      return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
-  }
-  return '';
-}
-String _getGitHash() {
-  switch(_getCiProvider()) {
-    case CiProviders.cirrus:
-      return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
-    case CiProviders.luci:
-      return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
-  }
-  return '';
-}
-Future<void> _processTestOutput(
-  FlutterCompactFormatter formatter,
-  Stream<String> testOutput,
-  bq.TabledataResourceApi tableData,
-) async {
-  final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
-    print('Processing...');
-  });
-  await testOutput.forEach(formatter.processRawOutput);
-  heartbeat.cancel();
-  formatter.finish();
-  if (tableData == null || formatter.tests.isEmpty) {
-    return;
-  }
-  final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
-  final String authors = await _getAuthors();
-  request.rows = List<bq.TableDataInsertAllRequestRows>.from(
-    formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
-      bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
-        'json': <String, dynamic>{
-          'source': <String, dynamic>{
-            'provider': _getCiProviderName(),
-            'url': _getCiUrl(),
-            'platform': <String, dynamic>{
-              'os': Platform.operatingSystem,
-              'version': Platform.operatingSystemVersion,
-            },
-          },
-          'test': <String, dynamic>{
-            'name': result.name,
-            'result': result.status.toString(),
-            'file': result.path,
-            'line': result.line,
-            'column': result.column,
-            'time': result.totalTime,
-          },
-          'git': <String, dynamic>{
-            'author': authors,
-            'pull_request': _getPrNumber(),
-            'commit': _getGitHash(),
-            'organization': 'flutter',
-            'repository': 'flutter',
-          },
-          'error': result.status != TestStatus.failed ? null : <String, dynamic>{
-            'message': result.errorMessage,
-            'stack_trace': result.stackTrace,
-          },
-          'information': result.messages,
-        },
-      }),
-    ),
-    growable: false,
-  );
-  final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
-  if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
-    print('${red}BigQuery insert errors:');
-    print(response.toJson());
-    print(reset);
-  }
-}
-class EvalResult {
-  EvalResult({
-    this.stdout,
-    this.stderr,
-    this.exitCode = 0,
-  });
-  final String stdout;
-  final String stderr;
-  final int exitCode;
-}
-/// The number of Cirrus jobs that run web tests in parallel.
-///
-/// WARNING: if you change this number, also change .cirrus.yml
-/// and make sure it runs _all_ shards.
-const int _kWebShardCount = 6;
-Future<void> _runFlutterWebTest(String workingDirectory, {
-  List<String> tests,
-}) async {
-  List<String> allTests = <String>[];
-  for (String testDirPath in tests) {
-    final Directory testDir = Directory(path.join(workingDirectory, testDirPath));
-    allTests.addAll(
-      testDir.listSync(recursive: true)
-        .whereType<File>()
-        .where((File file) => file.path.endsWith('_test.dart'))
-        .map<String>((File file) => path.relative(file.path, from: workingDirectory))
-        .where((String filePath) => !kWebTestFileBlacklist.contains(filePath)),
-    );
-  }
-  // If a shard is specified only run tests in that shard.
-  final int webShard = int.tryParse(Platform.environment['WEB_SHARD'] ?? 'n/a');
-  if (webShard != null) {
-    if (webShard >= _kWebShardCount) {
-      throw 'WEB_SHARD must be <= _kWebShardCount, but was $webShard';
-    }
-    final List<String> shard = <String>[];
-    for (int i = webShard; i < allTests.length; i += _kWebShardCount) {
-      shard.add(allTests[i]);
-    }
-    allTests = shard;
-  }
-  print(allTests.join('\n'));
-  print('${allTests.length} tests total');
-  // Maximum number of tests to run in a single `flutter test`. We found that
-  // large batches can get flaky, possibly because we reuse a single instance
-  // of the browser, and after many tests the browser's state gets corrupted.
-  const int kBatchSize = 20;
-  List<String> batch = <String>[];
-  for (int i = 0; i < allTests.length; i += 1) {
-    final String testFilePath = allTests[i];
-    batch.add(testFilePath);
-    if (batch.length == kBatchSize || i == allTests.length - 1) {
-      await _runFlutterWebTestBatch(workingDirectory, batch: batch);
-      batch = <String>[];
-    }
-  }
-}
-Future<void> _runFlutterWebTestBatch(String workingDirectory, {
-  List<String> batch,
-}) async {
-  final List<String> args = <String>[
-    'test',
-    if (_getCiProvider() == CiProviders.cirrus)
-      '--concurrency=1',  // do not parallelize on Cirrus to reduce flakiness
-    '-v',
-    '--platform=chrome',
-    ...?flutterTestArgs,
-    ...batch,
-  ];
-  // TODO(jonahwilliams): fix relative path issues to make this unecessary.
-  final Directory oldCurrent = Directory.current;
-  Directory.current = Directory(path.join(flutterRoot, 'packages', 'flutter'));
-  try {
-    await runCommand(
-      flutter,
-      args,
-      workingDirectory: workingDirectory,
-      expectFlaky: false,
-      environment: <String, String>{
-        'FLUTTER_WEB': 'true',
-        'FLUTTER_LOW_RESOURCE_MODE': 'true',
-      },
-    );
-  } finally {
-    Directory.current = oldCurrent;
-  }
-}
 Future<void> _runFlutterTest(String workingDirectory, {
  String script,
  bool expectFailure = false,
@@ -790,8 +621,7 @@ Future<void> _runFlutterTest(String workingDirectory, {
  Map<String, String> environment,
  List<String> tests = const <String>[],
 }) async {
-  assert(!printOutput || outputChecker == null,
+  assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
-      'Output either can be printed or checked but not both');
  final List<String> args = <String>[
    'test',
@@ -800,16 +630,15 @@ Future<void> _runFlutterTest(String workingDirectory, {
  ];
  final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
-  if (shouldProcessOutput) {
+  if (shouldProcessOutput)
    args.add('--machine');
-  }
  if (script != null) {
    final String fullScriptPath = path.join(workingDirectory, script);
    if (!FileSystemEntity.isFileSync(fullScriptPath)) {
-      print('Could not find test: $fullScriptPath');
+      print('${red}Could not find test$reset: $green$fullScriptPath$reset');
-      print('Working directory: $workingDirectory');
+      print('Working directory: $cyan$workingDirectory$reset');
-      print('Script: $script');
+      print('Script: $green$script$reset');
      if (!printOutput)
        print('This is one of the tests that does not normally print output.');
      if (skip)
@@ -876,110 +705,290 @@ Future<void> _runFlutterTest(String workingDirectory, {
  }
 }
-// the optional `file` argument is an override for testing
+Map<String, String> _initGradleEnvironment() {
-@visibleForTesting
+  final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
-Future<bool> verifyVersion(String filename, [File file]) async {
+      ? Platform.environment['ANDROID_SDK_ROOT']
-  final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
+      : Platform.environment['ANDROID_HOME'];
-  file ??= File(filename);
+  if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
-  final String version = await file.readAsString();
+    print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT (or ANDROID_HOME).$reset');
-  if (!file.existsSync()) {
+    exit(1);
-    print('$redLine');
-    print('The version logic failed to create the Flutter version file.');
-    print('$redLine');
-    return false;
-  }
-  if (version == '0.0.0-unknown') {
-    print('$redLine');
-    print('The version logic failed to determine the Flutter version.');
-    print('$redLine');
-    return false;
-  }
-  if (!version.contains(pattern)) {
-    print('$redLine');
-    print('The version logic generated an invalid version string: "$version".');
-    print('$redLine');
-    return false;
  }
-  return true;
+  return <String, String>{
+    'ANDROID_HOME': androidSdkRoot,
+    'ANDROID_SDK_ROOT': androidSdkRoot,
+  };
 }
-Future<void> _runIntegrationTests() async {
+final Map<String, String> gradleEnvironment = _initGradleEnvironment();
-  final String subShard = Platform.environment['SUBSHARD'];
+Future<void> _runHostOnlyDeviceLabTests() async {
-  switch (subShard) {
+  if (Platform.isWindows) {
-    case 'gradle1':
+    // TODO(ianh): remove when https://github.com/flutter/flutter/issues/36311 fixed by https://github.com/flutter/flutter/pull/42709
-    case 'gradle2':
+    return;
-      // This runs some gradle integration tests if the subshard is Android.
-      await _androidGradleTests(subShard);
-      break;
-    default:
-      await _runDevicelabTest('dartdocs');
-      if (Platform.isLinux) {
-        await _runDevicelabTest('flutter_create_offline_test_linux');
-      } else if (Platform.isWindows) {
-        await _runDevicelabTest('flutter_create_offline_test_windows');
-      } else if (Platform.isMacOS) {
-        await _runDevicelabTest('flutter_create_offline_test_mac');
-        await _runDevicelabTest('plugin_lint_mac');
-// TODO(jmagman): Re-enable once flakiness is resolved.
-//        await _runDevicelabTest('module_test_ios');
  }
+  // Please don't add more tests here. We should not be using the devicelab
+  // logic to run tests outside devicelab, that's just confusing.
+  // Instead, create tests that are not devicelab tests, and run those.
+  // TODO(ianh): Move the tests that are not running on devicelab any more out
+  // of the device lab directory.
+  // List the tests to run.
+  // We split these into subshards. The tests are randomly distributed into
+  // those subshards so as to get a uniform distribution of costs, but the
+  // seed is fixed so that issues are reproducible.
+  final List<ShardRunner> tests = <ShardRunner>[
+    // Keep this in alphabetical order.
+    () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    if (Platform.isMacOS) () => _runDevicelabTest('flutter_create_offline_test_mac'),
+    if (Platform.isLinux) () => _runDevicelabTest('flutter_create_offline_test_linux'),
+    if (Platform.isWindows) () => _runDevicelabTest('flutter_create_offline_test_windows'),
+    // TODO(ianh): Fails on macOS looking for "dexdump", https://github.com/flutter/flutter/issues/42494
+    if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true),
+    // TODO(jmagman): Re-enable once flakiness is resolved, https://github.com/flutter/flutter/issues/37525
+    // if (Platform.isMacOS) () => _runDevicelabTest('module_test_ios'),
+    if (Platform.isMacOS) () => _runDevicelabTest('plugin_lint_mac'),
+    () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: false),
+    () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: true),
+  ]..shuffle(math.Random(0));
+  final int testsPerShard = tests.length ~/ kDeviceLabShardCount;
+  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
+  for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) {
+    String last = '';
+    List<ShardRunner> sublist;
+    if (subshard < kDeviceLabShardCount - 1) {
+      sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard);
+    } else {
+      sublist = tests.sublist(subshard * testsPerShard, tests.length);
+      // We make sure the last shard ends in _last so it's easier to catch mismatches
+      // between `.cirrus.yml` and `test.dart`.
+      last = '_last';
+    }
+    subshards['$subshard$last'] = () async {
+      for (ShardRunner test in sublist)
+        await test();
+    };
  }
+  await selectSubshard(subshards);
 }
-Future<void> _runDevicelabTest(String testName, {Map<String, String> env}) async {
+Future<void> _runDevicelabTest(String testName, {
+  Map<String, String> environment,
+  bool testEmbeddingV2 = false,
+}) async {
  await runCommand(
    dart,
    <String>['bin/run.dart', '-t', testName],
    workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
-    environment: env,
+    environment: <String, String>{
+      ...?environment,
+      if (testEmbeddingV2)
+        'ENABLE_ANDROID_EMBEDDING_V2': 'true',
+    },
  );
 }
-String get androidSdkRoot {
+void deleteFile(String path) {
-  final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
+  // This is technically a race condition but nobody else should be running
-      ? Platform.environment['ANDROID_SDK_ROOT']
+  // while this script runs, so we should be ok. (Sadly recursive:true does not
-      : Platform.environment['ANDROID_HOME'];
+  // obviate the need for existsSync, at least on Windows.)
-  if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
+  final File file = File(path);
+  if (file.existsSync())
+    file.deleteSync();
+}
+enum CiProviders {
+  cirrus,
+  luci,
+}
+Future<void> _processTestOutput(
+  FlutterCompactFormatter formatter,
+  Stream<String> testOutput,
+  bq.TabledataResourceApi tableData,
+) async {
+  final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
+    print('Processing...');
+  });
+  await testOutput.forEach(formatter.processRawOutput);
+  heartbeat.cancel();
+  formatter.finish();
+  if (tableData == null || formatter.tests.isEmpty) {
+    return;
+  }
+  final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
+  final String authors = await _getAuthors();
+  request.rows = List<bq.TableDataInsertAllRequestRows>.from(
+    formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
+      bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
+        'json': <String, dynamic>{
+          'source': <String, dynamic>{
+            'provider': ciProviderName,
+            'url': ciUrl,
+            'platform': <String, dynamic>{
+              'os': Platform.operatingSystem,
+              'version': Platform.operatingSystemVersion,
+            },
+          },
+          'test': <String, dynamic>{
+            'name': result.name,
+            'result': result.status.toString(),
+            'file': result.path,
+            'line': result.line,
+            'column': result.column,
+            'time': result.totalTime,
+          },
+          'git': <String, dynamic>{
+            'author': authors,
+            'pull_request': prNumber,
+            'commit': gitHash,
+            'organization': 'flutter',
+            'repository': 'flutter',
+          },
+          'error': result.status != TestStatus.failed ? null : <String, dynamic>{
+            'message': result.errorMessage,
+            'stack_trace': result.stackTrace,
+          },
+          'information': result.messages,
+        },
+      }),
+    ),
+    growable: false,
+  );
+  final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
+  if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
+    print('${red}BigQuery insert errors:');
+    print(response.toJson());
+    print(reset);
+  }
+}
+CiProviders get ciProvider {
+  if (Platform.environment['CIRRUS_CI'] == 'true') {
+    return CiProviders.cirrus;
+  }
+  if (Platform.environment['LUCI_CONTEXT'] != null) {
+    return CiProviders.luci;
+  }
  return null;
+}
+String get ciProviderName {
+  switch (ciProvider) {
+    case CiProviders.cirrus:
+      return 'cirrusci';
+    case CiProviders.luci:
+      return 'luci';
  }
-  return androidSdkRoot;
+  return 'unknown';
 }
-Future<void> _androidGradleTests(String subShard) async {
+int get prNumber {
-  // TODO(dnfield): gradlew is crashing on the cirrus image and it's not clear why.
+  switch (ciProvider) {
-  if (androidSdkRoot == null || Platform.isWindows) {
+    case CiProviders.cirrus:
-    print('No Android SDK detected or on Windows, skipping Android gradle test.');
+      return Platform.environment['CIRRUS_PR'] == null
-    return;
+          ? -1
+          : int.tryParse(Platform.environment['CIRRUS_PR']);
+    case CiProviders.luci:
+      return -1; // LUCI doesn't know about this.
  }
-  final Map<String, String> defaultEnv = <String, String>{
+  return -1;
-    'ANDROID_HOME': androidSdkRoot,
+}
-    'ANDROID_SDK_ROOT': androidSdkRoot,
-    'ENABLE_ANDROID_EMBEDDING_V2': Platform.environment['ENABLE_ANDROID_EMBEDDING_V2'] ?? '',
+Future<String> _getAuthors() async {
-  };
+  final String exe = Platform.isWindows ? '.exe' : '';
-  if (subShard == 'gradle1') {
+  final String author = await runAndGetStdout(
-    await _runDevicelabTest('gradle_plugin_light_apk_test', env: defaultEnv);
+    'git$exe', <String>['-c', 'log.showSignature=false', 'log', gitHash, '--pretty="%an <%ae>"'],
-    await _runDevicelabTest('gradle_plugin_fat_apk_test', env: defaultEnv);
+    workingDirectory: flutterRoot,
-    await _runDevicelabTest('gradle_r8_test', env: defaultEnv);
+  ).first;
-    await _runDevicelabTest('gradle_non_android_plugin_test', env: defaultEnv);
+  return author;
-    await _runDevicelabTest('gradle_jetifier_test', env: defaultEnv);
+}
-  }
-  if (subShard == 'gradle2') {
+String get ciUrl {
-    await _runDevicelabTest('gradle_plugin_bundle_test', env: defaultEnv);
+  switch (ciProvider) {
-    await _runDevicelabTest('module_test', env: defaultEnv);
+    case CiProviders.cirrus:
-    await _runDevicelabTest('module_host_with_custom_build_test', env: defaultEnv);
+      return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
-    await _runDevicelabTest('build_aar_module_test', env: defaultEnv);
+    case CiProviders.luci:
-    await _runDevicelabTest('plugin_test', env: defaultEnv);
+      return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
  }
+  return '';
 }
-Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard');
+String get gitHash {
-Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard');
+  switch(ciProvider) {
+    case CiProviders.cirrus:
+      return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
+    case CiProviders.luci:
+      return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
+  }
+  return '';
+}
-Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name) async {
+/// Checks the given file's contents to determine if they match the allowed
-  final String item = Platform.environment[key];
+/// pattern for version strings.
-  if (item != null) {
+///
+/// Returns null if the contents are good. Returns a string if they are bad.
+/// The string is an error message.
+Future<String> verifyVersion(File file) async {
+  final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
+  final String version = await file.readAsString();
+  if (!file.existsSync())
+    return 'The version logic failed to create the Flutter version file.';
+  if (version == '0.0.0-unknown')
+    return 'The version logic failed to determine the Flutter version.';
+  if (!version.contains(pattern))
+    return 'The version logic generated an invalid version string: "$version".';
+  return null;
+}
+/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
+/// the shard and subshard (parsing it in the form shard-subshard-platform, ignoring
+/// the platform).
+///
+/// However, for local testing you can just set the SHARD and SUBSHARD
+/// environment variables. For example, to run all the framework tests you can
+/// just set SHARD=framework_tests. To run specifically the third subshard of
+/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
+Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard', 0);
+Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard', 1);
+const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
+Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
+  String item = Platform.environment[key];
+  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
+    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME].split('-');
+    assert(positionInTaskName < parts.length);
+    item = parts[positionInTaskName];
+  }
+  if (item == null) {
+    for (String currentItem in items.keys) {
+      print('$bold$key=$currentItem$reset');
+      await items[currentItem]();
+      print('');
+    }
+  } else {
    if (!items.containsKey(item)) {
      print('${red}Invalid $name: $item$reset');
      print('The available ${name}s are: ${items.keys.join(", ")}');
@@ -987,11 +996,5 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam
    }
    print('$bold$key=$item$reset');
    await items[item]();
-  } else {
-    for (String currentItem in items.keys) {
-      print('$bold$key=$currentItem$reset');
-      await items[currentItem]();
-      print('');
-    }
  }
 }
--- a/dev/bots/test/bot_test.dart
+++ b/dev/bots/test/bot_test.dart
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+import 'dart:io';
+import 'common.dart';
+void main() {
+  test('BOT variable is set on bots', () {
+    expect(Platform.environment['BOT'], 'true');
+  });
+}
--- a/dev/bots/test/sdk_directory_has_space_test.dart
+++ b/dev/bots/test/sdk_directory_has_space_test.dart
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+import 'dart:io';
+import 'package:path/path.dart' as path;
+import 'common.dart';
+void main() {
+  test('We are in a directory with a space in it', () async {
+    // The Flutter SDK should be in a directory with a space in it, to make sure
+    // our tools support that.
+    final String expectedName = Platform.environment['FLUTTER_SDK_PATH_WITH_SPACE'];
+    expect(expectedName, 'flutter sdk');
+    expect(expectedName, contains(' '));
+    final List<String> parts = path.split(Directory.current.absolute.path);
+    expect(parts.reversed.take(3), <String>['bots', 'dev', expectedName]);
+  });
+}
--- a/dev/bots/test/test_test.dart
+++ b/dev/bots/test/test_test.dart
@@ -29,7 +29,11 @@ void main() {
      ];
      for (String version in valid_versions) {
        when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version));
-        expect(await verifyVersion(version, file), isTrue, reason: '$version is invalid');
+        expect(
+          await verifyVersion(file),
+          isNull,
+          reason: '$version is valid but verifyVersionFile said it was bad',
+        );
      }
    });
@@ -41,10 +45,15 @@ void main() {
        '1.2.3-pre',
        '1.2.3-pre.1+hotfix.1',
        '  1.2.3',
+        '1.2.3-hotfix.1',
      ];
      for (String version in invalid_versions) {
        when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version));
-        expect(await verifyVersion(version, file), isFalse);
+        expect(
+          await verifyVersion(file),
+          'The version logic generated an invalid version string: "$version".',
+          reason: '$version is invalid but verifyVersionFile said it was fine',
+        );
      }
    });
  });

--- a/packages/flutter_tools/dart_test.yaml
+++ b/packages/flutter_tools/dart_test.yaml
-tags:
+# package:test configuration
-  "no_coverage":
+# https://github.com/dart-lang/test/blob/master/pkgs/test/doc/configuration.md
-  "create":
-  "integration":
+# Some of our tests take an absurdly long time to run, and on some
+# hosts they can take even longer due to the host suddenly being
+# overloaded. For this reason, we set the test timeout to
+# significantly more than it would be by default, and we never set the
+# timeouts in the tests themselves.
+timeout: 15m
--- a/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart
+++ b/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart
@@ -135,7 +135,7 @@ void main() {
      return _updateIdeConfig(
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
    testUsingContext('creates non-existent files', () async {
      final Map<String, String> templateManifest = _getManifest(
@@ -155,7 +155,7 @@ void main() {
      return _updateIdeConfig(
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
    testUsingContext('overwrites existing files with --overwrite', () async {
      final Map<String, String> templateManifest = _getManifest(
@@ -181,7 +181,7 @@ void main() {
        args: <String>['--overwrite'],
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
    testUsingContext('only adds new templates without --overwrite', () async {
      final Map<String, String> templateManifest = _getManifest(
@@ -212,7 +212,7 @@ void main() {
        args: <String>['--update-templates'],
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
    testUsingContext('update all templates with --overwrite', () async {
      final Map<String, String> templateManifest = _getManifest(
@@ -239,7 +239,7 @@ void main() {
        args: <String>['--update-templates', '--overwrite'],
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
    testUsingContext('removes deleted imls with --overwrite', () async {
      final Map<String, String> templateManifest = _getManifest(
@@ -275,7 +275,7 @@ void main() {
        args: <String>['--update-templates', '--overwrite'],
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
    testUsingContext('removes deleted imls with --overwrite, including empty parent dirs', () async {
      final Map<String, String> templateManifest = _getManifest(
@@ -316,7 +316,7 @@ void main() {
        args: <String>['--update-templates', '--overwrite'],
        expectedContents: expectedContents,
      );
-    }, timeout: const Timeout.factor(2.0));
+    });
  });
 }
--- a/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart
+++ b/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart
@@ -16,9 +16,6 @@ import 'package:flutter_tools/src/runner/flutter_command.dart';
 import '../../src/common.dart';
 import '../../src/context.dart';
-/// Test case timeout for tests involving project analysis.
-const Timeout allowForSlowAnalyzeTests = Timeout.factor(5.0);
 final Generator _kNoColorTerminalPlatform = () => FakePlatform.fromPlatform(const LocalPlatform())..stdoutSupportsAnsi = false;
 final Map<Type, Generator> noColorTerminalOverride = <Type, Generator>{
  Platform: _kNoColorTerminalPlatform,
@@ -54,7 +51,7 @@ void main() {
        ],
      );
      expect(libMain.existsSync(), isTrue);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -65,7 +62,7 @@ void main() {
        arguments: <String>['analyze'],
        statusTextContains: <String>['No issues found!'],
      );
-    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -80,6 +77,7 @@ void main() {
    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
    // Analyze in the current directory - no arguments
    testUsingContext('working directory with errors', () async {
      // Break the code to produce the "The parameter 'onPressed' is required" hint
@@ -110,7 +108,7 @@ void main() {
        exitMessageContains: '2 issues found.',
        toolExit: true,
      );
-    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
      ...noColorTerminalOverride,
    });
@@ -140,7 +138,7 @@ void main() {
        exitMessageContains: '3 issues found.',
        toolExit: true,
      );
-    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
      ...noColorTerminalOverride
    });

--- a/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart
+++ b/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart
@@ -75,7 +75,7 @@ void main() {
    expect(await command.usageValues,
        containsPair(CustomDimensions.commandBuildBundleIsModule, 'true'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('bundle getUsage indicate that project is not a module', () async {
    final String projectPath = await createProject(tempDir,
@@ -85,7 +85,7 @@ void main() {
    expect(await command.usageValues,
        containsPair(CustomDimensions.commandBuildBundleIsModule, 'false'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('bundle getUsage indicate the target platform', () async {
    final String projectPath = await createProject(tempDir,
@@ -95,7 +95,7 @@ void main() {
    expect(await command.usageValues,
        containsPair(CustomDimensions.commandBuildBundleTargetPlatform, 'android-arm'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('bundle fails to build for Windows if feature is disabled', () async {
    fs.file('lib/main.dart').createSync(recursive: true);

--- a/packages/flutter_tools/test/commands.shard/permeable/create_test.dart
+++ b/packages/flutter_tools/test/commands.shard/permeable/create_test.dart
@@ -78,7 +78,7 @@ void main() {
      ],
    );
    return _runFlutterTest(projectDir);
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -96,7 +96,7 @@ void main() {
        'ios/Runner/GeneratedPluginRegistrant.h',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -118,11 +118,9 @@ void main() {
      'ios/',
    ]);
    return _runFlutterTest(projectDir);
-  },
+  }, overrides: <Type, Generator>{
-    timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
    Pub: () => const Pub(),
-    },
+  });
-  );
  testUsingContext('cannot create a project if non-empty non-project directory exists with .metadata', () async {
    await projectDir.absolute.childDirectory('blag').create(recursive: true);
@@ -138,7 +136,7 @@ void main() {
          '.ios/',
        ]),
      throwsToolExit(message: 'Sorry, unable to detect the type of project to recreate'));
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
    ...noColorTerminalOverride,
  });
@@ -164,7 +162,7 @@ void main() {
        '.ios/',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -189,7 +187,7 @@ void main() {
        '.ios/',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -214,7 +212,7 @@ void main() {
        'lib/flutter_project.dart',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -245,7 +243,7 @@ void main() {
        'test/widget_test.dart',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -267,7 +265,7 @@ void main() {
        'ios/Runner/main.m',
      ],
    );
-  }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -297,7 +295,7 @@ void main() {
      ],
    );
    return _runFlutterTest(projectDir);
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -319,7 +317,7 @@ void main() {
      ],
    );
    return _runFlutterTest(projectDir.childDirectory('example'));
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -346,7 +344,7 @@ void main() {
        'example/ios/Runner/main.m',
      ],
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('plugin project with custom org', () async {
    return _createProject(
@@ -366,7 +364,7 @@ void main() {
        'example/android/app/src/main/java/com/example/flutter_project_example/MainActivity.java',
      ],
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('plugin project with valid custom project name', () async {
    return _createProject(
@@ -386,7 +384,7 @@ void main() {
        'example/android/app/src/main/java/com/example/flutter_project_example/MainActivity.java',
      ],
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('plugin project with invalid custom project name', () async {
    expect(
@@ -396,7 +394,7 @@ void main() {
      ),
      throwsToolExit(message: '"xyz.xyz" is not a valid Dart package name.'),
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('legacy app project with-driver-test', () async {
    return _createAndAnalyzeProject(
@@ -404,7 +402,7 @@ void main() {
      <String>['--with-driver-test', '--template=app'],
      <String>['lib/main.dart'],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -439,7 +437,7 @@ void main() {
      'android/',
      'ios/',
    ]);
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -463,7 +461,7 @@ void main() {
    final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
    expect(actualContents.contains('useAndroidX'), true);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('non androidx app project', () async {
    Cache.flutterRoot = '../..';
@@ -484,7 +482,7 @@ void main() {
    final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
    expect(actualContents.contains('useAndroidX'), false);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('androidx is used by default in a module project', () async {
    Cache.flutterRoot = '../..';
@@ -501,7 +499,7 @@ void main() {
      project.usesAndroidX,
      true,
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('non androidx module', () async {
    Cache.flutterRoot = '../..';
@@ -518,7 +516,7 @@ void main() {
      project.usesAndroidX,
      false,
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('androidx is used by default in a plugin project', () async {
    Cache.flutterRoot = '../..';
@@ -539,7 +537,7 @@ void main() {
    final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
    expect(actualContents.contains('useAndroidX'), true);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('non androidx plugin project', () async {
    Cache.flutterRoot = '../..';
@@ -560,7 +558,7 @@ void main() {
    final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
    expect(actualContents.contains('useAndroidX'), false);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('app supports macOS if requested', () async {
    Cache.flutterRoot = '../..';
@@ -573,7 +571,7 @@ void main() {
    await runner.run(<String>['create', '--no-pub', '--macos', projectDir.path]);
    expect(projectDir.childDirectory('macos').childDirectory('Runner.xcworkspace').existsSync(), true);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('app does not include macOS by default', () async {
    Cache.flutterRoot = '../..';
@@ -586,7 +584,7 @@ void main() {
    await runner.run(<String>['create', '--no-pub', projectDir.path]);
    expect(projectDir.childDirectory('macos').childDirectory('Runner.xcworkspace').existsSync(), false);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('plugin supports macOS if requested', () async {
    Cache.flutterRoot = '../..';
@@ -599,7 +597,7 @@ void main() {
    await runner.run(<String>['create', '--no-pub', '--template=plugin', '--macos', projectDir.path]);
    expect(projectDir.childDirectory('macos').childFile('flutter_project.podspec').existsSync(), true);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('plugin does not include macOS by default', () async {
    Cache.flutterRoot = '../..';
@@ -612,7 +610,7 @@ void main() {
    await runner.run(<String>['create', '--no-pub', '--template=plugin', projectDir.path]);
    expect(projectDir.childDirectory('macos').childFile('flutter_project.podspec').existsSync(), false);
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('has correct content and formatting with module template', () async {
    Cache.flutterRoot = '../..';
@@ -708,7 +706,7 @@ void main() {
  }, overrides: <Type, Generator>{
    FlutterVersion: () => mockFlutterVersion,
    Platform: _kNoColorTerminalPlatform,
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('has correct content and formatting with app template', () async {
    Cache.flutterRoot = '../..';
@@ -779,7 +777,7 @@ void main() {
  }, overrides: <Type, Generator>{
    FlutterVersion: () => mockFlutterVersion,
    Platform: _kNoColorTerminalPlatform,
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('has correct application id for android and bundle id for ios', () async {
    Cache.flutterRoot = '../..';
@@ -827,7 +825,7 @@ void main() {
  }, overrides: <Type, Generator>{
    FlutterVersion: () => mockFlutterVersion,
    Platform: _kNoColorTerminalPlatform,
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen default template over existing project', () async {
    Cache.flutterRoot = '../..';
@@ -841,7 +839,7 @@ void main() {
    final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
    expect(metadata, contains('project_type: app\n'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen default template over existing app project with no metadta and detect the type', () async {
    Cache.flutterRoot = '../..';
@@ -858,7 +856,7 @@ void main() {
    final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
    expect(metadata, contains('project_type: app\n'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen app template over existing app project and detect the type', () async {
    Cache.flutterRoot = '../..';
@@ -872,7 +870,7 @@ void main() {
    final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
    expect(metadata, contains('project_type: app\n'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen template over existing module project and detect the type', () async {
    Cache.flutterRoot = '../..';
@@ -886,7 +884,7 @@ void main() {
    final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
    expect(metadata, contains('project_type: module\n'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen default template over existing plugin project and detect the type', () async {
    Cache.flutterRoot = '../..';
@@ -900,7 +898,7 @@ void main() {
    final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
    expect(metadata, contains('project_type: plugin'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen default template over existing package project and detect the type', () async {
    Cache.flutterRoot = '../..';
@@ -914,7 +912,7 @@ void main() {
    final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
    expect(metadata, contains('project_type: package'));
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen module .android/ folder, reusing custom org', () async {
    await _createProject(
@@ -930,7 +928,7 @@ void main() {
        '.android/app/src/main/java/com/bar/foo/flutter_project/host/MainActivity.java',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -947,7 +945,7 @@ void main() {
      await project.ios.productBundleIdentifier,
      'com.bar.foo.flutterProject',
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -974,7 +972,7 @@ void main() {
        'android/app/src/main/java/com/example/flutter_project/MainActivity.java',
      ],
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen app ios/ folder, reusing custom org', () async {
    await _createProject(
@@ -989,7 +987,7 @@ void main() {
      await project.ios.productBundleIdentifier,
      'com.bar.foo.flutterProject',
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('can re-gen plugin ios/ and example/ folders, reusing custom org', () async {
    await _createProject(
@@ -1022,7 +1020,7 @@ void main() {
      await project.example.ios.productBundleIdentifier,
      'com.bar.foo.flutterProjectExample',
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  testUsingContext('fails to re-gen without specified org when org is ambiguous', () async {
    await _createProject(
@@ -1040,7 +1038,7 @@ void main() {
      () => _createProject(projectDir, <String>[], <String>[]),
      throwsToolExit(message: 'Ambiguous organization'),
    );
-  }, timeout: allowForCreateFlutterProject);
+  });
  // Verify that we help the user correct an option ordering issue
  testUsingContext('produces sensible error message', () async {
@@ -1102,7 +1100,7 @@ void main() {
        'ios/Runner/GeneratedPluginRegistrant.h',
      ],
    );
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    Pub: () => const Pub(),
  });
@@ -1128,7 +1126,6 @@ void main() {
      expect(loggingProcessManager.commands.first, contains(matches(r'dart-sdk[\\/]bin[\\/]pub')));
      expect(loggingProcessManager.commands.first, contains('--offline'));
    },
-    timeout: allowForCreateFlutterProject,
    overrides: <Type, Generator>{
      ProcessManager: () => loggingProcessManager,
      Pub: () => const Pub(),
@@ -1147,7 +1144,6 @@ void main() {
      expect(loggingProcessManager.commands.first, contains(matches(r'dart-sdk[\\/]bin[\\/]pub')));
      expect(loggingProcessManager.commands.first, isNot(contains('--offline')));
    },
-    timeout: allowForCreateFlutterProject,
    overrides: <Type, Generator>{
      ProcessManager: () => loggingProcessManager,
      Pub: () => const Pub(),
@@ -1168,7 +1164,7 @@ void main() {
    );
    expect(projectDir.childDirectory('lib').childFile('main.dart').readAsStringSync(),
      contains('void main() {}'));
-  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+  }, overrides: <Type, Generator>{
    HttpClientFactory: () => () => MockHttpClient(200, result: 'void main() {}'),
  });

--- a/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart
+++ b/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart
@@ -207,7 +207,7 @@ void main() {
      expectDependenciesResolved(projectPath);
      expectZeroPluginsInjected(projectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -220,7 +220,7 @@ void main() {
      expectDependenciesResolved(projectPath);
      expectZeroPluginsInjected(projectPath);
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -234,7 +234,7 @@ void main() {
      expect(await getCommand.usageValues,
             containsPair(CustomDimensions.commandPackagesNumberPlugins, '0'));
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -248,7 +248,7 @@ void main() {
      expect(await getCommand.usageValues,
             containsPair(CustomDimensions.commandPackagesProjectModule, 'false'));
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -262,7 +262,7 @@ void main() {
      expect(await getCommand.usageValues,
             containsPair(CustomDimensions.commandPackagesProjectModule, 'true'));
-    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -275,7 +275,7 @@ void main() {
      expectDependenciesResolved(projectPath);
      expectZeroPluginsInjected(projectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -288,7 +288,7 @@ void main() {
      expectDependenciesResolved(projectPath);
      expectModulePluginInjected(projectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
@@ -309,7 +309,7 @@ void main() {
      expectDependenciesResolved(exampleProjectPath);
      expectPluginInjected(exampleProjectPath);
-    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
+    }, overrides: <Type, Generator>{
      Pub: () => const Pub(),
    });
  });

--- a/packages/flutter_tools/test/general.shard/commands/build_aar_test.dart
+++ b/packages/flutter_tools/test/general.shard/commands/build_aar_test.dart
@@ -57,7 +57,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
    testUsingContext('indicate that project is a plugin', () async {
      final String projectPath = await createProject(tempDir,
@@ -69,7 +69,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
    testUsingContext('indicate the target platform', () async {
      final String projectPath = await createProject(tempDir,
@@ -82,7 +82,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
  });
  group('Gradle', () {

--- a/packages/flutter_tools/test/general.shard/commands/build_apk_test.dart
+++ b/packages/flutter_tools/test/general.shard/commands/build_apk_test.dart
@@ -48,7 +48,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
    testUsingContext('split per abi', () async {
      final String projectPath = await createProject(tempDir,
@@ -65,7 +65,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
    testUsingContext('build type', () async {
      final String projectPath = await createProject(tempDir,
@@ -92,7 +92,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
  });
  group('Gradle', () {
@@ -223,8 +223,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      GradleUtils: () => GradleUtils(),
      ProcessManager: () => mockProcessManager,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('shrinking is disabled when --no-shrink is passed', () async {
      final String projectPath = await createProject(tempDir,
@@ -255,8 +254,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      GradleUtils: () => GradleUtils(),
      ProcessManager: () => mockProcessManager,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('guides the user when the shrinker fails', () async {
      final String projectPath = await createProject(tempDir,
@@ -312,8 +310,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      ProcessManager: () => mockProcessManager,
      Usage: () => mockUsage,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('reports when the app isn\'t using AndroidX', () async {
      final String projectPath = await createProject(tempDir,
@@ -366,8 +363,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      ProcessManager: () => mockProcessManager,
      Usage: () => mockUsage,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('reports when the app is using AndroidX', () async {
      final String projectPath = await createProject(tempDir,
@@ -422,8 +418,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      ProcessManager: () => mockProcessManager,
      Usage: () => mockUsage,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
  });
 }

--- a/packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart
+++ b/packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart
@@ -48,7 +48,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
    testUsingContext('build type', () async {
      final String projectPath = await createProject(tempDir,
@@ -75,7 +75,7 @@ void main() {
    }, overrides: <Type, Generator>{
      AndroidBuilder: () => FakeAndroidBuilder(),
-    }, timeout: allowForCreateFlutterProject);
+    });
  });
  group('Gradle', () {
@@ -212,8 +212,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      GradleUtils: () => GradleUtils(),
      ProcessManager: () => mockProcessManager,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('shrinking is disabled when --no-shrink is passed', () async {
      final String projectPath = await createProject(
@@ -246,8 +245,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      GradleUtils: () => GradleUtils(),
      ProcessManager: () => mockProcessManager,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('guides the user when the shrinker fails', () async {
      final String projectPath = await createProject(tempDir,
@@ -303,8 +301,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      ProcessManager: () => mockProcessManager,
      Usage: () => mockUsage,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('reports when the app isn\'t using AndroidX', () async {
      final String projectPath = await createProject(tempDir,
@@ -357,8 +354,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      ProcessManager: () => mockProcessManager,
      Usage: () => mockUsage,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
    testUsingContext('reports when the app is using AndroidX', () async {
      final String projectPath = await createProject(tempDir,
@@ -413,8 +409,7 @@ flutter:
      FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
      ProcessManager: () => mockProcessManager,
      Usage: () => mockUsage,
-    },
+    });
-    timeout: allowForCreateFlutterProject);
  });
 }

--- a/packages/flutter_tools/test/integration.shard/daemon_mode_test.dart
+++ b/packages/flutter_tools/test/integration.shard/daemon_mode_test.dart
@@ -2,13 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
 import 'dart:convert';
+import 'dart:io';
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -21,49 +17,26 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 void main() {
-  group('daemon_mode', () {
+  test('device.getDevices', () async {
-    Directory tempDir;
+    final Directory tempDir = createResolvedTempDirectorySync('daemon_mode_test.');
-    final BasicProject _project = BasicProject();
-    Process process;
-    setUp(() async {
+    final BasicProject _project = BasicProject();
-      tempDir = createResolvedTempDirectorySync('daemon_mode_test.');
    await _project.setUpIn(tempDir);
-    });
-    tearDown(() async {
-      tryToDelete(tempDir);
-      process?.kill();
-    });
-    test('device.getDevices', () async {
+    final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter');
-      final String flutterBin =
-          fs.path.join(getFlutterRoot(), 'bin', 'flutter');
    const ProcessManager processManager = LocalProcessManager();
-      process = await processManager.start(
+    final Process process = await processManager.start(
      <String>[flutterBin, '--show-test-device', 'daemon'],
-          workingDirectory: tempDir.path);
+      workingDirectory: tempDir.path,
+    );
-      final StreamController<String> stdout =
-          StreamController<String>.broadcast();
-      transformToLines(process.stdout)
+    final StreamController<String> stdout = StreamController<String>.broadcast();
-          .listen((String line) => stdout.add(line));
+    transformToLines(process.stdout).listen((String line) => stdout.add(line));
+    final Stream<Map<String, dynamic>> stream = stdout
-      final Stream<Map<String, dynamic>> stream =
+      .stream
-        stdout.stream.where((String line) {
+      .map<Map<String, dynamic>>(parseFlutterResponse)
-          final Map<String, dynamic> response = parseFlutterResponse(line);
+      .where((Map<String, dynamic> value) => value != null);
-          // ignore 'Starting device daemon...'
-          if (response == null) {
-            return false;
-          }
-          // TODO(devoncarew): Remove this after #25440 lands.
-          if (response['event'] == 'daemon.showMessage') {
-            return false;
-          }
-          return true;
-        }).map(parseFlutterResponse);
    Map<String, dynamic> response = await stream.first;
    expect(response['event'], 'daemon.connected');
@@ -94,6 +67,8 @@ void main() {
    final dynamic result = response['result'];
    expect(result, isList);
    expect(result, isNotEmpty);
+    tryToDelete(tempDir);
+    process.kill();
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // This test uses the `flutter` tool, which could be blocked behind the startup lock for a long time.
 }
--- a/packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart
+++ b/packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart
@@ -2,11 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
+import 'dart:io';
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -16,23 +13,14 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 void main() {
-  group('debugger', () {
+  test('can step over statements', () async {
-    Directory tempDir;
+    final Directory tempDir = createResolvedTempDirectorySync('debugger_stepping_test.');
-    final SteppingProject _project = SteppingProject();
-    FlutterRunTestDriver _flutter;
-    setUp(() async {
+    final SteppingProject _project = SteppingProject();
-      tempDir = createResolvedTempDirectorySync('debugger_stepping_test.');
    await _project.setUpIn(tempDir);
-      _flutter = FlutterRunTestDriver(tempDir);
-    });
-    tearDown(() async {
+    final FlutterRunTestDriver _flutter = FlutterRunTestDriver(tempDir);
-      await _flutter.stop();
-      tryToDelete(tempDir);
-    });
-    test('can step over statements', () async {
    await _flutter.run(withDebugger: true, startPaused: true);
    await _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine);
    await _flutter.resume();
@@ -51,8 +39,11 @@ void main() {
      final int expectedLine = _project.lineForStep(i);
      expect(actualLine, equals(expectedLine),
-          reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine');
+        reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine'
+      );
    }
+    await _flutter.stop();
+    tryToDelete(tempDir);
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
--- a/packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart
+++ b/packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart
@@ -2,12 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
+import 'dart:io';
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -20,22 +16,21 @@ import 'test_data/tests_project.dart';
 import 'test_driver.dart';
 import 'test_utils.dart';
-void main() {
+void batch1() {
-  group('flutter run expression evaluation', () {
-    Directory tempDir;
  final BasicProject _project = BasicProject();
+  Directory tempDir;
  FlutterRunTestDriver _flutter;
-    setUp(() async {
+  Future<void> initProject() async {
    tempDir = createResolvedTempDirectorySync('run_expression_eval_test.');
    await _project.setUpIn(tempDir);
    _flutter = FlutterRunTestDriver(tempDir);
-    });
+  }
-    tearDown(() async {
+  Future<void> cleanProject() async {
    await _flutter.stop();
    tryToDelete(tempDir);
-    });
+  }
  Future<void> breakInBuildMethod(FlutterTestDriver flutter) async {
    await _flutter.breakAt(
@@ -51,89 +46,103 @@ void main() {
    );
  }
-    test('can evaluate trivial expressions in top level function', () async {
+  test('flutter run expression evaluation - can evaluate trivial expressions in top level function', () async {
+    await initProject();
    await _flutter.run(withDebugger: true);
    await breakInTopLevelFunction(_flutter);
    await evaluateTrivialExpressions(_flutter);
+    await cleanProject();
  });
-    test('can evaluate trivial expressions in build method', () async {
+  test('flutter run expression evaluation - can evaluate trivial expressions in build method', () async {
+    await initProject();
    await _flutter.run(withDebugger: true);
    await breakInBuildMethod(_flutter);
    await evaluateTrivialExpressions(_flutter);
+    await cleanProject();
  });
-    test('can evaluate complex expressions in top level function', () async {
+  test('flutter run expression evaluation - can evaluate complex expressions in top level function', () async {
+    await initProject();
    await _flutter.run(withDebugger: true);
    await breakInTopLevelFunction(_flutter);
    await evaluateComplexExpressions(_flutter);
+    await cleanProject();
  });
-    test('can evaluate complex expressions in build method', () async {
+  test('flutter run expression evaluation - can evaluate complex expressions in build method', () async {
+    await initProject();
    await _flutter.run(withDebugger: true);
    await breakInBuildMethod(_flutter);
    await evaluateComplexExpressions(_flutter);
+    await cleanProject();
  });
-    test('can evaluate expressions returning complex objects in top level function', () async {
+  test('flutter run expression evaluation - can evaluate expressions returning complex objects in top level function', () async {
+    await initProject();
    await _flutter.run(withDebugger: true);
    await breakInTopLevelFunction(_flutter);
    await evaluateComplexReturningExpressions(_flutter);
+    await cleanProject();
  });
-    test('can evaluate expressions returning complex objects in build method', () async {
+  test('flutter run expression evaluation - can evaluate expressions returning complex objects in build method', () async {
+    await initProject();
    await _flutter.run(withDebugger: true);
    await breakInBuildMethod(_flutter);
    await evaluateComplexReturningExpressions(_flutter);
+    await cleanProject();
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
+}
-  group('flutter test expression evaluation', () {
+void batch2() {
-    Directory tempDir;
  final TestsProject _project = TestsProject();
+  Directory tempDir;
  FlutterTestTestDriver _flutter;
-    setUp(() async {
+  Future<void> initProject() async {
    tempDir = createResolvedTempDirectorySync('test_expression_eval_test.');
    await _project.setUpIn(tempDir);
    _flutter = FlutterTestTestDriver(tempDir);
-    });
+  }
-    tearDown(() async {
+  Future<void> cleanProject() async {
-      await _flutter.quit();
+    await _flutter?.quit();
    tryToDelete(tempDir);
-    });
+  }
-    test('can evaluate trivial expressions in a test', () async {
+  test('flutter test expression evaluation - can evaluate trivial expressions in a test', () async {
+    await initProject();
    await _flutter.test(
      withDebugger: true,
      beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
    );
    await _flutter.waitForPause();
    await evaluateTrivialExpressions(_flutter);
-      await _flutter.resume();
+    await cleanProject();
  });
-    test('can evaluate complex expressions in a test', () async {
+  test('flutter test expression evaluation - can evaluate complex expressions in a test', () async {
+    await initProject();
    await _flutter.test(
      withDebugger: true,
      beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
    );
    await _flutter.waitForPause();
    await evaluateComplexExpressions(_flutter);
-      await _flutter.resume();
+    await cleanProject();
  });
-    test('can evaluate expressions returning complex objects in a test', () async {
+  test('flutter test expression evaluation - can evaluate expressions returning complex objects in a test', () async {
+    await initProject();
    await _flutter.test(
      withDebugger: true,
      beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
    );
    await _flutter.waitForPause();
    await evaluateComplexReturningExpressions(_flutter);
-      await _flutter.resume();
+    await cleanProject();
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
 Future<void> evaluateTrivialExpressions(FlutterTestDriver flutter) async {
@@ -164,3 +173,8 @@ Future<void> evaluateComplexReturningExpressions(FlutterTestDriver flutter) asyn
  final InstanceRef res = await flutter.evaluate(resp.id, r'"$year-$month-$day"');
  expect(res.valueAsString, equals('${now.year}-${now.month}-${now.day}'));
 }
+void main() {
+  batch1();
+  batch2();
+}
--- a/packages/flutter_tools/test/integration.shard/flutter_attach_test.dart
+++ b/packages/flutter_tools/test/integration.shard/flutter_attach_test.dart
@@ -2,11 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
+import 'dart:io';
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -33,7 +30,6 @@ void main() {
    tryToDelete(tempDir);
  });
-  group('attached process', () {
  test('writes pid-file', () async {
    final File pidFile = tempDir.childFile('test.pid');
    await _flutterRun.run(withDebugger: true);
@@ -43,11 +39,13 @@ void main() {
    );
    expect(pidFile.existsSync(), isTrue);
  });
  test('can hot reload', () async {
    await _flutterRun.run(withDebugger: true);
    await _flutterAttach.attach(_flutterRun.vmServicePort);
    await _flutterAttach.hotReload();
  });
  test('can detach, reattach, hot reload', () async {
    await _flutterRun.run(withDebugger: true);
    await _flutterAttach.attach(_flutterRun.vmServicePort);
@@ -55,6 +53,7 @@ void main() {
    await _flutterAttach.attach(_flutterRun.vmServicePort);
    await _flutterAttach.hotReload();
  });
  test('killing process behaves the same as detach ', () async {
    await _flutterRun.run(withDebugger: true);
    await _flutterAttach.attach(_flutterRun.vmServicePort);
@@ -63,5 +62,4 @@ void main() {
    await _flutterAttach.attach(_flutterRun.vmServicePort);
    await _flutterAttach.hotReload();
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
--- a/packages/flutter_tools/test/integration.shard/flutter_run_test.dart
+++ b/packages/flutter_tools/test/integration.shard/flutter_run_test.dart
@@ -2,11 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/io.dart';
@@ -18,7 +13,6 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 void main() {
-  group('flutter_run', () {
  Directory tempDir;
  final BasicProject _project = BasicProject();
  FlutterRunTestDriver _flutter;
@@ -34,7 +28,7 @@ void main() {
    tryToDelete(tempDir);
  });
-    test('reports an error if an invalid device is supplied', () async {
+  test('flutter run reports an error if an invalid device is supplied', () async {
    // This test forces flutter to check for all possible devices to catch issues
    // like https://github.com/flutter/flutter/issues/21418 which were skipped
    // over because other integration tests run using flutter-tester which short-cuts
@@ -55,10 +49,9 @@ void main() {
    }
  });
-    test('writes pid-file', () async {
+  test('flutter run writes pid-file', () async {
    final File pidFile = tempDir.childFile('test.pid');
    await _flutter.run(pidFile: pidFile);
    expect(pidFile.existsSync(), isTrue);
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
--- a/packages/flutter_tools/test/integration.shard/hot_reload_test.dart
+++ b/packages/flutter_tools/test/integration.shard/hot_reload_test.dart
@@ -2,11 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
 import 'package:file/file.dart';
@@ -20,7 +15,6 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 void main() {
-  group('hot reload tests', () {
  Directory tempDir;
  final HotReloadProject _project = HotReloadProject();
  FlutterRunTestDriver _flutter;
@@ -154,5 +148,4 @@ void main() {
    await _flutter.resume();
    await subscription.cancel();
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
--- a/packages/flutter_tools/test/integration.shard/lifetime_test.dart
+++ b/packages/flutter_tools/test/integration.shard/lifetime_test.dart
@@ -2,11 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-// Integration tests which invoke flutter instead of unit testing the code
-// will not produce meaningful coverage information - we can measure coverage
-// from the isolate running the test, but not from the isolate started via
-// the command line process.
-@Tags(<String>['no_coverage'])
 import 'dart:async';
 import 'package:file/file.dart';
@@ -23,7 +18,6 @@ import 'test_utils.dart';
 const Duration requiredLifespan = Duration(seconds: 5);
 void main() {
-  group('flutter run', () {
  final BasicProject _project = BasicProject();
  FlutterRunTestDriver _flutter;
  Directory tempDir;
@@ -39,16 +33,15 @@ void main() {
    tryToDelete(tempDir);
  });
-    test('does not terminate when a debugger is attached', () async {
+  test('flutter run does not terminate when a debugger is attached', () async {
    await _flutter.run(withDebugger: true);
    await Future<void>.delayed(requiredLifespan);
    expect(_flutter.hasExited, equals(false));
  });
-    test('does not terminate when a debugger is attached and pause-on-exceptions', () async {
+  test('fluter run does not terminate when a debugger is attached and pause-on-exceptions', () async {
    await _flutter.run(withDebugger: true, pauseOnExceptions: true);
    await Future<void>.delayed(requiredLifespan);
    expect(_flutter.hasExited, equals(false));
  });
-  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
--- a/packages/flutter_tools/test/src/common.dart
+++ b/packages/flutter_tools/test/src/common.dart
@@ -127,13 +127,6 @@ Future<String> createProject(Directory temp, { List<String> arguments }) async {
  return projectPath;
 }
-/// Test case timeout for tests involving remote calls to `pub get` or similar.
-const Timeout allowForRemotePubInvocation = Timeout.factor(10.0);
-/// Test case timeout for tests involving creating a Flutter project with
-/// `--no-pub`. Use [allowForRemotePubInvocation] when creation involves `pub`.
-const Timeout allowForCreateFlutterProject = Timeout.factor(3.0);
 Future<void> expectToolExitLater(Future<dynamic> future, Matcher messageMatcher) async {
  try {
    await future;

--- a/packages/flutter_tools/test/src/context.dart
+++ b/packages/flutter_tools/test/src/context.dart
@@ -48,7 +48,6 @@ typedef ContextInitializer = void Function(AppContext testContext);
 void testUsingContext(
  String description,
  dynamic testMethod(), {
-  Timeout timeout,
  Map<Type, Generator> overrides = const <Type, Generator>{},
  bool initializeFlutterRoot = true,
  String testOn,
@@ -137,8 +136,7 @@ void testUsingContext(
        },
      );
    });
-  }, timeout: timeout ?? const Timeout(Duration(seconds: 60)),
+  }, testOn: testOn, skip: skip);
-      testOn: testOn, skip: skip);
 }
 void _printBufferedErrors(AppContext testContext) {