Shard Cirrus build_tests (#56735)

.cirrus.yml

@@ -236,7 +236,17 @@ task:
     - name: web_tests-7_last-linux # last Web shard must end with _last
       << : *WEB_SHARD_TEMPLATE
 
-    - name: build_tests-linux
+    - name: build_tests-0-linux
+      environment:
+        # With 1 CPU and 4G of RAM, as of October 2019, build_tests-linux would get OOM-killed.
+        # Increasing the RAM to 12G allowed it to finish in about 30 minutes, any extra CPU (tried 2
+        # and 4) reduced that to just over 20 minutes. 6G was enough not to get OOM-killed.
+        CPU: 2
+        MEMORY: 6G
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+
+    - name: build_tests-1_last-linux
       environment:
         # With 1 CPU and 4G of RAM, as of October 2019, build_tests-linux would get OOM-killed.
         # Increasing the RAM to 12G allowed it to finish in about 30 minutes, any extra CPU (tried 2
@@ -289,7 +299,7 @@ task:
     - name: firebase_test_lab_tests-1-linux
       <<: *FIREBASE_SHARD_TEMPLATE
 
-    - name: firebase_test_lab_tests-2-linux
+    - name: firebase_test_lab_tests-2_last-linux
       <<: *FIREBASE_SHARD_TEMPLATE
 
     - name: web_smoke_test
@@ -436,7 +446,18 @@ task:
 
     # TODO(ianh): Enable Web tests on Windows
 
-    - name: build_tests-windows
+    - name: build_tests-0-windows
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
+      environment:
+        # As of December 2019, the build_tests-windows shard requires 6 GB RAM to pass.
+        # Emperically, using 6 CPUs and 10 GB RAM yielded optimal results (~33 minutes);
+        # bumping beyond these limits yielded no extra gain.
+        CPU: 6
+        MEMORY: 10G
+      script:
+        - dart --enable-asserts dev\bots\test.dart
+
+    - name: build_tests-1_last-windows
       only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941
       environment:
         # As of December 2019, the build_tests-windows shard requires 6 GB RAM to pass.
@@ -577,7 +598,12 @@ task:
 
     # TODO(ianh): Enable Web tests on macOS.
 
-    - name: build_tests-macos
+    - name: build_tests-0-macos
+      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
+      script:
+        - dart --enable-asserts ./dev/bots/test.dart
+
+    - name: build_tests-1_last-macos
       only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940
       script:
         - dart --enable-asserts ./dev/bots/test.dart

dev/bots/test.dart

@@ -61,6 +61,12 @@ final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTE
 /// and make sure it runs _all_ shards.
 const int kDeviceLabShardCount = 4;
 
+/// The number of Cirrus jobs that run build tests in parallel.
+///
+/// WARNING: if you change this number, also change .cirrus.yml
+/// and make sure it runs _all_ shards.
+const int kBuildTestShardCount = 2;
+
 /// The number of Cirrus jobs that run Web tests in parallel.
 ///
 /// The default is 8 shards. Typically .cirrus.yml would define the
@@ -336,45 +342,55 @@ Future<void> _runBuildTests() async {
   final List<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync()
     ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
     ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery')));
-  for (final FileSystemEntity fileEntity in exampleDirectories) {
-    // Only verify caching with flutter gallery.
-    final bool verifyCaching = fileEntity.path.contains('flutter_gallery');
-    if (fileEntity is! Directory) {
-      continue;
-    }
-    final String examplePath = fileEntity.path;
-    final bool hasNullSafety = File(path.join(examplePath, 'null_safety')).existsSync();
-    final List<String> additionalArgs = hasNullSafety
-      ? <String>['--enable-experiment', 'non-nullable']
-      : <String>[];
-    if (Directory(path.join(examplePath, 'android')).existsSync()) {
-      await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
-      await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
-    } else {
-      print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
-    }
-    if (Platform.isMacOS) {
-      if (Directory(path.join(examplePath, 'ios')).existsSync()) {
-        await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
-        await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
-      } else {
-        print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
-      }
-    }
-  }
 
   final String branch = Platform.environment['CIRRUS_BRANCH'];
-  if (branch != 'beta' && branch != 'stable') {
-    // Web compilation tests.
-    await _flutterBuildDart2js(
-      path.join('dev', 'integration_tests', 'web'),
-      path.join('lib', 'main.dart'),
-    );
-    // Should not fail to compile with dart:io.
-    await _flutterBuildDart2js(
-      path.join('dev', 'integration_tests', 'web_compile_tests'),
-      path.join('lib', 'dart_io_import.dart'),
-    );
+  // The tests are randomly distributed into subshards so as to get a uniform
+  // distribution of costs, but the seed is fixed so that issues are reproducible.
+  final List<ShardRunner> tests = <ShardRunner>[
+    for (final FileSystemEntity exampleDirectory in exampleDirectories)
+        () => _runExampleProjectBuildTests(exampleDirectory),
+    if (branch != 'beta' && branch != 'stable')
+      ...<ShardRunner>[
+        // Web compilation tests.
+          () =>  _flutterBuildDart2js(
+          path.join('dev', 'integration_tests', 'web'),
+          path.join('lib', 'main.dart'),
+        ),
+        // Should not fail to compile with dart:io.
+          () =>  _flutterBuildDart2js(
+          path.join('dev', 'integration_tests', 'web_compile_tests'),
+          path.join('lib', 'dart_io_import.dart'),
+        ),
+      ],
+  ]..shuffle(math.Random(0));
+
+  await _selectIndexedSubshard(tests, kBuildTestShardCount);
+}
+
+Future<void> _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async {
+  // Only verify caching with flutter gallery.
+  final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
+  if (exampleDirectory is! Directory) {
+    return;
+  }
+  final String examplePath = exampleDirectory.path;
+  final bool hasNullSafety = File(path.join(examplePath, 'null_safety')).existsSync();
+  final List<String> additionalArgs = hasNullSafety
+    ? <String>['--enable-experiment', 'non-nullable']
+    : <String>[];
+  if (Directory(path.join(examplePath, 'android')).existsSync()) {
+    await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
+    await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
+  } else {
+    print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
+  }
+  if (Platform.isMacOS) {
+    if (Directory(path.join(examplePath, 'ios')).existsSync()) {
+      await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
+      await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
+    } else {
+      print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
+    }
   }
 }
 
@@ -646,21 +662,15 @@ Future<void> _runFirebaseTestLabTests() async {
     'abstract_method_smoke_test',
     'android_embedding_v2_smoke_test',
   ];
-  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
 
   final String firebaseScript = path.join(flutterRoot, 'dev', 'bots', 'firebase_testlab.sh');
   final String integrationTestDirectory = path.join(flutterRoot, 'dev', 'integration_tests');
 
-  for (int index = 0; index < integrationTests.length; index += 1) {
-    final String integrationTestPath = path.join(integrationTestDirectory, integrationTests[index]);
-    subshards['$index'] = () => runCommand(
-      firebaseScript,
-      <String>[ integrationTestPath ],
-      workingDirectory: flutterRoot,
-    );
-  }
+  final List<ShardRunner> tests = integrationTests.map((String integrationTest) =>
+    () => runCommand(firebaseScript, <String>[ path.join(integrationTestDirectory, integrationTest) ])
+  ).toList();
 
-  await selectSubshard(subshards);
+  await _selectIndexedSubshard(tests, integrationTests.length);
 }
 
 Future<void> _runFrameworkCoverage() async {
@@ -1169,27 +1179,7 @@ Future<void> _runHostOnlyDeviceLabTests() async {
     if (Platform.isLinux) () => _runDevicelabTest('web_benchmarks_canvaskit', environment: kChromeVariables),
   ]..shuffle(math.Random(0));
 
-  final int testsPerShard = tests.length ~/ kDeviceLabShardCount;
-  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
-
-  for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) {
-    String last = '';
-    List<ShardRunner> sublist;
-    if (subshard < kDeviceLabShardCount - 1) {
-      sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard);
-    } else {
-      sublist = tests.sublist(subshard * testsPerShard, tests.length);
-      // We make sure the last shard ends in _last so it's easier to catch mismatches
-      // between `.cirrus.yml` and `test.dart`.
-      last = '_last';
-    }
-    subshards['$subshard$last'] = () async {
-      for (final ShardRunner test in sublist)
-        await test();
-    };
-  }
-
-  await selectSubshard(subshards);
+  await _selectIndexedSubshard(tests, kDeviceLabShardCount);
 }
 
 Future<void> _runDevicelabTest(String testName, {
@@ -1366,6 +1356,37 @@ Future<String> verifyVersion(File file) async {
   return null;
 }
 
+/// Parse (zero-)index-named subshards and equally distribute [tests]
+/// between them. Last shard should end in "_last" to catch mismatches
+/// between `.cirrus.yml` and `test.dart`. See [selectShard] for naming details.
+///
+/// Examples:
+/// build_tests-0-linux
+/// build_tests-1-linux
+/// build_tests-2_last-linux
+Future<void> _selectIndexedSubshard(List<ShardRunner> tests, int numberOfShards) async {
+  final int testsPerShard = tests.length ~/ numberOfShards;
+  final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
+
+  for (int subshard = 0; subshard < numberOfShards; subshard += 1) {
+    String last = '';
+    List<ShardRunner> sublist;
+    if (subshard < numberOfShards - 1) {
+      sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard);
+    } else {
+      sublist = tests.sublist(subshard * testsPerShard, tests.length);
+      // We make sure the last shard ends in _last.
+      last = '_last';
+    }
+    subshards['$subshard$last'] = () async {
+      for (final ShardRunner test in sublist)
+        await test();
+    };
+  }
+
+  await selectSubshard(subshards);
+}
+
 /// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
 /// the shard and sub-shard (parsing it in the form shard-subshard-platform, ignoring
 /// the platform).