consolidate all web integration tests under the same shard (#82307)

* consolidate all web integration tests under the same shard

.cirrus.yml

@@ -140,16 +140,6 @@ task:
         - (cd dev/bots; pub get)
         - dart --enable-asserts ./dev/bots/test.dart
 
-    - name: web_integration_tests
-      only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''"
-      environment:
-        # As of October 2019, the Web shards needed more than 6G of RAM.
-        CPU: 2
-        MEMORY: 8G
-        CHROME_NO_SANDBOX: true
-      script:
-        - dart --enable-asserts ./dev/bots/test.dart
-
     - name: docs-linux # linux-only
       environment:
         CPU: 4

dev/bots/service_worker_test.dart

@@ -12,6 +12,7 @@ import 'package:shelf/shelf.dart';
 import 'browser.dart';
 import 'run_command.dart';
 import 'test/common.dart';
+import 'utils.dart';
 
 final String _bat = Platform.isWindows ? '.bat' : '';
 final String _flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
@@ -53,10 +54,22 @@ Future<void> _rebuildApp({ @required int version }) async {
   );
 }
 
+/// A drop-in replacement for `package:test` expect that can run outside the
+/// test zone.
+void expect(Object actual, Object expected) {
+  final Matcher matcher = wrapMatcher(expected);
+  final Map<Object, Object> matchState = <Object, Object>{};
+  if (matcher.matches(actual, matchState)) {
+    return;
+  }
+  final StringDescription mismatchDescription = StringDescription();
+  matcher.describeMismatch(actual, mismatchDescription, matchState, true);
+  throw TestFailure(mismatchDescription.toString());
+}
+
 Future<void> runWebServiceWorkerTest({
   @required bool headless,
 }) async {
-  test('flutter_service_worker.js', () async {
   await _rebuildApp(version: 1);
 
   final Map<String, int> requestedPathCounts = <String, int>{};
@@ -85,10 +98,16 @@ Future<void> runWebServiceWorkerTest({
   Future<void> startAppServer({
     @required String cacheControl,
   }) async {
+    final int serverPort = await findAvailablePort();
+    final int browserDebugPort = await findAvailablePort();
     server = await AppServer.start(
       headless: headless,
       cacheControl: cacheControl,
-        appUrl: 'http://localhost:8080/index.html',
+      // TODO(yjbanov): use a better port disambiguation strategy than trying
+      //                to guess what ports other tests use.
+      appUrl: 'http://localhost:$serverPort/index.html',
+      serverPort: serverPort,
+      browserDebugPort: browserDebugPort,
       appDirectory: _appBuildDirectory,
       additionalRequestHandlers: <Handler>[
         (Request request) {
@@ -262,6 +281,4 @@ Future<void> runWebServiceWorkerTest({
     await _setAppVersion(1);
     await server?.stop();
   }
-    // This is a long test. The default 30 seconds is not enough.
-  }, timeout: const Timeout(Duration(minutes: 10)));
 }

dev/bots/test.dart

@@ -108,8 +108,9 @@ Future<void> main(List<String> args) async {
       'tool_tests': _runToolTests,
       'web_tool_tests': _runToolTests,
       'tool_integration_tests': _runIntegrationToolTests,
+      // All the unit/widget tests run using `flutter test --platform=chrome`
       'web_tests': _runWebUnitTests,
-      'web_integration_tests': _runWebIntegrationTests,
+      // All web integration tests
       'web_long_running_tests': _runWebLongRunningTests,
       'flutter_plugins': _runFlutterPluginsTests,
       'skp_generator': _runSkpGeneratorTests,
@@ -850,6 +851,34 @@ Future<void> _runWebLongRunningTests() async {
     () => _runGalleryE2eWebTest('release'),
     () => _runGalleryE2eWebTest('release', canvasKit: true),
     () => runWebServiceWorkerTest(headless: true),
+    () => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
+    () => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
+    () => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
+    () => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
+    () => _runWebDebugTest('lib/stack_trace.dart'),
+    () => _runWebDebugTest('lib/framework_stack_trace.dart'),
+    () => _runWebDebugTest('lib/web_directory_loading.dart'),
+    () => _runWebDebugTest('test/test.dart'),
+    () => _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true),
+    () => _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true),
+    () => _runWebDebugTest('lib/web_define_loading.dart',
+      additionalArguments: <String>[
+        '--dart-define=test.valueA=Example,A',
+        '--dart-define=test.valueB=Value',
+      ]
+    ),
+    () => _runWebReleaseTest('lib/web_define_loading.dart',
+      additionalArguments: <String>[
+        '--dart-define=test.valueA=Example,A',
+        '--dart-define=test.valueB=Value',
+      ]
+    ),
+    () => _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
+      '--sound-null-safety',
+    ]),
+    () => _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
+      '--sound-null-safety',
+    ]),
   ];
   await _ensureChromeDriverIsRunning();
   await _runShardRunnerIndexOfTotalSubshard(tests);
@@ -1069,48 +1098,6 @@ Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false })
   print('${green}Integration test passed.$reset');
 }
 
-/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-/// !!! WARNING WARNING WARNING WARNING WARNING WARNING!!!
-/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-///
-/// Do not put any more tests here. This shard is not properly subsharded.
-/// Adding more tests here will linearly increase the runtime of the shard
-/// making the overall Flutter CI build longer. Consider adding tests to
-/// [_runWebLongRunningTests] instead (increasing subshard count if necessary).
-///
-// TODO(yjbanov): increase subshard count in _runWebLongRunningTests and retire
-//                this shard.
-Future<void> _runWebIntegrationTests() async {
-  await _runWebStackTraceTest('profile', 'lib/stack_trace.dart');
-  await _runWebStackTraceTest('release', 'lib/stack_trace.dart');
-  await _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart');
-  await _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart');
-  await _runWebDebugTest('lib/stack_trace.dart');
-  await _runWebDebugTest('lib/framework_stack_trace.dart');
-  await _runWebDebugTest('lib/web_directory_loading.dart');
-  await _runWebDebugTest('test/test.dart');
-  await _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true);
-  await _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true);
-  await _runWebDebugTest('lib/web_define_loading.dart',
-    additionalArguments: <String>[
-      '--dart-define=test.valueA=Example,A',
-      '--dart-define=test.valueB=Value',
-    ]
-  );
-  await _runWebReleaseTest('lib/web_define_loading.dart',
-    additionalArguments: <String>[
-      '--dart-define=test.valueA=Example,A',
-      '--dart-define=test.valueB=Value',
-    ]
-  );
-  await _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
-    '--sound-null-safety',
-  ]);
-  await _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
-    '--sound-null-safety',
-  ]);
-}
-
 Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
   final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
   final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
@@ -1137,9 +1124,13 @@ Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
   );
 
   // Run the app.
+  final int serverPort = await findAvailablePort();
+  final int browserDebugPort = await findAvailablePort();
   final String result = await evalTestAppInChrome(
-    appUrl: 'http://localhost:8080/index.html',
+    appUrl: 'http://localhost:$serverPort/index.html',
     appDirectory: appBuildDirectory,
+    serverPort: serverPort,
+    browserDebugPort: browserDebugPort,
   );
 
   if (result.contains('--- TEST SUCCEEDED ---')) {
@@ -1181,9 +1172,13 @@ Future<void> _runWebReleaseTest(String target, {
   );
 
   // Run the app.
+  final int serverPort = await findAvailablePort();
+  final int browserDebugPort = await findAvailablePort();
   final String result = await evalTestAppInChrome(
-    appUrl: 'http://localhost:8080/index.html',
+    appUrl: 'http://localhost:$serverPort/index.html',
     appDirectory: appBuildDirectory,
+    serverPort: serverPort,
+    browserDebugPort: browserDebugPort,
   );
 
   if (result.contains('--- TEST SUCCEEDED ---')) {

dev/bots/utils.dart

@@ -70,3 +70,24 @@ String prettyPrintDuration(Duration duration) {
 void printProgress(String action, String workingDir, String command) {
   print('$clock $action: cd $cyan$workingDir$reset; $green$command$reset');
 }
+
+int _portCounter = 8080;
+
+/// Finds the next available local port.
+Future<int> findAvailablePort() async {
+  while (!await _isPortAvailable(_portCounter)) {
+    _portCounter += 1;
+  }
+  return _portCounter++;
+}
+
+Future<bool> _isPortAvailable(int port) async {
+  try {
+    final RawSocket socket = await RawSocket.connect('localhost', port);
+    socket.shutdown(SocketDirection.both);
+    await socket.close();
+    return false;
+  } on SocketException {
+    return true;
+  }
+}

dev/prod_builders.json

@@ -523,27 +523,33 @@
       "flaky": false
     },
     {
-      "name": "Linux web_integration_tests",
+      "name": "Linux web_long_running_tests_1_5",
       "repo": "flutter",
-      "task_name": "linux_web_integration_tests",
+      "task_name": "linux_web_long_running_tests_1_5",
       "flaky": false
     },
     {
-      "name": "Linux web_long_running_tests_1_3",
+      "name": "Linux web_long_running_tests_2_5",
       "repo": "flutter",
-      "task_name": "linux_web_long_running_tests_1_3",
+      "task_name": "linux_web_long_running_tests_2_5",
       "flaky": false
     },
     {
-      "name": "Linux web_long_running_tests_2_3",
+      "name": "Linux web_long_running_tests_3_5",
       "repo": "flutter",
-      "task_name": "linux_web_long_running_tests_2_3",
+      "task_name": "linux_web_long_running_tests_3_5",
       "flaky": false
     },
     {
-      "name": "Linux web_long_running_tests_3_3",
+      "name": "Linux web_long_running_tests_4_5",
       "repo": "flutter",
-      "task_name": "linux_web_long_running_tests_3_3",
+      "task_name": "linux_web_long_running_tests_4_5",
+      "flaky": false
+    },
+    {
+      "name": "Linux web_long_running_tests_5_5",
+      "repo": "flutter",
+      "task_name": "linux_web_long_running_tests_5_5",
       "flaky": false
     },
     {

dev/try_builders.json

@@ -305,37 +305,37 @@
       "run_if": ["dev/", "packages/", "bin/"]
     },
     {
-      "name": "Linux web_integration_tests",
+      "name": "Linux web_long_running_tests_1_5",
       "repo": "flutter",
-      "task_name": "linux_web_integration_tests",
+      "task_name": "web_long_running_tests_1_5",
       "enabled": true,
-      "run_if": [
-        "dev/",
-        "packages/flutter/",
-        "packages/flutter_test/",
-        "packages/flutter_tools/",
-        "packages/flutter_web_plugins/",
-        "bin/"
-      ]
+      "run_if": ["dev/", "packages/", "bin/"]
+    },
+    {
+      "name": "Linux web_long_running_tests_2_5",
+      "repo": "flutter",
+      "task_name": "web_long_running_tests_2_5",
+      "enabled": true,
+      "run_if": ["dev/", "packages/", "bin/"]
     },
     {
-      "name": "Linux web_long_running_tests_1_3",
+      "name": "Linux web_long_running_tests_3_5",
       "repo": "flutter",
-      "task_name": "web_long_running_tests_1_3",
+      "task_name": "web_long_running_tests_3_5",
       "enabled": true,
       "run_if": ["dev/", "packages/", "bin/"]
     },
     {
-      "name": "Linux web_long_running_tests_2_3",
+      "name": "Linux web_long_running_tests_4_5",
       "repo": "flutter",
-      "task_name": "web_long_running_tests_2_3",
+      "task_name": "web_long_running_tests_4_5",
       "enabled": true,
       "run_if": ["dev/", "packages/", "bin/"]
     },
     {
-      "name": "Linux web_long_running_tests_3_3",
+      "name": "Linux web_long_running_tests_5_5",
       "repo": "flutter",
-      "task_name": "web_long_running_tests_1_3",
+      "task_name": "web_long_running_tests_5_5",
       "enabled": true,
       "run_if": ["dev/", "packages/", "bin/"]
     },