Move web integration tool tests to web.shard (#70226)

* Move web integration tool tests to web.shard

Web integration tool tests depend on DDC changes in SDK. This change
moves them to a separate shard and subshard so CI bot configurations
can run them separately.

In particular, with will allow running those tests on dart CI flutter
HHH web bot instead of a non-web one, allowing early detection and easy
classification of issues caused by SDK changes as VM- or Web related.

* Enabled verbose mode for flaky web_tool_tests

* Split out the test changes to be commited first

dev/bots/test.dart

@@ -125,6 +125,7 @@ Future<void> main(List<String> args) async {
       'framework_tests': _runFrameworkTests,
       'tool_coverage': _runToolCoverage,
       'tool_tests': _runToolTests,
+      'web_tool_tests': _runWebToolTests,
       'web_tests': _runWebUnitTests,
       'web_integration_tests': _runWebIntegrationTests,
       'web_long_running_tests': _runWebLongRunningTests,
@@ -313,6 +314,7 @@ Future<void> _runToolCoverage() async {
 
 Future<void> _runToolTests() async {
   const String kDotShard = '.shard';
+  const String kWeb = 'web';
   const String kTest = 'test';
   final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
 
@@ -321,6 +323,7 @@ Future<void> _runToolTests() async {
       .listSync()
       .map<String>((FileSystemEntity entry) => entry.path)
       .where((String name) => name.endsWith(kDotShard))
+      .where((String name) => path.basenameWithoutExtension(name) != kWeb)
       .map<String>((String name) => path.basenameWithoutExtension(name)),
     // The `dynamic` on the next line is because Map.fromIterable isn't generic.
     value: (dynamic subshard) => () async {
@@ -337,10 +340,27 @@ Future<void> _runToolTests() async {
       );
     },
   );
-  // Prevent web tests from running if not explicitly requested.
-  if (Platform.environment[CIRRUS_TASK_NAME] == null) {
-    subshards.remove('web');
+
+  await selectSubshard(subshards);
+}
+
+Future<void> _runWebToolTests() async {
+  const String kDotShard = '.shard';
+  const String kWeb = 'web';
+  const String kTest = 'test';
+  final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
+
+  final Map<String, ShardRunner> subshards = <String, ShardRunner>{
+      kWeb:
+      () async {
+        await _pubRunTest(
+          toolsPath,
+          forceSingleCore: true,
+          testPaths: <String>[path.join(kTest, '$kWeb$kDotShard', '')],
+          enableFlutterToolAsserts: true,
+        );
       }
+  };
 
   await selectSubshard(subshards);
 }

packages/flutter_tools/test/web.shard/README.md

+# Web integration tests
+
+These tests are not hermetic, and use the actual Flutter SDK. While
+they don't require actual devices, they run `flutter_tester` to test
+Dart web debug services (dwds)  and Flutter integration.
+
+Use this command to run (from the `flutter_tools` directory):
+
+```shell
+../../bin/cache/dart-sdk/bin/pub run test test/web.shard
+```
+
+These tests are expensive to run and do not give meaningful coverage
+information for the flutter tool (since they are black-box tests that
+run the tool as a subprocess, rather than being unit tests). For this
+reason, they are in a separate shard when running on continuous
+integration and are not run when calculating coverage.

packages/flutter_tools/test/integration.shard/debugger_stepping_web_test.dart → packages/flutter_tools/test/web.shard/debugger_stepping_web_test.dart

@@ -6,10 +6,10 @@ import 'dart:io';
 
 import 'package:file/file.dart';
 
+import '../integration.shard/test_data/stepping_project.dart';
+import '../integration.shard/test_driver.dart';
+import '../integration.shard/test_utils.dart';
 import '../src/common.dart';
-import 'test_data/stepping_project.dart';
-import 'test_driver.dart';
-import 'test_utils.dart';
 
 void main() {
   Directory tempDirectory;
@@ -25,7 +25,9 @@ void main() {
 
     flutter = FlutterRunTestDriver(tempDirectory);
 
-    await flutter.run(withDebugger: true, startPaused: true, chrome: true);
+    await flutter.run(
+      withDebugger: true, startPaused: true, chrome: true,
+      additionalCommandArgs: <String>['--verbose']);
     await flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine);
     await flutter.resume();
     await flutter.waitForPause(); // Now we should be on the breakpoint.

packages/flutter_tools/test/web.shard/expression_evaluation_web_test.dart

@@ -33,7 +33,8 @@ void batch1() {
     // No need to start paused as all breakpoint would be eventually reached.
     return  _flutter.run(
       withDebugger: true, chrome: true,
-      expressionEvaluation: expressionEvaluation);
+      expressionEvaluation: expressionEvaluation,
+      additionalCommandArgs: <String>['--verbose']);
   }
 
   Future<void> breakInBuildMethod(FlutterTestDriver flutter) async {
@@ -139,7 +140,8 @@ void batch2() {
     return  _flutter.run(
       withDebugger: true, chrome: true,
       expressionEvaluation: expressionEvaluation,
-      startPaused: true, script: _project.testFilePath);
+      startPaused: true, script: _project.testFilePath,
+      additionalCommandArgs: <String>['--verbose']);
   }
 
   testWithoutContext('flutter test expression evaluation - error if expression evaluation disabled', () async {

packages/flutter_tools/test/web.shard/hot_reload_web_test.dart

@@ -24,11 +24,12 @@ void main() {
 
   tearDown(() async {
     await flutter?.stop();
+    await flutter?.done;
     tryToDelete(tempDir);
   });
 
   testWithoutContext('hot restart works without error', () async {
-    await flutter.run(chrome: true);
+    await flutter.run(chrome: true, additionalCommandArgs: <String>['--verbose']);
     await flutter.hotRestart();
   });
 
@@ -40,7 +41,7 @@ void main() {
         completer.complete();
       }
     });
-    await flutter.run(chrome: true);
+    await flutter.run(chrome: true, additionalCommandArgs: <String>['--verbose']);
     project.uncommentHotReloadPrint();
     try {
       await flutter.hotRestart();
@@ -58,7 +59,8 @@ void main() {
         completer.complete();
       }
     });
-    await flutter.run(chrome: true, additionalCommandArgs: <String>['--dart-define=FLUTTER_WEB_USE_SKIA=true']);
+    await flutter.run(chrome: true,
+      additionalCommandArgs: <String>['--dart-define=FLUTTER_WEB_USE_SKIA=true', '--verbose']);
     project.uncommentHotReloadPrint();
     try {
       await flutter.hotRestart();

packages/flutter_tools/test/integration.shard/web_run_test.dart → packages/flutter_tools/test/web.shard/web_run_test.dart

@@ -4,10 +4,11 @@
 
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
+
+import '../integration.shard/test_data/basic_project.dart';
+import '../integration.shard/test_driver.dart';
+import '../integration.shard/test_utils.dart';
 import '../src/common.dart';
-import 'test_data/basic_project.dart';
-import 'test_driver.dart';
-import 'test_utils.dart';
 
 void main() {
   Directory tempDir;
@@ -26,6 +27,6 @@ void main() {
   });
 
   testWithoutContext('flutter run works on web devices with a unary main function', () async {
-    await flutter.run(chrome: true);
+    await flutter.run(chrome: true, additionalCommandArgs: <String>['--verbose']);
   }, skip: platform.isMacOS);
 }