Support using lightweight Flutter Engines to run tests (#141726)

This PR implements the functionality described above and hides it behind
the `--experimental-faster-testing` flag of `flutter test`.

### The following are some performance measurements from test runs
conducted on GitHub Actions

run 1 logs:
https://github.com/derekxu16/flutter_test_ci/actions/runs/8008029772/attempts/1
run 2 logs:
https://github.com/derekxu16/flutter_test_ci/actions/runs/8008029772/attempts/2
run 3 logs:
https://github.com/derekxu16/flutter_test_ci/actions/runs/8008029772/attempts/3

**length of `flutter test --reporter=expanded test/animation
test/foundation` step**

run 1: 54s
run 2: 52s
run 3: 56s

average: 54s

**length of `flutter test --experimental-faster-testing
--reporter=expanded test/animation test/foundation` step**

run 1: 27s
run 2: 27s
run 3: 29s

average: 27.67s (~48.77% shorter than 54s)

**length of `flutter test --reporter=expanded test/animation
test/foundation test/gestures test/painting test/physics test/rendering
test/scheduler test/semantics test/services` step**

run 1: 260s
run 2: 270s
run 3: 305s

average: 278.33s


**length of `flutter test --experimental-faster-testing
--reporter=expanded test/animation test/foundation test/gestures
test/painting test/physics test/rendering test/scheduler test/semantics
test/services` step**

from a clean build (right after deleting the build folder):

run 1: 215s
run 2: 227s
run 3: 245s

average: 229s (~17.72% shorter than 278.33s)

Note that in reality, `test/material` was not passed to `flutter test`
in the trials below. All of the test files under `test/material` except
for `test/material/icons_test.dart` were listed out individually

**length of `flutter test --reporter=expanded test/material` step**

run 1: 408s
run 2: 421s
run 3: 451s

average: 426.67s

**length of `flutter test --experimental-faster-testing
--reporter=expanded test/material` step**

run 1: 382s
run 2: 373s
run 3: 400s

average: 385s (~9.77% shorter than 426.67s)

---------
Co-authored-by: Dan Field <dnfield@google.com>

packages/flutter_test/lib/src/binding.dart

@@ -195,6 +195,7 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
   TestWidgetsFlutterBinding() : platformDispatcher = TestPlatformDispatcher(
     platformDispatcher: PlatformDispatcher.instance,
   ) {
+    platformDispatcher.defaultRouteNameTestValue = '/';
     debugPrint = debugPrintOverride;
     debugDisableShadows = disableShadows;
   }
@@ -246,6 +247,7 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
   void reset() {
     _restorationManager?.dispose();
     _restorationManager = null;
+    platformDispatcher.defaultRouteNameTestValue = '/';
     resetGestureBinding();
     testTextInput.reset();
     if (registerTestTextInput) {

packages/flutter_tools/lib/src/commands/test.dart

@@ -80,6 +80,11 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
     addEnableImpellerFlag(verboseHelp: verboseHelp);
 
     argParser
+      ..addFlag('experimental-faster-testing',
+        negatable: false,
+        hide: !verboseHelp,
+        help: 'Run each test in a separate lightweight Flutter Engine to speed up testing.'
+      )
       ..addMultiOption('name',
         help: 'A regular expression matching substrings of the names of tests to run.',
         valueHelp: 'regexp',
@@ -350,6 +355,23 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
       );
     }
 
+    bool experimentalFasterTesting = boolArg('experimental-faster-testing');
+    if (experimentalFasterTesting) {
+      if (_isIntegrationTest || isWeb) {
+        experimentalFasterTesting = false;
+        globals.printStatus(
+          '--experimental-faster-testing was parsed but will be ignored. This '
+          'option is not supported when running integration tests or web tests.',
+        );
+      } else if (_testFileUris.length == 1) {
+        experimentalFasterTesting = false;
+        globals.printStatus(
+          '--experimental-faster-testing was parsed but will be ignored. This '
+          'option should not be used when running a single test file.',
+        );
+      }
+    }
+
     final bool startPaused = boolArg('start-paused');
     if (startPaused && _testFileUris.length != 1) {
       throwToolExit(
@@ -402,6 +424,13 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
       // Running with concurrency will result in deploying multiple test apps
       // on the connected device concurrently, which is not supported.
       jobs = 1;
+    } else if (experimentalFasterTesting) {
+      if (argResults!.wasParsed('concurrency')) {
+        globals.printStatus(
+          '-j/--concurrency was parsed but will be ignored. This option is not '
+          'compatible with --experimental-faster-testing.',
+        );
+      }
     }
 
     final int? shardIndex = int.tryParse(stringArg('shard-index') ?? '');
@@ -425,6 +454,25 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
           'If you set --shard-index you need to also set --total-shards.');
     }
 
+    final bool enableVmService = boolArg('enable-vmservice');
+    if (experimentalFasterTesting && enableVmService) {
+      globals.printStatus(
+        '--enable-vmservice was parsed but will be ignored. This option is not '
+        'compatible with --experimental-faster-testing.',
+      );
+    }
+
+    final bool ipv6 = boolArg('ipv6');
+    if (experimentalFasterTesting && ipv6) {
+      // [ipv6] is set when the user desires for the test harness server to use
+      // IPv6, but a test harness server will not be started at all when
+      // [experimentalFasterTesting] is set.
+      globals.printStatus(
+        '--ipv6 was parsed but will be ignored. This option is not compatible '
+        'with --experimental-faster-testing.',
+      );
+    }
+
     final bool machine = boolArg('machine');
     CoverageCollector? collector;
     if (boolArg('coverage') || boolArg('merge-coverage') ||
@@ -487,35 +535,61 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
     }
 
     final Stopwatch? testRunnerTimeRecorderStopwatch = testTimeRecorder?.start(TestTimePhases.TestRunner);
-    final int result = await testRunner.runTests(
-      testWrapper,
-      _testFileUris.toList(),
-      debuggingOptions: debuggingOptions,
-      names: names,
-      plainNames: plainNames,
-      tags: tags,
-      excludeTags: excludeTags,
-      watcher: watcher,
-      enableVmService: collector != null || startPaused || boolArg('enable-vmservice'),
-      ipv6: boolArg('ipv6'),
-      machine: machine,
-      updateGoldens: boolArg('update-goldens'),
-      concurrency: jobs,
-      testAssetDirectory: testAssetDirectory,
-      flutterProject: flutterProject,
-      web: isWeb,
-      randomSeed: stringArg('test-randomize-ordering-seed'),
-      reporter: stringArg('reporter'),
-      fileReporter: stringArg('file-reporter'),
-      timeout: stringArg('timeout'),
-      runSkipped: boolArg('run-skipped'),
-      shardIndex: shardIndex,
-      totalShards: totalShards,
-      integrationTestDevice: integrationTestDevice,
-      integrationTestUserIdentifier: stringArg(FlutterOptions.kDeviceUser),
-      testTimeRecorder: testTimeRecorder,
-      nativeAssetsBuilder: nativeAssetsBuilder,
-    );
+    final int result;
+    if (experimentalFasterTesting) {
+      assert(!isWeb && !_isIntegrationTest && _testFileUris.length > 1);
+      result = await testRunner.runTestsBySpawningLightweightEngines(
+        _testFileUris.toList(),
+        debuggingOptions: debuggingOptions,
+        names: names,
+        plainNames: plainNames,
+        tags: tags,
+        excludeTags: excludeTags,
+        machine: machine,
+        updateGoldens: boolArg('update-goldens'),
+        concurrency: jobs,
+        testAssetDirectory: testAssetDirectory,
+        flutterProject: flutterProject,
+        randomSeed: stringArg('test-randomize-ordering-seed'),
+        reporter: stringArg('reporter'),
+        fileReporter: stringArg('file-reporter'),
+        timeout: stringArg('timeout'),
+        runSkipped: boolArg('run-skipped'),
+        shardIndex: shardIndex,
+        totalShards: totalShards,
+        testTimeRecorder: testTimeRecorder,
+      );
+    } else {
+      result = await testRunner.runTests(
+        testWrapper,
+        _testFileUris.toList(),
+        debuggingOptions: debuggingOptions,
+        names: names,
+        plainNames: plainNames,
+        tags: tags,
+        excludeTags: excludeTags,
+        watcher: watcher,
+        enableVmService: collector != null || startPaused || enableVmService,
+        ipv6: ipv6,
+        machine: machine,
+        updateGoldens: boolArg('update-goldens'),
+        concurrency: jobs,
+        testAssetDirectory: testAssetDirectory,
+        flutterProject: flutterProject,
+        web: isWeb,
+        randomSeed: stringArg('test-randomize-ordering-seed'),
+        reporter: stringArg('reporter'),
+        fileReporter: stringArg('file-reporter'),
+        timeout: stringArg('timeout'),
+        runSkipped: boolArg('run-skipped'),
+        shardIndex: shardIndex,
+        totalShards: totalShards,
+        integrationTestDevice: integrationTestDevice,
+        integrationTestUserIdentifier: stringArg(FlutterOptions.kDeviceUser),
+        testTimeRecorder: testTimeRecorder,
+        nativeAssetsBuilder: nativeAssetsBuilder,
+      );
+    }
     testTimeRecorder?.stop(TestTimePhases.TestRunner, testRunnerTimeRecorderStopwatch!);
 
     if (collector != null) {

packages/flutter_tools/pubspec.yaml

@@ -110,6 +110,7 @@ dev_dependencies:
   pubspec_parse: 1.2.3
 
   checked_yaml: 2.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  ffi: 2.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   json_annotation: 4.8.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   node_preamble: 2.0.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   test: 1.25.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -118,4 +119,4 @@ dartdoc:
   # Exclude this package from the hosted API docs.
   nodoc: true
 
-# PUBSPEC CHECKSUM: a00d
+# PUBSPEC CHECKSUM: 588f

packages/flutter_tools/test/integration.shard/test_test.dart

@@ -130,6 +130,23 @@ void main() {
     );
   });
 
+  testWithoutContext('flutter test should run a test when its name matches a regexp when --experimental-faster-testing is set',
+      () async {
+    final ProcessResult result = await _runFlutterTest(
+      null,
+      automatedTestsDirectory,
+      flutterTestDirectory,
+      extraArguments: const <String>[
+        '--experimental-faster-testing',
+        '--name=inc.*de',
+      ],
+    );
+    expect(
+      result,
+      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
+    );
+  });
+
   testWithoutContext('flutter test should run a test when its name contains a string', () async {
     final ProcessResult result = await _runFlutterTest(
       'filtering',
@@ -143,6 +160,22 @@ void main() {
     );
   });
 
+  testWithoutContext('flutter test should run a test when its name contains a string when --experimental-faster-testing is set', () async {
+    final ProcessResult result = await _runFlutterTest(
+      null,
+      automatedTestsDirectory,
+      flutterTestDirectory,
+      extraArguments: const <String>[
+        '--experimental-faster-testing',
+        '--plain-name=include',
+      ],
+    );
+    expect(
+      result,
+      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
+    );
+  });
+
   testWithoutContext('flutter test should run a test with a given tag', () async {
     final ProcessResult result = await _runFlutterTest(
       'filtering_tag',
@@ -156,6 +189,23 @@ void main() {
     );
   });
 
+  testWithoutContext('flutter test should run a test with a given tag when --experimental-faster-testing is set', () async {
+    final ProcessResult result = await _runFlutterTest(
+      null,
+      automatedTestsDirectory,
+      flutterTestDirectory,
+      extraArguments: const <String>[
+        '--experimental-faster-testing',
+        '--tags=include-tag',
+      ],
+    );
+    expect(
+      result,
+      ProcessResultMatcher(stdoutPattern: RegExp(r'\+\d+: All tests passed!')),
+    );
+  });
+
+
   testWithoutContext('flutter test should not run a test with excluded tag', () async {
     final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory,
         extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
@@ -176,6 +226,22 @@ void main() {
     );
   });
 
+  testWithoutContext('flutter test should run all tests when tags are unspecified when --experimental-faster-testing is set', () async {
+    final ProcessResult result = await _runFlutterTest(
+      null,
+      automatedTestsDirectory,
+      flutterTestDirectory,
+      extraArguments: const <String>['--experimental-faster-testing'],
+    );
+    expect(
+      result,
+      ProcessResultMatcher(
+        exitCode: 1,
+        stdoutPattern: RegExp(r'\+\d+ -\d+: Some tests failed\.'),
+      ),
+    );
+  });
+
   testWithoutContext('flutter test should run a widgetTest with a given tag', () async {
     final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
         extraArguments: const <String>['--tags', 'include-tag']);
@@ -238,6 +304,75 @@ void main() {
     expect(result, const ProcessResultMatcher());
   });
 
+  testWithoutContext('flutter test should test runs to completion when --experimental-faster-testing is set', () async {
+    final ProcessResult result = await _runFlutterTest(
+      null,
+      automatedTestsDirectory,
+      '$flutterTestDirectory/child_directory',
+      extraArguments: const <String>[
+        '--experimental-faster-testing',
+        '--verbose',
+      ],
+    );
+    final String stdout = (result.stdout as String).replaceAll('\r', '\n');
+    expect(stdout, contains(RegExp(r'\+\d+: All tests passed\!')));
+    expect(stdout, contains('Starting flutter_tester process with command'));
+    if ((result.stderr as String).isNotEmpty) {
+      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
+    }
+    expect(result, const ProcessResultMatcher());
+  });
+
+  testWithoutContext('flutter test should ignore --experimental-faster-testing when only a single test file is specified', () async {
+    final ProcessResult result = await _runFlutterTest(
+      'trivial',
+      automatedTestsDirectory,
+      flutterTestDirectory,
+      extraArguments: const <String>[
+        '--experimental-faster-testing',
+        '--verbose'
+      ],
+    );
+    final String stdout = (result.stdout as String).replaceAll('\r', '\n');
+    expect(
+      stdout,
+      contains('--experimental-faster-testing was parsed but will be ignored. '
+          'This option should not be used when running a single test file.'),
+    );
+    expect(stdout, contains(RegExp(r'\+\d+: All tests passed\!')));
+    expect(stdout, contains('test 0: Starting flutter_tester process with command'));
+    expect(stdout, contains('test 0: deleting temporary directory'));
+    expect(stdout, contains('test 0: finished'));
+    expect(stdout, contains('test package returned with exit code 0'));
+    if ((result.stderr as String).isNotEmpty) {
+      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
+    }
+    expect(result, const ProcessResultMatcher());
+  });
+
+   testWithoutContext('flutter test should ignore --experimental-faster-testing when running integration tests', () async {
+    final ProcessResult result = await _runFlutterTest(
+      'trivial_widget',
+      automatedTestsDirectory,
+      integrationTestDirectory,
+      extraArguments: <String>[
+        ...integrationTestExtraArgs,
+        '--experimental-faster-testing',
+      ],
+    );
+    final String stdout = (result.stdout as String).replaceAll('\r', '\n');
+    expect(
+      stdout,
+      contains('--experimental-faster-testing was parsed but will be ignored. '
+          'This option is not supported when running integration tests or web '
+          'tests.'),
+    );
+    if ((result.stderr as String).isNotEmpty) {
+      fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
+    }
+    expect(result, const ProcessResultMatcher());
+  });
+
   testWithoutContext('flutter test should run all tests inside of a directory with no trailing slash', () async {
     final ProcessResult result = await _runFlutterTest(null, automatedTestsDirectory, '$flutterTestDirectory/child_directory',
       extraArguments: const <String>['--verbose']);