Add support for running tests through debug-adapter (#92587)

* Add support for running tests through debug-adapter

* Improve comments about stdout + remove pedantic

packages/flutter_tools/lib/src/commands/debug_adapter.dart

@@ -28,6 +28,13 @@ class DebugAdapterCommand extends FlutterCommand {
   DebugAdapterCommand({ bool verboseHelp = false}) : hidden = !verboseHelp {
     usesIpv6Flag(verboseHelp: verboseHelp);
     addDdsOptions(verboseHelp: verboseHelp);
+    argParser
+      .addFlag(
+        'test',
+        defaultsTo: false,
+        help: 'Whether to use the "flutter test" debug adapter to run tests'
+            ' and emit custom events for test progress/results.',
+      );
   }
 
   @override
@@ -54,6 +61,7 @@ class DebugAdapterCommand extends FlutterCommand {
       platform: globals.platform,
       ipv6: ipv6,
       enableDds: enableDds,
+      test: boolArg('test') ?? false,
     );
 
     await server.channel.closed;

packages/flutter_tools/lib/src/debug_adapters/README.md

@@ -4,7 +4,14 @@ This document is Flutter-specific. For information on the standard Dart DAP impl
 
 Flutter includes support for debugging using [the Debug Adapter Protocol](https://microsoft.github.io/debug-adapter-protocol/) as an alternative to using the [VM Service](https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md) directly, simplying the integration for new editors.
 
-The debug adapter is started with the `flutter debug-adapter` command and is intended to be consumed by DAP-compliant tools such as Flutter-specific extensions for editors, or configured by users whose editors include generic configurable DAP clients.
+The debug adapters are started with the `flutter debug-adapter` command and are intended to be consumed by DAP-compliant tools such as Flutter-specific extensions for editors, or configured by users whose editors include generic configurable DAP clients.
+
+Two adapters are available:
+
+- `flutter debug_adapter`
+- `flutter debug_adapter --test`
+
+The standard adapter will run applications using `flutter run` while the `--test` adapter will cause scripts to be run using `flutter test` and will emit custom `dart.testNotification` events (described in the [Dart DAP documentation](https://github.com/dart-lang/sdk/blob/main/pkg/dds/tool/dap/README.md#darttestnotification)).
 
 Because in the DAP protocol the client speaks first, running this command from the terminal will result in no output (nor will the process terminate). This is expected behaviour.
 

packages/flutter_tools/lib/src/debug_adapters/flutter_adapter.dart

@@ -164,9 +164,7 @@ class FlutterDebugAdapter extends DartDebugAdapter<FlutterLaunchRequestArguments
     final List<String> toolArgs = <String>[
       'run',
       '--machine',
-      if (debug) ...<String>[
-        '--start-paused',
-      ],
+      if (debug) '--start-paused',
     ];
     final List<String> processArgs = <String>[
       ...toolArgs,

packages/flutter_tools/lib/src/debug_adapters/flutter_test_adapter.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:dds/dap.dart' hide PidTracker, PackageConfigUtils;
+import 'package:vm_service/vm_service.dart' as vm;
+
+import '../base/file_system.dart';
+import '../base/io.dart';
+import '../base/platform.dart';
+import '../cache.dart';
+import '../convert.dart';
+import 'flutter_adapter_args.dart';
+import 'mixins.dart';
+
+/// A DAP Debug Adapter for running and debugging Flutter tests.
+class FlutterTestDebugAdapter extends DartDebugAdapter<FlutterLaunchRequestArguments, FlutterAttachRequestArguments>
+    with PidTracker, PackageConfigUtils, TestAdapter {
+  FlutterTestDebugAdapter(
+    ByteStreamServerChannel channel, {
+    required this.fileSystem,
+    required this.platform,
+    bool ipv6 = false,
+    bool enableDds = true,
+    bool enableAuthCodes = true,
+    Logger? logger,
+  }) : super(
+          channel,
+          ipv6: ipv6,
+          enableDds: enableDds,
+          enableAuthCodes: enableAuthCodes,
+          logger: logger,
+        );
+
+  @override
+  FileSystem fileSystem;
+  Platform platform;
+  Process? _process;
+
+  @override
+  final FlutterLaunchRequestArguments Function(Map<String, Object?> obj)
+      parseLaunchArgs = FlutterLaunchRequestArguments.fromJson;
+
+  @override
+  final FlutterAttachRequestArguments Function(Map<String, Object?> obj)
+      parseAttachArgs = FlutterAttachRequestArguments.fromJson;
+
+  /// Whether the VM Service closing should be used as a signal to terminate the debug session.
+  ///
+  /// Since we do not support attaching for tests, this is always false.
+  @override
+  bool get terminateOnVmServiceClose => false;
+
+  /// Called by [attachRequest] to request that we actually connect to the app to be debugged.
+  @override
+  Future<void> attachImpl() async {
+    sendOutput('console', '\nAttach is not currently supported');
+    handleSessionTerminate();
+  }
+
+  @override
+  Future<void> debuggerConnected(vm.VM vmInfo) async {
+    // Capture the PID from the VM Service so that we can terminate it when
+    // cleaning up. Terminating the process might not be enough as it could be
+    // just a shell script (e.g. pub on Windows) and may not pass the
+    // signal on correctly.
+    // See: https://github.com/Dart-Code/Dart-Code/issues/907
+    final int? pid = vmInfo.pid;
+    if (pid != null) {
+      pidsToTerminate.add(pid);
+    }
+  }
+
+  /// Called by [disconnectRequest] to request that we forcefully shut down the app being run (or in the case of an attach, disconnect).
+  ///
+  /// Client IDEs/editors should send a terminateRequest before a
+  /// disconnectRequest to allow a graceful shutdown. This method must terminate
+  /// quickly and therefore may leave orphaned processes.
+  @override
+  Future<void> disconnectImpl() async {
+    terminatePids(ProcessSignal.sigkill);
+  }
+
+  /// Called by [launchRequest] to request that we actually start the tests to be run/debugged.
+  ///
+  /// For debugging, this should start paused, connect to the VM Service, set
+  /// breakpoints, and resume.
+  @override
+  Future<void> launchImpl() async {
+    final FlutterLaunchRequestArguments args = this.args as FlutterLaunchRequestArguments;
+    final String flutterToolPath = fileSystem.path.join(Cache.flutterRoot!, 'bin', platform.isWindows ? 'flutter.bat' : 'flutter');
+
+    final bool debug = !(args.noDebug ?? false);
+    final String? program = args.program;
+
+    final List<String> toolArgs = <String>[
+      'test',
+      '--machine',
+      if (debug) '--start-paused',
+    ];
+    final List<String> processArgs = <String>[
+      ...toolArgs,
+      ...?args.toolArgs,
+      if (program != null) program,
+      ...?args.args,
+    ];
+
+    // Find the package_config file for this script. This is used by the
+    // debugger to map package: URIs to file paths to check whether they're in
+    // the editors workspace (args.cwd/args.additionalProjectPaths) so they can
+    // be correctly classes as "my code", "sdk" or "external packages".
+    // TODO(dantup): Remove this once https://github.com/dart-lang/sdk/issues/45530
+    // is done as it will not be necessary.
+    final String? possibleRoot = program == null
+        ? args.cwd
+        : fileSystem.path.isAbsolute(program)
+            ? fileSystem.path.dirname(program)
+            : fileSystem.path.dirname(
+                fileSystem.path.normalize(fileSystem.path.join(args.cwd ?? '', args.program)));
+    if (possibleRoot != null) {
+      final File? packageConfig = findPackageConfigFile(possibleRoot);
+      if (packageConfig != null) {
+        usePackageConfigFile(packageConfig);
+      }
+    }
+
+    logger?.call('Spawning $flutterToolPath with $processArgs in ${args.cwd}');
+    final Process process = await Process.start(
+      flutterToolPath,
+      processArgs,
+      workingDirectory: args.cwd,
+    );
+    _process = process;
+    pidsToTerminate.add(process.pid);
+
+    process.stdout.transform(ByteToLineTransformer()).listen(_handleStdout);
+    process.stderr.listen(_handleStderr);
+    unawaited(process.exitCode.then(_handleExitCode));
+
+    // Delay responding until the debugger is connected.
+    if (debug) {
+      await debuggerInitialized;
+    }
+  }
+
+  /// Called by [terminateRequest] to request that we gracefully shut down the app being run (or in the case of an attach, disconnect).
+  @override
+  Future<void> terminateImpl() async {
+    terminatePids(ProcessSignal.sigterm);
+    await _process?.exitCode;
+  }
+
+  /// Handles the Flutter process exiting, terminating the debug session if it has not already begun terminating.
+  void _handleExitCode(int code) {
+    final String codeSuffix = code == 0 ? '' : ' ($code)';
+    logger?.call('Process exited ($code)');
+    handleSessionTerminate(codeSuffix);
+  }
+
+  /// Handles incoming JSON events from `flutter test --machine`.
+  bool _handleJsonEvent(String event, Map<String, Object?>? params) {
+    params ??= <String, Object?>{};
+    switch (event) {
+      case 'test.startedProcess':
+        _handleTestStartedProcess(params);
+        return true;
+    }
+
+    return false;
+  }
+
+  void _handleStderr(List<int> data) {
+    logger?.call('stderr: $data');
+    sendOutput('stderr', utf8.decode(data));
+  }
+
+  /// Handles stdout from the `flutter test --machine` process, decoding the JSON and calling the appropriate handlers.
+  void _handleStdout(String data) {
+    // Output to stdout from `flutter test --machine` is either:
+    //   1. JSON output from flutter_tools (eg. "test.startedProcess") which is
+    //      wrapped in [] brackets and has an event/params.
+    //   2. JSON output from package:test (not wrapped in brackets).
+    //   3. Non-JSON output (user messages, or flutter_tools printing things like
+    //      call stacks/error information).
+    logger?.call('stdout: $data');
+
+    Object? jsonData;
+    try {
+      jsonData = jsonDecode(data);
+    } on FormatException {
+      // If the output wasn't valid JSON, it was standard stdout that should
+      // be passed through to the user.
+      sendOutput('stdout', data);
+      return;
+    }
+
+    // Check for valid flutter_tools JSON output (1) first.
+    final Map<String, Object?>? flutterPayload = jsonData is List &&
+            jsonData.length == 1 &&
+            jsonData.first is Map<String, Object?>
+        ? jsonData.first as Map<String, Object?>
+        : null;
+    final Object? event = flutterPayload?['event'];
+    final Object? params = flutterPayload?['params'];
+
+    if (event is String && params is Map<String, Object?>?) {
+      _handleJsonEvent(event, params);
+    } else if (jsonData != null) {
+      // Handle package:test output (2).
+      sendTestEvents(jsonData);
+    } else {
+      // Other output should just be passed straight through.
+      sendOutput('stdout', data);
+    }
+  }
+
+  /// Handles the test.processStarted event from Flutter that provides the VM Service URL.
+  void _handleTestStartedProcess(Map<String, Object?> params) {
+    final String? vmServiceUriString = params['observatoryUri'] as String?;
+    // For no-debug mode, this event is still sent, but has a null URI.
+    if (vmServiceUriString == null) {
+      return;
+    }
+    final Uri vmServiceUri = Uri.parse(vmServiceUriString);
+    connectDebugger(vmServiceUri, resumeIfStarting: true);
+  }
+}

packages/flutter_tools/lib/src/debug_adapters/server.dart

@@ -10,6 +10,7 @@ import '../base/file_system.dart';
 import '../base/platform.dart';
 import '../debug_adapters/flutter_adapter.dart';
 import '../debug_adapters/flutter_adapter_args.dart';
+import 'flutter_test_adapter.dart';
 
 /// A DAP server that communicates over a [ByteStreamServerChannel], usually constructed from the processes stdin/stdout streams.
 ///
@@ -27,15 +28,23 @@ class DapServer {
     this.ipv6 = false,
     this.enableDds = true,
     this.enableAuthCodes = true,
+    bool test = false,
     this.logger,
   }) : channel = ByteStreamServerChannel(_input, _output, logger) {
-    adapter = FlutterDebugAdapter(channel,
-        fileSystem: fileSystem,
-        platform: platform,
-        ipv6: ipv6,
-        enableDds: enableDds,
-        enableAuthCodes: enableAuthCodes,
-        logger: logger);
+    adapter = test
+        ? FlutterTestDebugAdapter(channel,
+            fileSystem: fileSystem,
+            platform: platform,
+            ipv6: ipv6,
+            enableDds: enableDds,
+            enableAuthCodes: enableAuthCodes,
+            logger: logger)
+        : FlutterDebugAdapter(channel,
+            fileSystem: fileSystem,
+            platform: platform,
+            enableDds: enableDds,
+            enableAuthCodes: enableAuthCodes,
+            logger: logger);
   }
 
   final ByteStreamServerChannel channel;

packages/flutter_tools/test/integration.shard/debug_adapter/flutter_adapter_test.dart

@@ -27,7 +27,7 @@ void main() {
   });
 
   setUp(() async {
-    tempDir = createResolvedTempDirectorySync('debug_adapter_test.');
+    tempDir = createResolvedTempDirectorySync('flutter_adapter_test.');
     dap = await DapTestSession.setUp();
   });
 

packages/flutter_tools/test/integration.shard/debug_adapter/test_adapter_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'package:dds/src/dap/protocol_generated.dart';
+import 'package:file/file.dart';
+import 'package:flutter_tools/src/cache.dart';
+
+import '../../src/common.dart';
+import '../test_data/tests_project.dart';
+import '../test_utils.dart';
+import 'test_client.dart';
+import 'test_support.dart';
+
+void main() {
+  Directory tempDir;
+  /*late*/ DapTestSession dap;
+
+  setUpAll(() {
+    Cache.flutterRoot = getFlutterRoot();
+  });
+
+  setUp(() async {
+    tempDir = createResolvedTempDirectorySync('flutter_test_adapter_test.');
+    dap = await DapTestSession.setUp(additionalArgs: <String>['--test']);
+  });
+
+  tearDown(() async {
+    await dap.tearDown();
+    tryToDelete(tempDir);
+  });
+
+  test('can run in debug mode', () async {
+    final DapTestClient client = dap.client;
+    final TestsProject project = TestsProject();
+    await project.setUpIn(tempDir);
+
+    // Collect output and test events while running the script.
+    final TestEvents outputEvents = await client.collectTestOutput(
+      launch: () => client.launch(
+        program: project.testFilePath,
+        cwd: project.dir.path,
+      ),
+    );
+
+    // Check the printed output shows that the run finished, and it's exit
+    // code (which is 1 due to the failing test).
+    final String output = outputEvents.output.map((OutputEventBody e) => e.output).join();
+    expectLines(
+      output,
+      <Object>[
+        startsWith('Connecting to VM Service at'),
+        ..._testsProjectExpectedOutput
+      ],
+      allowExtras: true, // Allow for printed call stack etc.
+    );
+
+    _expectStandardTestsProjectResults(outputEvents);
+  });
+
+  test('can run in noDebug mode', () async {
+    final DapTestClient client = dap.client;
+    final TestsProject project = TestsProject();
+    await project.setUpIn(tempDir);
+
+    // Collect output and test events while running the script.
+    final TestEvents outputEvents = await client.collectTestOutput(
+      launch: () => client.launch(
+        program: project.testFilePath,
+        noDebug: true,
+        cwd: project.dir.path,
+      ),
+    );
+
+    // Check the printed output shows that the run finished, and it's exit
+    // code (which is 1 due to the failing test).
+    final String output = outputEvents.output.map((OutputEventBody e) => e.output).join();
+    expectLines(
+      output,
+      _testsProjectExpectedOutput,
+      allowExtras: true, // Allow for printed call stack etc.
+    );
+
+    _expectStandardTestsProjectResults(outputEvents);
+  });
+
+  test('can run a single test', () async {
+    final DapTestClient client = dap.client;
+    final TestsProject project = TestsProject();
+    await project.setUpIn(tempDir);
+
+    // Collect output and test events while running the script.
+    final TestEvents outputEvents = await client.collectTestOutput(
+      launch: () => client.launch(
+        program: project.testFilePath,
+        noDebug: true,
+        cwd: project.dir.path,
+        // It's up to the calling IDE to pass the correct args for 'dart test'
+        // if it wants to run a subset of tests.
+        args: <String>[
+          '--plain-name',
+          'can pass',
+        ],
+      ),
+    );
+
+    final List<Object> testsNames = outputEvents.testNotifications
+        .where((Map<String, Object>/*?*/ e) => e['type'] == 'testStart')
+        .map((Map<String, Object>/*?*/ e) => (e['test'] as Map<String, Object/*?*/>)['name'])
+        .toList();
+
+    expect(testsNames, contains('Flutter tests can pass'));
+    expect(testsNames, isNot(contains('Flutter tests can fail')));
+  });
+}
+
+/// Matchers for the expected console output of [TestsProject].
+final List<Object> _testsProjectExpectedOutput = <Object>[
+  // First test
+  '✓ Flutter tests can pass',
+  // Second test
+  '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════',
+  'The following TestFailure object was thrown running a test:',
+  '  Expected: false',
+  '  Actual: <true>',
+  '',
+  'The test description was: can fail',
+  '',
+  '✖ Flutter tests can fail',
+  // Exit
+  '',
+  'Exited (1).',
+];
+
+/// A helper that verifies a full set of expected test results for the
+/// [TestsProject] script.
+void _expectStandardTestsProjectResults(TestEvents events) {
+  // Check we recieved all expected test events passed through from
+  // package:test.
+  final List<Object> eventNames =
+      events.testNotifications.map((Map<String, Object/*?*/> e) => e['type']).toList();
+
+  // start/done should always be first/last.
+  expect(eventNames.first, equals('start'));
+  expect(eventNames.last, equals('done'));
+
+  // allSuites should have occurred after start.
+  expect(
+    eventNames,
+    containsAllInOrder(<String>['start', 'allSuites']),
+  );
+
+  // Expect two tests, with the failing one emitting an error.
+  expect(
+    eventNames,
+    containsAllInOrder(<String>[
+      'testStart',
+      'testDone',
+      'testStart',
+      'error',
+      'testDone',
+    ]),
+  );
+}

packages/flutter_tools/test/integration.shard/debug_adapter/test_client.dart

@@ -74,6 +74,12 @@ class DapTestClient {
     return _eventController.stream.where((Event e) => e.event == event);
   }
 
+  /// Returns a stream of 'dart.testNotification' custom events from the
+  /// package:test JSON reporter.
+  Stream<Map<String, Object?>> get testNotificationEvents =>
+      events('dart.testNotification')
+          .map((Event e) => e.body! as Map<String, Object?>);
+
   /// Sends a custom request to the debug adapter to trigger a Hot Reload.
   Future<Response> hotReload() {
     return custom('hotReload');
@@ -220,6 +226,17 @@ class DapTestClient {
   }
 }
 
+/// Useful events produced by the debug adapter during a debug session.
+class TestEvents {
+  TestEvents({
+    required this.output,
+    required this.testNotifications,
+  });
+
+  final List<OutputEventBody> output;
+  final List<Map<String, Object?>> testNotifications;
+}
+
 class _OutgoingRequest {
   _OutgoingRequest(this.completer, this.name, this.allowFailure);
 
@@ -273,4 +290,39 @@ extension DapTestClientExtension on DapTestClient {
         ? output.skipWhile((OutputEventBody output) => output.output.startsWith('Running "flutter pub get"')).toList()
         : output;
   }
+
+  /// Collects all output and test events until the program terminates.
+  ///
+  /// These results include all events in the order they are recieved, including
+  /// console, stdout, stderr and test notifications from the test JSON reporter.
+  ///
+  /// Only one of [start] or [launch] may be provided. Use [start] to customise
+  /// the whole start of the session (including initialise) or [launch] to only
+  /// customise the [launchRequest].
+  Future<TestEvents> collectTestOutput({
+    String? program,
+    String? cwd,
+    Future<Response> Function()? start,
+    Future<Object?> Function()? launch,
+  }) async {
+    assert(
+      start == null || launch == null,
+      'Only one of "start" or "launch" may be provided',
+    );
+
+    final Future<List<OutputEventBody>> outputEventsFuture = outputEvents.toList();
+    final Future<List<Map<String, Object?>>> testNotificationEventsFuture = testNotificationEvents.toList();
+
+    if (start != null) {
+      await start();
+    } else {
+      await this.start(program: program, cwd: cwd, launch: launch);
+    }
+
+    return TestEvents(
+      output: await outputEventsFuture,
+      testNotifications: await testNotificationEventsFuture,
+    );
+  }
+
 }

packages/flutter_tools/test/integration.shard/debug_adapter/test_support.dart

@@ -29,11 +29,22 @@ final bool verboseLogging = Platform.environment['DAP_TEST_VERBOSE'] == 'true';
 
 /// Expects the lines in [actual] to match the relevant matcher in [expected],
 /// ignoring differences in line endings and trailing whitespace.
-void expectLines(String actual, List<Object> expected) {
-  expect(
-    actual.replaceAll('\r\n', '\n').trim().split('\n'),
-    equals(expected),
-  );
+void expectLines(
+  String actual,
+  List<Object> expected, {
+  bool allowExtras = false,
+}) {
+  if (allowExtras) {
+    expect(
+      actual.replaceAll('\r\n', '\n').trim().split('\n'),
+      containsAllInOrder(expected),
+    );
+  } else {
+    expect(
+      actual.replaceAll('\r\n', '\n').trim().split('\n'),
+      equals(expected),
+    );
+  }
 }
 
 /// A helper class containing the DAP server/client for DAP integration tests.

packages/flutter_tools/test/integration.shard/test_data/tests_project.dart

@@ -34,8 +34,13 @@ class TestsProject extends Project {
   import 'package:flutter_test/flutter_test.dart';
 
   void main() {
-    testWidgets('Hello world test', (WidgetTester tester) async {
-      expect(true, isTrue); // BREAKPOINT
+    group('Flutter tests', () {
+      testWidgets('can pass', (WidgetTester tester) async {
+        expect(true, isTrue); // BREAKPOINT
+      });
+      testWidgets('can fail', (WidgetTester tester) async {
+        expect(true, isFalse);
+      });
     });
   }
   ''';