Run flutter tests through mini test engine when run directly (flutter run -t test_file) (#24930)

dev/automated_tests/flutter_run_test/flutter_run_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+  group('example', () {
+    test('passed', () {
+      expect(true, true);
+    });
+    test('failed', () {
+      expect(true, false);
+    });
+    test('skipped', () {
+      expect(true, false);
+    }, skip: true);
+  });
+}

dev/devicelab/bin/tasks/flutter_run_test.dart

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+final Directory flutterGalleryDir = dir(path.join(flutterDirectory.path, 'examples/hello_world'));
+final File runTestSource = File(path.join(
+  flutterDirectory.path, 'dev', 'automated_tests', 'flutter_run_test', 'flutter_run_test.dart',
+));
+const Pattern passedMessageMatch = '+0: example passed';
+const Pattern failedMessageMatch = '+1: example failed [E]';
+const Pattern skippedMessageMatch = '+1 -1: example skipped';
+const Pattern finishedMessageMatch = '+1 ~1 -1: Some tests failed.';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(createFlutterRunTask);
+}
+
+// verifies that the messages above are printed as a test script is executed.
+Future<TaskResult> createFlutterRunTask() async {
+  bool passedTest = false;
+  bool failedTest = false;
+  bool skippedTest = false;
+  bool finishedMessage = false;
+  final Device device = await devices.workingDevice;
+  await device.unlock();
+  final List<String> options = <String>[
+    '-t', runTestSource.absolute.path, '-d', device.deviceId,
+  ];
+  setLocalEngineOptionIfNecessary(options);
+  await inDirectory<void>(flutterGalleryDir, () async {
+    startProcess(
+      path.join(flutterDirectory.path, 'bin', 'flutter'),
+      <String>['run']..addAll(options),
+      environment: null
+    );
+    final Completer<void> finished = Completer<void>();
+    final StreamSubscription<void> subscription = device.logcat.listen((String line) {
+      // tests execute in order.
+      if (line.contains(passedMessageMatch)) {
+        passedTest = true;
+      } else if (line.contains(failedMessageMatch)) {
+        failedTest = true;
+      } else if (line.contains(skippedMessageMatch)) {
+        skippedTest = true;
+      } else if (line.contains(finishedMessageMatch)) {
+        finishedMessage = true;
+        finished.complete();
+      }
+    });
+    await finished.future.timeout(const Duration(minutes: 1));
+    subscription.cancel();
+  });
+  return passedTest && failedTest && skippedTest && finishedMessage
+    ? TaskResult.success(<String, dynamic>{})
+    : TaskResult.failure('Test did not execute as expected.');
+}
+

dev/devicelab/manifest.yaml

@@ -283,6 +283,13 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
+  flutter_run_test:
+    description: >
+      Tests the `flutter run -t` command with a testfile.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
   named_isolates_test:
     description: >
       Tests naming and attaching to specific isolates.

packages/flutter_test/lib/flutter_test.dart

@@ -56,6 +56,7 @@ export 'src/matchers.dart';
 export 'src/nonconst.dart';
 export 'src/stack_manipulation.dart';
 export 'src/test_async_utils.dart';
+export 'src/test_compat.dart';
 export 'src/test_exception_reporter.dart';
 export 'src/test_pointer.dart';
 export 'src/test_text_input.dart';

packages/flutter_test/lib/src/widget_tester.dart

@@ -19,12 +19,22 @@ import 'controller.dart';
 import 'finders.dart';
 import 'matchers.dart';
 import 'test_async_utils.dart';
+import 'test_compat.dart';
 import 'test_text_input.dart';
 
 /// Keep users from needing multiple imports to test semantics.
 export 'package:flutter/rendering.dart' show SemanticsHandle;
 
+/// Hide these imports so that they do not conflict with our own implementations in
+/// test_compat.dart. This handles setting up a declarer when one is not defined, which
+/// can happen when a test is executed via flutter_run.
 export 'package:test_api/test_api.dart' hide
+  test,
+  group,
+  setUpAll,
+  tearDownAll,
+  setUp,
+  tearDown,
   expect, // we have our own wrapper below
   TypeMatcher, // matcher's TypeMatcher conflicts with the one in the Flutter framework
   isInstanceOf; // we have our own wrapper in matchers.dart
@@ -63,7 +73,7 @@ void testWidgets(String description, WidgetTesterCallback callback, {
   final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized();
   final WidgetTester tester = WidgetTester._(binding);
   timeout ??= binding.defaultTestTimeout;
-  test_package.test(
+  test(
     description,
     () {
       tester._recordNumberOfSemanticsHandles();