[flutter_tools] Support Integration Tests (#76200)

dev/automated_tests/integration_test/exception_handling_expectation.txt

+<<skip until matching line>>
+══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════
+The following message was thrown running a test:
+Who lives, who dies, who tells your story\?
+
+When the exception was thrown, this was the stack:
+#0      main.<anonymous closure> \(.+[/\\]dev[/\\]automated_tests[/\\]integration_test[/\\]exception_handling_test\.dart:10:5\)
+<<skip until matching line>>
+The test description was:
+Exception handling in test harness - string
+<<skip until matching line>>
+  Test failed\. See exception logs above\.
+  The test description was: Exception handling in test harness - string
+<<skip until matching line>>
+══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════
+The following assertion was thrown running a test:
+Who lives, who dies, who tells your story\?
+
+When the exception was thrown, this was the stack:
+#0      main.<anonymous closure> \(.+[/\\]dev[/\\]automated_tests[/\\]integration_test[/\\]exception_handling_test\.dart:13:5\)
+<<skip until matching line>>
+The test description was:
+Exception handling in test harness - FlutterError
+<<skip until matching line>>
+  Test failed\. See exception logs above\.
+  The test description was: Exception handling in test harness - FlutterError
+<<skip until matching line>>
+══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════
+The following message was thrown running a test:
+Who lives, who dies, who tells your story\?
+
+When the exception was thrown, this was the stack:
+.*(TODO\(jiahaog\): Remove --no-chain-stack-traces and replace this with the actual stack once https://github.com/dart-lang/stack_trace/issues/106 is fixed)?
+<<skip until matching line>>
+The test description was:
+Exception handling in test harness - uncaught Future error
+<<skip until matching line>>
+  Test failed\. See exception logs above\.
+  The test description was: Exception handling in test harness - uncaught Future error
+<<skip until matching line>>
+.*..:.. \+0 -3: Some tests failed\. *

dev/automated_tests/integration_test/exception_handling_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+  testWidgets('Exception handling in test harness - string', (WidgetTester tester) async {
+    throw 'Who lives, who dies, who tells your story?';
+  });
+  testWidgets('Exception handling in test harness - FlutterError', (WidgetTester tester) async {
+    throw FlutterError('Who lives, who dies, who tells your story?');
+  });
+  testWidgets('Exception handling in test harness - uncaught Future error', (WidgetTester tester) async {
+    Future<void>.error('Who lives, who dies, who tells your story?');
+  });
+}

dev/automated_tests/integration_test/trivial_widget_expectation.txt

+<<skip until matching line>>
+[0-9]+:[0-9]+ [+]1: All tests passed!

dev/automated_tests/integration_test/trivial_widget_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+  testWidgets('A trivial widget test', (WidgetTester tester) async {});
+}

dev/automated_tests/pubspec.yaml

@@ -8,6 +8,8 @@ dependencies:
     sdk: flutter
   flutter_test:
     sdk: flutter
+  integration_test:
+    sdk: flutter
   test: 1.16.5
 
   _fe_analyzer_shared: 19.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -25,6 +25,7 @@ import '../test/test_wrapper.dart';
 
 import 'flutter_tester_device.dart';
 import 'font_config_manager.dart';
+import 'integration_test_device.dart';
 import 'test_compiler.dart';
 import 'test_config.dart';
 import 'test_device.dart';
@@ -60,6 +61,8 @@ FlutterPlatform installHook({
   FlutterProject flutterProject,
   String icudtlPath,
   PlatformPluginRegistration platformPluginRegistration,
+  Device integrationTestDevice,
+  String integrationTestUserIdentifier,
 }) {
   assert(testWrapper != null);
   assert(enableObservatory || (!debuggingOptions.startPaused && debuggingOptions.hostVmServicePort == null));
@@ -87,6 +90,8 @@ FlutterPlatform installHook({
     projectRootDirectory: projectRootDirectory,
     flutterProject: flutterProject,
     icudtlPath: icudtlPath,
+    integrationTestDevice: integrationTestDevice,
+    integrationTestUserIdentifier: integrationTestUserIdentifier,
   );
   platformPluginRegistration(platform);
   return platform;
@@ -107,6 +112,10 @@ FlutterPlatform installHook({
 ///
 /// The [updateGoldens] argument will set the [autoUpdateGoldens] global
 /// variable in the [flutter_test] package before invoking the test.
+///
+/// The [integrationTest] argument can be specified to generate the bootstrap
+/// for integration tests.
+///
 // NOTE: this API is used by the fuchsia source tree, do not add new
 // required or position parameters.
 String generateTestBootstrap({
@@ -117,6 +126,7 @@ String generateTestBootstrap({
   String languageVersionHeader = '',
   bool nullSafety = false,
   bool flutterTestDep = true,
+  bool integrationTest = false,
 }) {
   assert(testUrl != null);
   assert(host != null);
@@ -138,6 +148,12 @@ import 'dart:isolate';
   if (flutterTestDep) {
     buffer.write('''
 import 'package:flutter_test/flutter_test.dart';
+''');
+  }
+  if (integrationTest) {
+    buffer.write('''
+import 'package:integration_test/integration_test.dart';
+import 'dart:developer' as developer;
 ''');
   }
   buffer.write('''
@@ -159,6 +175,17 @@ StreamChannel<dynamic> serializeSuite(Function getMain()) {
   return RemoteListener.start(getMain);
 }
 
+Future<void> _testMain() async {
+''');
+  if (integrationTest) {
+    buffer.write('''
+  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+''');
+  }
+  buffer.write('''
+  return test.main();
+}
+
 /// Capture any top-level errors (mostly lazy syntax errors, since other are
 /// caught below) and report them to the parent isolate.
 void catchIsolateErrors() {
@@ -191,15 +218,36 @@ void main() {
   }
   if (testConfigFile != null) {
     buffer.write('''
-    return () => test_config.testExecutable(test.main);
+    return () => test_config.testExecutable(_testMain);
 ''');
   } else {
     buffer.write('''
-    return test.main;
+    return _testMain;
 ''');
   }
   buffer.write('''
   });
+''');
+  if (integrationTest) {
+    buffer.write('''
+  final callback = (method, params) async {
+    testChannel.sink.add(json.decode(params['$kIntegrationTestData'] as String));
+
+    // Result is ignored but null is not accepted here.
+    return developer.ServiceExtensionResponse.result('{}');
+  };
+
+  developer.registerExtension('$kIntegrationTestMethod', callback);
+
+  testChannel.stream.listen((x) {
+    developer.postEvent(
+      '$kIntegrationTestExtension',
+      {'$kIntegrationTestData': json.encode(x)},
+    );
+  });
+  ''');
+  } else {
+    buffer.write('''
   WebSocket.connect(server).then((WebSocket socket) {
     socket.map((dynamic message) {
       // We're only communicating with string encoded JSON.
@@ -207,8 +255,11 @@ void main() {
     }).pipe(testChannel.sink);
     socket.addStream(testChannel.stream.map(json.encode));
   });
-}
 ''');
+  }
+  buffer.write('''
+}
+  ''');
   return buffer.toString();
 }
 
@@ -230,6 +281,8 @@ class FlutterPlatform extends PlatformPlugin {
     this.projectRootDirectory,
     this.flutterProject,
     this.icudtlPath,
+    this.integrationTestDevice,
+    this.integrationTestUserIdentifier,
   }) : assert(shellPath != null);
 
   final String shellPath;
@@ -246,6 +299,15 @@ class FlutterPlatform extends PlatformPlugin {
   final FlutterProject flutterProject;
   final String icudtlPath;
 
+  /// The device to run the test on for Integration Tests.
+  ///
+  /// If this is null, the test will run as a regular test with the Flutter
+  /// Tester; otherwise it will run as a Integration Test on this device.
+  final Device integrationTestDevice;
+  bool get _isIntegrationTest => integrationTestDevice != null;
+
+  final String integrationTestUserIdentifier;
+
   final FontConfigManager _fontConfigManager = FontConfigManager();
 
   /// The test compiler produces dill files for each test main.
@@ -336,6 +398,14 @@ class FlutterPlatform extends PlatformPlugin {
   }
 
   TestDevice _createTestDevice(int ourTestCount) {
+    if (_isIntegrationTest) {
+      return IntegrationTestTestDevice(
+        id: ourTestCount,
+        debuggingOptions: debuggingOptions,
+        device: integrationTestDevice,
+        userIdentifier: integrationTestUserIdentifier,
+      );
+    }
     return FlutterTesterTestDevice(
       id: ourTestCount,
       platform: globals.platform,
@@ -380,26 +450,25 @@ class FlutterPlatform extends PlatformPlugin {
         mainDart = precompiledDillPath;
       } else if (precompiledDillFiles != null) {
         mainDart = precompiledDillFiles[testPath];
-      }
-      mainDart ??= _createListenerDart(finalizers, ourTestCount, testPath);
+      } else {
+        mainDart = _createListenerDart(finalizers, ourTestCount, testPath);
 
-      if (precompiledDillPath == null && precompiledDillFiles == null) {
-        // Lazily instantiate compiler so it is built only if it is actually used.
-        compiler ??= TestCompiler(debuggingOptions.buildInfo, flutterProject);
-        mainDart = await compiler.compile(globals.fs.file(mainDart).uri);
+        // Integration test device takes care of the compilation.
+        if (integrationTestDevice == null) {
+          // Lazily instantiate compiler so it is built only if it is actually used.
+          compiler ??= TestCompiler(debuggingOptions.buildInfo, flutterProject);
+          mainDart = await compiler.compile(globals.fs.file(mainDart).uri);
 
-        if (mainDart == null) {
-          testHarnessChannel.sink.addError('Compilation failed for testPath=$testPath');
-          return null;
+          if (mainDart == null) {
+            testHarnessChannel.sink.addError('Compilation failed for testPath=$testPath');
+            return null;
+          }
         }
       }
 
       globals.printTrace('test $ourTestCount: starting test device');
-
       final TestDevice testDevice = _createTestDevice(ourTestCount);
-      final Future<StreamChannel<String>> remoteChannelFuture = testDevice.start(
-        compiledEntrypointPath: mainDart,
-      );
+      final Future<StreamChannel<String>> remoteChannelFuture = testDevice.start(mainDart);
       finalizers.add(() async {
         globals.printTrace('test $ourTestCount: ensuring test device is terminated.');
         await testDevice.kill();
@@ -521,7 +590,8 @@ class FlutterPlatform extends PlatformPlugin {
       host: host,
       updateGoldens: updateGoldens,
       flutterTestDep: packageConfig['flutter_test'] != null,
-      languageVersionHeader: '// @dart=${languageVersion.major}.${languageVersion.minor}'
+      languageVersionHeader: '// @dart=${languageVersion.major}.${languageVersion.minor}',
+      integrationTest: _isIntegrationTest,
     );
   }
 

packages/flutter_tools/lib/src/test/flutter_tester_device.dart

@@ -71,8 +71,12 @@ class FlutterTesterTestDevice extends TestDevice {
   Process _process;
   HttpServer _server;
 
+  /// Starts the device.
+  ///
+  /// [entrypointPath] is the path to the entrypoint file which must be compiled
+  /// as a dill.
   @override
-  Future<StreamChannel<String>> start({@required String compiledEntrypointPath}) async {
+  Future<StreamChannel<String>> start(String entrypointPath) async {
     assert(!_exitCode.isCompleted);
     assert(_process == null);
     assert(_server == null);
@@ -113,7 +117,7 @@ class FlutterTesterTestDevice extends TestDevice {
       if (debuggingOptions.nullAssertions)
         '--dart-flags=--null_assertions',
       ...debuggingOptions.dartEntrypointArgs,
-      compiledEntrypointPath,
+      entrypointPath,
     ];
 
     // If the FLUTTER_TEST environment variable has been set, then pass it on

packages/flutter_tools/lib/src/test/integration_test_device.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'dart:async';
+
+import 'package:meta/meta.dart';
+import 'package:stream_channel/stream_channel.dart';
+import 'package:vm_service/vm_service.dart' as vm_service;
+
+import '../application_package.dart';
+import '../base/common.dart';
+import '../build_info.dart';
+import '../device.dart';
+import '../globals.dart' as globals;
+import '../vmservice.dart';
+import 'test_device.dart';
+
+const String kIntegrationTestExtension = 'Flutter.IntegrationTest';
+const String kIntegrationTestData = 'data';
+const String kIntegrationTestMethod = 'ext.flutter.integrationTest';
+
+class IntegrationTestTestDevice implements TestDevice {
+  IntegrationTestTestDevice({
+    @required this.id,
+    @required this.device,
+    @required this.debuggingOptions,
+    @required this.userIdentifier,
+  });
+
+  final int id;
+  final Device device;
+  final DebuggingOptions debuggingOptions;
+  final String userIdentifier;
+
+  ApplicationPackage _applicationPackage;
+  final Completer<void> _finished = Completer<void>();
+  final Completer<Uri> _gotProcessObservatoryUri = Completer<Uri>();
+
+  /// Starts the device.
+  ///
+  /// [entrypointPath] must be a path to an uncompiled source file.
+  @override
+  Future<StreamChannel<String>> start(String entrypointPath) async {
+    final TargetPlatform targetPlatform = await device.targetPlatform;
+    _applicationPackage = await ApplicationPackageFactory.instance.getPackageForPlatform(
+      targetPlatform,
+      buildInfo: debuggingOptions.buildInfo,
+    );
+
+    final LaunchResult launchResult = await device.startApp(
+      _applicationPackage,
+      mainPath: entrypointPath,
+      platformArgs: <String, dynamic>{},
+      debuggingOptions: debuggingOptions,
+      userIdentifier: userIdentifier,
+    );
+    if (!launchResult.started) {
+      throw TestDeviceException('Unable to start the app on the device.', StackTrace.current);
+    }
+    if (launchResult.observatoryUri == null) {
+      throw TestDeviceException('Observatory is not available on the test device.', StackTrace.current);
+    }
+
+    // No need to set up the log reader because the logs are captured and
+    // streamed to the package:test_core runner.
+
+    _gotProcessObservatoryUri.complete(launchResult.observatoryUri);
+
+    globals.printTrace('test $id: Connecting to vm service');
+    final FlutterVmService vmService = await connectToVmService(launchResult.observatoryUri).timeout(
+      const Duration(seconds: 5),
+      onTimeout: () => throw TimeoutException('Connecting to the VM Service timed out.'),
+    );
+
+    globals.printTrace('test $id: Finding the correct isolate with the integration test service extension');
+    final vm_service.IsolateRef isolateRef = await vmService.findExtensionIsolate(kIntegrationTestMethod);
+
+    await vmService.service.streamListen(vm_service.EventStreams.kExtension);
+    final Stream<String> remoteMessages = vmService.service.onExtensionEvent
+        .where((vm_service.Event e) => e.extensionKind == kIntegrationTestExtension)
+        .map((vm_service.Event e) => e.extensionData.data[kIntegrationTestData] as String);
+
+    final StreamChannelController<String> controller = StreamChannelController<String>();
+
+    controller.local.stream.listen((String event) {
+      vmService.service.callServiceExtension(
+        kIntegrationTestMethod,
+        isolateId: isolateRef.id,
+        args: <String, String>{
+          kIntegrationTestData: event,
+        },
+      );
+    });
+
+    unawaited(remoteMessages.pipe(controller.local.sink));
+    return controller.foreign;
+  }
+
+  @override
+  Future<Uri> get observatoryUri => _gotProcessObservatoryUri.future;
+
+  @override
+  Future<void> kill() async {
+    if (!await device.stopApp(_applicationPackage, userIdentifier: userIdentifier)) {
+      globals.printTrace('Could not stop the Integration Test app.');
+    }
+    if (!await device.uninstallApp(_applicationPackage, userIdentifier: userIdentifier)) {
+      globals.printTrace('Could not uninstall the Integration Test app.');
+    }
+
+    await device.dispose();
+    _finished.complete();
+  }
+
+  @override
+  Future<void> get finished => _finished.future;
+}

packages/flutter_tools/lib/src/test/runner.dart

@@ -53,6 +53,8 @@ abstract class FlutterTestRunner {
     bool runSkipped = false,
     int shardIndex,
     int totalShards,
+    Device integrationTestDevice,
+    String integrationTestUserIdentifier,
   });
 }
 
@@ -87,6 +89,8 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
     bool runSkipped = false,
     int shardIndex,
     int totalShards,
+    Device integrationTestDevice,
+    String integrationTestUserIdentifier,
   }) async {
     // Configure package:test to use the Flutter engine for child processes.
     final String shellPath = globals.artifacts.getArtifactPath(Artifact.flutterTester);
@@ -179,6 +183,13 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
       return exitCode;
     }
 
+    if (integrationTestDevice != null) {
+      // Without this, some async exceptions which are caught will surface when
+      // debugging tests.
+      // TODO(jiahaog): Remove this once https://github.com/dart-lang/stack_trace/issues/106 is fixed.
+      testArgs.add('--no-chain-stack-traces');
+    }
+
     testArgs
       ..add('--')
       ..addAll(testFiles);
@@ -201,6 +212,8 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
       projectRootDirectory: globals.fs.currentDirectory.uri,
       flutterProject: flutterProject,
       icudtlPath: icudtlPath,
+      integrationTestDevice: integrationTestDevice,
+      integrationTestUserIdentifier: integrationTestUserIdentifier,
     );
 
     try {

packages/flutter_tools/lib/src/test/test_device.dart

@@ -4,30 +4,30 @@
 
 import 'dart:async';
 
-import 'package:meta/meta.dart';
 import 'package:stream_channel/stream_channel.dart';
 
 /// A remote device where tests can be executed on.
 ///
 /// Reusability of an instance across multiple runs is not guaranteed for all
 /// implementations.
+///
+/// Methods may throw [TestDeviceException] if a problem is encountered.
 abstract class TestDevice {
   /// Starts the test device with the provided entrypoint.
   ///
   /// Returns a channel that can be used to communicate with the test process.
-  Future<StreamChannel<String>> start({@required String compiledEntrypointPath});
+  ///
+  /// It is up to the device to determine if [entrypointPath] is a precompiled
+  /// or raw source file.
+  Future<StreamChannel<String>> start(String entrypointPath);
 
   /// Should complete with null if the observatory is not enabled.
   Future<Uri> get observatoryUri;
 
   /// Terminates the test device.
-  ///
-  /// A [TestDeviceException] can be thrown if it did not stop gracefully.
   Future<void> kill();
 
   /// Waits for the test device to stop.
-  ///
-  /// A [TestDeviceException] can be thrown if it did not stop gracefully.
   Future<void> get finished;
 }
 

packages/flutter_tools/test/general.shard/device_test.dart

@@ -150,11 +150,11 @@ void main() {
   });
 
   group('Filter devices', () {
-    final FakeDevice ephemeralOne = FakeDevice('ephemeralOne', 'ephemeralOne', true);
-    final FakeDevice ephemeralTwo = FakeDevice('ephemeralTwo', 'ephemeralTwo', true);
-    final FakeDevice nonEphemeralOne = FakeDevice('nonEphemeralOne', 'nonEphemeralOne', false);
-    final FakeDevice nonEphemeralTwo = FakeDevice('nonEphemeralTwo', 'nonEphemeralTwo', false);
-    final FakeDevice unsupported = FakeDevice('unsupported', 'unsupported', true, false);
+    final FakeDevice ephemeralOne = FakeDevice('ephemeralOne', 'ephemeralOne');
+    final FakeDevice ephemeralTwo = FakeDevice('ephemeralTwo', 'ephemeralTwo');
+    final FakeDevice nonEphemeralOne = FakeDevice('nonEphemeralOne', 'nonEphemeralOne', ephemeral: false);
+    final FakeDevice nonEphemeralTwo = FakeDevice('nonEphemeralTwo', 'nonEphemeralTwo', ephemeral: false);
+    final FakeDevice unsupported = FakeDevice('unsupported', 'unsupported', isSupported: false);
     final FakeDevice webDevice = FakeDevice('webby', 'webby')
       ..targetPlatform = Future<TargetPlatform>.value(TargetPlatform.web_javascript);
     final FakeDevice fuchsiaDevice = FakeDevice('fuchsiay', 'fuchsiay')

packages/flutter_tools/test/general.shard/flutter_tester_device_test.dart

@@ -88,7 +88,7 @@ void main() {
     testUsingContext('as true when not originally set', () async {
       processManager.addCommand(flutterTestCommand('true'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -96,7 +96,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': 'true'};
       processManager.addCommand(flutterTestCommand('true'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -104,7 +104,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': 'false'};
       processManager.addCommand(flutterTestCommand('false'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -112,7 +112,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': 'neither true nor false'};
       processManager.addCommand(flutterTestCommand('neither true nor false'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -120,7 +120,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': null};
       processManager.addCommand(flutterTestCommand(null));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
   });
@@ -154,7 +154,7 @@ void main() {
     });
 
     testUsingContext('Can pass additional arguments to tester binary', () async {
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
 
       expect(processManager, hasNoRemainingExpectations);
     });
@@ -187,7 +187,7 @@ void main() {
     });
 
     testUsingContext('skips setting observatory port and uses the input port for for DDS instead', () async {
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       await device.observatoryUri;
 
       final Uri uri = await (device as TestFlutterTesterDevice).ddsServiceUriFuture();

packages/flutter_tools/test/general.shard/integration_test_device_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'package:flutter_tools/src/base/io.dart' as io;
+import 'package:flutter_tools/src/build_info.dart';
+import 'package:flutter_tools/src/device.dart';
+import 'package:flutter_tools/src/test/integration_test_device.dart';
+import 'package:flutter_tools/src/test/test_device.dart';
+import 'package:flutter_tools/src/vmservice.dart';
+import 'package:vm_service/vm_service.dart' as vm_service;
+
+import '../src/common.dart';
+import '../src/context.dart';
+import '../src/fake_devices.dart';
+
+final Map<String, Object> vm = <String, dynamic>{
+  'isolates': <dynamic>[
+    <String, dynamic>{
+      'type': '@Isolate',
+      'fixedId': true,
+      'id': 'isolates/242098474',
+      'name': 'main.dart:main()',
+      'number': 242098474,
+    },
+  ],
+};
+
+final vm_service.Isolate isolate = vm_service.Isolate(
+  id: '1',
+  pauseEvent: vm_service.Event(
+      kind: vm_service.EventKind.kResume,
+      timestamp: 0
+  ),
+  breakpoints: <vm_service.Breakpoint>[],
+  exceptionPauseMode: null,
+  libraries: <vm_service.LibraryRef>[
+    vm_service.LibraryRef(
+      id: '1',
+      uri: 'file:///hello_world/main.dart',
+      name: '',
+    ),
+  ],
+  livePorts: 0,
+  name: 'test',
+  number: '1',
+  pauseOnExit: false,
+  runnable: true,
+  startTime: 0,
+  isSystemIsolate: false,
+  isolateFlags: <vm_service.IsolateFlag>[],
+  extensionRPCs: <String>[kIntegrationTestMethod],
+);
+
+final FlutterView fakeFlutterView = FlutterView(
+  id: 'a',
+  uiIsolate: isolate,
+);
+
+final FakeVmServiceRequest listViewsRequest = FakeVmServiceRequest(
+  method: kListViewsMethod,
+  jsonResponse: <String, Object>{
+    'views': <Object>[
+      fakeFlutterView.toJson(),
+    ],
+  },
+);
+
+final Uri observatoryUri = Uri.parse('http://localhost:1234');
+
+void main() {
+  FakeVmServiceHost fakeVmServiceHost;
+  TestDevice testDevice;
+
+  setUp(() {
+    testDevice = IntegrationTestTestDevice(
+      id: 1,
+      device: FakeDevice(
+        'ephemeral',
+        'ephemeral',
+        type: PlatformType.android,
+        launchResult: LaunchResult.succeeded(observatoryUri: observatoryUri),
+      ),
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      userIdentifier: '',
+    );
+
+    fakeVmServiceHost = FakeVmServiceHost(requests: <VmServiceExpectation>[
+      const FakeVmServiceRequest(
+        method: 'streamListen',
+        args: <String, Object>{
+          'streamId': 'Isolate',
+        },
+      ),
+      listViewsRequest,
+      FakeVmServiceRequest(
+        method: 'getIsolate',
+        jsonResponse: isolate.toJson(),
+        args: <String, Object>{
+          'isolateId': '1',
+        },
+      ),
+      const FakeVmServiceRequest(
+        method: 'streamCancel',
+        args: <String, Object>{
+          'streamId': 'Isolate',
+        },
+      ),
+      const FakeVmServiceRequest(
+        method: 'streamListen',
+        args: <String, Object>{
+          'streamId': 'Extension',
+        },
+      ),
+    ]);
+  });
+
+  testUsingContext('Can start the entrypoint', () async {
+    await testDevice.start('entrypointPath');
+
+    expect(await testDevice.observatoryUri, observatoryUri);
+    expect(testDevice.finished, doesNotComplete);
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+
+  testUsingContext('Can kill the started device', () async {
+    await testDevice.start('entrypointPath');
+    await testDevice.kill();
+
+    expect(testDevice.finished, completes);
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+
+  testUsingContext('when the device starts without providing an observatory URI', () async {
+    final TestDevice testDevice = IntegrationTestTestDevice(
+      id: 1,
+      device: FakeDevice(
+        'ephemeral',
+        'ephemeral',
+        type: PlatformType.android,
+        launchResult: LaunchResult.succeeded(observatoryUri: null),
+      ),
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      userIdentifier: '',
+    );
+
+    expect(() => testDevice.start('entrypointPath'), throwsA(isA<TestDeviceException>()));
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+
+  testUsingContext('when the device fails to start', () async {
+    final TestDevice testDevice = IntegrationTestTestDevice(
+      id: 1,
+      device: FakeDevice(
+        'ephemeral',
+        'ephemeral',
+        type: PlatformType.android,
+        launchResult: LaunchResult.failed(),
+      ),
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      userIdentifier: '',
+    );
+
+    expect(() => testDevice.start('entrypointPath'), throwsA(isA<TestDeviceException>()));
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+}

packages/flutter_tools/test/integration.shard/test_test.dart

@@ -18,8 +18,13 @@ import 'test_utils.dart';
 final String automatedTestsDirectory = fileSystem.path.join('..', '..', 'dev', 'automated_tests');
 final String missingDependencyDirectory = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
 final String flutterTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'flutter_test');
+final String integrationTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'integration_test');
 final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', platform.isWindows ? 'flutter.bat' : 'flutter');
 
+// Running Integration Tests in the Flutter Tester will still exercise the same
+// flows specific to Integration Tests.
+final List<String> integrationTestExtraArgs = <String>['-d', 'flutter-tester'];
+
 void main() {
   setUpAll(() async {
     await processManager.run(
@@ -44,6 +49,10 @@ void main() {
     return _testFile('trivial_widget', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
   });
 
+  testWithoutContext('integration test should not have extraneous error messages', () async {
+    return _testFile('trivial_widget', automatedTestsDirectory, integrationTestDirectory, exitCode: isZero, extraArguments: integrationTestExtraArgs);
+  });
+
   testWithoutContext('flutter test set the working directory correctly', () async {
     return _testFile('working_directory', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
   });
@@ -52,6 +61,10 @@ void main() {
     return _testFile('exception_handling', automatedTestsDirectory, flutterTestDirectory);
   });
 
+  testWithoutContext('integration test should report nice errors for exceptions thrown within testWidgets()', () async {
+    return _testFile('exception_handling', automatedTestsDirectory, integrationTestDirectory, extraArguments: integrationTestExtraArgs);
+  });
+
   testWithoutContext('flutter test should report a nice error when a guarded function was called without await', () async {
     return _testFile('test_async_utils_guarded', automatedTestsDirectory, flutterTestDirectory);
   });

packages/flutter_tools/test/src/fake_devices.dart

@@ -4,6 +4,7 @@
 
 // @dart = 2.8
 
+import 'package:flutter_tools/src/application_package.dart';
 import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/device.dart';
 import 'package:flutter_tools/src/project.dart';
@@ -12,7 +13,7 @@ import 'package:flutter_tools/src/project.dart';
 /// (`Device.toJson()` and `--machine` flag for `devices` command)
 List<FakeDeviceJsonData> fakeDevices = <FakeDeviceJsonData>[
   FakeDeviceJsonData(
-    FakeDevice('ephemeral', 'ephemeral', true, true, PlatformType.android),
+    FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
     <String, Object>{
       'name': 'ephemeral',
       'id': 'ephemeral',
@@ -57,18 +58,51 @@ List<FakeDeviceJsonData> fakeDevices = <FakeDeviceJsonData>[
 
 /// Fake device to test `devices` command.
 class FakeDevice extends Device {
-  FakeDevice(this.name, String id, [bool ephemeral = true, this._isSupported = true, PlatformType type = PlatformType.web]) : super(
-      id,
-      platformType: type,
-      category: Category.mobile,
-      ephemeral: ephemeral,
-  );
+  FakeDevice(this.name, String id, {
+    bool ephemeral = true,
+    bool isSupported = true,
+    PlatformType type = PlatformType.web,
+    LaunchResult launchResult,
+  }) : _isSupported = isSupported,
+      _launchResult = launchResult ?? LaunchResult.succeeded(),
+      super(
+        id,
+        platformType: type,
+        category: Category.mobile,
+        ephemeral: ephemeral,
+      );
 
   final bool _isSupported;
+  final LaunchResult _launchResult;
 
   @override
   final String name;
 
+  @override
+  Future<LaunchResult> startApp(covariant ApplicationPackage package, {
+    String mainPath,
+    String route,
+    DebuggingOptions debuggingOptions,
+    Map<String, dynamic> platformArgs,
+    bool prebuiltApplication = false,
+    bool ipv6 = false,
+    String userIdentifier,
+  }) async => _launchResult;
+
+  @override
+  Future<bool> stopApp(covariant ApplicationPackage app, {
+    String userIdentifier,
+  }) async => true;
+
+  @override
+  Future<bool> uninstallApp(
+  covariant ApplicationPackage app, {
+    String userIdentifier,
+  }) async => true;
+
+  @override
+  Future<void> dispose() async {}
+
   @override
   Future<TargetPlatform> targetPlatform = Future<TargetPlatform>.value(TargetPlatform.android_arm);