[flutter_tools] Support Integration Tests (#76200)

dev/automated_tests/integration_test/exception_handling_expectation.txt

+<<skip until matching line>>
+══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════
+The following message was thrown running a test:
+Who lives, who dies, who tells your story\?
+
+When the exception was thrown, this was the stack:
+#0      main.<anonymous closure> \(.+[/\\]dev[/\\]automated_tests[/\\]integration_test[/\\]exception_handling_test\.dart:10:5\)
+<<skip until matching line>>
+The test description was:
+Exception handling in test harness - string
+<<skip until matching line>>
+  Test failed\. See exception logs above\.
+  The test description was: Exception handling in test harness - string
+<<skip until matching line>>
+══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════
+The following assertion was thrown running a test:
+Who lives, who dies, who tells your story\?
+
+When the exception was thrown, this was the stack:
+#0      main.<anonymous closure> \(.+[/\\]dev[/\\]automated_tests[/\\]integration_test[/\\]exception_handling_test\.dart:13:5\)
+<<skip until matching line>>
+The test description was:
+Exception handling in test harness - FlutterError
+<<skip until matching line>>
+  Test failed\. See exception logs above\.
+  The test description was: Exception handling in test harness - FlutterError
+<<skip until matching line>>
+══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════
+The following message was thrown running a test:
+Who lives, who dies, who tells your story\?
+
+When the exception was thrown, this was the stack:
+.*(TODO\(jiahaog\): Remove --no-chain-stack-traces and replace this with the actual stack once https://github.com/dart-lang/stack_trace/issues/106 is fixed)?
+<<skip until matching line>>
+The test description was:
+Exception handling in test harness - uncaught Future error
+<<skip until matching line>>
+  Test failed\. See exception logs above\.
+  The test description was: Exception handling in test harness - uncaught Future error
+<<skip until matching line>>
+.*..:.. \+0 -3: Some tests failed\. *

dev/automated_tests/integration_test/exception_handling_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+  testWidgets('Exception handling in test harness - string', (WidgetTester tester) async {
+    throw 'Who lives, who dies, who tells your story?';
+  });
+  testWidgets('Exception handling in test harness - FlutterError', (WidgetTester tester) async {
+    throw FlutterError('Who lives, who dies, who tells your story?');
+  });
+  testWidgets('Exception handling in test harness - uncaught Future error', (WidgetTester tester) async {
+    Future<void>.error('Who lives, who dies, who tells your story?');
+  });
+}

dev/automated_tests/integration_test/trivial_widget_expectation.txt

+<<skip until matching line>>
+[0-9]+:[0-9]+ [+]1: All tests passed!

dev/automated_tests/integration_test/trivial_widget_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+  testWidgets('A trivial widget test', (WidgetTester tester) async {});
+}

dev/automated_tests/pubspec.yaml

@@ -8,6 +8,8 @@ dependencies:
     sdk: flutter
   flutter_test:
     sdk: flutter
+  integration_test:
+    sdk: flutter
   test: 1.16.5
 
   _fe_analyzer_shared: 19.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

packages/flutter_tools/lib/src/commands/test.dart

@@ -6,6 +6,8 @@
 
 import 'dart:math' as math;
 
+import 'package:meta/meta.dart';
+
 import '../asset.dart';
 import '../base/common.dart';
 import '../base/file_system.dart';
@@ -23,7 +25,42 @@ import '../test/runner.dart';
 import '../test/test_wrapper.dart';
 import '../test/watcher.dart';
 
-class TestCommand extends FlutterCommand {
+/// The name of the directory where Integration Tests are placed.
+///
+/// When there are test files specified for the test command that are part of
+/// this directory, *relative to the package root*, the files will be executed
+/// as Integration Tests.
+const String _kIntegrationTestDirectory = 'integration_test';
+
+/// A command to run tests.
+///
+/// This command has two modes of execution:
+///
+/// ## Unit / Widget Tests
+///
+/// These tests run in the Flutter Tester, which is a desktop-based Flutter
+/// embedder. In this mode, tests are quick to compile and run.
+///
+/// By default, if no flags are passed to the `flutter test` command, the Tool
+/// will recursively find all files within the `test/` directory that end with
+/// the `*_test.dart` suffix, and run them in a single invocation.
+///
+/// See:
+/// - https://flutter.dev/docs/cookbook/testing/unit/introduction
+/// - https://flutter.dev/docs/cookbook/testing/widget/introduction
+///
+/// ## Integration Tests
+///
+/// These tests run in a connected Flutter Device, similar to `flutter run`. As
+/// a result, iteration is slower because device-based artifacts have to be
+/// built.
+///
+/// Integration tests should be placed in the `integration_test/` directory of
+/// your package. To run these tests, use `flutter test integration_test`.
+///
+/// See:
+/// - https://flutter.dev/docs/testing/integration-tests
+class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
   TestCommand({
     bool verboseHelp = false,
     this.testWrapper = const TestWrapper(),
@@ -36,6 +73,8 @@ class TestCommand extends FlutterCommand {
     addEnableExperimentation(hide: !verboseHelp);
     usesDartDefineOption();
     usesWebRendererOption();
+    usesDeviceUserOption();
+
     argParser
       ..addMultiOption('name',
         help: 'A regular expression matching substrings of the names of tests to run.',
@@ -108,7 +147,8 @@ class TestCommand extends FlutterCommand {
       ..addOption('concurrency',
         abbr: 'j',
         defaultsTo: math.max<int>(1, globals.platform.numberOfProcessors - 2).toString(),
-        help: 'The number of concurrent test processes to run.',
+        help: 'The number of concurrent test processes to run. This will be ignored '
+              'when running integration tests.',
         valueHelp: 'jobs',
       )
       ..addFlag('test-assets',
@@ -186,9 +226,18 @@ class TestCommand extends FlutterCommand {
   /// Interface for running the tester process.
   final FlutterTestRunner testRunner;
 
+  @visibleForTesting
+  bool get isIntegrationTest => _isIntegrationTest;
+  bool _isIntegrationTest = false;
+
+  List<String> _testFiles = <String>[];
+
   @override
   Future<Set<DevelopmentArtifact>> get requiredArtifacts async {
-    final Set<DevelopmentArtifact> results = <DevelopmentArtifact>{};
+    final Set<DevelopmentArtifact> results = _isIntegrationTest
+        // Use [DeviceBasedDevelopmentArtifacts].
+        ? await super.requiredArtifacts
+        : <DevelopmentArtifact>{};
     if (stringArg('platform') == 'chrome') {
       results.add(DevelopmentArtifact.web);
     }
@@ -201,6 +250,44 @@ class TestCommand extends FlutterCommand {
   @override
   String get description => 'Run Flutter unit tests for the current project.';
 
+  @override
+  Future<FlutterCommandResult> verifyThenRunCommand(String commandPath) {
+    _testFiles = argResults.rest.map<String>(globals.fs.path.absolute).toList();
+    if (_testFiles.isEmpty) {
+      // We don't scan the entire package, only the test/ subdirectory, so that
+      // files with names like like "hit_test.dart" don't get run.
+      final Directory testDir = globals.fs.directory('test');
+      if (!testDir.existsSync()) {
+        throwToolExit('Test directory "${testDir.path}" not found.');
+      }
+      _testFiles = _findTests(testDir).toList();
+      if (_testFiles.isEmpty) {
+        throwToolExit(
+          'Test directory "${testDir.path}" does not appear to contain any test files.\n'
+          'Test files must be in that directory and end with the pattern "_test.dart".'
+        );
+      }
+    } else {
+      _testFiles = <String>[
+        for (String path in _testFiles)
+          if (globals.fs.isDirectorySync(path))
+            ..._findTests(globals.fs.directory(path))
+          else
+            globals.fs.path.absolute(path)
+      ];
+    }
+
+    // This needs to be set before [super.verifyThenRunCommand] so that the
+    // correct [requiredArtifacts] can be identified before [run] takes place.
+    _isIntegrationTest = _shouldRunAsIntegrationTests(globals.fs.currentDirectory.absolute.path, _testFiles);
+
+    globals.logger.printTrace(
+      'Found ${_testFiles.length} files which will be executed as '
+      '${_isIntegrationTest ? 'Integration' : 'Widget'} Tests.',
+    );
+    return super.verifyThenRunCommand(commandPath);
+  }
+
   @override
   Future<FlutterCommandResult> runCommand() async {
     if (!globals.fs.isFileSync('pubspec.yaml')) {
@@ -233,45 +320,30 @@ class TestCommand extends FlutterCommand {
       await _buildTestAsset();
     }
 
-    List<String> files = argResults.rest.map<String>((String testPath) => globals.fs.path.absolute(testPath)).toList();
-
     final bool startPaused = boolArg('start-paused');
-    if (startPaused && files.length != 1) {
+    if (startPaused && _testFiles.length != 1) {
       throwToolExit(
         'When using --start-paused, you must specify a single test file to run.',
         exitCode: 1,
       );
     }
 
-    final int jobs = int.tryParse(stringArg('concurrency'));
+    int jobs = int.tryParse(stringArg('concurrency'));
     if (jobs == null || jobs <= 0 || !jobs.isFinite) {
       throwToolExit(
         'Could not parse -j/--concurrency argument. It must be an integer greater than zero.'
       );
     }
-
-    if (files.isEmpty) {
-      // We don't scan the entire package, only the test/ subdirectory, so that
-      // files with names like like "hit_test.dart" don't get run.
-      final Directory testDir = globals.fs.directory('test');
-      if (!testDir.existsSync()) {
-        throwToolExit('Test directory "${testDir.path}" not found.');
-      }
-      files = _findTests(testDir).toList();
-      if (files.isEmpty) {
-        throwToolExit(
-            'Test directory "${testDir.path}" does not appear to contain any test files.\n'
-            'Test files must be in that directory and end with the pattern "_test.dart".'
+    if (_isIntegrationTest) {
+      if (argResults.wasParsed('concurrency')) {
+        globals.logger.printStatus(
+          '-j/--concurrency was parsed but will be ignored, this option is not '
+          'supported when running Integration Tests.',
         );
       }
-    } else {
-      files = <String>[
-        for (String path in files)
-          if (globals.fs.isDirectorySync(path))
-            ..._findTests(globals.fs.directory(path))
-          else
-            path,
-      ];
+      // Running with concurrency will result in deploying multiple test apps
+      // on the connected device concurrently, which is not supported.
+      jobs = 1;
     }
 
     final int shardIndex = int.tryParse(stringArg('shard-index') ?? '');
@@ -319,13 +391,42 @@ class TestCommand extends FlutterCommand {
       buildInfo,
       startPaused: startPaused,
       disableServiceAuthCodes: boolArg('disable-service-auth-codes'),
+      // On iOS >=14, keeping this enabled will leave a prompt on the screen.
+      disablePortPublication: true,
       disableDds: disableDds,
       nullAssertions: boolArg(FlutterOptions.kNullAssertions),
     );
 
+    Device integrationTestDevice;
+    if (_isIntegrationTest) {
+      integrationTestDevice = await findTargetDevice();
+
+      if (integrationTestDevice == null) {
+        throwToolExit(
+          'No devices are connected. '
+          'Ensure that `flutter doctor` shows at least one connected device',
+        );
+      }
+      if (integrationTestDevice.platformType == PlatformType.web) {
+        // TODO(jiahaog): Support web. https://github.com/flutter/flutter/pull/74236
+        throwToolExit('Web devices are not supported for integration tests yet.');
+      }
+
+      if (buildInfo.packageConfig['integration_test'] == null) {
+        throwToolExit(
+          'Error: cannot run without a dependency on "package:integration_test". '
+          'Ensure the following lines are present in your pubspec.yaml:'
+          '\n\n'
+          'dev_dependencies:\n'
+          '  integration_test:\n'
+          '    sdk: flutter\n',
+        );
+      }
+    }
+
     final int result = await testRunner.runTests(
       testWrapper,
-      files,
+      _testFiles,
       debuggingOptions: debuggingOptions,
       names: names,
       plainNames: plainNames,
@@ -346,6 +447,8 @@ class TestCommand extends FlutterCommand {
       runSkipped: boolArg('run-skipped'),
       shardIndex: shardIndex,
       totalShards: totalShards,
+      integrationTestDevice: integrationTestDevice,
+      integrationTestUserIdentifier: stringArg(FlutterOptions.kDeviceUser),
     );
 
     if (collector != null) {
@@ -398,9 +501,36 @@ class TestCommand extends FlutterCommand {
   }
 }
 
+/// Searches [directory] and returns files that end with `_test.dart` as
+/// absolute paths.
 Iterable<String> _findTests(Directory directory) {
   return directory.listSync(recursive: true, followLinks: false)
       .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart') &&
       globals.fs.isFileSync(entity.path))
       .map((FileSystemEntity entity) => globals.fs.path.absolute(entity.path));
 }
+
+/// Returns true if there are files that are Integration Tests.
+///
+/// The [currentDirectory] and [testFiles] parameters here must be provided as
+/// absolute paths.
+///
+/// Throws an exception if there are both Integration Tests and Widget Tests
+/// found in [testFiles].
+bool _shouldRunAsIntegrationTests(String currentDirectory, List<String> testFiles) {
+  final String integrationTestDirectory = globals.fs.path.join(currentDirectory, _kIntegrationTestDirectory);
+
+  if (testFiles.every((String absolutePath) => !absolutePath.startsWith(integrationTestDirectory))) {
+    return false;
+  }
+
+  if (testFiles.every((String absolutePath) => absolutePath.startsWith(integrationTestDirectory))) {
+    return true;
+  }
+
+  throwToolExit(
+    'Integration tests and unit tests cannot be run in a single invocation.'
+    ' Use separate invocations of `flutter test` to run integration tests'
+    ' and unit tests.'
+  );
+}

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -25,6 +25,7 @@ import '../test/test_wrapper.dart';
 
 import 'flutter_tester_device.dart';
 import 'font_config_manager.dart';
+import 'integration_test_device.dart';
 import 'test_compiler.dart';
 import 'test_config.dart';
 import 'test_device.dart';
@@ -60,6 +61,8 @@ FlutterPlatform installHook({
   FlutterProject flutterProject,
   String icudtlPath,
   PlatformPluginRegistration platformPluginRegistration,
+  Device integrationTestDevice,
+  String integrationTestUserIdentifier,
 }) {
   assert(testWrapper != null);
   assert(enableObservatory || (!debuggingOptions.startPaused && debuggingOptions.hostVmServicePort == null));
@@ -87,6 +90,8 @@ FlutterPlatform installHook({
     projectRootDirectory: projectRootDirectory,
     flutterProject: flutterProject,
     icudtlPath: icudtlPath,
+    integrationTestDevice: integrationTestDevice,
+    integrationTestUserIdentifier: integrationTestUserIdentifier,
   );
   platformPluginRegistration(platform);
   return platform;
@@ -107,6 +112,10 @@ FlutterPlatform installHook({
 ///
 /// The [updateGoldens] argument will set the [autoUpdateGoldens] global
 /// variable in the [flutter_test] package before invoking the test.
+///
+/// The [integrationTest] argument can be specified to generate the bootstrap
+/// for integration tests.
+///
 // NOTE: this API is used by the fuchsia source tree, do not add new
 // required or position parameters.
 String generateTestBootstrap({
@@ -117,6 +126,7 @@ String generateTestBootstrap({
   String languageVersionHeader = '',
   bool nullSafety = false,
   bool flutterTestDep = true,
+  bool integrationTest = false,
 }) {
   assert(testUrl != null);
   assert(host != null);
@@ -138,6 +148,12 @@ import 'dart:isolate';
   if (flutterTestDep) {
     buffer.write('''
 import 'package:flutter_test/flutter_test.dart';
+''');
+  }
+  if (integrationTest) {
+    buffer.write('''
+import 'package:integration_test/integration_test.dart';
+import 'dart:developer' as developer;
 ''');
   }
   buffer.write('''
@@ -159,6 +175,17 @@ StreamChannel<dynamic> serializeSuite(Function getMain()) {
   return RemoteListener.start(getMain);
 }
 
+Future<void> _testMain() async {
+''');
+  if (integrationTest) {
+    buffer.write('''
+  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+''');
+  }
+  buffer.write('''
+  return test.main();
+}
+
 /// Capture any top-level errors (mostly lazy syntax errors, since other are
 /// caught below) and report them to the parent isolate.
 void catchIsolateErrors() {
@@ -191,15 +218,36 @@ void main() {
   }
   if (testConfigFile != null) {
     buffer.write('''
-    return () => test_config.testExecutable(test.main);
+    return () => test_config.testExecutable(_testMain);
 ''');
   } else {
     buffer.write('''
-    return test.main;
+    return _testMain;
 ''');
   }
   buffer.write('''
   });
+''');
+  if (integrationTest) {
+    buffer.write('''
+  final callback = (method, params) async {
+    testChannel.sink.add(json.decode(params['$kIntegrationTestData'] as String));
+
+    // Result is ignored but null is not accepted here.
+    return developer.ServiceExtensionResponse.result('{}');
+  };
+
+  developer.registerExtension('$kIntegrationTestMethod', callback);
+
+  testChannel.stream.listen((x) {
+    developer.postEvent(
+      '$kIntegrationTestExtension',
+      {'$kIntegrationTestData': json.encode(x)},
+    );
+  });
+  ''');
+  } else {
+    buffer.write('''
   WebSocket.connect(server).then((WebSocket socket) {
     socket.map((dynamic message) {
       // We're only communicating with string encoded JSON.
@@ -207,8 +255,11 @@ void main() {
     }).pipe(testChannel.sink);
     socket.addStream(testChannel.stream.map(json.encode));
   });
-}
 ''');
+  }
+  buffer.write('''
+}
+  ''');
   return buffer.toString();
 }
 
@@ -230,6 +281,8 @@ class FlutterPlatform extends PlatformPlugin {
     this.projectRootDirectory,
     this.flutterProject,
     this.icudtlPath,
+    this.integrationTestDevice,
+    this.integrationTestUserIdentifier,
   }) : assert(shellPath != null);
 
   final String shellPath;
@@ -246,6 +299,15 @@ class FlutterPlatform extends PlatformPlugin {
   final FlutterProject flutterProject;
   final String icudtlPath;
 
+  /// The device to run the test on for Integration Tests.
+  ///
+  /// If this is null, the test will run as a regular test with the Flutter
+  /// Tester; otherwise it will run as a Integration Test on this device.
+  final Device integrationTestDevice;
+  bool get _isIntegrationTest => integrationTestDevice != null;
+
+  final String integrationTestUserIdentifier;
+
   final FontConfigManager _fontConfigManager = FontConfigManager();
 
   /// The test compiler produces dill files for each test main.
@@ -336,6 +398,14 @@ class FlutterPlatform extends PlatformPlugin {
   }
 
   TestDevice _createTestDevice(int ourTestCount) {
+    if (_isIntegrationTest) {
+      return IntegrationTestTestDevice(
+        id: ourTestCount,
+        debuggingOptions: debuggingOptions,
+        device: integrationTestDevice,
+        userIdentifier: integrationTestUserIdentifier,
+      );
+    }
     return FlutterTesterTestDevice(
       id: ourTestCount,
       platform: globals.platform,
@@ -380,26 +450,25 @@ class FlutterPlatform extends PlatformPlugin {
         mainDart = precompiledDillPath;
       } else if (precompiledDillFiles != null) {
         mainDart = precompiledDillFiles[testPath];
-      }
-      mainDart ??= _createListenerDart(finalizers, ourTestCount, testPath);
+      } else {
+        mainDart = _createListenerDart(finalizers, ourTestCount, testPath);
 
-      if (precompiledDillPath == null && precompiledDillFiles == null) {
-        // Lazily instantiate compiler so it is built only if it is actually used.
-        compiler ??= TestCompiler(debuggingOptions.buildInfo, flutterProject);
-        mainDart = await compiler.compile(globals.fs.file(mainDart).uri);
+        // Integration test device takes care of the compilation.
+        if (integrationTestDevice == null) {
+          // Lazily instantiate compiler so it is built only if it is actually used.
+          compiler ??= TestCompiler(debuggingOptions.buildInfo, flutterProject);
+          mainDart = await compiler.compile(globals.fs.file(mainDart).uri);
 
-        if (mainDart == null) {
-          testHarnessChannel.sink.addError('Compilation failed for testPath=$testPath');
-          return null;
+          if (mainDart == null) {
+            testHarnessChannel.sink.addError('Compilation failed for testPath=$testPath');
+            return null;
+          }
         }
       }
 
       globals.printTrace('test $ourTestCount: starting test device');
-
       final TestDevice testDevice = _createTestDevice(ourTestCount);
-      final Future<StreamChannel<String>> remoteChannelFuture = testDevice.start(
-        compiledEntrypointPath: mainDart,
-      );
+      final Future<StreamChannel<String>> remoteChannelFuture = testDevice.start(mainDart);
       finalizers.add(() async {
         globals.printTrace('test $ourTestCount: ensuring test device is terminated.');
         await testDevice.kill();
@@ -521,7 +590,8 @@ class FlutterPlatform extends PlatformPlugin {
       host: host,
       updateGoldens: updateGoldens,
       flutterTestDep: packageConfig['flutter_test'] != null,
-      languageVersionHeader: '// @dart=${languageVersion.major}.${languageVersion.minor}'
+      languageVersionHeader: '// @dart=${languageVersion.major}.${languageVersion.minor}',
+      integrationTest: _isIntegrationTest,
     );
   }
 

packages/flutter_tools/lib/src/test/flutter_tester_device.dart

@@ -71,8 +71,12 @@ class FlutterTesterTestDevice extends TestDevice {
   Process _process;
   HttpServer _server;
 
+  /// Starts the device.
+  ///
+  /// [entrypointPath] is the path to the entrypoint file which must be compiled
+  /// as a dill.
   @override
-  Future<StreamChannel<String>> start({@required String compiledEntrypointPath}) async {
+  Future<StreamChannel<String>> start(String entrypointPath) async {
     assert(!_exitCode.isCompleted);
     assert(_process == null);
     assert(_server == null);
@@ -113,7 +117,7 @@ class FlutterTesterTestDevice extends TestDevice {
       if (debuggingOptions.nullAssertions)
         '--dart-flags=--null_assertions',
       ...debuggingOptions.dartEntrypointArgs,
-      compiledEntrypointPath,
+      entrypointPath,
     ];
 
     // If the FLUTTER_TEST environment variable has been set, then pass it on

packages/flutter_tools/lib/src/test/integration_test_device.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'dart:async';
+
+import 'package:meta/meta.dart';
+import 'package:stream_channel/stream_channel.dart';
+import 'package:vm_service/vm_service.dart' as vm_service;
+
+import '../application_package.dart';
+import '../base/common.dart';
+import '../build_info.dart';
+import '../device.dart';
+import '../globals.dart' as globals;
+import '../vmservice.dart';
+import 'test_device.dart';
+
+const String kIntegrationTestExtension = 'Flutter.IntegrationTest';
+const String kIntegrationTestData = 'data';
+const String kIntegrationTestMethod = 'ext.flutter.integrationTest';
+
+class IntegrationTestTestDevice implements TestDevice {
+  IntegrationTestTestDevice({
+    @required this.id,
+    @required this.device,
+    @required this.debuggingOptions,
+    @required this.userIdentifier,
+  });
+
+  final int id;
+  final Device device;
+  final DebuggingOptions debuggingOptions;
+  final String userIdentifier;
+
+  ApplicationPackage _applicationPackage;
+  final Completer<void> _finished = Completer<void>();
+  final Completer<Uri> _gotProcessObservatoryUri = Completer<Uri>();
+
+  /// Starts the device.
+  ///
+  /// [entrypointPath] must be a path to an uncompiled source file.
+  @override
+  Future<StreamChannel<String>> start(String entrypointPath) async {
+    final TargetPlatform targetPlatform = await device.targetPlatform;
+    _applicationPackage = await ApplicationPackageFactory.instance.getPackageForPlatform(
+      targetPlatform,
+      buildInfo: debuggingOptions.buildInfo,
+    );
+
+    final LaunchResult launchResult = await device.startApp(
+      _applicationPackage,
+      mainPath: entrypointPath,
+      platformArgs: <String, dynamic>{},
+      debuggingOptions: debuggingOptions,
+      userIdentifier: userIdentifier,
+    );
+    if (!launchResult.started) {
+      throw TestDeviceException('Unable to start the app on the device.', StackTrace.current);
+    }
+    if (launchResult.observatoryUri == null) {
+      throw TestDeviceException('Observatory is not available on the test device.', StackTrace.current);
+    }
+
+    // No need to set up the log reader because the logs are captured and
+    // streamed to the package:test_core runner.
+
+    _gotProcessObservatoryUri.complete(launchResult.observatoryUri);
+
+    globals.printTrace('test $id: Connecting to vm service');
+    final FlutterVmService vmService = await connectToVmService(launchResult.observatoryUri).timeout(
+      const Duration(seconds: 5),
+      onTimeout: () => throw TimeoutException('Connecting to the VM Service timed out.'),
+    );
+
+    globals.printTrace('test $id: Finding the correct isolate with the integration test service extension');
+    final vm_service.IsolateRef isolateRef = await vmService.findExtensionIsolate(kIntegrationTestMethod);
+
+    await vmService.service.streamListen(vm_service.EventStreams.kExtension);
+    final Stream<String> remoteMessages = vmService.service.onExtensionEvent
+        .where((vm_service.Event e) => e.extensionKind == kIntegrationTestExtension)
+        .map((vm_service.Event e) => e.extensionData.data[kIntegrationTestData] as String);
+
+    final StreamChannelController<String> controller = StreamChannelController<String>();
+
+    controller.local.stream.listen((String event) {
+      vmService.service.callServiceExtension(
+        kIntegrationTestMethod,
+        isolateId: isolateRef.id,
+        args: <String, String>{
+          kIntegrationTestData: event,
+        },
+      );
+    });
+
+    unawaited(remoteMessages.pipe(controller.local.sink));
+    return controller.foreign;
+  }
+
+  @override
+  Future<Uri> get observatoryUri => _gotProcessObservatoryUri.future;
+
+  @override
+  Future<void> kill() async {
+    if (!await device.stopApp(_applicationPackage, userIdentifier: userIdentifier)) {
+      globals.printTrace('Could not stop the Integration Test app.');
+    }
+    if (!await device.uninstallApp(_applicationPackage, userIdentifier: userIdentifier)) {
+      globals.printTrace('Could not uninstall the Integration Test app.');
+    }
+
+    await device.dispose();
+    _finished.complete();
+  }
+
+  @override
+  Future<void> get finished => _finished.future;
+}

packages/flutter_tools/lib/src/test/runner.dart

@@ -53,6 +53,8 @@ abstract class FlutterTestRunner {
     bool runSkipped = false,
     int shardIndex,
     int totalShards,
+    Device integrationTestDevice,
+    String integrationTestUserIdentifier,
   });
 }
 
@@ -87,6 +89,8 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
     bool runSkipped = false,
     int shardIndex,
     int totalShards,
+    Device integrationTestDevice,
+    String integrationTestUserIdentifier,
   }) async {
     // Configure package:test to use the Flutter engine for child processes.
     final String shellPath = globals.artifacts.getArtifactPath(Artifact.flutterTester);
@@ -179,6 +183,13 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
       return exitCode;
     }
 
+    if (integrationTestDevice != null) {
+      // Without this, some async exceptions which are caught will surface when
+      // debugging tests.
+      // TODO(jiahaog): Remove this once https://github.com/dart-lang/stack_trace/issues/106 is fixed.
+      testArgs.add('--no-chain-stack-traces');
+    }
+
     testArgs
       ..add('--')
       ..addAll(testFiles);
@@ -201,6 +212,8 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
       projectRootDirectory: globals.fs.currentDirectory.uri,
       flutterProject: flutterProject,
       icudtlPath: icudtlPath,
+      integrationTestDevice: integrationTestDevice,
+      integrationTestUserIdentifier: integrationTestUserIdentifier,
     );
 
     try {

packages/flutter_tools/lib/src/test/test_device.dart

@@ -4,30 +4,30 @@
 
 import 'dart:async';
 
-import 'package:meta/meta.dart';
 import 'package:stream_channel/stream_channel.dart';
 
 /// A remote device where tests can be executed on.
 ///
 /// Reusability of an instance across multiple runs is not guaranteed for all
 /// implementations.
+///
+/// Methods may throw [TestDeviceException] if a problem is encountered.
 abstract class TestDevice {
   /// Starts the test device with the provided entrypoint.
   ///
   /// Returns a channel that can be used to communicate with the test process.
-  Future<StreamChannel<String>> start({@required String compiledEntrypointPath});
+  ///
+  /// It is up to the device to determine if [entrypointPath] is a precompiled
+  /// or raw source file.
+  Future<StreamChannel<String>> start(String entrypointPath);
 
   /// Should complete with null if the observatory is not enabled.
   Future<Uri> get observatoryUri;
 
   /// Terminates the test device.
-  ///
-  /// A [TestDeviceException] can be thrown if it did not stop gracefully.
   Future<void> kill();
 
   /// Waits for the test device to stop.
-  ///
-  /// A [TestDeviceException] can be thrown if it did not stop gracefully.
   Future<void> get finished;
 }
 

packages/flutter_tools/test/commands.shard/hermetic/test_test.dart

@@ -24,11 +24,14 @@ import 'package:process/process.dart';
 
 import '../../src/common.dart';
 import '../../src/context.dart';
+import '../../src/fake_devices.dart';
 import '../../src/testbed.dart';
 
 const String _pubspecContents = '''
 dev_dependencies:
   flutter_test:
+    sdk: flutter
+  integration_test:
     sdk: flutter''';
 final String _packageConfigContents = json.encode(<String, Object>{
   'configVersion': 2,
@@ -38,7 +41,13 @@ final String _packageConfigContents = json.encode(<String, Object>{
       'rootUri': 'file:///path/to/pubcache/.pub-cache/hosted/pub.dartlang.org/test_api-0.2.19',
       'packageUri': 'lib/',
       'languageVersion': '2.12'
-    }
+    },
+    <String, String>{
+      'name': 'integration_test',
+      'rootUri': 'file:///path/to/flutter/packages/integration_test',
+      'packageUri': 'lib/',
+      'languageVersion': '2.12'
+    },
   ],
   'generated': '2021-02-24T07:55:20.084834Z',
   'generator': 'pub',
@@ -51,13 +60,16 @@ void main() {
 
   setUp(() {
     fs = MemoryFileSystem.test();
-    fs.file('pubspec.yaml').createSync();
-    fs.file('pubspec.yaml').writeAsStringSync(_pubspecContents);
-    (fs.directory('.dart_tool')
+    fs.file('/package/pubspec.yaml').createSync(recursive: true);
+    fs.file('/package/pubspec.yaml').writeAsStringSync(_pubspecContents);
+    (fs.directory('/package/.dart_tool')
         .childFile('package_config.json')
       ..createSync(recursive: true))
         .writeAsString(_packageConfigContents);
-    fs.directory('test').childFile('some_test.dart').createSync(recursive: true);
+    fs.directory('/package/test').childFile('some_test.dart').createSync(recursive: true);
+    fs.directory('/package/integration_test').childFile('some_integration_test.dart').createSync(recursive: true);
+
+    fs.currentDirectory = '/package';
   });
 
   testUsingContext('Missing dependencies in pubspec',
@@ -80,6 +92,43 @@ void main() {
     ProcessManager: () => FakeProcessManager.any(),
   });
 
+  testUsingContext('Missing dependencies in pubspec for integration tests',
+      () async {
+    // Only use the flutter_test dependency, integration_test is deliberately
+    // absent.
+    fs.file('pubspec.yaml').writeAsStringSync('''
+dev_dependencies:
+  flutter_test:
+    sdk: flutter
+    ''');
+    fs.directory('.dart_tool').childFile('package_config.json').writeAsStringSync(json.encode(<String, Object>{
+      'configVersion': 2,
+      'packages': <Map<String, Object>>[
+        <String, String>{
+          'name': 'test_api',
+          'rootUri': 'file:///path/to/pubcache/.pub-cache/hosted/pub.dartlang.org/test_api-0.2.19',
+          'packageUri': 'lib/',
+          'languageVersion': '2.12'
+        },
+      ],
+      'generated': '2021-02-24T07:55:20.084834Z',
+      'generator': 'pub',
+      'generatorVersion': '2.13.0-68.0.dev'
+    }));
+    final FakePackageTest fakePackageTest = FakePackageTest();
+    final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+    final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+    expect(() => commandRunner.run(const <String>[
+      'test',
+      '--no-pub',
+      'integration_test'
+    ]), throwsToolExit());
+  }, overrides: <Type, Generator>{
+    FileSystem: () => fs,
+    ProcessManager: () => FakeProcessManager.any(),
+  });
+
   testUsingContext('Pipes test-randomize-ordering-seed to package:test',
       () async {
     final FakePackageTest fakePackageTest = FakePackageTest();
@@ -268,6 +317,250 @@ void main() {
     ProcessManager: () => FakeProcessManager.any(),
     Cache: () => Cache.test(processManager: FakeProcessManager.any()),
   });
+
+  testUsingContext('Pipes different args when running Integration Tests', () async {
+    final FakePackageTest fakePackageTest = FakePackageTest();
+
+    final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+    final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+    await commandRunner.run(const <String>[
+      'test',
+      '--no-pub',
+      'integration_test',
+    ]);
+
+    expect(fakePackageTest.lastArgs, contains('--concurrency=1'));
+    expect(fakePackageTest.lastArgs, contains('--no-chain-stack-traces'));
+  }, overrides: <Type, Generator>{
+    FileSystem: () => fs,
+    ProcessManager: () => FakeProcessManager.any(),
+    DeviceManager: () => _FakeDeviceManager(<Device>[
+      FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
+    ]),
+  });
+
+  testUsingContext('Overrides concurrency when running Integration Tests', () async {
+    final FakePackageTest fakePackageTest = FakePackageTest();
+
+    final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+    final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+    await commandRunner.run(const <String>[
+      'test',
+      '--no-pub',
+      '--concurrency=100',
+      'integration_test',
+    ]);
+
+    expect(fakePackageTest.lastArgs, contains('--concurrency=1'));
+  }, overrides: <Type, Generator>{
+    FileSystem: () => fs,
+    ProcessManager: () => FakeProcessManager.any(),
+    DeviceManager: () => _FakeDeviceManager(<Device>[
+      FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
+    ]),
+  });
+
+  group('Detecting Integration Tests', () {
+    testUsingContext('when integration_test is not passed', () async {
+      final FakePackageTest fakePackageTest = FakePackageTest();
+
+      final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+      ]);
+
+      expect(testCommand.isIntegrationTest, false);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+      DeviceManager: () => _FakeDeviceManager(<Device>[
+        FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
+      ]),
+    });
+
+    testUsingContext('when integration_test is passed', () async {
+      final FakePackageTest fakePackageTest = FakePackageTest();
+
+      final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+        'integration_test',
+      ]);
+
+      expect(testCommand.isIntegrationTest, true);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+      DeviceManager: () => _FakeDeviceManager(<Device>[
+        FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
+      ]),
+    });
+
+    testUsingContext('when relative path to integration test is passed', () async {
+      final FakePackageTest fakePackageTest = FakePackageTest();
+
+      final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+        'integration_test/some_integration_test.dart',
+      ]);
+
+      expect(testCommand.isIntegrationTest, true);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+      DeviceManager: () => _FakeDeviceManager(<Device>[
+        FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
+      ]),
+    });
+
+    testUsingContext('when absolute path to integration test is passed', () async {
+      final FakePackageTest fakePackageTest = FakePackageTest();
+
+      final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+        '/package/integration_test/some_integration_test.dart',
+      ]);
+
+      expect(testCommand.isIntegrationTest, true);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+      DeviceManager: () => _FakeDeviceManager(<Device>[
+        FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
+      ]),
+    });
+
+    testUsingContext('when both test and integration test are passed', () async {
+      final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0);
+
+      final TestCommand testCommand = TestCommand(testRunner: testRunner);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      expect(() => commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+        'test/some_test.dart',
+        'integration_test/some_integration_test.dart',
+      ]), throwsToolExit());
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+    });
+  });
+
+  group('Required artifacts', () {
+    testUsingContext('for default invocation', () async {
+      final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0);
+
+      final TestCommand testCommand = TestCommand(testRunner: testRunner);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+      ]);
+
+      expect(await testCommand.requiredArtifacts, isEmpty);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+    });
+
+    testUsingContext('when platform is chrome', () async {
+      final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0);
+
+      final TestCommand testCommand = TestCommand(testRunner: testRunner);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+        '--platform=chrome'
+      ]);
+
+      expect(await testCommand.requiredArtifacts, <DevelopmentArtifact>[DevelopmentArtifact.web]);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+    });
+
+    testUsingContext('when running integration tests', () async {
+      final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0);
+
+      final TestCommand testCommand = TestCommand(testRunner: testRunner);
+      final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+      await commandRunner.run(const <String>[
+        'test',
+        '--no-pub',
+        'integration_test',
+      ]);
+
+      expect(await testCommand.requiredArtifacts, <DevelopmentArtifact>[
+        DevelopmentArtifact.universal,
+        DevelopmentArtifact.androidGenSnapshot,
+      ]);
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      ProcessManager: () => FakeProcessManager.any(),
+      DeviceManager: () => _FakeDeviceManager(<Device>[
+        FakeDevice('ephemeral', 'ephemeral', ephemeral: true, isSupported: true, type: PlatformType.android),
+      ]),
+    });
+  });
+
+  testUsingContext('Integration tests when no devices are connected', () async {
+    final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0);
+
+    final TestCommand testCommand = TestCommand(testRunner: testRunner);
+    final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+    expect(() => commandRunner.run(const <String>[
+      'test',
+      '--no-pub',
+      'integration_test',
+    ]), throwsToolExit());
+  }, overrides: <Type, Generator>{
+    FileSystem: () => fs,
+    ProcessManager: () => FakeProcessManager.any(),
+    DeviceManager: () => _FakeDeviceManager(<Device>[]),
+  });
+
+  // TODO(jiahaog): Remove this when web is supported. https://github.com/flutter/flutter/pull/74236
+  testUsingContext('Integration tests when only web devices are connected', () async {
+    final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0);
+
+    final TestCommand testCommand = TestCommand(testRunner: testRunner);
+    final CommandRunner<void> commandRunner = createTestCommandRunner(testCommand);
+
+    expect(() => commandRunner.run(const <String>[
+      'test',
+      '--no-pub',
+      'integration_test',
+    ]), throwsToolExit());
+  }, overrides: <Type, Generator>{
+    FileSystem: () => fs,
+    ProcessManager: () => FakeProcessManager.any(),
+    DeviceManager: () => _FakeDeviceManager(<Device>[
+      FakeDevice('ephemeral', 'ephemeral', ephemeral: true, isSupported: true, type: PlatformType.web),
+    ]),
+  });
 }
 
 class FakeFlutterTestRunner implements FlutterTestRunner {
@@ -308,6 +601,8 @@ class FakeFlutterTestRunner implements FlutterTestRunner {
     bool runSkipped = false,
     int shardIndex,
     int totalShards,
+    Device integrationTestDevice,
+    String integrationTestUserIdentifier,
   }) async {
     lastEnableObservatoryValue = enableObservatory;
     return exitCode;
@@ -328,3 +623,15 @@ class FakePackageTest implements TestWrapper {
     FutureOr<PlatformPlugin> Function() platforms,
   ) {}
 }
+
+class _FakeDeviceManager extends DeviceManager {
+  _FakeDeviceManager(this._connectedDevices);
+
+  final List<Device> _connectedDevices;
+
+  @override
+  Future<List<Device>> getAllConnectedDevices() async => _connectedDevices;
+
+  @override
+  List<DeviceDiscovery> get deviceDiscoverers => <DeviceDiscovery>[];
+}

packages/flutter_tools/test/general.shard/device_test.dart

@@ -150,11 +150,11 @@ void main() {
   });
 
   group('Filter devices', () {
-    final FakeDevice ephemeralOne = FakeDevice('ephemeralOne', 'ephemeralOne', true);
-    final FakeDevice ephemeralTwo = FakeDevice('ephemeralTwo', 'ephemeralTwo', true);
-    final FakeDevice nonEphemeralOne = FakeDevice('nonEphemeralOne', 'nonEphemeralOne', false);
-    final FakeDevice nonEphemeralTwo = FakeDevice('nonEphemeralTwo', 'nonEphemeralTwo', false);
-    final FakeDevice unsupported = FakeDevice('unsupported', 'unsupported', true, false);
+    final FakeDevice ephemeralOne = FakeDevice('ephemeralOne', 'ephemeralOne');
+    final FakeDevice ephemeralTwo = FakeDevice('ephemeralTwo', 'ephemeralTwo');
+    final FakeDevice nonEphemeralOne = FakeDevice('nonEphemeralOne', 'nonEphemeralOne', ephemeral: false);
+    final FakeDevice nonEphemeralTwo = FakeDevice('nonEphemeralTwo', 'nonEphemeralTwo', ephemeral: false);
+    final FakeDevice unsupported = FakeDevice('unsupported', 'unsupported', isSupported: false);
     final FakeDevice webDevice = FakeDevice('webby', 'webby')
       ..targetPlatform = Future<TargetPlatform>.value(TargetPlatform.web_javascript);
     final FakeDevice fuchsiaDevice = FakeDevice('fuchsiay', 'fuchsiay')

packages/flutter_tools/test/general.shard/flutter_tester_device_test.dart

@@ -88,7 +88,7 @@ void main() {
     testUsingContext('as true when not originally set', () async {
       processManager.addCommand(flutterTestCommand('true'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -96,7 +96,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': 'true'};
       processManager.addCommand(flutterTestCommand('true'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -104,7 +104,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': 'false'};
       processManager.addCommand(flutterTestCommand('false'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -112,7 +112,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': 'neither true nor false'};
       processManager.addCommand(flutterTestCommand('neither true nor false'));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
 
@@ -120,7 +120,7 @@ void main() {
       platform.environment = <String, String>{'FLUTTER_TEST': null};
       processManager.addCommand(flutterTestCommand(null));
 
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       expect(processManager.hasRemainingExpectations, isFalse);
     });
   });
@@ -154,7 +154,7 @@ void main() {
     });
 
     testUsingContext('Can pass additional arguments to tester binary', () async {
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
 
       expect(processManager, hasNoRemainingExpectations);
     });
@@ -187,7 +187,7 @@ void main() {
     });
 
     testUsingContext('skips setting observatory port and uses the input port for for DDS instead', () async {
-      await device.start(compiledEntrypointPath: 'example.dill');
+      await device.start('example.dill');
       await device.observatoryUri;
 
       final Uri uri = await (device as TestFlutterTesterDevice).ddsServiceUriFuture();

packages/flutter_tools/test/general.shard/integration_test_device_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'package:flutter_tools/src/base/io.dart' as io;
+import 'package:flutter_tools/src/build_info.dart';
+import 'package:flutter_tools/src/device.dart';
+import 'package:flutter_tools/src/test/integration_test_device.dart';
+import 'package:flutter_tools/src/test/test_device.dart';
+import 'package:flutter_tools/src/vmservice.dart';
+import 'package:vm_service/vm_service.dart' as vm_service;
+
+import '../src/common.dart';
+import '../src/context.dart';
+import '../src/fake_devices.dart';
+
+final Map<String, Object> vm = <String, dynamic>{
+  'isolates': <dynamic>[
+    <String, dynamic>{
+      'type': '@Isolate',
+      'fixedId': true,
+      'id': 'isolates/242098474',
+      'name': 'main.dart:main()',
+      'number': 242098474,
+    },
+  ],
+};
+
+final vm_service.Isolate isolate = vm_service.Isolate(
+  id: '1',
+  pauseEvent: vm_service.Event(
+      kind: vm_service.EventKind.kResume,
+      timestamp: 0
+  ),
+  breakpoints: <vm_service.Breakpoint>[],
+  exceptionPauseMode: null,
+  libraries: <vm_service.LibraryRef>[
+    vm_service.LibraryRef(
+      id: '1',
+      uri: 'file:///hello_world/main.dart',
+      name: '',
+    ),
+  ],
+  livePorts: 0,
+  name: 'test',
+  number: '1',
+  pauseOnExit: false,
+  runnable: true,
+  startTime: 0,
+  isSystemIsolate: false,
+  isolateFlags: <vm_service.IsolateFlag>[],
+  extensionRPCs: <String>[kIntegrationTestMethod],
+);
+
+final FlutterView fakeFlutterView = FlutterView(
+  id: 'a',
+  uiIsolate: isolate,
+);
+
+final FakeVmServiceRequest listViewsRequest = FakeVmServiceRequest(
+  method: kListViewsMethod,
+  jsonResponse: <String, Object>{
+    'views': <Object>[
+      fakeFlutterView.toJson(),
+    ],
+  },
+);
+
+final Uri observatoryUri = Uri.parse('http://localhost:1234');
+
+void main() {
+  FakeVmServiceHost fakeVmServiceHost;
+  TestDevice testDevice;
+
+  setUp(() {
+    testDevice = IntegrationTestTestDevice(
+      id: 1,
+      device: FakeDevice(
+        'ephemeral',
+        'ephemeral',
+        type: PlatformType.android,
+        launchResult: LaunchResult.succeeded(observatoryUri: observatoryUri),
+      ),
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      userIdentifier: '',
+    );
+
+    fakeVmServiceHost = FakeVmServiceHost(requests: <VmServiceExpectation>[
+      const FakeVmServiceRequest(
+        method: 'streamListen',
+        args: <String, Object>{
+          'streamId': 'Isolate',
+        },
+      ),
+      listViewsRequest,
+      FakeVmServiceRequest(
+        method: 'getIsolate',
+        jsonResponse: isolate.toJson(),
+        args: <String, Object>{
+          'isolateId': '1',
+        },
+      ),
+      const FakeVmServiceRequest(
+        method: 'streamCancel',
+        args: <String, Object>{
+          'streamId': 'Isolate',
+        },
+      ),
+      const FakeVmServiceRequest(
+        method: 'streamListen',
+        args: <String, Object>{
+          'streamId': 'Extension',
+        },
+      ),
+    ]);
+  });
+
+  testUsingContext('Can start the entrypoint', () async {
+    await testDevice.start('entrypointPath');
+
+    expect(await testDevice.observatoryUri, observatoryUri);
+    expect(testDevice.finished, doesNotComplete);
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+
+  testUsingContext('Can kill the started device', () async {
+    await testDevice.start('entrypointPath');
+    await testDevice.kill();
+
+    expect(testDevice.finished, completes);
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+
+  testUsingContext('when the device starts without providing an observatory URI', () async {
+    final TestDevice testDevice = IntegrationTestTestDevice(
+      id: 1,
+      device: FakeDevice(
+        'ephemeral',
+        'ephemeral',
+        type: PlatformType.android,
+        launchResult: LaunchResult.succeeded(observatoryUri: null),
+      ),
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      userIdentifier: '',
+    );
+
+    expect(() => testDevice.start('entrypointPath'), throwsA(isA<TestDeviceException>()));
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+
+  testUsingContext('when the device fails to start', () async {
+    final TestDevice testDevice = IntegrationTestTestDevice(
+      id: 1,
+      device: FakeDevice(
+        'ephemeral',
+        'ephemeral',
+        type: PlatformType.android,
+        launchResult: LaunchResult.failed(),
+      ),
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      userIdentifier: '',
+    );
+
+    expect(() => testDevice.start('entrypointPath'), throwsA(isA<TestDeviceException>()));
+  }, overrides: <Type, Generator>{
+    VMServiceConnector: () => (Uri httpUri, {
+      ReloadSources reloadSources,
+      Restart restart,
+      CompileExpression compileExpression,
+      GetSkSLMethod getSkSLMethod,
+      PrintStructuredErrorLogMethod printStructuredErrorLogMethod,
+      io.CompressionOptions compression,
+      Device device,
+    }) async => fakeVmServiceHost.vmService,
+  });
+}

packages/flutter_tools/test/integration.shard/test_test.dart

@@ -18,8 +18,13 @@ import 'test_utils.dart';
 final String automatedTestsDirectory = fileSystem.path.join('..', '..', 'dev', 'automated_tests');
 final String missingDependencyDirectory = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
 final String flutterTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'flutter_test');
+final String integrationTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'integration_test');
 final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', platform.isWindows ? 'flutter.bat' : 'flutter');
 
+// Running Integration Tests in the Flutter Tester will still exercise the same
+// flows specific to Integration Tests.
+final List<String> integrationTestExtraArgs = <String>['-d', 'flutter-tester'];
+
 void main() {
   setUpAll(() async {
     await processManager.run(
@@ -44,6 +49,10 @@ void main() {
     return _testFile('trivial_widget', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
   });
 
+  testWithoutContext('integration test should not have extraneous error messages', () async {
+    return _testFile('trivial_widget', automatedTestsDirectory, integrationTestDirectory, exitCode: isZero, extraArguments: integrationTestExtraArgs);
+  });
+
   testWithoutContext('flutter test set the working directory correctly', () async {
     return _testFile('working_directory', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
   });
@@ -52,6 +61,10 @@ void main() {
     return _testFile('exception_handling', automatedTestsDirectory, flutterTestDirectory);
   });
 
+  testWithoutContext('integration test should report nice errors for exceptions thrown within testWidgets()', () async {
+    return _testFile('exception_handling', automatedTestsDirectory, integrationTestDirectory, extraArguments: integrationTestExtraArgs);
+  });
+
   testWithoutContext('flutter test should report a nice error when a guarded function was called without await', () async {
     return _testFile('test_async_utils_guarded', automatedTestsDirectory, flutterTestDirectory);
   });

packages/flutter_tools/test/src/fake_devices.dart

@@ -4,6 +4,7 @@
 
 // @dart = 2.8
 
+import 'package:flutter_tools/src/application_package.dart';
 import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/device.dart';
 import 'package:flutter_tools/src/project.dart';
@@ -12,7 +13,7 @@ import 'package:flutter_tools/src/project.dart';
 /// (`Device.toJson()` and `--machine` flag for `devices` command)
 List<FakeDeviceJsonData> fakeDevices = <FakeDeviceJsonData>[
   FakeDeviceJsonData(
-    FakeDevice('ephemeral', 'ephemeral', true, true, PlatformType.android),
+    FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android),
     <String, Object>{
       'name': 'ephemeral',
       'id': 'ephemeral',
@@ -57,18 +58,51 @@ List<FakeDeviceJsonData> fakeDevices = <FakeDeviceJsonData>[
 
 /// Fake device to test `devices` command.
 class FakeDevice extends Device {
-  FakeDevice(this.name, String id, [bool ephemeral = true, this._isSupported = true, PlatformType type = PlatformType.web]) : super(
-      id,
-      platformType: type,
-      category: Category.mobile,
-      ephemeral: ephemeral,
-  );
+  FakeDevice(this.name, String id, {
+    bool ephemeral = true,
+    bool isSupported = true,
+    PlatformType type = PlatformType.web,
+    LaunchResult launchResult,
+  }) : _isSupported = isSupported,
+      _launchResult = launchResult ?? LaunchResult.succeeded(),
+      super(
+        id,
+        platformType: type,
+        category: Category.mobile,
+        ephemeral: ephemeral,
+      );
 
   final bool _isSupported;
+  final LaunchResult _launchResult;
 
   @override
   final String name;
 
+  @override
+  Future<LaunchResult> startApp(covariant ApplicationPackage package, {
+    String mainPath,
+    String route,
+    DebuggingOptions debuggingOptions,
+    Map<String, dynamic> platformArgs,
+    bool prebuiltApplication = false,
+    bool ipv6 = false,
+    String userIdentifier,
+  }) async => _launchResult;
+
+  @override
+  Future<bool> stopApp(covariant ApplicationPackage app, {
+    String userIdentifier,
+  }) async => true;
+
+  @override
+  Future<bool> uninstallApp(
+  covariant ApplicationPackage app, {
+    String userIdentifier,
+  }) async => true;
+
+  @override
+  Future<void> dispose() async {}
+
   @override
   Future<TargetPlatform> targetPlatform = Future<TargetPlatform>.value(TargetPlatform.android_arm);