Tests for Engine ensuring debug-mode apps are attached on iOS. (#37043)

This PR contains the tests for flutter/engine#10186.

dev/devicelab/bin/tasks/system_debug_ios.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/ios.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+import 'service_extensions_test.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.ios;
+  await task(() async {
+    final Device device = await devices.workingDevice;
+    await device.unlock();
+    final String deviceId = device.deviceId;
+
+    final Directory testDirectory =
+        dir('${flutterDirectory.path}/dev/integration_tests/ui');
+    await inDirectory<void>(testDirectory, () async {
+      await flutter('packages', options: <String>['get']);
+      await prepareProvisioningCertificates(testDirectory.path);
+
+      await checkNoWarningHostLaunch(deviceId);
+      await checkNoWarningXcodeLaunch(deviceId);
+      await checkWarningHomeScreenLaunch(deviceId);
+    });
+    return TaskResult.success(<String, dynamic>{});
+  });
+}
+
+const String expectedWarning =
+    'Launching a debug-mode app from the home screen may cause problems.';
+
+// When a debug-mode app is launched from the host there should be no warnings.
+Future<void> checkNoWarningHostLaunch(String deviceId) async {
+  final String output = await evalFlutter('drive', options: <String>[
+    '--debug',
+    '--verbose',
+    '--verbose-system-logs',
+    '-d',
+    deviceId,
+    'lib/empty.dart'
+  ]);
+
+  expect(!output.contains(expectedWarning));
+}
+
+// When a debug-mode app is launched from Xcode there should be no warnings. The
+// Xcode launch is simulated by keeping LLDB attached throughout the lifetime of
+// the app.
+Future<void> checkNoWarningXcodeLaunch(String deviceId) async {
+  await flutter('build',
+      options: <String>['ios', '--debug', '--verbose', 'lib/exit.dart']);
+
+  final String output = await eval(
+      '${flutterDirectory.path}/bin/cache/artifacts/ios-deploy/ios-deploy',
+      <String>[
+        '--bundle',
+        'build/ios/iphoneos/Runner.app',
+        '-d', // Actually start the app in LLDB, don't just install it.
+        '--noninteractive',
+        '--args',
+        '--verbose-logging',
+      ]);
+
+  expect(output.contains('success') && !output.contains(expectedWarning));
+}
+
+// When a debug-mode app is launched from the home screen there should be a
+// warning every ~100 launches. We lower the threshold from to 1 via
+// "--verbose-system-logs" and simulate a home-screen-launch by setting an
+// environment variable. The environment variable forces "flutter drive" to not
+// pass a flag which it normally passes to debug-mode apps, imitating launchd,
+// which doesn't pass any command-line flags.
+Future<void> checkWarningHomeScreenLaunch(String deviceId) async {
+  final String output = await evalFlutter('drive', options: <String>[
+    '--debug',
+    '--verbose',
+    '--verbose-system-logs',
+    '-d',
+    deviceId,
+    'lib/empty.dart'
+  ], environment: <String, String>{
+    'FLUTTER_TOOLS_DEBUG_WITHOUT_CHECKED_MODE': 'true'
+  });
+  expect(output.contains(expectedWarning));
+}
\ No newline at end of file

dev/devicelab/manifest.yaml

@@ -496,6 +496,14 @@ tasks:
     required_agent_capabilities: ["mac/ios"]
     timeout_in_minutes: 20
 
+  system_debug_ios:
+    description: >
+      Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps.
+    stage: devicelab_ios
+    required_agent_capabilities: ["mac/ios"]
+    timeout_in_minutes: 10
+    flaky: true
+
   ios_sample_catalog_generator:
     description: >
       Builds sample catalog markdown pages and iOS screenshots

dev/integration_tests/ui/lib/empty.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/material.dart';
+import 'package:flutter_driver/driver_extension.dart';
+
+/// This application does nothing but show a empty screen.
+void main() {
+  enableFlutterDriverExtension();
+  runApp(Empty());
+}
+
+class Empty extends StatelessWidget {
+  @override
+  Widget build(BuildContext context) => Container();
+}

dev/integration_tests/ui/lib/exit.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+/// This application exits immediately.
+void main() => exit(0);

dev/integration_tests/ui/test_driver/empty_test.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_driver/flutter_driver.dart';
+import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
+
+// Connect and disconnect from the empty app.
+void main() {
+  group('FlutterDriver', () {
+    FlutterDriver driver;
+
+    setUpAll(() async {
+      driver = await FlutterDriver.connect();
+    });
+
+    tearDownAll(() async {
+      await driver.close();
+    });
+
+    test('empty', () async {}, timeout: const Timeout(Duration(minutes: 1)));
+  });
+}

packages/flutter_tools/lib/src/commands/drive.dart

@@ -85,6 +85,8 @@ class DriveCommand extends RunCommandBase {
   Device get device => _device;
   bool get shouldBuild => argResults['build'];
 
+  bool get verboseSystemLogs => argResults['verbose-system-logs'];
+
   /// Subscription to log messages printed on the device or simulator.
   // ignore: cancel_subscriptions
   StreamSubscription<String> _deviceLogSubscription;
@@ -264,6 +266,7 @@ Future<LaunchResult> _startApp(DriveCommand command) async {
       command.getBuildInfo(),
       startPaused: true,
       observatoryPort: command.observatoryPort,
+      verboseSystemLogs: command.verboseSystemLogs
     ),
     platformArgs: platformArgs,
     prebuiltApplication: !command.shouldBuild,

packages/flutter_tools/lib/src/ios/devices.dart

@@ -312,7 +312,13 @@ class IOSDevice extends Device {
     if (debuggingOptions.useTestFonts)
       launchArguments.add('--use-test-fonts');
 
-    if (debuggingOptions.debuggingEnabled) {
+    // "--enable-checked-mode" and "--verify-entry-points" should always be
+    // passed when we launch debug build via "ios-deploy". However, we don't
+    // pass them if a certain environment variable is set to enable the
+    // "system_debug_ios" integration test in the CI, which simulates a
+    // home-screen launch.
+    if (debuggingOptions.debuggingEnabled &&
+        platform.environment['FLUTTER_TOOLS_DEBUG_WITHOUT_CHECKED_MODE'] != 'true') {
       launchArguments.add('--enable-checked-mode');
       launchArguments.add('--verify-entry-points');
     }