Retry connecting to device in CI after lost connection (#133769)

Sometimes `ios-deploy` loses connection to the device after installing, starting debugserver, and launching. This is shown with an error message like:
```
Process 579 exited with status = -1 (0xffffffff) lost connection
```
This happens frequently in our CI system: https://github.com/flutter/flutter/issues/120808

Usually in CI, on retry it'll work and pass - so this is an attempt to retry without failing the test first. It's not guaranteed to fix since we're unable to recreate this error locally.

packages/flutter_tools/lib/src/ios/devices.dart

@@ -537,34 +537,12 @@ class IOSDevice extends Device {
       int installationResult = 1;
       if (debuggingOptions.debuggingEnabled) {
         _logger.printTrace('Debugging is enabled, connecting to vmService');
-        final DeviceLogReader deviceLogReader = getLogReader(
-          app: package,
-          usingCISystem: debuggingOptions.usingCISystem,
-        );
-
-        // If the device supports syslog reading, prefer launching the app without
-        // attaching the debugger to avoid the overhead of the unnecessary extra running process.
-        if (majorSdkVersion >= IOSDeviceLogReader.minimumUniversalLoggingSdkVersion) {
-          iosDeployDebugger = _iosDeploy.prepareDebuggerForLaunch(
-            deviceId: id,
-            bundlePath: bundle.path,
-            appDeltaDirectory: package.appDeltaDirectory,
+        vmServiceDiscovery = _setupDebuggerAndVmServiceDiscovery(
+          package: package,
+          bundle: bundle,
+          debuggingOptions: debuggingOptions,
           launchArguments: launchArguments,
-            interfaceType: connectionInterface,
-            uninstallFirst: debuggingOptions.uninstallFirst,
-          );
-          if (deviceLogReader is IOSDeviceLogReader) {
-            deviceLogReader.debuggerStream = iosDeployDebugger;
-          }
-        }
-        // Don't port foward if debugging with a wireless device.
-        vmServiceDiscovery = ProtocolDiscovery.vmService(
-          deviceLogReader,
-          portForwarder: isWirelesslyConnected ? null : portForwarder,
-          hostPort: debuggingOptions.hostVmServicePort,
-          devicePort: debuggingOptions.deviceVmServicePort,
           ipv6: ipv6,
-          logger: _logger,
         );
       }
 
@@ -589,10 +567,7 @@ class IOSDevice extends Device {
         installationResult = await iosDeployDebugger!.launchAndAttach() ? 0 : 1;
       }
       if (installationResult != 0) {
-        _logger.printError('Could not run ${bundle.path} on $id.');
-        _logger.printError('Try launching Xcode and selecting "Product > Run" to fix the problem:');
-        _logger.printError('  open ios/Runner.xcworkspace');
-        _logger.printError('');
+        _printInstallError(bundle);
         await dispose();
         return LaunchResult.failed();
       }
@@ -704,6 +679,31 @@ class IOSDevice extends Device {
           );
         } else {
           localUri = await vmServiceDiscovery?.uri;
+          // If the `ios-deploy` debugger loses connection before it finds the
+          // Dart Service VM url, try starting the debugger and launching the
+          // app again.
+          if (localUri == null &&
+              debuggingOptions.usingCISystem &&
+              iosDeployDebugger != null &&
+              iosDeployDebugger!.lostConnection) {
+            _logger.printStatus('Lost connection to device. Trying to connect again...');
+            await dispose();
+            vmServiceDiscovery = _setupDebuggerAndVmServiceDiscovery(
+              package: package,
+              bundle: bundle,
+              debuggingOptions: debuggingOptions,
+              launchArguments: launchArguments,
+              ipv6: ipv6,
+              skipInstall: true,
+            );
+            installationResult = await iosDeployDebugger!.launchAndAttach() ? 0 : 1;
+            if (installationResult != 0) {
+              _printInstallError(bundle);
+              await dispose();
+              return LaunchResult.failed();
+            }
+            localUri = await vmServiceDiscovery.uri;
+          }
         }
       }
       timer.cancel();
@@ -723,6 +723,53 @@ class IOSDevice extends Device {
     }
   }
 
+  void _printInstallError(Directory bundle) {
+    _logger.printError('Could not run ${bundle.path} on $id.');
+    _logger.printError('Try launching Xcode and selecting "Product > Run" to fix the problem:');
+    _logger.printError('  open ios/Runner.xcworkspace');
+    _logger.printError('');
+  }
+
+  ProtocolDiscovery _setupDebuggerAndVmServiceDiscovery({
+    required IOSApp package,
+    required Directory bundle,
+    required DebuggingOptions debuggingOptions,
+    required List<String> launchArguments,
+    required bool ipv6,
+    bool skipInstall = false,
+  }) {
+    final DeviceLogReader deviceLogReader = getLogReader(
+      app: package,
+      usingCISystem: debuggingOptions.usingCISystem,
+    );
+
+    // If the device supports syslog reading, prefer launching the app without
+    // attaching the debugger to avoid the overhead of the unnecessary extra running process.
+    if (majorSdkVersion >= IOSDeviceLogReader.minimumUniversalLoggingSdkVersion) {
+      iosDeployDebugger = _iosDeploy.prepareDebuggerForLaunch(
+        deviceId: id,
+        bundlePath: bundle.path,
+        appDeltaDirectory: package.appDeltaDirectory,
+        launchArguments: launchArguments,
+        interfaceType: connectionInterface,
+        uninstallFirst: debuggingOptions.uninstallFirst,
+        skipInstall: skipInstall,
+      );
+      if (deviceLogReader is IOSDeviceLogReader) {
+        deviceLogReader.debuggerStream = iosDeployDebugger;
+      }
+    }
+    // Don't port foward if debugging with a wireless device.
+    return ProtocolDiscovery.vmService(
+      deviceLogReader,
+      portForwarder: isWirelesslyConnected ? null : portForwarder,
+      hostPort: debuggingOptions.hostVmServicePort,
+      devicePort: debuggingOptions.deviceVmServicePort,
+      ipv6: ipv6,
+      logger: _logger,
+    );
+  }
+
   /// Starting with Xcode 15 and iOS 17, `ios-deploy` stopped working due to
   /// the new CoreDevice connectivity stack. Previously, `ios-deploy` was used
   /// to install the app, launch the app, and start `debugserver`.

packages/flutter_tools/lib/src/ios/ios_deploy.dart

@@ -129,6 +129,7 @@ class IOSDeploy {
     required DeviceConnectionInterface interfaceType,
     Directory? appDeltaDirectory,
     required bool uninstallFirst,
+    bool skipInstall = false,
   }) {
     appDeltaDirectory?.createSync(recursive: true);
     // Interactive debug session to support sending the lldb detach command.
@@ -148,6 +149,8 @@ class IOSDeploy {
       ],
       if (uninstallFirst)
         '--uninstall',
+      if (skipInstall)
+        '--noinstall',
       '--debug',
       if (interfaceType != DeviceConnectionInterface.wireless)
         '--no-wifi',
@@ -327,6 +330,14 @@ class IOSDeployDebugger {
   /// The future should be completed once the backtraces are logged.
   Completer<void>? _processResumeCompleter;
 
+  // Process 525 exited with status = -1 (0xffffffff) lost connection
+  static final RegExp _lostConnectionPattern = RegExp(r'exited with status = -1 \(0xffffffff\) lost connection');
+
+  /// Whether ios-deploy received a message matching [_lostConnectionPattern],
+  /// indicating that it lost connection to the device.
+  bool get lostConnection => _lostConnection;
+  bool _lostConnection = false;
+
   /// Launch the app on the device, and attach the debugger.
   ///
   /// Returns whether or not the debugger successfully attached.
@@ -448,6 +459,9 @@ class IOSDeployDebugger {
           // The app exited or crashed, so exit. Continue passing debugging
           // messages to the log reader until it exits to capture crash dumps.
           _logger.printTrace(line);
+          if (line.contains(_lostConnectionPattern)) {
+            _lostConnection = true;
+          }
           exit();
           return;
         }

packages/flutter_tools/test/general.shard/ios/ios_device_start_prebuilt_test.dart

@@ -75,6 +75,7 @@ FakeCommand attachDebuggerCommand({
   String stdout = '(lldb)     run\nsuccess',
   Completer<void>? completer,
   bool isWirelessDevice = false,
+  bool skipInstall = false,
 }) {
   return FakeCommand(
     command: <String>[
@@ -87,6 +88,8 @@ FakeCommand attachDebuggerCommand({
       '123',
       '--bundle',
       '/',
+      if (skipInstall)
+        '--noinstall',
       '--debug',
       if (!isWirelessDevice) '--no-wifi',
       '--args',
@@ -339,6 +342,86 @@ void main() {
     MDnsVmServiceDiscovery: () => FakeMDnsVmServiceDiscovery(),
   });
 
+  testWithoutContext('IOSDevice.startApp retries when ios-deploy loses connection the first time in CI', () async {
+    final BufferLogger logger = BufferLogger.test();
+    final FileSystem fileSystem = MemoryFileSystem.test();
+    final Completer<void> completer = Completer<void>();
+    final FakeProcessManager processManager = FakeProcessManager.list(<FakeCommand>[
+      attachDebuggerCommand(
+        stdout: '(lldb)     run\nsuccess\nProcess 525 exited with status = -1 (0xffffffff) lost connection',
+      ),
+      attachDebuggerCommand(
+        stdout: '(lldb)     run\nsuccess\nThe Dart VM service is listening on http://127.0.0.1:456',
+        completer: completer,
+        skipInstall: true,
+      ),
+    ]);
+    final IOSDevice device = setUpIOSDevice(
+      processManager: processManager,
+      fileSystem: fileSystem,
+      logger: logger,
+    );
+    final IOSApp iosApp = PrebuiltIOSApp(
+      projectBundleId: 'app',
+      bundleName: 'Runner',
+      uncompressedBundle: fileSystem.currentDirectory,
+      applicationPackage: fileSystem.currentDirectory,
+    );
+
+    device.portForwarder = const NoOpDevicePortForwarder();
+
+    final LaunchResult launchResult = await device.startApp(iosApp,
+      prebuiltApplication: true,
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+        usingCISystem: true,
+      ),
+      platformArgs: <String, dynamic>{},
+    );
+    completer.complete();
+
+    expect(processManager, hasNoRemainingExpectations);
+    expect(launchResult.started, true);
+    expect(launchResult.hasVmService, true);
+    expect(await device.stopApp(iosApp), false);
+  });
+
+  testWithoutContext('IOSDevice.startApp does not retry when ios-deploy loses connection if not in CI', () async {
+    final BufferLogger logger = BufferLogger.test();
+    final FileSystem fileSystem = MemoryFileSystem.test();
+    final FakeProcessManager processManager = FakeProcessManager.list(<FakeCommand>[
+      attachDebuggerCommand(
+        stdout: '(lldb)     run\nsuccess\nProcess 525 exited with status = -1 (0xffffffff) lost connection',
+      ),
+    ]);
+    final IOSDevice device = setUpIOSDevice(
+      processManager: processManager,
+      fileSystem: fileSystem,
+      logger: logger,
+    );
+    final IOSApp iosApp = PrebuiltIOSApp(
+      projectBundleId: 'app',
+      bundleName: 'Runner',
+      uncompressedBundle: fileSystem.currentDirectory,
+      applicationPackage: fileSystem.currentDirectory,
+    );
+
+    device.portForwarder = const NoOpDevicePortForwarder();
+
+    final LaunchResult launchResult = await device.startApp(iosApp,
+      prebuiltApplication: true,
+      debuggingOptions: DebuggingOptions.enabled(
+        BuildInfo.debug,
+      ),
+      platformArgs: <String, dynamic>{},
+    );
+
+    expect(processManager, hasNoRemainingExpectations);
+    expect(launchResult.started, false);
+    expect(launchResult.hasVmService, false);
+    expect(await device.stopApp(iosApp), false);
+  });
+
   testWithoutContext('IOSDevice.startApp succeeds in release mode', () async {
     final FileSystem fileSystem = MemoryFileSystem.test();
     final FakeProcessManager processManager = FakeProcessManager.list(<FakeCommand>[