[devicelab] reboot attached devices after 30 test runs (#69447)

dev/devicelab/lib/framework/adb.dart

@@ -131,6 +131,9 @@ abstract class Device {
   /// Assumes the device doesn't have a secure unlock pattern.
   Future<void> unlock();
 
+  /// Attempt to reboot the phone.
+  Future<void> reboot();
+
   /// Emulate a tap on the touch screen.
   Future<void> tap(int x, int y);
 
@@ -575,6 +578,11 @@ class AndroidDevice extends Device {
   String toString() {
     return '$deviceId $deviceInfo';
   }
+
+  @override
+  Future<void> reboot() {
+    return adb(<String>['reboot']);
+  }
 }
 
 class IosDeviceDiscovery implements DeviceDiscovery {
@@ -740,6 +748,11 @@ class IosDevice extends Device {
 
   @override
   Future<void> stop(String packageName) async {}
+
+  @override
+  Future<void> reboot() {
+    return Process.run('idevicesyslog', <String>['reboot', '-u', deviceId]);
+  }
 }
 
 /// Fuchsia device.
@@ -783,6 +796,11 @@ class FuchsiaDevice extends Device {
   Stream<String> get logcat {
     throw UnimplementedError();
   }
+
+  @override
+  Future<void> reboot() async {
+    // Unsupported.
+  }
 }
 
 /// Path to the `adb` executable.
@@ -846,6 +864,11 @@ class FakeDevice extends Device {
 
   @override
   Future<void> stop(String packageName) async {}
+
+  @override
+  Future<void> reboot() async {
+    // Unsupported.
+  }
 }
 
 class FakeDeviceDiscovery implements DeviceDiscovery {

dev/devicelab/lib/framework/framework.dart

@@ -12,10 +12,20 @@ import 'package:path/path.dart' as path;
 import 'package:logging/logging.dart';
 import 'package:stack_trace/stack_trace.dart';
 
-import 'running_processes.dart';
+import 'adb.dart';
 import 'task_result.dart';
 import 'utils.dart';
 
+/// Identifiers for devices that should never be rebooted.
+final Set<String> noRebootForbidList = <String>{
+  '822ef7958bba573829d85eef4df6cbdd86593730', // 32bit iPhone requires manual intervention on reboot.
+};
+
+/// The maximum number of test runs before a device must be rebooted.
+///
+/// This number was chosen arbitrarily.
+const int maxiumRuns = 30;
+
 /// Represents a unit of work performed in the CI environment that can
 /// succeed, fail and be retried independently of others.
 typedef TaskFunction = Future<TaskResult> Function();
@@ -80,27 +90,15 @@ class _TaskRunner {
     try {
       _taskStarted = true;
       print('Running task with a timeout of $taskTimeout.');
-      final String exe = Platform.isWindows ? '.exe' : '';
-      section('Checking running Dart$exe processes');
-      final Set<RunningProcessInfo> beforeRunningDartInstances = await getRunningProcesses(
-        processName: 'dart$exe',
-      ).toSet();
-      final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
-      beforeRunningDartInstances.forEach(print);
-      for (final RunningProcessInfo info in allProcesses) {
-        if (info.commandLine.contains('iproxy')) {
-          print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
-        }
-      }
-
       print('enabling configs for macOS, Linux, Windows, and Web...');
       final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
         'config',
+        '-v',
         '--enable-macos-desktop',
         '--enable-windows-desktop',
         '--enable-linux-desktop',
         '--enable-web'
-      ]);
+      ], canFail: true);
       if (configResult != 0) {
         print('Failed to enable configuration, tasks may not run.');
       }
@@ -109,34 +107,7 @@ class _TaskRunner {
       if (taskTimeout != null)
         futureResult = futureResult.timeout(taskTimeout);
 
-      TaskResult result = await futureResult;
-
-      section('Checking running Dart$exe processes after task...');
-      final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
-        processName: 'dart$exe',
-      ).toList();
-      for (final RunningProcessInfo info in afterRunningDartInstances) {
-        if (!beforeRunningDartInstances.contains(info)) {
-          print('$info was leaked by this test.');
-          if (result is TaskResultCheckProcesses) {
-            result = TaskResult.failure('This test leaked dart processes');
-          }
-          final bool killed = await killProcess(info.pid);
-          if (!killed) {
-            print('Failed to kill process ${info.pid}.');
-          } else {
-            print('Killed process id ${info.pid}.');
-          }
-        }
-      }
-      final Set<RunningProcessInfo> allEndProcesses = await getRunningProcesses().toSet();
-      for (final RunningProcessInfo info in allEndProcesses) {
-        if (allProcesses.contains(info)) {
-          continue;
-        }
-        print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
-      }
-
+      final TaskResult result = await futureResult;
       _completer.complete(result);
       return result;
     } on TimeoutException catch (err, stackTrace) {
@@ -147,10 +118,40 @@ class _TaskRunner {
     } finally {
       print('Cleaning up after task...');
       await forceQuitRunningProcesses();
+      await checkForRebootRequired();
       _closeKeepAlivePort();
     }
   }
 
+  Future<void> checkForRebootRequired() async {
+    try {
+      final Device device = await devices.workingDevice.timeout(const Duration(seconds: 15));
+      if (noRebootForbidList.contains(device.deviceId)) {
+        return;
+      }
+      final File rebootFile = _rebootFile();
+      int runCount;
+      if (rebootFile.existsSync()) {
+        runCount = int.tryParse(rebootFile.readAsStringSync().trim());
+      } else {
+        runCount = 0;
+      }
+      if (runCount < maxiumRuns) {
+        rebootFile
+          ..createSync()
+          ..writeAsStringSync((runCount + 1).toString());
+        return;
+      }
+      rebootFile.deleteSync();
+      print('Rebooting ${device.deviceId}');
+      await device.reboot();
+    } on TimeoutException {
+      // Could not find device in order to reboot.
+    } on DeviceException {
+      // No attached device needed to reboot.
+    }
+  }
+
   /// Causes the Dart VM to stay alive until a request to run the task is
   /// received via the VM service protocol.
   void keepVmAliveUntilTaskRunRequested() {
@@ -199,3 +200,13 @@ class _TaskRunner {
     return completer.future;
   }
 }
+
+File _rebootFile() {
+  if (Platform.isLinux || Platform.isMacOS) {
+    return File(path.join(Platform.environment['HOME'], '.reboot-count'));
+  }
+  if (!Platform.isWindows) {
+    throw StateError('Unexpected platform ${Platform.operatingSystem}');
+  }
+  return File(path.join(Platform.environment['USERPROFILE'], '.reboot-count'));
+}