[macOS] Add run release test in devicelab (#100526)

Adds a test that invokes flutter run in release mode on macOS desktop,
waits for successful launch and the flutter command list, then sends the
'q' command to quit the running app.

This adds an integration test for https://github.com/flutter/flutter/pull/100504.

Issue: https://github.com/flutter/flutter/issues/100348 (fix)
Issue: https://github.com/flutter/flutter/issues/97978 (partial fix)
Issue: https://github.com/flutter/flutter/issues/97977 (partial fix)
Umbrella issue: https://github.com/flutter/flutter/issues/60113

.ci.yaml

@@ -3679,10 +3679,30 @@ targets:
       task_name: native_ui_tests_macos
     scheduler: luci
     runIf:
-    - dev/**
-    - packages/flutter_tools/**
-    - bin/**
-    - .ci.yaml
+      - dev/**
+      - packages/flutter_tools/**
+      - bin/**
+      - .ci.yaml
+
+  - name: Mac run_release_test_macos
+    recipe: devicelab/devicelab_drone
+    bringup: true
+    timeout: 60
+    properties:
+      dependencies: >-
+        [
+          {"dependency": "xcode"},
+          {"dependency": "gems"}
+        ]
+      tags: >
+        ["devicelab","hostonly"]
+      task_name: run_release_test_macos
+    scheduler: luci
+    runIf:
+      - dev/**
+      - packages/flutter_tools/**
+      - bin/**
+      - .ci.yaml
 
   - name: Windows build_aar_module_test
     recipe: devicelab/devicelab_drone

TESTOWNERS

@@ -200,6 +200,7 @@
 /dev/devicelab/bin/tasks/gradle_plugin_light_apk_test.dart @stuartmorgan @flutter/plugin
 /dev/devicelab/bin/tasks/module_test_ios.dart @jmagman @flutter/tool
 /dev/devicelab/bin/tasks/plugin_lint_mac.dart @stuartmorgan @flutter/plugin
+/dev/devicelab/bin/tasks/run_release_test_macos.dart @cbracken @flutter/tool
 /dev/devicelab/bin/tasks/windows_home_scroll_perf__timeline_summary.dart @jonahwilliams @flutter/engine
 
 ## Host only framework tests

dev/devicelab/bin/tasks/run_release_test_macos.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter_devicelab/common.dart';
+import 'package:flutter_devicelab/framework/devices.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:path/path.dart' as path;
+
+/// Basic launch test for desktop operating systems.
+void main() {
+  task(() async {
+    deviceOperatingSystem = DeviceOperatingSystem.macos;
+    final Device device = await devices.workingDevice;
+    // TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201
+    // Switch to dev/integration_tests/ui once we have CocoaPods working on M1 Macs.
+    final Directory appDir = dir(path.join(flutterDirectory.path, 'examples/hello_world'));
+    await inDirectory(appDir, () async {
+      final Completer<void> ready = Completer<void>();
+      final List<String> stdout = <String>[];
+      final List<String> stderr = <String>[];
+
+      print('run: starting...');
+      final List<String> options = <String>[
+        '--release',
+        '-d',
+        device.deviceId,
+      ];
+      final Process run = await startFlutter(
+        'run',
+        options: options,
+        isBot: false,
+      );
+      int? runExitCode;
+      run.stdout
+        .transform<String>(utf8.decoder)
+        .transform<String>(const LineSplitter())
+        .listen((String line) {
+          print('run:stdout: $line');
+          if (
+            !line.startsWith('Building flutter tool...') &&
+            !line.startsWith('Running "flutter pub get" in ui...') &&
+            !line.startsWith('Resolving dependencies...') &&
+            // Catch engine piped output from unrelated concurrent Flutter apps
+            !line.contains(RegExp(r'[A-Z]\/flutter \([0-9]+\):')) &&
+            // Empty lines could be due to the progress spinner breaking up.
+            line.length > 1
+          ) {
+            stdout.add(line);
+          }
+          if (line.contains('Quit (terminate the application on the device).')) {
+            ready.complete();
+          }
+        });
+      run.stderr
+        .transform<String>(utf8.decoder)
+        .transform<String>(const LineSplitter())
+        .listen((String line) {
+          print('run:stderr: $line');
+          stderr.add(line);
+        });
+      unawaited(run.exitCode.then<void>((int exitCode) { runExitCode = exitCode; }));
+      await Future.any<dynamic>(<Future<dynamic>>[ ready.future, run.exitCode ]);
+      if (runExitCode != null) {
+        throw 'Failed to run test app; runner unexpected exited, with exit code $runExitCode.';
+      }
+      run.stdin.write('q');
+
+      await run.exitCode;
+
+      if (stderr.isNotEmpty) {
+        throw 'flutter run --release had output on standard error.';
+      }
+
+      _findNextMatcherInList(
+        stdout,
+        (String line) => line.startsWith('Launching lib/main.dart on ') && line.endsWith(' in release mode...'),
+        'Launching lib/main.dart on',
+      );
+
+      _findNextMatcherInList(
+        stdout,
+        (String line) => line.contains('Quit (terminate the application on the device).'),
+        'q Quit (terminate the application on the device)',
+      );
+
+      _findNextMatcherInList(
+        stdout,
+        (String line) => line == 'Application finished.',
+        'Application finished.',
+      );
+    });
+    return TaskResult.success(null);
+  });
+}
+
+void _findNextMatcherInList(
+  List<String> list,
+  bool Function(String testLine) matcher,
+  String errorMessageExpectedLine
+) {
+  final List<String> copyOfListForErrorMessage = List<String>.from(list);
+
+  while (list.isNotEmpty) {
+    final String nextLine = list.first;
+    list.removeAt(0);
+
+    if (matcher(nextLine)) {
+      return;
+    }
+  }
+
+  throw '''
+Did not find expected line
+
+$errorMessageExpectedLine
+
+in flutter run --release stdout
+
+$copyOfListForErrorMessage
+  ''';
+}

dev/devicelab/lib/framework/devices.dart

@@ -52,7 +52,16 @@ String? _findMatchId(List<String> idList, String idPattern) {
 DeviceDiscovery get devices => DeviceDiscovery();
 
 /// Device operating system the test is configured to test.
-enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake, windows }
+enum DeviceOperatingSystem {
+  android,
+  androidArm,
+  androidArm64,
+  fake,
+  fuchsia,
+  ios,
+  macos,
+  windows,
+}
 
 /// Device OS to test on.
 DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android;
@@ -71,6 +80,8 @@ abstract class DeviceDiscovery {
         return IosDeviceDiscovery();
       case DeviceOperatingSystem.fuchsia:
         return FuchsiaDeviceDiscovery();
+      case DeviceOperatingSystem.macos:
+        return MacosDeviceDiscovery();
       case DeviceOperatingSystem.windows:
         return WindowsDeviceDiscovery();
       case DeviceOperatingSystem.fake:
@@ -342,6 +353,40 @@ class AndroidDeviceDiscovery implements DeviceDiscovery {
   }
 }
 
+class MacosDeviceDiscovery implements DeviceDiscovery {
+  factory MacosDeviceDiscovery() {
+    return _instance ??= MacosDeviceDiscovery._();
+  }
+
+  MacosDeviceDiscovery._();
+
+  static MacosDeviceDiscovery? _instance;
+
+  static const MacosDevice _device = MacosDevice();
+
+  @override
+  Future<Map<String, HealthCheckResult>> checkDevices() async {
+    return <String, HealthCheckResult>{};
+  }
+
+  @override
+  Future<void> chooseWorkingDevice() async { }
+
+  @override
+  Future<void> chooseWorkingDeviceById(String deviceId) async { }
+
+  @override
+  Future<List<String>> discoverDevices() async {
+    return <String>['macos'];
+  }
+
+  @override
+  Future<void> performPreflightTasks() async { }
+
+  @override
+  Future<Device> get workingDevice  async => _device;
+}
+
 class WindowsDeviceDiscovery implements DeviceDiscovery {
   factory WindowsDeviceDiscovery() {
     return _instance ??= WindowsDeviceDiscovery._();
@@ -374,7 +419,6 @@ class WindowsDeviceDiscovery implements DeviceDiscovery {
 
   @override
   Future<Device> get workingDevice  async => _device;
-
 }
 
 class FuchsiaDeviceDiscovery implements DeviceDiscovery {
@@ -996,6 +1040,58 @@ class IosDevice extends Device {
   }
 }
 
+class MacosDevice extends Device {
+  const MacosDevice();
+
+  @override
+  String get deviceId => 'macos';
+
+  @override
+  Future<Map<String, dynamic>> getMemoryStats(String packageName) async {
+    return <String, dynamic>{};
+  }
+
+  @override
+  Future<void> home() async { }
+
+  @override
+  Future<bool> isAsleep() async {
+    return false;
+  }
+
+  @override
+  Future<bool> isAwake() async {
+    return true;
+  }
+
+  @override
+  Stream<String> get logcat => const Stream<String>.empty();
+
+  @override
+  Future<void> clearLogs() async {}
+
+  @override
+  Future<void> reboot() async { }
+
+  @override
+  Future<void> sendToSleep() async { }
+
+  @override
+  Future<void> stop(String packageName) async { }
+
+  @override
+  Future<void> tap(int x, int y) async { }
+
+  @override
+  Future<void> togglePower() async { }
+
+  @override
+  Future<void> unlock() async { }
+
+  @override
+  Future<void> wakeUp() async { }
+}
+
 class WindowsDevice extends Device {
   const WindowsDevice();
 

dev/devicelab/lib/framework/utils.dart

@@ -471,15 +471,40 @@ Future<int> flutter(String command, {
     canFail: canFail, environment: environment);
 }
 
+/// Starts a Flutter subprocess.
+///
+/// The first argument is the flutter command to run.
+///
+/// The second argument is the list of arguments to provide on the command line.
+/// This argument can be null, indicating no arguments (same as the empty list).
+///
+/// The `environment` argument can be provided to configure environment variables
+/// that will be made available to the subprocess. The `BOT` environment variable
+/// is always set and overrides any value provided in the `environment` argument.
+/// The `isBot` argument controls the value of the `BOT` variable. It will either
+/// be "true", if `isBot` is true (the default), or "false" if it is false.
+///
+/// The `isBot` argument controls whether the `BOT` environment variable is set
+/// to `true` or `false` and is used by the `flutter` tool to determine how
+/// verbose to be and whether to enable analytics by default.
+///
+/// Information regarding the execution of the subprocess is printed to the
+/// console.
+///
+/// The actual process executes asynchronously. A handle to the subprocess is
+/// returned in the form of a [Future] that completes to a [Process] object.
 Future<Process> startFlutter(String command, {
   List<String> options = const <String>[],
   Map<String, String> environment = const <String, String>{},
+  bool isBot = true, // set to false to pretend not to be on a bot (e.g. to test user-facing outputs)
 }) {
+  assert(isBot != null);
   final List<String> args = flutterCommandArgs(command, options);
   return startProcess(
     path.join(flutterDirectory.path, 'bin', 'flutter'),
     args,
     environment: environment,
+    isBot: isBot,
   );
 }
 

dev/devicelab/lib/tasks/perf_tests.dart

@@ -622,9 +622,10 @@ class StartupTest {
           ]);
           applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos');
           break;
-        case DeviceOperatingSystem.windows:
-        case DeviceOperatingSystem.fuchsia:
         case DeviceOperatingSystem.fake:
+        case DeviceOperatingSystem.fuchsia:
+        case DeviceOperatingSystem.macos:
+        case DeviceOperatingSystem.windows:
           break;
       }
 
@@ -738,9 +739,10 @@ class DevtoolsStartupTest {
           ]);
           applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos');
           break;
-        case DeviceOperatingSystem.windows:
-        case DeviceOperatingSystem.fuchsia:
         case DeviceOperatingSystem.fake:
+        case DeviceOperatingSystem.fuchsia:
+        case DeviceOperatingSystem.macos:
+        case DeviceOperatingSystem.windows:
           break;
       }
 
@@ -1348,12 +1350,14 @@ class CompileTest {
         if (reportPackageContentSizes)
           metrics.addAll(await getSizesFromApk(apkPath));
         break;
-      case DeviceOperatingSystem.windows:
-        throw Exception('Unsupported option for Windows devices');
-      case DeviceOperatingSystem.fuchsia:
-        throw Exception('Unsupported option for Fuchsia devices');
       case DeviceOperatingSystem.fake:
         throw Exception('Unsupported option for fake devices');
+      case DeviceOperatingSystem.fuchsia:
+        throw Exception('Unsupported option for Fuchsia devices');
+      case DeviceOperatingSystem.macos:
+        throw Exception('Unsupported option for macOS devices');
+      case DeviceOperatingSystem.windows:
+        throw Exception('Unsupported option for Windows devices');
     }
 
     metrics.addAll(<String, dynamic>{
@@ -1386,12 +1390,14 @@ class CompileTest {
         options.insert(0, 'apk');
         options.add('--target-platform=android-arm64');
         break;
-      case DeviceOperatingSystem.windows:
-        throw Exception('Unsupported option for Windows devices');
-      case DeviceOperatingSystem.fuchsia:
-        throw Exception('Unsupported option for Fuchsia devices');
       case DeviceOperatingSystem.fake:
         throw Exception('Unsupported option for fake devices');
+      case DeviceOperatingSystem.fuchsia:
+        throw Exception('Unsupported option for Fuchsia devices');
+      case DeviceOperatingSystem.macos:
+        throw Exception('Unsupported option for Fuchsia devices');
+      case DeviceOperatingSystem.windows:
+        throw Exception('Unsupported option for Windows devices');
     }
     watch.start();
     await flutter('build', options: options);