Run flutter_gallery macOS native tests on presubmit (#89620)

.ci.yaml

@@ -3182,6 +3182,25 @@ targets:
       task_name: native_ui_tests_ios
     scheduler: luci
 
+  - name: Mac native_ui_tests_macos
+    recipe: devicelab/devicelab_drone
+    bringup: true
+    timeout: 60
+    properties:
+      dependencies: >-
+        [
+          {"dependency": "xcode"},
+          {"dependency": "gems"}
+        ]
+      tags: >
+        ["devicelab","hostonly"]
+      task_name: native_ui_tests_macos
+    scheduler: luci
+    runIf:
+    - dev/**
+    - packages/flutter_tools/**
+    - bin/**
+
   - name: Windows build_aar_module_test
     recipe: devicelab/devicelab_drone
     timeout: 60

TESTOWNERS

@@ -173,6 +173,7 @@
 /dev/devicelab/bin/tasks/module_host_with_custom_build_test.dart @zanderso @flutter/tool
 /dev/devicelab/bin/tasks/module_test.dart @zanderso @flutter/tool
 /dev/devicelab/bin/tasks/native_ui_tests_ios.dart @jmagman @flutter/engine
+/dev/devicelab/bin/tasks/native_ui_tests_macos.dart @cbracken @flutter/desktop
 /dev/devicelab/bin/tasks/plugin_test.dart @stuartmorgan @flutter/plugin
 /dev/devicelab/bin/tasks/technical_debt__cost.dart @HansMuller @flutter/framework
 /dev/devicelab/bin/tasks/web_benchmarks_canvaskit.dart @yjbanov @flutter/web

dev/devicelab/bin/tasks/native_ui_tests_ios.dart

@@ -2,14 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:io';
-
 import 'package:flutter_devicelab/framework/devices.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/framework/host_agent.dart';
+import 'package:flutter_devicelab/framework/ios.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
 import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:path/path.dart' as path;
 
 Future<void> main() async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
@@ -38,58 +35,7 @@ Future<void> main() async {
     section('Run platform unit tests');
 
     final Device device = await devices.workingDevice;
-    final Map<String, String> environment = Platform.environment;
-    // If not running on CI, inject the Flutter team code signing properties.
-    final String developmentTeam = environment['FLUTTER_XCODE_DEVELOPMENT_TEAM'] ?? 'S8QB4VV633';
-    final String? codeSignStyle = environment['FLUTTER_XCODE_CODE_SIGN_STYLE'];
-    final String? provisioningProfile = environment['FLUTTER_XCODE_PROVISIONING_PROFILE_SPECIFIER'];
-
-    final String resultBundleTemp = Directory.systemTemp.createTempSync('flutter_native_ui_tests_ios32_xcresult.').path;
-    final String resultBundlePath = path.join(resultBundleTemp, 'result');
-    final int testResultExit = await exec(
-      'xcodebuild',
-      <String>[
-        '-workspace',
-        'Runner.xcworkspace',
-        '-scheme',
-        'Runner',
-        '-configuration',
-        'Release',
-        '-destination',
-        'id=${device.deviceId}',
-        '-resultBundlePath',
-        resultBundlePath,
-        'test',
-        'COMPILER_INDEX_STORE_ENABLE=NO',
-        'DEVELOPMENT_TEAM=$developmentTeam',
-        if (codeSignStyle != null)
-          'CODE_SIGN_STYLE=$codeSignStyle',
-        if (provisioningProfile != null)
-          'PROVISIONING_PROFILE_SPECIFIER=$provisioningProfile',
-      ],
-      workingDirectory: path.join(projectDirectory, 'ios'),
-      canFail: true,
-    );
-
-    if (testResultExit != 0) {
-      final Directory? dumpDirectory = hostAgent.dumpDirectory;
-      if (dumpDirectory != null) {
-        // Zip the test results to the artifacts directory for upload.
-        final String zipPath = path.join(dumpDirectory.path,
-            'native_ui_tests_ios32-${DateTime.now().toLocal().toIso8601String()}.zip');
-        await exec(
-          'zip',
-          <String>[
-            '-r',
-            '-9',
-            zipPath,
-            'result.xcresult',
-          ],
-          workingDirectory: resultBundleTemp,
-          canFail: true, // Best effort to get the logs.
-        );
-      }
-
+    if (!await runXcodeTests(projectDirectory, device.deviceId, 'native_ui_tests_ios')) {
       return TaskResult.failure('Platform unit tests failed');
     }
 

dev/devicelab/bin/tasks/native_ui_tests_macos.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/ios.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+Future<void> main() async {
+  await task(() async {
+    final String projectDirectory = '${flutterDirectory.path}/dev/integration_tests/flutter_gallery';
+
+    await inDirectory(projectDirectory, () async {
+      section('Build gallery app');
+
+      await flutter(
+        'build',
+        options: <String>[
+          'macos',
+          '-v',
+          '--debug',
+        ],
+      );
+    });
+
+    section('Run platform unit tests');
+
+    if (!await runXcodeTests(projectDirectory, 'platform=macOS', 'native_ui_tests_macos')) {
+      return TaskResult.failure('Platform unit tests failed');
+    }
+
+    return TaskResult.success(null);
+  });
+}

dev/devicelab/lib/framework/ios.dart

@@ -3,7 +3,11 @@
 // found in the LICENSE file.
 
 import 'dart:convert';
+import 'dart:io';
 
+import 'package:path/path.dart' as path;
+
+import 'host_agent.dart';
 import 'utils.dart';
 
 typedef SimulatorFunction = Future<void> Function(String deviceId);
@@ -135,3 +139,60 @@ Future<void> removeIOSimulator(String deviceId) async {
     );
   }
 }
+
+Future<bool> runXcodeTests(String projectDirectory, String deviceId, String testName) async {
+  final Map<String, String> environment = Platform.environment;
+  // If not running on CI, inject the Flutter team code signing properties.
+  final String developmentTeam = environment['FLUTTER_XCODE_DEVELOPMENT_TEAM'] ?? 'S8QB4VV633';
+  final String? codeSignStyle = environment['FLUTTER_XCODE_CODE_SIGN_STYLE'];
+  final String? provisioningProfile = environment['FLUTTER_XCODE_PROVISIONING_PROFILE_SPECIFIER'];
+
+  final String resultBundleTemp = Directory.systemTemp.createTempSync('flutter_xcresult.').path;
+  final String resultBundlePath = path.join(resultBundleTemp, 'result');
+  final int testResultExit = await exec(
+    'xcodebuild',
+    <String>[
+      '-workspace',
+      'Runner.xcworkspace',
+      '-scheme',
+      'Runner',
+      '-configuration',
+      'Release',
+      '-destination',
+      'id=$deviceId',
+      '-resultBundlePath',
+      resultBundlePath,
+      'test',
+      'COMPILER_INDEX_STORE_ENABLE=NO',
+      'DEVELOPMENT_TEAM=$developmentTeam',
+      if (codeSignStyle != null)
+        'CODE_SIGN_STYLE=$codeSignStyle',
+      if (provisioningProfile != null)
+        'PROVISIONING_PROFILE_SPECIFIER=$provisioningProfile',
+    ],
+    workingDirectory: path.join(projectDirectory, 'ios'),
+    canFail: true,
+  );
+
+  if (testResultExit != 0) {
+    final Directory? dumpDirectory = hostAgent.dumpDirectory;
+    if (dumpDirectory != null) {
+      // Zip the test results to the artifacts directory for upload.
+      final String zipPath = path.join(dumpDirectory.path,
+          '$testName-${DateTime.now().toLocal().toIso8601String()}.zip');
+      await exec(
+        'zip',
+        <String>[
+          '-r',
+          '-9',
+          zipPath,
+          'result.xcresult',
+        ],
+        workingDirectory: resultBundleTemp,
+        canFail: true, // Best effort to get the logs.
+      );
+    }
+    return false;
+  }
+  return true;
+}

dev/integration_tests/flutter_gallery/macos/Podfile

@@ -31,6 +31,10 @@ target 'Runner' do
   use_modular_headers!
 
   flutter_install_all_macos_pods File.dirname(File.realpath(__FILE__))
+
+  target 'RunnerTests' do
+    inherit! :search_paths
+  end
 end
 
 post_install do |installer|

dev/integration_tests/flutter_gallery/macos/Podfile.lock

@@ -25,11 +25,11 @@ EXTERNAL SOURCES:
     :path: Flutter/ephemeral/.symlinks/plugins/url_launcher_macos/macos
 
 SPEC CHECKSUMS:
-  connectivity_macos: 9f30e9d0e67a0bc08a0c563ee82310b51ca6e818
+  connectivity_macos: 5dae6ee11d320fac7c05f0d08bd08fc32b5514d9
   FlutterMacOS: 57701585bf7de1b3fc2bb61f6378d73bbdea8424
   Reachability: 33e18b67625424e47b6cde6d202dce689ad7af96
   url_launcher_macos: 45af3d61de06997666568a7149c1be98b41c95d4
 
-PODFILE CHECKSUM: 6eac6b3292e5142cfc23bdeb71848a40ec51c14c
+PODFILE CHECKSUM: b5c36ba411e4471a03727d0463fa17be341876c1
 
-COCOAPODS: 1.10.1
+COCOAPODS: 1.10.2

dev/integration_tests/flutter_gallery/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

@@ -37,6 +37,16 @@
          </BuildableReference>
       </MacroExpansion>
       <Testables>
+         <TestableReference
+            skipped = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "F7BDBB0A26E829B700DE9315"
+               BuildableName = "RunnerTests.xctest"
+               BlueprintName = "RunnerTests"
+               ReferencedContainer = "container:Runner.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
       </Testables>
    </TestAction>
    <LaunchAction

dev/integration_tests/flutter_gallery/macos/RunnerTests/RunnerTests.m

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import <FlutterMacOS/FlutterMacOS.h>
+#import <XCTest/XCTest.h>
+
+@interface RunnerTests : XCTestCase
+@end
+
+@implementation RunnerTests
+
+- (void)testMenu {
+  NSMenu *applicationMenu = ((FlutterAppDelegate *)NSApplication.sharedApplication.delegate).applicationMenu;
+  XCTAssertEqual(applicationMenu.numberOfItems, 11);
+  XCTAssertEqualObjects([applicationMenu itemAtIndex:0].title, @"About flutter_gallery");
+
+  NSMenu *mainMenu = NSApplication.sharedApplication.mainMenu;
+  XCTAssertEqual([mainMenu indexOfItemWithSubmenu:applicationMenu], 0);
+
+  XCTAssertEqual([mainMenu itemWithTitle:@"Edit"].submenu.numberOfItems, 19);
+  XCTAssertEqual([mainMenu itemWithTitle:@"View"].submenu.numberOfItems, 1);
+  XCTAssertEqual([mainMenu itemWithTitle:@"Window"].submenu.numberOfItems, 6);
+
+  XCTAssertNil(NSApplication.sharedApplication.helpMenu);
+}
+
+@end