Restore and fix flavors_test_macos (#139841)

Partially resolves[^1] https://github.com/flutter/flutter/issues/139774.

Effectively reverts https://github.com/flutter/flutter/pull/125581.

The main change here is that I deleted and recreated the macos Xcode project for this integration test (hence the large diff). I tried fixing the existing project first, but it was set up quite differently, and—for whatever reason—the integration test would get stuck trying to load `dev/integration_tests/flavors/integration_test/integration_test.dart`.

I verified that this works locally, but I don't know if it's possible to run this on the devicelab try pool to verify that it works on devicelab hardware.

[^1]: I would not close the issue until 1) this PR lands, 2) the integration test consistently passes on CI, and 3) macOS support for flavors is publicly documented.

.ci.yaml

@@ -3156,6 +3156,21 @@ targets:
       - bin/**
       - .ci.yaml
 
+  - name: Mac flavors_test_macos
+    bringup: true
+    presubmit: false
+    recipe: devicelab/devicelab_drone
+    timeout: 60
+    properties:
+      dependencies: >-
+        [
+          {"dependency": "xcode", "version": "14c18"},
+          {"dependency": "gems", "version": "v3.3.14"}
+        ]
+      tags: >
+        ["devicelab", "hostonly", "mac"]
+      task_name: flavors_test_macos
+
   - name: Mac_benchmark flutter_gallery_macos__compile
     presubmit: false
     recipe: devicelab/devicelab_drone

dev/devicelab/bin/tasks/flavors_test_macos.dart

@@ -17,22 +17,27 @@ Future<void> main() async {
     final TaskResult installTestsResult = await inDirectory(
       '${flutterDirectory.path}/dev/integration_tests/flavors',
       () async {
+        await flutter(
+          'install',
+          options: <String>['--flavor', 'paid', '-d', 'macos'],
+        );
+        await flutter(
+          'install',
+          options: <String>['--flavor', 'paid', '--uninstall-only', '-d', 'macos'],
+        );
         final StringBuffer stderr = StringBuffer();
-
         await evalFlutter(
-          'install',
+          'build',
           canFail: true,
           stderr: stderr,
-          options: <String>[
-            '--d', 'macos',
-            '--flavor', 'free'
-          ],
+          options: <String>['macos', '--flavor', 'bogus'],
         );
 
         final String stderrString = stderr.toString();
-        if (!stderrString.contains('Host and target are the same. Nothing to install.')) {
+        print(stderrString);
+        if (!stderrString.contains('The Xcode project defines schemes:')) {
           print(stderrString);
-          return TaskResult.failure('Installing a macOS app on macOS should no-op');
+          return TaskResult.failure('Should not succeed with bogus flavor');
         }
 
         return TaskResult.success(null);

dev/integration_tests/flavors/.gitignore

+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.buildlog/
+.history
+.svn/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+**/doc/api/
+**/ios/Flutter/.last_build_id
+.dart_tool/
+.flutter-plugins
+.flutter-plugins-dependencies
+.pub-cache/
+.pub/
+/build/
+
+# Symbolication related
+app.*.symbols
+
+# Obfuscation related
+app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release

dev/integration_tests/flavors/.metadata

+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: "e0caf9ca0b7c592bd7b00d5413fae534738bb385"
+  channel: "[user-branch]"
+
+project_type: app
+
+# Tracks metadata for the flutter migrate command
+migration:
+  platforms:
+    - platform: root
+      create_revision: e0caf9ca0b7c592bd7b00d5413fae534738bb385
+      base_revision: e0caf9ca0b7c592bd7b00d5413fae534738bb385
+    - platform: macos
+      create_revision: e0caf9ca0b7c592bd7b00d5413fae534738bb385
+      base_revision: e0caf9ca0b7c592bd7b00d5413fae534738bb385
+
+  # User provided section
+
+  # List of Local paths (relative to this file) that should be
+  # ignored by the migrate tool.
+  #
+  # Files that are not part of the templates will be ignored by default.
+  unmanaged_files:
+    - 'lib/main.dart'
+    - 'ios/Runner.xcodeproj/project.pbxproj'

dev/integration_tests/flavors/macos/Flutter/Flutter-Debug.xcconfig

+#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"
+#include "ephemeral/Flutter-Generated.xcconfig"

dev/integration_tests/flavors/macos/Flutter/Flutter-Release.xcconfig

+#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"
+#include "ephemeral/Flutter-Generated.xcconfig"

dev/integration_tests/flavors/macos/Podfile

@@ -4,10 +4,9 @@
 ENV['COCOAPODS_DISABLE_STATS'] = 'true'
 
 project 'Runner', {
-  'Debug Free' => :debug,
-  'Debug Paid' => :debug,
-  'Release Free' => :release,
-  'Release Paid' => :release,
+  'Debug' => :debug,
+  'Profile' => :release,
+  'Release' => :release,
 }
 
 def flutter_root
@@ -27,12 +26,14 @@ require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelpe
 
 flutter_macos_podfile_setup
 
-target 'Free App' do
-  flutter_install_all_macos_pods File.dirname(File.realpath(__FILE__))
-end
+target 'Runner' do
+  use_frameworks!
+  use_modular_headers!
 
-target 'Paid App' do
   flutter_install_all_macos_pods File.dirname(File.realpath(__FILE__))
+  target 'RunnerTests' do
+    inherit! :search_paths
+  end
 end
 
 post_install do |installer|

dev/integration_tests/flavors/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1430"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+               BuildableName = "Debug Free.app"
+               BlueprintName = "Runner"
+               ReferencedContainer = "container:Runner.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <MacroExpansion>
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
+            ReferencedContainer = "container:Runner.xcodeproj">
+         </BuildableReference>
+      </MacroExpansion>
+      <Testables>
+         <TestableReference
+            skipped = "NO"
+            parallelizable = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "331C80D4294CF70F00263BE5"
+               BuildableName = "RunnerTests.xctest"
+               BlueprintName = "RunnerTests"
+               ReferencedContainer = "container:Runner.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
+            ReferencedContainer = "container:Runner.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Profile"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
+            ReferencedContainer = "container:Runner.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

dev/integration_tests/flavors/macos/Runner.xcodeproj/xcshareddata/xcschemes/free.xcscheme

 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "1430"
-   version = "1.3">
+   LastUpgradeVersion = "1500"
+   version = "1.7">
    <BuildAction
       parallelizeBuildables = "YES"
       buildImplicitDependencies = "YES">
@@ -15,32 +15,22 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-               BuildableName = "Free App.app"
-               BlueprintName = "Free App"
+               BuildableName = "Debug Free.app"
+               BlueprintName = "Runner"
                ReferencedContainer = "container:Runner.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
       </BuildActionEntries>
    </BuildAction>
    <TestAction
-      buildConfiguration = "Debug Free"
+      buildConfiguration = "Debug-free"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      shouldUseLaunchSchemeArgsEnv = "YES">
-      <MacroExpansion>
-         <BuildableReference
-            BuildableIdentifier = "primary"
-            BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-            BuildableName = "Free App.app"
-            BlueprintName = "Free App"
-            ReferencedContainer = "container:Runner.xcodeproj">
-         </BuildableReference>
-      </MacroExpansion>
-      <Testables>
-      </Testables>
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      shouldAutocreateTestPlan = "YES">
    </TestAction>
    <LaunchAction
-      buildConfiguration = "Debug Free"
+      buildConfiguration = "Release-free"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       launchStyle = "0"
@@ -54,14 +44,14 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-            BuildableName = "Free App.app"
-            BlueprintName = "Free App"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>
       </BuildableProductRunnable>
    </LaunchAction>
    <ProfileAction
-      buildConfiguration = "Debug Free"
+      buildConfiguration = "Release-free"
       shouldUseLaunchSchemeArgsEnv = "YES"
       savedToolIdentifier = ""
       useCustomWorkingDirectory = "NO"
@@ -71,17 +61,17 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-            BuildableName = "Free App.app"
-            BlueprintName = "Free App"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>
       </BuildableProductRunnable>
    </ProfileAction>
    <AnalyzeAction
-      buildConfiguration = "Debug Free">
+      buildConfiguration = "Debug-free">
    </AnalyzeAction>
    <ArchiveAction
-      buildConfiguration = "Release Free"
+      buildConfiguration = "Release-free"
       revealArchiveInOrganizer = "YES">
    </ArchiveAction>
 </Scheme>

dev/integration_tests/flavors/macos/Runner.xcodeproj/xcshareddata/xcschemes/paid.xcscheme

 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "1400"
-   version = "1.3">
+   LastUpgradeVersion = "1500"
+   version = "1.7">
    <BuildAction
       parallelizeBuildables = "YES"
       buildImplicitDependencies = "YES">
@@ -14,24 +14,23 @@
             buildForAnalyzing = "YES">
             <BuildableReference
                BuildableIdentifier = "primary"
-               BlueprintIdentifier = "F383D4A52938029D00598432"
-               BuildableName = "Paid App.app"
-               BlueprintName = "Paid App"
+               BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+               BuildableName = "Debug Free.app"
+               BlueprintName = "Runner"
                ReferencedContainer = "container:Runner.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
       </BuildActionEntries>
    </BuildAction>
    <TestAction
-      buildConfiguration = "Debug Paid"
+      buildConfiguration = "Debug-free"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      shouldUseLaunchSchemeArgsEnv = "YES">
-      <Testables>
-      </Testables>
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      shouldAutocreateTestPlan = "YES">
    </TestAction>
    <LaunchAction
-      buildConfiguration = "Debug Paid"
+      buildConfiguration = "Release-paid"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       launchStyle = "0"
@@ -44,15 +43,15 @@
          runnableDebuggingMode = "0">
          <BuildableReference
             BuildableIdentifier = "primary"
-            BlueprintIdentifier = "F383D4A52938029D00598432"
-            BuildableName = "Paid App.app"
-            BlueprintName = "Paid App"
+            BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>
       </BuildableProductRunnable>
    </LaunchAction>
    <ProfileAction
-      buildConfiguration = "Release Paid"
+      buildConfiguration = "Release-free"
       shouldUseLaunchSchemeArgsEnv = "YES"
       savedToolIdentifier = ""
       useCustomWorkingDirectory = "NO"
@@ -61,18 +60,18 @@
          runnableDebuggingMode = "0">
          <BuildableReference
             BuildableIdentifier = "primary"
-            BlueprintIdentifier = "F383D4A52938029D00598432"
-            BuildableName = "Paid App.app"
-            BlueprintName = "Paid App"
+            BlueprintIdentifier = "33CC10EC2044A3C60003C045"
+            BuildableName = "Debug Free.app"
+            BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>
       </BuildableProductRunnable>
    </ProfileAction>
    <AnalyzeAction
-      buildConfiguration = "Debug Paid">
+      buildConfiguration = "Debug-free">
    </AnalyzeAction>
    <ArchiveAction
-      buildConfiguration = "Release Paid"
+      buildConfiguration = "Release-free"
       revealArchiveInOrganizer = "YES">
    </ArchiveAction>
 </Scheme>

dev/integration_tests/flavors/macos/Runner/Configs/Debug.xcconfig

-// This xcconfig used for both the paid and free flavor, but the plugins are not different between
-// them. Pick one.
-#include? "Pods/Target Support Files/Pods-Free App/Pods-Free App.debug free.xcconfig"
+#include "../../Flutter/Flutter-Debug.xcconfig"
+#include "Warnings.xcconfig"

dev/integration_tests/flavors/macos/Runner/Configs/Release.xcconfig

-// This xcconfig used for both the paid and free flavor, but the plugins are not different between
-// them. Pick one.
-#include? "Pods/Target Support Files/Pods-Paid App/Pods-Paid App.release free.xcconfig"
+#include "../../Flutter/Flutter-Release.xcconfig"
+#include "Warnings.xcconfig"

dev/integration_tests/flavors/macos/Runner/Info.plist

@@ -4,6 +4,8 @@
 <dict>
 	<key>CFBundleDevelopmentRegion</key>
 	<string>$(DEVELOPMENT_LANGUAGE)</string>
+	<key>CFBundleDisplayName</key>
+	<string>$(PRODUCT_NAME)</string>
 	<key>CFBundleExecutable</key>
 	<string>$(EXECUTABLE_NAME)</string>
 	<key>CFBundleIconFile</key>
@@ -20,8 +22,6 @@
 	<string>$(FLUTTER_BUILD_NAME)</string>
 	<key>CFBundleVersion</key>
 	<string>$(FLUTTER_BUILD_NUMBER)</string>
-	<key>Flavor</key>
-	<string>${PRODUCT_FLAVOR}</string>
 	<key>LSMinimumSystemVersion</key>
 	<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
 	<key>NSHumanReadableCopyright</key>

dev/integration_tests/flavors/macos/Runner/MainFlutterWindow.swift

@@ -24,7 +24,8 @@ class MainFlutterWindow: NSWindow {
     let channel = FlutterMethodChannel(name: "flavor",
                                        binaryMessenger: registrar.messenger)
     channel.setMethodCallHandler({ (call, result) in
-      let flavor = Bundle.main.infoDictionary?["Flavor"] as? String
+      let bundleIdentifier = Bundle.main.infoDictionary?["CFBundleName"] as? String
+      let flavor = bundleIdentifier?.split(separator: " ").last?.lowercased();
       result(flavor)
     })
   }

dev/integration_tests/flavors/macos/RunnerTests/RunnerTests.swift

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import FlutterMacOS
+import Cocoa
+import XCTest
+
+class RunnerTests: XCTestCase {
+
+  func testExample() {
+    // If you add code to the Runner application, consider adding tests here.
+    // See https://developer.apple.com/documentation/xctest for more information about using XCTest.
+  }
+
+}