Dynamically add integration_tests and screenshots to native iOS test results (#95704)

packages/integration_test/example/integration_test/_extended_test_io.dart

@@ -25,6 +25,24 @@ void main() {
     // Build our app.
     app.main();
 
+    // Pump a frame.
+    await tester.pumpAndSettle();
+
+    // Verify that platform version is retrieved.
+    expect(
+      find.byWidgetPredicate(
+        (Widget widget) =>
+            widget is Text &&
+            widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
+      ),
+      findsOneWidget,
+    );
+  });
+
+  testWidgets('verify screenshot', (WidgetTester tester) async {
+    // Build our app.
+    app.main();
+
     // On Android, this is required prior to taking the screenshot.
     await binding.convertFlutterSurfaceToImage();
 
@@ -39,15 +57,5 @@ void main() {
     expect(secondPng.isNotEmpty, isTrue);
 
     expect(listEquals(firstPng, secondPng), isTrue);
-
-    // Verify that platform version is retrieved.
-    expect(
-      find.byWidgetPredicate(
-        (Widget widget) =>
-            widget is Text &&
-            widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
-      ),
-      findsOneWidget,
-    );
   });
 }

packages/integration_test/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

@@ -20,6 +20,20 @@
                ReferencedContainer = "container:Runner.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "NO"
+            buildForProfiling = "NO"
+            buildForArchiving = "NO"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "769541C723A0351900E5C350"
+               BuildableName = "RunnerTests.xctest"
+               BlueprintName = "RunnerTests"
+               ReferencedContainer = "container:Runner.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
       </BuildActionEntries>
    </BuildAction>
    <TestAction

packages/integration_test/example/ios/RunnerTests/RunnerTests.m

@@ -5,4 +5,89 @@
 @import XCTest;
 @import integration_test;
 
+#pragma mark - Dynamic tests
+
 INTEGRATION_TEST_IOS_RUNNER(RunnerTests)
+
+@interface RunnerTests (DynamicTests)
+@end
+
+@implementation RunnerTests (DynamicTests)
+
+- (void)setUp {
+  // Verify tests have been dynamically added from FLUTTER_TARGET=integration_test/extended_test.dart
+  XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyScreenshot")]);
+  XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyText")]);
+  XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"screenshotPlaceholder")]);
+}
+
+@end
+
+#pragma mark - Fake test results
+
+@interface IntegrationTestPlugin ()
+- (instancetype)initForRegistration;
+@end
+
+@interface FLTIntegrationTestRunner ()
+@property IntegrationTestPlugin *integrationTestPlugin;
+@end
+
+@interface FakeIntegrationTestPlugin : IntegrationTestPlugin
+@property(nonatomic, nullable) NSDictionary<NSString *, NSString *> *testResults;
+@end
+
+@implementation FakeIntegrationTestPlugin
+@synthesize testResults;
+
+- (void)setupChannels:(id<FlutterBinaryMessenger>)binaryMessenger {
+}
+
+@end
+
+#pragma mark - Behavior tests
+
+@interface IntegrationTestTests : XCTestCase
+@end
+
+@implementation IntegrationTestTests
+
+- (void)testDeprecatedIntegrationTest {
+  NSString *testResult;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+  BOOL testPass = [[IntegrationTestIosTest new] testIntegrationTest:&testResult];
+#pragma clang diagnostic pop
+  XCTAssertTrue(testPass, @"%@", testResult);
+}
+
+- (void)testMethodNamesFromDartTests {
+  XCTAssertEqualObjects([FLTIntegrationTestRunner
+                         testCaseNameFromDartTestName:@"this is a test"], @"testThisIsATest");
+  XCTAssertEqualObjects([FLTIntegrationTestRunner
+                         testCaseNameFromDartTestName:@"VALIDATE multi-point 🚀 UNICODE123: 😁"], @"testValidateMultiPointUnicode123");
+  XCTAssertEqualObjects([FLTIntegrationTestRunner
+                         testCaseNameFromDartTestName:@"!UPPERCASE:\\ lower_seperate?"], @"testUppercaseLowerSeperate");
+}
+
+- (void)testDuplicatedDartTests {
+  FakeIntegrationTestPlugin *fakePlugin = [[FakeIntegrationTestPlugin alloc] initForRegistration];
+  // These are unique test names in dart, but would result in duplicate
+  // XCTestCase names when the emojis are stripped.
+  fakePlugin.testResults = @{@"unique": @"dart test failure", @"emoji 🐢": @"success", @"emoji 🐇": @"failure"};
+
+  FLTIntegrationTestRunner *runner = [[FLTIntegrationTestRunner alloc] init];
+  runner.integrationTestPlugin = fakePlugin;
+
+  NSMutableDictionary<NSString *, NSString *> *failuresByTestName = [[NSMutableDictionary alloc] init];
+  [runner testIntegrationTestWithResults:^(SEL nativeTestSelector, BOOL success, NSString *failureMessage) {
+    NSString *testName = NSStringFromSelector(nativeTestSelector);
+    XCTAssertFalse([failuresByTestName.allKeys containsObject:testName]);
+    failuresByTestName[testName] = failureMessage;
+  }];
+  XCTAssertEqualObjects(failuresByTestName,
+                        (@{@"testUnique": @"dart test failure",
+                           @"testDuplicateTestNames": @"Cannot test \"emoji 🐇\", duplicate XCTestCase tests named testEmoji"}));
+}
+
+@end

packages/integration_test/ios/Classes/FLTIntegrationTestRunner.h

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@import Foundation;
+
+@class UIImage;
+
+NS_ASSUME_NONNULL_BEGIN
+
+typedef void (^FLTIntegrationTestResults)(SEL nativeTestSelector, BOOL success, NSString *_Nullable failureMessage);
+
+@interface FLTIntegrationTestRunner : NSObject
+
+/**
+ * Any screenshots captured by the plugin.
+ */
+@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
+
+/**
+ * Starts dart tests and waits for results.
+ *
+ * @param testResult Will be called once per every completed dart test.
+ */
+- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult;
+
+/**
+ * An appropriate XCTest method name based on the dart test name.
+ *
+ * Example: dart test "verify widget-ABC123" becomes "testVerifyWidgetABC123"
+ */
++ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName;
+
+@end
+
+NS_ASSUME_NONNULL_END

packages/integration_test/ios/Classes/FLTIntegrationTestRunner.m

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "FLTIntegrationTestRunner.h"
+
+#import "IntegrationTestPlugin.h"
+
+@import ObjectiveC.runtime;
+@import UIKit;
+
+@interface FLTIntegrationTestRunner ()
+
+@property IntegrationTestPlugin *integrationTestPlugin;
+
+@end
+
+@implementation FLTIntegrationTestRunner
+
+- (instancetype)init {
+  self = [super init];
+  _integrationTestPlugin = [IntegrationTestPlugin instance];
+
+  return self;
+}
+
+- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult {
+  IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
+  UIViewController *rootViewController = UIApplication.sharedApplication.delegate.window.rootViewController;
+  if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
+    testResult(NSSelectorFromString(@"testSetup"), NO, @"rootViewController was not expected FlutterViewController");
+  }
+  FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
+  [integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
+
+  // Spin the runloop.
+  while (!integrationTestPlugin.testResults) {
+    [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]];
+  }
+
+  NSMutableSet<NSString *> *testCaseNames = [[NSMutableSet alloc] init];
+
+  [integrationTestPlugin.testResults enumerateKeysAndObjectsUsingBlock:^(NSString *test, NSString *result, BOOL *stop) {
+    NSString *testSelectorName = [[self class] testCaseNameFromDartTestName:test];
+
+    // Validate Objective-C test names are unique after sanitization.
+    if ([testCaseNames containsObject:testSelectorName]) {
+      NSString *reason = [NSString stringWithFormat:@"Cannot test \"%@\", duplicate XCTestCase tests named %@", test, testSelectorName];
+      testResult(NSSelectorFromString(@"testDuplicateTestNames"), NO, reason);
+      *stop = YES;
+      return;
+    }
+    [testCaseNames addObject:testSelectorName];
+    SEL testSelector = NSSelectorFromString(testSelectorName);
+
+    if ([result isEqualToString:@"success"]) {
+      testResult(testSelector, YES, nil);
+    } else {
+      testResult(testSelector, NO, result);
+    }
+  }];
+}
+
+- (NSDictionary<NSString *,UIImage *> *)capturedScreenshotsByName {
+  return self.integrationTestPlugin.capturedScreenshotsByName;
+}
+
++ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName {
+  NSString *capitalizedString = dartTestName.localizedCapitalizedString;
+  // Objective-C method names must be alphanumeric.
+  NSCharacterSet *disallowedCharacters = NSCharacterSet.alphanumericCharacterSet.invertedSet;
+  // Remove disallowed characters.
+  NSString *upperCamelTestName = [[capitalizedString componentsSeparatedByCharactersInSet:disallowedCharacters] componentsJoinedByString:@""];
+  return [NSString stringWithFormat:@"test%@", upperCamelTestName];
+}
+
+@end

packages/integration_test/ios/Classes/IntegrationTestIosTest.h

@@ -2,16 +2,14 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#import <Foundation/Foundation.h>
+@import Foundation;
+@import ObjectiveC.runtime;
 
 NS_ASSUME_NONNULL_BEGIN
 
-@protocol FLTIntegrationTestScreenshotDelegate;
-
+DEPRECATED_MSG_ATTRIBUTE("Use FLTIntegrationTestRunner instead.")
 @interface IntegrationTestIosTest : NSObject
 
-- (instancetype)initWithScreenshotDelegate:(nullable id<FLTIntegrationTestScreenshotDelegate>)delegate NS_DESIGNATED_INITIALIZER;
-
 /**
  * Initiate dart tests and wait for results.  @c testResult will be set to a string describing the results.
  *
@@ -21,26 +19,48 @@ NS_ASSUME_NONNULL_BEGIN
 
 @end
 
+// For every Flutter dart test, dynamically generate an Objective-C method mirroring the test results
+// so it is reported as a native XCTest run result.
+// If the Flutter dart tests have captured screenshots, add them to the XCTest bundle.
 #define INTEGRATION_TEST_IOS_RUNNER(__test_class)                                           \
-  @interface __test_class : XCTestCase<FLTIntegrationTestScreenshotDelegate>                \
+  @interface __test_class : XCTestCase                                                      \
   @end                                                                                      \
                                                                                             \
   @implementation __test_class                                                              \
                                                                                             \
-  - (void)testIntegrationTest {                                                             \
-    NSString *testResult;                                                                   \
-    IntegrationTestIosTest *integrationTestIosTest = integrationTestIosTest = [[IntegrationTestIosTest alloc] initWithScreenshotDelegate:self]; \
-    BOOL testPass = [integrationTestIosTest testIntegrationTest:&testResult];               \
-    XCTAssertTrue(testPass, @"%@", testResult);                                             \
-  }                                                                                         \
-                                                                                            \
-  - (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(NSString *)name {         \
+  + (NSArray<NSInvocation *> *)testInvocations {                                            \
+    FLTIntegrationTestRunner *integrationTestRunner = [[FLTIntegrationTestRunner alloc] init]; \
+    NSMutableArray<NSInvocation *> *testInvocations = [[NSMutableArray alloc] init];        \
+    [integrationTestRunner testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *failureMessage) { \
+      IMP assertImplementation = imp_implementationWithBlock(^(id _self) {                  \
+        XCTAssertTrue(success, @"%@", failureMessage);                                      \
+      });                                                                                   \
+      class_addMethod(self, testSelector, assertImplementation, "v@:");                     \
+      NSMethodSignature *signature = [self instanceMethodSignatureForSelector:testSelector]; \
+      NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature];    \
+      invocation.selector = testSelector;                                                   \
+      [testInvocations addObject:invocation];                                               \
+    }];                                                                                     \
+    NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName = integrationTestRunner.capturedScreenshotsByName; \
+    if (capturedScreenshotsByName.count > 0) {                                              \
+      IMP screenshotImplementation = imp_implementationWithBlock(^(id _self) {              \
+        [capturedScreenshotsByName enumerateKeysAndObjectsUsingBlock:^(NSString *name, UIImage *screenshot, BOOL *stop) { \
           XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot];       \
           attachment.lifetime = XCTAttachmentLifetimeKeepAlways;                            \
           if (name != nil) {                                                                \
             attachment.name = name;                                                         \
           }                                                                                 \
-    [self addAttachment:attachment];                                                        \
+          [_self addAttachment:attachment];                                                 \
+        }];                                                                                 \
+      });                                                                                   \
+      SEL attachmentSelector = NSSelectorFromString(@"screenshotPlaceholder");              \
+      class_addMethod(self, attachmentSelector, screenshotImplementation, "v@:");           \
+      NSMethodSignature *attachmentSignature = [self instanceMethodSignatureForSelector:attachmentSelector]; \
+      NSInvocation *attachmentInvocation = [NSInvocation invocationWithMethodSignature:attachmentSignature]; \
+      attachmentInvocation.selector = attachmentSelector;                                   \
+      [testInvocations addObject:attachmentInvocation];                                     \
+    }                                                                                       \
+    return testInvocations;                                                                 \
   }                                                                                         \
                                                                                             \
   @end

packages/integration_test/ios/Classes/IntegrationTestIosTest.m

@@ -3,61 +3,40 @@
 // found in the LICENSE file.
 
 #import "IntegrationTestIosTest.h"
+
 #import "IntegrationTestPlugin.h"
+#import "FLTIntegrationTestRunner.h"
 
-@interface IntegrationTestIosTest()
-@property (nonatomic) IntegrationTestPlugin *integrationTestPlugin;
-@end
+#pragma mark - Deprecated
 
-@implementation IntegrationTestIosTest
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-implementations"
 
-- (instancetype)initWithScreenshotDelegate:(id<FLTIntegrationTestScreenshotDelegate>)delegate {
-  self = [super init];
-  _integrationTestPlugin = [IntegrationTestPlugin instance];
-  _integrationTestPlugin.screenshotDelegate = delegate;
-  return self;
-}
-
-- (instancetype)init {
-  return [self initWithScreenshotDelegate:nil];
-}
+@implementation IntegrationTestIosTest
 
 - (BOOL)testIntegrationTest:(NSString **)testResult {
-  IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
-
-  UIViewController *rootViewController =
-      [[[[UIApplication sharedApplication] delegate] window] rootViewController];
-  if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
-    NSLog(@"expected FlutterViewController as rootViewController.");
-    return NO;
-  }
-  FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
-  [integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
-  while (!integrationTestPlugin.testResults) {
-    CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.f, NO);
-  }
-  NSDictionary<NSString *, NSString *> *testResults = integrationTestPlugin.testResults;
-  NSMutableArray<NSString *> *passedTests = [NSMutableArray array];
-  NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
   NSLog(@"==================== Test Results =====================");
-  for (NSString *test in testResults.allKeys) {
-    NSString *result = testResults[test];
-    if ([result isEqualToString:@"success"]) {
-      NSLog(@"%@ passed.", test);
-      [passedTests addObject:test];
+  NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
+  NSMutableArray<NSString *> *testNames = [NSMutableArray array];
+  [[FLTIntegrationTestRunner new] testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *message) {
+    NSString *testName = NSStringFromSelector(testSelector);
+    [testNames addObject:testName];
+    if (success) {
+      NSLog(@"%@ passed.", testName);
     } else {
-      NSLog(@"%@ failed: %@", test, result);
-      [failedTests addObject:test];
-    }
+      NSLog(@"%@ failed: %@", testName, message);
+      [failedTests addObject:testName];
     }
+  }];
   NSLog(@"================== Test Results End ====================");
   BOOL testPass = failedTests.count == 0;
-  if (!testPass && testResult) {
+  if (!testPass && testResult != NULL) {
     *testResult =
         [NSString stringWithFormat:@"Detected failed integration test(s) %@ among %@",
-                                   failedTests.description, testResults.allKeys.description];
+                                   failedTests.description, testNames.description];
   }
   return testPass;
 }
 
 @end
+#pragma clang diagnostic pop

packages/integration_test/ios/Classes/IntegrationTestPlugin.h

@@ -6,13 +6,6 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
-@protocol FLTIntegrationTestScreenshotDelegate
-
-/** This will be called when a dart integration test triggers a window screenshot with  @c takeScreenshot. */
-- (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(nullable NSString *)name;
-
-@end
-
 /** A Flutter plugin that's responsible for communicating the test results back
  * to iOS XCTest. */
 @interface IntegrationTestPlugin : NSObject <FlutterPlugin>
@@ -23,6 +16,11 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property(nonatomic, readonly, nullable) NSDictionary<NSString *, NSString *> *testResults;
 
+/**
+ * Mapping of screenshot images by suggested names, captured by the dart tests.
+ */
+@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
+
 /** Fetches the singleton instance of the plugin. */
 + (IntegrationTestPlugin *)instance;
 
@@ -30,8 +28,6 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (instancetype)init NS_UNAVAILABLE;
 
-@property(weak, nonatomic) id<FLTIntegrationTestScreenshotDelegate> screenshotDelegate;
-
 @end
 
 NS_ASSUME_NONNULL_END

packages/integration_test/ios/Classes/IntegrationTestPlugin.m

@@ -2,10 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-@import UIKit;
-
 #import "IntegrationTestPlugin.h"
 
+@import UIKit;
+
 static NSString *const kIntegrationTestPluginChannel = @"plugins.flutter.io/integration_test";
 static NSString *const kMethodTestFinished = @"allTestsFinished";
 static NSString *const kMethodScreenshot = @"captureScreenshot";
@@ -16,10 +16,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
 
 @property(nonatomic, readwrite) NSDictionary<NSString *, NSString *> *testResults;
 
+- (instancetype)init NS_DESIGNATED_INITIALIZER;
+
 @end
 
 @implementation IntegrationTestPlugin {
   NSDictionary<NSString *, NSString *> *_testResults;
+  NSMutableDictionary<NSString *, UIImage *> *_capturedScreenshotsByName;
 }
 
 + (IntegrationTestPlugin *)instance {
@@ -32,7 +35,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
 }
 
 - (instancetype)initForRegistration {
-  return [super init];
+  return [self init];
+}
+
+- (instancetype)init {
+  self = [super init];
+  _capturedScreenshotsByName = [NSMutableDictionary new];
+  return self;
 }
 
 + (void)registerWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar {
@@ -59,7 +68,7 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
     // If running as a native Xcode test, attach to test.
     UIImage *screenshot = [self capturePngScreenshot];
     NSString *name = call.arguments[@"name"];
-    [self.screenshotDelegate didTakeScreenshot:screenshot attachmentName:name];
+    _capturedScreenshotsByName[name] = screenshot;
 
     // Also pass back along the channel for the driver to handle.
     NSData *pngData = UIImagePNGRepresentation(screenshot);