Unverified Commit 88327e3b authored by Jenn Magder's avatar Jenn Magder Committed by GitHub

Dynamically add integration_tests and screenshots to native iOS test results (#95704)

parent bbc68cd2
...@@ -25,6 +25,24 @@ void main() { ...@@ -25,6 +25,24 @@ void main() {
// Build our app. // Build our app.
app.main(); app.main();
// Pump a frame.
await tester.pumpAndSettle();
// Verify that platform version is retrieved.
expect(
find.byWidgetPredicate(
(Widget widget) =>
widget is Text &&
widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
),
findsOneWidget,
);
});
testWidgets('verify screenshot', (WidgetTester tester) async {
// Build our app.
app.main();
// On Android, this is required prior to taking the screenshot. // On Android, this is required prior to taking the screenshot.
await binding.convertFlutterSurfaceToImage(); await binding.convertFlutterSurfaceToImage();
...@@ -39,15 +57,5 @@ void main() { ...@@ -39,15 +57,5 @@ void main() {
expect(secondPng.isNotEmpty, isTrue); expect(secondPng.isNotEmpty, isTrue);
expect(listEquals(firstPng, secondPng), isTrue); expect(listEquals(firstPng, secondPng), isTrue);
// Verify that platform version is retrieved.
expect(
find.byWidgetPredicate(
(Widget widget) =>
widget is Text &&
widget.data!.startsWith('Platform: ${Platform.operatingSystem}'),
),
findsOneWidget,
);
}); });
} }
...@@ -20,6 +20,20 @@ ...@@ -20,6 +20,20 @@
ReferencedContainer = "container:Runner.xcodeproj"> ReferencedContainer = "container:Runner.xcodeproj">
</BuildableReference> </BuildableReference>
</BuildActionEntry> </BuildActionEntry>
<BuildActionEntry
buildForTesting = "YES"
buildForRunning = "NO"
buildForProfiling = "NO"
buildForArchiving = "NO"
buildForAnalyzing = "YES">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "769541C723A0351900E5C350"
BuildableName = "RunnerTests.xctest"
BlueprintName = "RunnerTests"
ReferencedContainer = "container:Runner.xcodeproj">
</BuildableReference>
</BuildActionEntry>
</BuildActionEntries> </BuildActionEntries>
</BuildAction> </BuildAction>
<TestAction <TestAction
......
...@@ -5,4 +5,89 @@ ...@@ -5,4 +5,89 @@
@import XCTest; @import XCTest;
@import integration_test; @import integration_test;
#pragma mark - Dynamic tests
INTEGRATION_TEST_IOS_RUNNER(RunnerTests) INTEGRATION_TEST_IOS_RUNNER(RunnerTests)
@interface RunnerTests (DynamicTests)
@end
@implementation RunnerTests (DynamicTests)
- (void)setUp {
// Verify tests have been dynamically added from FLUTTER_TARGET=integration_test/extended_test.dart
XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyScreenshot")]);
XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"testVerifyText")]);
XCTAssertTrue([self respondsToSelector:NSSelectorFromString(@"screenshotPlaceholder")]);
}
@end
#pragma mark - Fake test results
@interface IntegrationTestPlugin ()
- (instancetype)initForRegistration;
@end
@interface FLTIntegrationTestRunner ()
@property IntegrationTestPlugin *integrationTestPlugin;
@end
@interface FakeIntegrationTestPlugin : IntegrationTestPlugin
@property(nonatomic, nullable) NSDictionary<NSString *, NSString *> *testResults;
@end
@implementation FakeIntegrationTestPlugin
@synthesize testResults;
- (void)setupChannels:(id<FlutterBinaryMessenger>)binaryMessenger {
}
@end
#pragma mark - Behavior tests
@interface IntegrationTestTests : XCTestCase
@end
@implementation IntegrationTestTests
- (void)testDeprecatedIntegrationTest {
NSString *testResult;
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
BOOL testPass = [[IntegrationTestIosTest new] testIntegrationTest:&testResult];
#pragma clang diagnostic pop
XCTAssertTrue(testPass, @"%@", testResult);
}
- (void)testMethodNamesFromDartTests {
XCTAssertEqualObjects([FLTIntegrationTestRunner
testCaseNameFromDartTestName:@"this is a test"], @"testThisIsATest");
XCTAssertEqualObjects([FLTIntegrationTestRunner
testCaseNameFromDartTestName:@"VALIDATE multi-point 🚀 UNICODE123: 😁"], @"testValidateMultiPointUnicode123");
XCTAssertEqualObjects([FLTIntegrationTestRunner
testCaseNameFromDartTestName:@"!UPPERCASE:\\ lower_seperate?"], @"testUppercaseLowerSeperate");
}
- (void)testDuplicatedDartTests {
FakeIntegrationTestPlugin *fakePlugin = [[FakeIntegrationTestPlugin alloc] initForRegistration];
// These are unique test names in dart, but would result in duplicate
// XCTestCase names when the emojis are stripped.
fakePlugin.testResults = @{@"unique": @"dart test failure", @"emoji 🐢": @"success", @"emoji 🐇": @"failure"};
FLTIntegrationTestRunner *runner = [[FLTIntegrationTestRunner alloc] init];
runner.integrationTestPlugin = fakePlugin;
NSMutableDictionary<NSString *, NSString *> *failuresByTestName = [[NSMutableDictionary alloc] init];
[runner testIntegrationTestWithResults:^(SEL nativeTestSelector, BOOL success, NSString *failureMessage) {
NSString *testName = NSStringFromSelector(nativeTestSelector);
XCTAssertFalse([failuresByTestName.allKeys containsObject:testName]);
failuresByTestName[testName] = failureMessage;
}];
XCTAssertEqualObjects(failuresByTestName,
(@{@"testUnique": @"dart test failure",
@"testDuplicateTestNames": @"Cannot test \"emoji 🐇\", duplicate XCTestCase tests named testEmoji"}));
}
@end
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@import Foundation;
@class UIImage;
NS_ASSUME_NONNULL_BEGIN
typedef void (^FLTIntegrationTestResults)(SEL nativeTestSelector, BOOL success, NSString *_Nullable failureMessage);
@interface FLTIntegrationTestRunner : NSObject
/**
* Any screenshots captured by the plugin.
*/
@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
/**
* Starts dart tests and waits for results.
*
* @param testResult Will be called once per every completed dart test.
*/
- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult;
/**
* An appropriate XCTest method name based on the dart test name.
*
* Example: dart test "verify widget-ABC123" becomes "testVerifyWidgetABC123"
*/
+ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName;
@end
NS_ASSUME_NONNULL_END
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import "FLTIntegrationTestRunner.h"
#import "IntegrationTestPlugin.h"
@import ObjectiveC.runtime;
@import UIKit;
@interface FLTIntegrationTestRunner ()
@property IntegrationTestPlugin *integrationTestPlugin;
@end
@implementation FLTIntegrationTestRunner
- (instancetype)init {
self = [super init];
_integrationTestPlugin = [IntegrationTestPlugin instance];
return self;
}
- (void)testIntegrationTestWithResults:(NS_NOESCAPE FLTIntegrationTestResults)testResult {
IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
UIViewController *rootViewController = UIApplication.sharedApplication.delegate.window.rootViewController;
if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
testResult(NSSelectorFromString(@"testSetup"), NO, @"rootViewController was not expected FlutterViewController");
}
FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
[integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
// Spin the runloop.
while (!integrationTestPlugin.testResults) {
[NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:1.0]];
}
NSMutableSet<NSString *> *testCaseNames = [[NSMutableSet alloc] init];
[integrationTestPlugin.testResults enumerateKeysAndObjectsUsingBlock:^(NSString *test, NSString *result, BOOL *stop) {
NSString *testSelectorName = [[self class] testCaseNameFromDartTestName:test];
// Validate Objective-C test names are unique after sanitization.
if ([testCaseNames containsObject:testSelectorName]) {
NSString *reason = [NSString stringWithFormat:@"Cannot test \"%@\", duplicate XCTestCase tests named %@", test, testSelectorName];
testResult(NSSelectorFromString(@"testDuplicateTestNames"), NO, reason);
*stop = YES;
return;
}
[testCaseNames addObject:testSelectorName];
SEL testSelector = NSSelectorFromString(testSelectorName);
if ([result isEqualToString:@"success"]) {
testResult(testSelector, YES, nil);
} else {
testResult(testSelector, NO, result);
}
}];
}
- (NSDictionary<NSString *,UIImage *> *)capturedScreenshotsByName {
return self.integrationTestPlugin.capturedScreenshotsByName;
}
+ (NSString *)testCaseNameFromDartTestName:(NSString *)dartTestName {
NSString *capitalizedString = dartTestName.localizedCapitalizedString;
// Objective-C method names must be alphanumeric.
NSCharacterSet *disallowedCharacters = NSCharacterSet.alphanumericCharacterSet.invertedSet;
// Remove disallowed characters.
NSString *upperCamelTestName = [[capitalizedString componentsSeparatedByCharactersInSet:disallowedCharacters] componentsJoinedByString:@""];
return [NSString stringWithFormat:@"test%@", upperCamelTestName];
}
@end
...@@ -2,16 +2,14 @@ ...@@ -2,16 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
#import <Foundation/Foundation.h> @import Foundation;
@import ObjectiveC.runtime;
NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_BEGIN
@protocol FLTIntegrationTestScreenshotDelegate; DEPRECATED_MSG_ATTRIBUTE("Use FLTIntegrationTestRunner instead.")
@interface IntegrationTestIosTest : NSObject @interface IntegrationTestIosTest : NSObject
- (instancetype)initWithScreenshotDelegate:(nullable id<FLTIntegrationTestScreenshotDelegate>)delegate NS_DESIGNATED_INITIALIZER;
/** /**
* Initiate dart tests and wait for results. @c testResult will be set to a string describing the results. * Initiate dart tests and wait for results. @c testResult will be set to a string describing the results.
* *
...@@ -21,26 +19,48 @@ NS_ASSUME_NONNULL_BEGIN ...@@ -21,26 +19,48 @@ NS_ASSUME_NONNULL_BEGIN
@end @end
// For every Flutter dart test, dynamically generate an Objective-C method mirroring the test results
// so it is reported as a native XCTest run result.
// If the Flutter dart tests have captured screenshots, add them to the XCTest bundle.
#define INTEGRATION_TEST_IOS_RUNNER(__test_class) \ #define INTEGRATION_TEST_IOS_RUNNER(__test_class) \
@interface __test_class : XCTestCase<FLTIntegrationTestScreenshotDelegate> \ @interface __test_class : XCTestCase \
@end \ @end \
\ \
@implementation __test_class \ @implementation __test_class \
\ \
- (void)testIntegrationTest { \ + (NSArray<NSInvocation *> *)testInvocations { \
NSString *testResult; \ FLTIntegrationTestRunner *integrationTestRunner = [[FLTIntegrationTestRunner alloc] init]; \
IntegrationTestIosTest *integrationTestIosTest = integrationTestIosTest = [[IntegrationTestIosTest alloc] initWithScreenshotDelegate:self]; \ NSMutableArray<NSInvocation *> *testInvocations = [[NSMutableArray alloc] init]; \
BOOL testPass = [integrationTestIosTest testIntegrationTest:&testResult]; \ [integrationTestRunner testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *failureMessage) { \
XCTAssertTrue(testPass, @"%@", testResult); \ IMP assertImplementation = imp_implementationWithBlock(^(id _self) { \
} \ XCTAssertTrue(success, @"%@", failureMessage); \
\ }); \
- (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(NSString *)name { \ class_addMethod(self, testSelector, assertImplementation, "v@:"); \
XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot]; \ NSMethodSignature *signature = [self instanceMethodSignatureForSelector:testSelector]; \
attachment.lifetime = XCTAttachmentLifetimeKeepAlways; \ NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; \
if (name != nil) { \ invocation.selector = testSelector; \
attachment.name = name; \ [testInvocations addObject:invocation]; \
}]; \
NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName = integrationTestRunner.capturedScreenshotsByName; \
if (capturedScreenshotsByName.count > 0) { \
IMP screenshotImplementation = imp_implementationWithBlock(^(id _self) { \
[capturedScreenshotsByName enumerateKeysAndObjectsUsingBlock:^(NSString *name, UIImage *screenshot, BOOL *stop) { \
XCTAttachment *attachment = [XCTAttachment attachmentWithImage:screenshot]; \
attachment.lifetime = XCTAttachmentLifetimeKeepAlways; \
if (name != nil) { \
attachment.name = name; \
} \
[_self addAttachment:attachment]; \
}]; \
}); \
SEL attachmentSelector = NSSelectorFromString(@"screenshotPlaceholder"); \
class_addMethod(self, attachmentSelector, screenshotImplementation, "v@:"); \
NSMethodSignature *attachmentSignature = [self instanceMethodSignatureForSelector:attachmentSelector]; \
NSInvocation *attachmentInvocation = [NSInvocation invocationWithMethodSignature:attachmentSignature]; \
attachmentInvocation.selector = attachmentSelector; \
[testInvocations addObject:attachmentInvocation]; \
} \ } \
[self addAttachment:attachment]; \ return testInvocations; \
} \ } \
\ \
@end @end
......
...@@ -3,61 +3,40 @@ ...@@ -3,61 +3,40 @@
// found in the LICENSE file. // found in the LICENSE file.
#import "IntegrationTestIosTest.h" #import "IntegrationTestIosTest.h"
#import "IntegrationTestPlugin.h" #import "IntegrationTestPlugin.h"
#import "FLTIntegrationTestRunner.h"
@interface IntegrationTestIosTest() #pragma mark - Deprecated
@property (nonatomic) IntegrationTestPlugin *integrationTestPlugin;
@end
@implementation IntegrationTestIosTest #pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-implementations"
- (instancetype)initWithScreenshotDelegate:(id<FLTIntegrationTestScreenshotDelegate>)delegate { @implementation IntegrationTestIosTest
self = [super init];
_integrationTestPlugin = [IntegrationTestPlugin instance];
_integrationTestPlugin.screenshotDelegate = delegate;
return self;
}
- (instancetype)init {
return [self initWithScreenshotDelegate:nil];
}
- (BOOL)testIntegrationTest:(NSString **)testResult { - (BOOL)testIntegrationTest:(NSString **)testResult {
IntegrationTestPlugin *integrationTestPlugin = self.integrationTestPlugin;
UIViewController *rootViewController =
[[[[UIApplication sharedApplication] delegate] window] rootViewController];
if (![rootViewController isKindOfClass:[FlutterViewController class]]) {
NSLog(@"expected FlutterViewController as rootViewController.");
return NO;
}
FlutterViewController *flutterViewController = (FlutterViewController *)rootViewController;
[integrationTestPlugin setupChannels:flutterViewController.engine.binaryMessenger];
while (!integrationTestPlugin.testResults) {
CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.f, NO);
}
NSDictionary<NSString *, NSString *> *testResults = integrationTestPlugin.testResults;
NSMutableArray<NSString *> *passedTests = [NSMutableArray array];
NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
NSLog(@"==================== Test Results ====================="); NSLog(@"==================== Test Results =====================");
for (NSString *test in testResults.allKeys) { NSMutableArray<NSString *> *failedTests = [NSMutableArray array];
NSString *result = testResults[test]; NSMutableArray<NSString *> *testNames = [NSMutableArray array];
if ([result isEqualToString:@"success"]) { [[FLTIntegrationTestRunner new] testIntegrationTestWithResults:^(SEL testSelector, BOOL success, NSString *message) {
NSLog(@"%@ passed.", test); NSString *testName = NSStringFromSelector(testSelector);
[passedTests addObject:test]; [testNames addObject:testName];
if (success) {
NSLog(@"%@ passed.", testName);
} else { } else {
NSLog(@"%@ failed: %@", test, result); NSLog(@"%@ failed: %@", testName, message);
[failedTests addObject:test]; [failedTests addObject:testName];
} }
} }];
NSLog(@"================== Test Results End ===================="); NSLog(@"================== Test Results End ====================");
BOOL testPass = failedTests.count == 0; BOOL testPass = failedTests.count == 0;
if (!testPass && testResult) { if (!testPass && testResult != NULL) {
*testResult = *testResult =
[NSString stringWithFormat:@"Detected failed integration test(s) %@ among %@", [NSString stringWithFormat:@"Detected failed integration test(s) %@ among %@",
failedTests.description, testResults.allKeys.description]; failedTests.description, testNames.description];
} }
return testPass; return testPass;
} }
@end @end
#pragma clang diagnostic pop
...@@ -6,13 +6,6 @@ ...@@ -6,13 +6,6 @@
NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_BEGIN
@protocol FLTIntegrationTestScreenshotDelegate
/** This will be called when a dart integration test triggers a window screenshot with @c takeScreenshot. */
- (void)didTakeScreenshot:(UIImage *)screenshot attachmentName:(nullable NSString *)name;
@end
/** A Flutter plugin that's responsible for communicating the test results back /** A Flutter plugin that's responsible for communicating the test results back
* to iOS XCTest. */ * to iOS XCTest. */
@interface IntegrationTestPlugin : NSObject <FlutterPlugin> @interface IntegrationTestPlugin : NSObject <FlutterPlugin>
...@@ -23,6 +16,11 @@ NS_ASSUME_NONNULL_BEGIN ...@@ -23,6 +16,11 @@ NS_ASSUME_NONNULL_BEGIN
*/ */
@property(nonatomic, readonly, nullable) NSDictionary<NSString *, NSString *> *testResults; @property(nonatomic, readonly, nullable) NSDictionary<NSString *, NSString *> *testResults;
/**
* Mapping of screenshot images by suggested names, captured by the dart tests.
*/
@property (copy, readonly) NSDictionary<NSString *, UIImage *> *capturedScreenshotsByName;
/** Fetches the singleton instance of the plugin. */ /** Fetches the singleton instance of the plugin. */
+ (IntegrationTestPlugin *)instance; + (IntegrationTestPlugin *)instance;
...@@ -30,8 +28,6 @@ NS_ASSUME_NONNULL_BEGIN ...@@ -30,8 +28,6 @@ NS_ASSUME_NONNULL_BEGIN
- (instancetype)init NS_UNAVAILABLE; - (instancetype)init NS_UNAVAILABLE;
@property(weak, nonatomic) id<FLTIntegrationTestScreenshotDelegate> screenshotDelegate;
@end @end
NS_ASSUME_NONNULL_END NS_ASSUME_NONNULL_END
...@@ -2,10 +2,10 @@ ...@@ -2,10 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
@import UIKit;
#import "IntegrationTestPlugin.h" #import "IntegrationTestPlugin.h"
@import UIKit;
static NSString *const kIntegrationTestPluginChannel = @"plugins.flutter.io/integration_test"; static NSString *const kIntegrationTestPluginChannel = @"plugins.flutter.io/integration_test";
static NSString *const kMethodTestFinished = @"allTestsFinished"; static NSString *const kMethodTestFinished = @"allTestsFinished";
static NSString *const kMethodScreenshot = @"captureScreenshot"; static NSString *const kMethodScreenshot = @"captureScreenshot";
...@@ -16,10 +16,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage"; ...@@ -16,10 +16,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
@property(nonatomic, readwrite) NSDictionary<NSString *, NSString *> *testResults; @property(nonatomic, readwrite) NSDictionary<NSString *, NSString *> *testResults;
- (instancetype)init NS_DESIGNATED_INITIALIZER;
@end @end
@implementation IntegrationTestPlugin { @implementation IntegrationTestPlugin {
NSDictionary<NSString *, NSString *> *_testResults; NSDictionary<NSString *, NSString *> *_testResults;
NSMutableDictionary<NSString *, UIImage *> *_capturedScreenshotsByName;
} }
+ (IntegrationTestPlugin *)instance { + (IntegrationTestPlugin *)instance {
...@@ -32,7 +35,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage"; ...@@ -32,7 +35,13 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
} }
- (instancetype)initForRegistration { - (instancetype)initForRegistration {
return [super init]; return [self init];
}
- (instancetype)init {
self = [super init];
_capturedScreenshotsByName = [NSMutableDictionary new];
return self;
} }
+ (void)registerWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar { + (void)registerWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar {
...@@ -59,7 +68,7 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage"; ...@@ -59,7 +68,7 @@ static NSString *const kMethodRevertImage = @"revertFlutterImage";
// If running as a native Xcode test, attach to test. // If running as a native Xcode test, attach to test.
UIImage *screenshot = [self capturePngScreenshot]; UIImage *screenshot = [self capturePngScreenshot];
NSString *name = call.arguments[@"name"]; NSString *name = call.arguments[@"name"];
[self.screenshotDelegate didTakeScreenshot:screenshot attachmentName:name]; _capturedScreenshotsByName[name] = screenshot;
// Also pass back along the channel for the driver to handle. // Also pass back along the channel for the driver to handle.
NSData *pngData = UIImagePNGRepresentation(screenshot); NSData *pngData = UIImagePNGRepresentation(screenshot);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment