Unverified Commit 93fb2586 authored by Casey Hillers's avatar Casey Hillers Committed by GitHub

[devicelab] Migrate Gallery to BuildTestTask (#77956)

parent 14b8194b
......@@ -333,11 +333,18 @@ Future<void> _runBuildTests() async {
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
final List<String> devicelabBuildTasks = <String>[
'flutter_gallery__transition_perf',
'flutter_gallery_ios__transition_perf',
];
// The tests are randomly distributed into subshards so as to get a uniform
// distribution of costs, but the seed is fixed so that issues are reproducible.
final List<ShardRunner> tests = <ShardRunner>[
for (final FileSystemEntity exampleDirectory in exampleDirectories)
() => _runExampleProjectBuildTests(exampleDirectory),
for (String devicelabBuildTask in devicelabBuildTasks)
() => _runDeviceLabBuildTask(devicelabBuildTask),
...<ShardRunner>[
// Web compilation tests.
() => _flutterBuildDart2js(
......@@ -355,6 +362,26 @@ Future<void> _runBuildTests() async {
await _runShardRunnerIndexOfTotalSubshard(tests);
}
Future<void> _runDeviceLabBuildTask(String task) async {
// Run the ios tasks
if (!Platform.isMacOS && task.contains('_ios_')) {
return;
}
final String targetPlatform = (task.contains('_ios_')) ? 'ios' : 'android';
await runCommand(dart, <String>[
'run', path.join('bin', 'test_runner.dart'),
'test',
'--task', task,
'--task-args', 'build',
'--task-args', 'target-platform=$targetPlatform',
],
workingDirectory: path.join('dev', 'devicelab'));
}
Future<void> _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async {
// Only verify caching with flutter gallery.
final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
......
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionTest());
await task(createGalleryTransitionTest(args));
}
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionE2ETest());
await task(createGalleryTransitionE2ETest(args));
}
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionE2ETest());
await task(createGalleryTransitionE2ETest(args));
}
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionE2ETest());
await task(createGalleryTransitionE2ETest(args));
}
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionHybridTest());
await task(createGalleryTransitionHybridTest(args));
}
......@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(() async {
final TaskResult withoutSemantics = await createGalleryTransitionTest()();
final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)();
final TaskResult withoutSemantics = await createGalleryTransitionTest(args)();
final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)();
if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) {
String message = 'Lack of data';
if (withSemantics.benchmarkScoreKeys.isEmpty) {
......
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionTest());
await task(createGalleryTransitionTest(args));
}
......@@ -65,7 +65,6 @@ class TestCommand extends Command<void> {
final List<String> taskArgsRaw = argResults['task-args'] as List<String>;
// Prepend '--' to convert args to options when passed to task
final List<String> taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList();
print(taskArgs);
await runTasks(
<String>[argResults['task'] as String],
deviceId: argResults['device-id'] as String,
......
......@@ -54,6 +54,26 @@ DeviceDiscovery get devices => DeviceDiscovery();
/// Device operating system the test is configured to test.
enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake }
/// Helper function to allow passing the target platform as a task arg instead
/// of hardcoding it in the task.
DeviceOperatingSystem deviceOperatingSystemFromString(String os) {
switch (os) {
case 'android':
return DeviceOperatingSystem.android;
case 'android_arm':
return DeviceOperatingSystem.androidArm;
case 'android_arm64':
return DeviceOperatingSystem.androidArm64;
case 'fake':
return DeviceOperatingSystem.fake;
case 'fuchsia':
return DeviceOperatingSystem.fuchsia;
case 'ios':
return DeviceOperatingSystem.ios;
}
throw UnimplementedError('$os is not defined in function deviceOperatingSystemFromString');
}
/// Device OS to test on.
DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android;
......
......@@ -19,16 +19,23 @@ abstract class BuildTestTask {
applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
buildOnly = argResults[kBuildOnlyFlag] as bool;
testOnly = argResults[kTestOnlyFlag] as bool;
if (argResults.wasParsed(kTargetPlatformOption)) {
// Override deviceOperatingSystem to prevent extra utilities from being used.
targetPlatform = deviceOperatingSystemFromString(argResults[kTargetPlatformOption] as String);
_originalDeviceOperatingSystem = deviceOperatingSystem;
deviceOperatingSystem = DeviceOperatingSystem.fake;
}
}
static const String kApplicationBinaryPathOption = 'application-binary-path';
static const String kBuildOnlyFlag = 'build';
static const String kTargetPlatformOption = 'target-platform';
static const String kTestOnlyFlag = 'test';
final ArgParser argParser = ArgParser()
..addOption(kApplicationBinaryPathOption)
..addFlag(kBuildOnlyFlag)
..addOption(kTargetPlatformOption)
..addFlag(kTestOnlyFlag);
/// Args passed from the test runner via "--task-arg".
......@@ -37,6 +44,15 @@ abstract class BuildTestTask {
/// If true, skip [test].
bool buildOnly = false;
/// The [DeviceOperatingSystem] being targeted for this build.
///
/// If passed, no connected device checks are run as the current connected device
/// will be set as [DeviceOperatingSystem.fake].
DeviceOperatingSystem targetPlatform;
/// Original [deviceOperatingSystem] if [targetPlatform] is given.
DeviceOperatingSystem _originalDeviceOperatingSystem;
/// If true, skip [build].
bool testOnly = false;
......@@ -59,7 +75,7 @@ abstract class BuildTestTask {
await flutter('clean');
}
section('BUILDING APPLICATION');
await flutter('build', options: getBuildArgs(deviceOperatingSystem));
await flutter('build', options: getBuildArgs());
});
}
......@@ -68,21 +84,25 @@ abstract class BuildTestTask {
///
/// This assumes that [applicationBinaryPath] exists.
Future<TaskResult> test() async {
// Ensure deviceOperatingSystem is the one set in bin/task.
if (deviceOperatingSystem == DeviceOperatingSystem.fake) {
deviceOperatingSystem = _originalDeviceOperatingSystem;
}
final Device device = await devices.workingDevice;
await device.unlock();
await inDirectory<void>(workingDirectory, () async {
section('DRIVE START');
await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
await flutter('drive', options: getTestArgs(device.deviceId));
});
return parseTaskResult();
}
/// Args passed to flutter build to build the application under test.
List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
List<String> getBuildArgs() => throw UnimplementedError('getBuildArgs is not implemented');
/// Args passed to flutter drive to test the built application.
List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
List<String> getTestArgs(String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
/// Logic to construct [TaskResult] from this test's results.
Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
......@@ -106,7 +126,7 @@ abstract class BuildTestTask {
}
if (!testOnly) {
build();
await build();
}
if (buildOnly) {
......
......@@ -11,13 +11,17 @@ import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
import 'build_test_task.dart';
TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
TaskFunction createGalleryTransitionTest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,);
}
TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
TaskFunction createGalleryTransitionE2ETest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(
args,
testFile: semanticsEnabled
? 'transitions_perf_e2e_with_semantics'
: 'transitions_perf_e2e',
......@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
transitionDurationFile: null,
timelineTraceFile: null,
driverFile: 'transitions_perf_e2e_test',
workingDirectory: galleryDirectory,
);
}
TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
TaskFunction createGalleryTransitionHybridTest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(
args,
semanticsEnabled: semanticsEnabled,
driverFile: semanticsEnabled
? 'transitions_perf_hybrid_with_semantics_test'
: 'transitions_perf_hybrid_test',
workingDirectory: galleryDirectory,
);
}
class GalleryTransitionTest {
GalleryTransitionTest({
class GalleryTransitionTest extends BuildTestTask {
GalleryTransitionTest(List<String> args, {
this.semanticsEnabled = false,
this.testFile = 'transitions_perf',
this.needFullTimeline = true,
......@@ -48,7 +54,8 @@ class GalleryTransitionTest {
this.timelineTraceFile = 'transitions.timeline',
this.transitionDurationFile = 'transition_durations.timeline',
this.driverFile,
});
Directory workingDirectory,
}) : super(args, workingDirectory: workingDirectory);
final bool semanticsEnabled;
final bool needFullTimeline;
......@@ -58,59 +65,58 @@ class GalleryTransitionTest {
final String transitionDurationFile;
final String driverFile;
Future<TaskResult> call() async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
await inDirectory<void>(galleryDirectory, () async {
String applicationBinaryPath;
if (deviceOperatingSystem == DeviceOperatingSystem.android) {
section('BUILDING APPLICATION');
await flutter(
'build',
options: <String>[
'apk',
'--no-android-gradle-daemon',
@override
List<String> getBuildArgs() {
switch (targetPlatform) {
case DeviceOperatingSystem.android:
return <String>[
'apk',
'--no-android-gradle-daemon',
'--profile',
'-t',
'test_driver/$testFile.dart',
'--target-platform',
'android-arm,android-arm64',
];
case DeviceOperatingSystem.ios:
return <String>[
'ios',
// Skip codesign on presubmit checks
if (targetPlatform != null)
'--no-codesign',
'--profile',
'-t',
'test_driver/$testFile.dart',
'--target-platform',
'android-arm,android-arm64',
],
);
applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
];
default:
throw Exception('$deviceOperatingSystem has no build configuration');
}
}
final String testDriver = driverFile ?? (semanticsEnabled
? '${testFile}_with_semantics_test'
: '${testFile}_test');
section('DRIVE START');
await flutter('drive', options: <String>[
@override
List<String> getTestArgs(String deviceId) {
final String testDriver = driverFile ?? (semanticsEnabled
? '${testFile}_with_semantics_test'
: '${testFile}_test');
return <String>[
'--profile',
if (needFullTimeline)
'--trace-startup',
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath'
else
...<String>[
'-t',
'test_driver/$testFile.dart',
],
'--driver',
'test_driver/$testDriver.dart',
'-d',
deviceId,
]);
});
'--use-application-binary="${getApplicationBinaryPath()}"',
'--driver', 'test_driver/$testDriver.dart',
'-d', deviceId,
];
}
@override
Future<TaskResult> parseTaskResult() async {
final Map<String, dynamic> summary = json.decode(
file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
) as Map<String, dynamic>;
if (transitionDurationFile != null) {
final Map<String, dynamic> original = json.decode(
file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
) as Map<String, dynamic>;
final Map<String, List<int>> transitions = <String, List<int>>{};
for (final String key in original.keys) {
......@@ -123,9 +129,9 @@ class GalleryTransitionTest {
return TaskResult.success(summary,
detailFiles: <String>[
if (transitionDurationFile != null)
'${galleryDirectory.path}/build/$transitionDurationFile.json',
'${workingDirectory.path}/build/$transitionDurationFile.json',
if (timelineTraceFile != null)
'${galleryDirectory.path}/build/$timelineTraceFile.json'
'${workingDirectory.path}/build/$timelineTraceFile.json'
],
benchmarkScoreKeys: <String>[
if (transitionDurationFile != null)
......@@ -141,6 +147,22 @@ class GalleryTransitionTest {
],
);
}
@override
String getApplicationBinaryPath() {
if (applicationBinaryPath != null) {
return applicationBinaryPath;
}
switch (targetPlatform) {
case DeviceOperatingSystem.android:
return 'build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.ios:
return 'build/ios/iphoneos/Flutter Gallery.app';
default:
throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem');
}
}
}
int _countMissedTransitions(Map<String, List<int>> transitions) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment