[devicelab] Migrate Gallery to BuildTestTask (#77956)

dev/bots/test.dart

@@ -333,11 +333,18 @@ Future<void> _runBuildTests() async {
     ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
     ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
 
+  final List<String> devicelabBuildTasks = <String>[
+    'flutter_gallery__transition_perf',
+    'flutter_gallery_ios__transition_perf',
+  ];
+
   // The tests are randomly distributed into subshards so as to get a uniform
   // distribution of costs, but the seed is fixed so that issues are reproducible.
   final List<ShardRunner> tests = <ShardRunner>[
     for (final FileSystemEntity exampleDirectory in exampleDirectories)
       () => _runExampleProjectBuildTests(exampleDirectory),
+    for (String devicelabBuildTask in devicelabBuildTasks)
+      () => _runDeviceLabBuildTask(devicelabBuildTask),
     ...<ShardRunner>[
       // Web compilation tests.
       () => _flutterBuildDart2js(
@@ -355,6 +362,26 @@ Future<void> _runBuildTests() async {
   await _runShardRunnerIndexOfTotalSubshard(tests);
 }
 
+
+Future<void> _runDeviceLabBuildTask(String task) async {
+  // Run the ios tasks
+  if (!Platform.isMacOS && task.contains('_ios_')) {
+    return;
+  }
+
+  final String targetPlatform = (task.contains('_ios_')) ? 'ios' : 'android';
+
+  await runCommand(dart, <String>[
+    'run', path.join('bin', 'test_runner.dart'),
+    'test',
+    '--task', task,
+    '--task-args', 'build',
+    '--task-args', 'target-platform=$targetPlatform',
+  ],
+  workingDirectory: path.join('dev', 'devicelab'));
+}
+
+
 Future<void> _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async {
   // Only verify caching with flutter gallery.
   final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');

dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionTest());
+  await task(createGalleryTransitionTest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionHybridTest());
+  await task(createGalleryTransitionHybridTest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart

@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
   await task(() async {
-    final TaskResult withoutSemantics = await createGalleryTransitionTest()();
-    final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)();
+    final TaskResult withoutSemantics = await createGalleryTransitionTest(args)();
+    final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)();
     if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) {
       String message = 'Lack of data';
       if (withSemantics.benchmarkScoreKeys.isEmpty) {

dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionTest());
+  await task(createGalleryTransitionTest(args));
 }

dev/devicelab/lib/command/test.dart

@@ -65,7 +65,6 @@ class TestCommand extends Command<void> {
     final List<String> taskArgsRaw = argResults['task-args'] as List<String>;
     // Prepend '--' to convert args to options when passed to task
     final List<String> taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList();
-    print(taskArgs);
     await runTasks(
       <String>[argResults['task'] as String],
       deviceId: argResults['device-id'] as String,

dev/devicelab/lib/framework/adb.dart

@@ -54,6 +54,26 @@ DeviceDiscovery get devices => DeviceDiscovery();
 /// Device operating system the test is configured to test.
 enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake }
 
+/// Helper function to allow passing the target platform as a task arg instead
+/// of hardcoding it in the task.
+DeviceOperatingSystem deviceOperatingSystemFromString(String os) {
+  switch (os) {
+    case 'android':
+      return DeviceOperatingSystem.android;
+    case 'android_arm':
+      return DeviceOperatingSystem.androidArm;
+    case 'android_arm64':
+      return DeviceOperatingSystem.androidArm64;
+    case 'fake':
+      return DeviceOperatingSystem.fake;
+    case 'fuchsia':
+      return DeviceOperatingSystem.fuchsia;
+    case 'ios':
+      return DeviceOperatingSystem.ios;
+  }
+  throw UnimplementedError('$os is not defined in function deviceOperatingSystemFromString');
+}
+
 /// Device OS to test on.
 DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android;
 

dev/devicelab/lib/tasks/build_test_task.dart

@@ -19,16 +19,23 @@ abstract class BuildTestTask {
     applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
     buildOnly = argResults[kBuildOnlyFlag] as bool;
     testOnly = argResults[kTestOnlyFlag] as bool;
-
+    if (argResults.wasParsed(kTargetPlatformOption)) {
+      // Override deviceOperatingSystem to prevent extra utilities from being used.
+      targetPlatform = deviceOperatingSystemFromString(argResults[kTargetPlatformOption] as String);
+      _originalDeviceOperatingSystem = deviceOperatingSystem;
+      deviceOperatingSystem = DeviceOperatingSystem.fake;
+    }
   }
 
   static const String kApplicationBinaryPathOption = 'application-binary-path';
   static const String kBuildOnlyFlag = 'build';
+  static const String kTargetPlatformOption = 'target-platform';
   static const String kTestOnlyFlag = 'test';
 
   final ArgParser argParser = ArgParser()
     ..addOption(kApplicationBinaryPathOption)
     ..addFlag(kBuildOnlyFlag)
+    ..addOption(kTargetPlatformOption)
     ..addFlag(kTestOnlyFlag);
 
   /// Args passed from the test runner via "--task-arg".
@@ -37,6 +44,15 @@ abstract class BuildTestTask {
   /// If true, skip [test].
   bool buildOnly = false;
 
+  /// The [DeviceOperatingSystem] being targeted for this build.
+  ///
+  /// If passed, no connected device checks are run as the current connected device
+  /// will be set as [DeviceOperatingSystem.fake].
+  DeviceOperatingSystem targetPlatform;
+
+  /// Original [deviceOperatingSystem] if [targetPlatform] is given.
+  DeviceOperatingSystem _originalDeviceOperatingSystem;
+
   /// If true, skip [build].
   bool testOnly = false;
 
@@ -59,7 +75,7 @@ abstract class BuildTestTask {
         await flutter('clean');
       }
       section('BUILDING APPLICATION');
-      await flutter('build', options: getBuildArgs(deviceOperatingSystem));
+      await flutter('build', options: getBuildArgs());
     });
 
   }
@@ -68,21 +84,25 @@ abstract class BuildTestTask {
   ///
   /// This assumes that [applicationBinaryPath] exists.
   Future<TaskResult> test() async {
+    // Ensure deviceOperatingSystem is the one set in bin/task.
+    if (deviceOperatingSystem == DeviceOperatingSystem.fake) {
+      deviceOperatingSystem = _originalDeviceOperatingSystem;
+    }
     final Device device = await devices.workingDevice;
     await device.unlock();
     await inDirectory<void>(workingDirectory, () async {
       section('DRIVE START');
-      await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
+      await flutter('drive', options: getTestArgs(device.deviceId));
     });
 
     return parseTaskResult();
   }
 
   /// Args passed to flutter build to build the application under test.
-  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
+  List<String> getBuildArgs() => throw UnimplementedError('getBuildArgs is not implemented');
 
   /// Args passed to flutter drive to test the built application.
-  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
+  List<String> getTestArgs(String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
 
   /// Logic to construct [TaskResult] from this test's results.
   Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
@@ -106,7 +126,7 @@ abstract class BuildTestTask {
     }
 
     if (!testOnly) {
-      build();
+      await build();
     }
 
     if (buildOnly) {

dev/devicelab/lib/tasks/gallery.dart

@@ -11,13 +11,17 @@ import '../framework/adb.dart';
 import '../framework/framework.dart';
 import '../framework/task_result.dart';
 import '../framework/utils.dart';
+import 'build_test_task.dart';
 
-TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
-  return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
+final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
+
+TaskFunction createGalleryTransitionTest(List<String> args, {bool semanticsEnabled = false}) {
+  return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,);
 }
 
-TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
+TaskFunction createGalleryTransitionE2ETest(List<String> args, {bool semanticsEnabled = false}) {
   return GalleryTransitionTest(
+    args,
     testFile: semanticsEnabled
         ? 'transitions_perf_e2e_with_semantics'
         : 'transitions_perf_e2e',
@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
     transitionDurationFile: null,
     timelineTraceFile: null,
     driverFile: 'transitions_perf_e2e_test',
+    workingDirectory: galleryDirectory,
   );
 }
 
-TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
+TaskFunction createGalleryTransitionHybridTest(List<String> args, {bool semanticsEnabled = false}) {
   return GalleryTransitionTest(
+    args,
     semanticsEnabled: semanticsEnabled,
     driverFile: semanticsEnabled
         ? 'transitions_perf_hybrid_with_semantics_test'
         : 'transitions_perf_hybrid_test',
+    workingDirectory: galleryDirectory,
   );
 }
 
-class GalleryTransitionTest {
-
-  GalleryTransitionTest({
+class GalleryTransitionTest extends BuildTestTask {
+  GalleryTransitionTest(List<String> args, {
     this.semanticsEnabled = false,
     this.testFile = 'transitions_perf',
     this.needFullTimeline = true,
@@ -48,7 +54,8 @@ class GalleryTransitionTest {
     this.timelineTraceFile = 'transitions.timeline',
     this.transitionDurationFile = 'transition_durations.timeline',
     this.driverFile,
-  });
+    Directory workingDirectory,
+  }) : super(args, workingDirectory: workingDirectory);
 
   final bool semanticsEnabled;
   final bool needFullTimeline;
@@ -58,59 +65,58 @@ class GalleryTransitionTest {
   final String transitionDurationFile;
   final String driverFile;
 
-  Future<TaskResult> call() async {
-    final Device device = await devices.workingDevice;
-    await device.unlock();
-    final String deviceId = device.deviceId;
-    final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
-    await inDirectory<void>(galleryDirectory, () async {
-      String applicationBinaryPath;
-      if (deviceOperatingSystem == DeviceOperatingSystem.android) {
-        section('BUILDING APPLICATION');
-        await flutter(
-          'build',
-          options: <String>[
-            'apk',
-            '--no-android-gradle-daemon',
+  @override
+  List<String> getBuildArgs() {
+      switch (targetPlatform) {
+        case DeviceOperatingSystem.android:
+          return <String>[
+              'apk',
+              '--no-android-gradle-daemon',
+              '--profile',
+              '-t',
+              'test_driver/$testFile.dart',
+              '--target-platform',
+              'android-arm,android-arm64',
+            ];
+        case DeviceOperatingSystem.ios:
+          return <String>[
+            'ios',
+            // Skip codesign on presubmit checks
+            if (targetPlatform != null)
+              '--no-codesign',
             '--profile',
             '-t',
             'test_driver/$testFile.dart',
-            '--target-platform',
-            'android-arm,android-arm64',
-          ],
-        );
-        applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
+          ];
+        default:
+          throw Exception('$deviceOperatingSystem has no build configuration');
       }
+    }
 
-      final String testDriver = driverFile ?? (semanticsEnabled
-          ? '${testFile}_with_semantics_test'
-          : '${testFile}_test');
-      section('DRIVE START');
-      await flutter('drive', options: <String>[
+  @override
+  List<String> getTestArgs(String deviceId) {
+    final String testDriver = driverFile ?? (semanticsEnabled
+      ? '${testFile}_with_semantics_test'
+      : '${testFile}_test');
+    return <String>[
         '--profile',
         if (needFullTimeline)
           '--trace-startup',
-        if (applicationBinaryPath != null)
-          '--use-application-binary=$applicationBinaryPath'
-        else
-          ...<String>[
-            '-t',
-            'test_driver/$testFile.dart',
-          ],
-        '--driver',
-        'test_driver/$testDriver.dart',
-        '-d',
-        deviceId,
-      ]);
-    });
+        '--use-application-binary="${getApplicationBinaryPath()}"',
+        '--driver', 'test_driver/$testDriver.dart',
+        '-d', deviceId,
+      ];
+  }
 
+  @override
+  Future<TaskResult> parseTaskResult() async {
     final Map<String, dynamic> summary = json.decode(
-      file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
+      file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
     ) as Map<String, dynamic>;
 
     if (transitionDurationFile != null) {
       final Map<String, dynamic> original = json.decode(
-        file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
+        file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
       ) as Map<String, dynamic>;
       final Map<String, List<int>> transitions = <String, List<int>>{};
       for (final String key in original.keys) {
@@ -123,9 +129,9 @@ class GalleryTransitionTest {
     return TaskResult.success(summary,
       detailFiles: <String>[
         if (transitionDurationFile != null)
-          '${galleryDirectory.path}/build/$transitionDurationFile.json',
+          '${workingDirectory.path}/build/$transitionDurationFile.json',
         if (timelineTraceFile != null)
-          '${galleryDirectory.path}/build/$timelineTraceFile.json'
+          '${workingDirectory.path}/build/$timelineTraceFile.json'
       ],
       benchmarkScoreKeys: <String>[
         if (transitionDurationFile != null)
@@ -141,6 +147,22 @@ class GalleryTransitionTest {
       ],
     );
   }
+
+  @override
+  String getApplicationBinaryPath() {
+    if (applicationBinaryPath != null) {
+      return applicationBinaryPath;
+    }
+
+    switch (targetPlatform) {
+      case DeviceOperatingSystem.android:
+        return 'build/app/outputs/flutter-apk/app-profile.apk';
+      case DeviceOperatingSystem.ios:
+        return 'build/ios/iphoneos/Flutter Gallery.app';
+      default:
+        throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem');
+    }
+  }
 }
 
 int _countMissedTransitions(Map<String, List<int>> transitions) {