Reland [devicelab] Migrate Gallery to BuildTestTask (#78361)

dev/bots/test.dart

@@ -323,11 +323,18 @@ Future<void> _runBuildTests() async {
     ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
     ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
 
+  final List<String> devicelabBuildTasks = <String>[
+    'flutter_gallery__transition_perf',
+    'flutter_gallery_ios__transition_perf',
+  ];
+
   // The tests are randomly distributed into subshards so as to get a uniform
   // distribution of costs, but the seed is fixed so that issues are reproducible.
   final List<ShardRunner> tests = <ShardRunner>[
     for (final FileSystemEntity exampleDirectory in exampleDirectories)
       () => _runExampleProjectBuildTests(exampleDirectory),
+    for (String devicelabBuildTask in devicelabBuildTasks)
+      () => _runDeviceLabBuildTask(devicelabBuildTask),
     ...<ShardRunner>[
       // Web compilation tests.
       () => _flutterBuildDart2js(
@@ -345,6 +352,26 @@ Future<void> _runBuildTests() async {
   await _runShardRunnerIndexOfTotalSubshard(tests);
 }
 
+
+Future<void> _runDeviceLabBuildTask(String task) async {
+  // Run the ios tasks
+  if (!Platform.isMacOS && task.contains('_ios_')) {
+    return;
+  }
+
+  final String targetPlatform = (task.contains('_ios_')) ? 'ios' : 'android';
+
+  await runCommand(dart, <String>[
+    'run', path.join('bin', 'test_runner.dart'),
+    'test',
+    '--task', task,
+    '--task-args', 'build',
+    '--task-args', 'target-platform=$targetPlatform',
+  ],
+  workingDirectory: path.join('dev', 'devicelab'));
+}
+
+
 Future<void> _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async {
   // Only verify caching with flutter gallery.
   final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
@@ -640,7 +667,7 @@ Future<void> _runFrameworkTests() async {
   Future<void> runMisc() async {
     print('${green}Running package tests$reset for directories other than packages/flutter');
     await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'));
-    await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'));
+    await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), forceSingleCore: true);
     await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'));
     await _pubRunTest(path.join(flutterRoot, 'dev', 'tools'), forceSingleCore: true);
     await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'));

dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionTest());
+  await task(createGalleryTransitionTest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionHybridTest());
+  await task(createGalleryTransitionHybridTest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart

@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
   await task(() async {
-    final TaskResult withoutSemantics = await createGalleryTransitionTest()();
-    final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)();
+    final TaskResult withoutSemantics = await createGalleryTransitionTest(args)();
+    final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)();
     if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) {
       String message = 'Lack of data';
       if (withSemantics.benchmarkScoreKeys.isEmpty) {

dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionTest());
+  await task(createGalleryTransitionTest(args));
 }

dev/devicelab/bin/tasks/smoke_test_build_test.dart

@@ -14,18 +14,23 @@ Future<void> main(List<String> args) async {
 }
 
 class FakeBuildTestTask extends BuildTestTask {
-  FakeBuildTestTask(List<String> args) : super(args, runFlutterClean: false) {
-    deviceOperatingSystem = DeviceOperatingSystem.fake;
-  }
+  FakeBuildTestTask(List<String> args) : super(args, runFlutterClean: false);
 
   @override
   // In prod, tasks always run some unit of work and the test framework assumes
   // there will be some work done when managing the isolate. To fake this, add a delay.
-  Future<void> build() => Future<void>.delayed(const Duration(milliseconds: 500));
+  Future<void> build() async {
+    if (targetPlatform != DeviceOperatingSystem.fake) {
+      throw Exception('Only DeviceOperatingSystem.fake is supported');
+    }
+  }
 
   @override
   Future<TaskResult> test() async {
-    await Future<void>.delayed(const Duration(milliseconds: 500));
+    if (targetPlatform != DeviceOperatingSystem.fake) {
+      throw Exception('Only DeviceOperatingSystem.fake is supported');
+    }
+
     return TaskResult.success(<String, String>{'benchmark': 'data'});
   }
 }

dev/devicelab/bin/tasks/smoke_test_build_test_target_platform.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+
+import 'smoke_test_build_test.dart';
+
+/// Smoke test of a build test task with [deviceOperatingSystem] set. This should
+/// only pass in tests run with target platform set to fake.
+Future<void> main(List<String> args) async {
+  deviceOperatingSystem = DeviceOperatingSystem.ios;
+  await task(FakeBuildTestTask(args));
+}

dev/devicelab/lib/command/test.dart

@@ -65,7 +65,6 @@ class TestCommand extends Command<void> {
     final List<String> taskArgsRaw = argResults['task-args'] as List<String>;
     // Prepend '--' to convert args to options when passed to task
     final List<String> taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList();
-    print(taskArgs);
     await runTasks(
       <String>[argResults['task'] as String],
       deviceId: argResults['device-id'] as String,

dev/devicelab/lib/framework/adb.dart

@@ -54,6 +54,26 @@ DeviceDiscovery get devices => DeviceDiscovery();
 /// Device operating system the test is configured to test.
 enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake }
 
+/// Helper function to allow passing the target platform as a task arg instead
+/// of hardcoding it in the task.
+DeviceOperatingSystem deviceOperatingSystemFromString(String os) {
+  switch (os) {
+    case 'android':
+      return DeviceOperatingSystem.android;
+    case 'android_arm':
+      return DeviceOperatingSystem.androidArm;
+    case 'android_arm64':
+      return DeviceOperatingSystem.androidArm64;
+    case 'fake':
+      return DeviceOperatingSystem.fake;
+    case 'fuchsia':
+      return DeviceOperatingSystem.fuchsia;
+    case 'ios':
+      return DeviceOperatingSystem.ios;
+  }
+  throw UnimplementedError('$os is not defined in function deviceOperatingSystemFromString');
+}
+
 /// Device OS to test on.
 DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android;
 

dev/devicelab/lib/framework/framework.dart

@@ -20,6 +20,7 @@ import 'utils.dart';
 /// Identifiers for devices that should never be rebooted.
 final Set<String> noRebootForbidList = <String>{
   '822ef7958bba573829d85eef4df6cbdd86593730', // 32bit iPhone requires manual intervention on reboot.
+  'FAKE_SUCCESS', // Fake device id used in unit tests
 };
 
 /// The maximum number of test runs before a device must be rebooted.
@@ -61,8 +62,8 @@ class _TaskRunner {
   _TaskRunner(this.task) {
     registerExtension('ext.cocoonRunTask',
         (String method, Map<String, String> parameters) async {
-      final Duration taskTimeout = parameters.containsKey('timeoutInMinutes')
-        ? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
+      final Duration taskTimeout = parameters.containsKey('timeoutInMinutes') || parameters.containsKey('timeoutInSeconds')
+        ? Duration(minutes: int.parse(parameters['timeoutInMinutes'] ?? '0'), seconds: int.parse(parameters['timeoutInSeconds'] ?? '0'))
         : null;
       // This is only expected to be passed in unit test runs so they do not
       // kill the Dart process that is running them and waste time running config.
@@ -170,8 +171,12 @@ class _TaskRunner {
       print(stackTrace);
       return TaskResult.failure('Task timed out after $taskTimeout');
     } finally {
-      await checkForRebootRequired();
-      await forceQuitRunningProcesses();
+      if (runProcessCleanup) {
+        await checkForRebootRequired();
+        await forceQuitRunningProcesses();
+      } else {
+        print('Skipped checkForRebootRequired and forceQuitRunningProcesses');
+      }
       _closeKeepAlivePort();
     }
   }

dev/devicelab/lib/tasks/build_test_task.dart

@@ -19,16 +19,25 @@ abstract class BuildTestTask {
     applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
     buildOnly = argResults[kBuildOnlyFlag] as bool;
     testOnly = argResults[kTestOnlyFlag] as bool;
-
+    if (argResults.wasParsed(kTargetPlatformOption)) {
+      // Override deviceOperatingSystem to prevent extra utilities from being used.
+      targetPlatform = deviceOperatingSystemFromString(argResults[kTargetPlatformOption] as String);
+      _originalDeviceOperatingSystem = deviceOperatingSystem;
+      deviceOperatingSystem = DeviceOperatingSystem.fake;
+    } else {
+      targetPlatform = deviceOperatingSystem;
+    }
   }
 
   static const String kApplicationBinaryPathOption = 'application-binary-path';
   static const String kBuildOnlyFlag = 'build';
+  static const String kTargetPlatformOption = 'target-platform';
   static const String kTestOnlyFlag = 'test';
 
   final ArgParser argParser = ArgParser()
     ..addOption(kApplicationBinaryPathOption)
     ..addFlag(kBuildOnlyFlag)
+    ..addOption(kTargetPlatformOption)
     ..addFlag(kTestOnlyFlag);
 
   /// Args passed from the test runner via "--task-arg".
@@ -37,6 +46,15 @@ abstract class BuildTestTask {
   /// If true, skip [test].
   bool buildOnly = false;
 
+  /// The [DeviceOperatingSystem] being targeted for this build.
+  ///
+  /// If passed, no connected device checks are run as the current connected device
+  /// will be set as [DeviceOperatingSystem.fake].
+  DeviceOperatingSystem targetPlatform;
+
+  /// Original [deviceOperatingSystem] if [targetPlatform] is given.
+  DeviceOperatingSystem _originalDeviceOperatingSystem;
+
   /// If true, skip [build].
   bool testOnly = false;
 
@@ -59,7 +77,7 @@ abstract class BuildTestTask {
         await flutter('clean');
       }
       section('BUILDING APPLICATION');
-      await flutter('build', options: getBuildArgs(deviceOperatingSystem));
+      await flutter('build', options: getBuildArgs());
     });
 
   }
@@ -68,21 +86,25 @@ abstract class BuildTestTask {
   ///
   /// This assumes that [applicationBinaryPath] exists.
   Future<TaskResult> test() async {
+    // Ensure deviceOperatingSystem is the one set in bin/task.
+    if (deviceOperatingSystem == DeviceOperatingSystem.fake) {
+      deviceOperatingSystem = _originalDeviceOperatingSystem;
+    }
     final Device device = await devices.workingDevice;
     await device.unlock();
     await inDirectory<void>(workingDirectory, () async {
       section('DRIVE START');
-      await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
+      await flutter('drive', options: getTestArgs(device.deviceId));
     });
 
     return parseTaskResult();
   }
 
   /// Args passed to flutter build to build the application under test.
-  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
+  List<String> getBuildArgs() => throw UnimplementedError('getBuildArgs is not implemented');
 
   /// Args passed to flutter drive to test the built application.
-  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
+  List<String> getTestArgs(String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
 
   /// Logic to construct [TaskResult] from this test's results.
   Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
@@ -106,7 +128,7 @@ abstract class BuildTestTask {
     }
 
     if (!testOnly) {
-      build();
+      await build();
     }
 
     if (buildOnly) {

dev/devicelab/lib/tasks/gallery.dart

@@ -11,13 +11,17 @@ import '../framework/framework.dart';
 import '../framework/host_agent.dart';
 import '../framework/task_result.dart';
 import '../framework/utils.dart';
+import 'build_test_task.dart';
 
-TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
-  return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
+final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
+
+TaskFunction createGalleryTransitionTest(List<String> args, {bool semanticsEnabled = false}) {
+  return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,);
 }
 
-TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
+TaskFunction createGalleryTransitionE2ETest(List<String> args, {bool semanticsEnabled = false}) {
   return GalleryTransitionTest(
+    args,
     testFile: semanticsEnabled
         ? 'transitions_perf_e2e_with_semantics'
         : 'transitions_perf_e2e',
@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
     transitionDurationFile: null,
     timelineTraceFile: null,
     driverFile: 'transitions_perf_e2e_test',
+    workingDirectory: galleryDirectory,
   );
 }
 
-TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
+TaskFunction createGalleryTransitionHybridTest(List<String> args, {bool semanticsEnabled = false}) {
   return GalleryTransitionTest(
+    args,
     semanticsEnabled: semanticsEnabled,
     driverFile: semanticsEnabled
         ? 'transitions_perf_hybrid_with_semantics_test'
         : 'transitions_perf_hybrid_test',
+    workingDirectory: galleryDirectory,
   );
 }
 
-class GalleryTransitionTest {
-
-  GalleryTransitionTest({
+class GalleryTransitionTest extends BuildTestTask {
+  GalleryTransitionTest(List<String> args, {
     this.semanticsEnabled = false,
     this.testFile = 'transitions_perf',
     this.needFullTimeline = true,
@@ -48,7 +54,8 @@ class GalleryTransitionTest {
     this.timelineTraceFile = 'transitions.timeline',
     this.transitionDurationFile = 'transition_durations.timeline',
     this.driverFile,
-  });
+    Directory workingDirectory,
+  }) : super(args, workingDirectory: workingDirectory);
 
   final bool semanticsEnabled;
   final bool needFullTimeline;
@@ -58,55 +65,55 @@ class GalleryTransitionTest {
   final String transitionDurationFile;
   final String driverFile;
 
-  Future<TaskResult> call() async {
-    final Device device = await devices.workingDevice;
-    await device.unlock();
-    final String deviceId = device.deviceId;
-    final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
-    await inDirectory<void>(galleryDirectory, () async {
-      String applicationBinaryPath;
-      if (deviceOperatingSystem == DeviceOperatingSystem.android) {
-        section('BUILDING APPLICATION');
-        await flutter(
-          'build',
-          options: <String>[
-            'apk',
-            '--no-android-gradle-daemon',
+  final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
+
+  @override
+  List<String> getBuildArgs() {
+      switch (targetPlatform) {
+        case DeviceOperatingSystem.android:
+          return <String>[
+              'apk',
+              '--no-android-gradle-daemon',
+              '--profile',
+              '-t',
+              'test_driver/$testFile.dart',
+              '--target-platform',
+              'android-arm,android-arm64',
+            ];
+        case DeviceOperatingSystem.ios:
+          return <String>[
+            'ios',
+            // Skip codesign on presubmit checks
+            if (targetPlatform != null)
+              '--no-codesign',
             '--profile',
             '-t',
             'test_driver/$testFile.dart',
-            '--target-platform',
-            'android-arm,android-arm64',
-          ],
-        );
-        applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
+          ];
+        default:
+          throw Exception('$targetPlatform has no build configuration');
       }
+    }
 
-      final String testDriver = driverFile ?? (semanticsEnabled
-          ? '${testFile}_with_semantics_test'
-          : '${testFile}_test');
-      section('DRIVE START');
-      await flutter('drive', options: <String>[
+  @override
+  List<String> getTestArgs(String deviceId) {
+    final String testDriver = driverFile ?? (semanticsEnabled
+      ? '${testFile}_with_semantics_test'
+      : '${testFile}_test');
+    return <String>[
         '--profile',
         if (needFullTimeline)
           '--trace-startup',
-        if (applicationBinaryPath != null)
-          '--use-application-binary=$applicationBinaryPath'
-        else
-          ...<String>[
-            '-t',
-            'test_driver/$testFile.dart',
-          ],
-        '--driver',
-        'test_driver/$testDriver.dart',
-        '-d',
-        deviceId,
-        '--screenshot',
-        hostAgent.dumpDirectory.path,
-      ]);
-    });
-
-    final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
+        '-t', 'test_driver/$testFile.dart',
+        '--use-application-binary="${getApplicationBinaryPath()}"',
+        '--driver', 'test_driver/$testDriver.dart',
+        '-d', deviceId,
+        '--screenshot', hostAgent.dumpDirectory.path,
+      ];
+  }
+
+  @override
+  Future<TaskResult> parseTaskResult() async {
     final Map<String, dynamic> summary = json.decode(
       file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
     ) as Map<String, dynamic>;
@@ -144,6 +151,22 @@ class GalleryTransitionTest {
       ],
     );
   }
+
+  @override
+  String getApplicationBinaryPath() {
+    if (applicationBinaryPath != null) {
+      return applicationBinaryPath;
+    }
+
+    switch (targetPlatform) {
+      case DeviceOperatingSystem.android:
+        return 'build/app/outputs/flutter-apk/app-profile.apk';
+      case DeviceOperatingSystem.ios:
+        return 'build/ios/iphoneos/Flutter Gallery.app';
+      default:
+        throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem');
+    }
+  }
 }
 
 int _countMissedTransitions(Map<String, List<int>> transitions) {

dev/devicelab/test/tasks/build_test_task_test.dart

@@ -11,7 +11,7 @@ void main() {
   final Map<String, String> isolateParams = <String, String>{
     'runFlutterConfig': 'false',
     'runProcessCleanup': 'false',
-    'timeoutInMinutes': '1',
+    'timeoutInSeconds': '5',
   };
 
   test('runs build and test when no args are passed', () async {
@@ -62,4 +62,23 @@ void main() {
     );
     expect(result.message, 'Task failed: Exception: Application binary path is only used for tests');
   });
+
+  test('sets target platform when given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test_target_platform',
+      taskArgs: <String>['--target-platform=fake'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'success');
+  });
+
+  test('defaults target platform to task deviceOperatingSystem', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test_target_platform',
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'Task failed: Exception: Only DeviceOperatingSystem.fake is supported');
+  });
 }