[devicelab] Separate build and test from Flutter gallery tests (#76415)

dev/devicelab/bin/run.dart

@@ -9,7 +9,6 @@ import 'package:args/args.dart';
 import 'package:path/path.dart' as path;
 
 import 'package:flutter_devicelab/framework/ab.dart';
-import 'package:flutter_devicelab/framework/cocoon.dart';
 import 'package:flutter_devicelab/framework/manifest.dart';
 import 'package:flutter_devicelab/framework/runner.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
@@ -105,46 +104,14 @@ Future<void> main(List<String> rawArgs) async {
   if (args.wasParsed('ab')) {
     await _runABTest();
   } else {
-    await _runTasks();
-  }
-}
-
-Future<void> _runTasks() async {
-  for (final String taskName in _taskNames) {
-    section('Running task "$taskName"');
-    final TaskResult result = await runTask(
-      taskName,
+    await runTasks(_taskNames,
       silent: silent,
-      localEngine: localEngine,
-      localEngineSrcPath: localEngineSrcPath,
+      exitOnFirstTestFailure: exitOnFirstTestFailure,
       deviceId: deviceId,
+      gitBranch: gitBranch,
+      luciBuilder: luciBuilder,
+      resultsPath: resultsPath,
     );
-
-    print('Task result:');
-    print(const JsonEncoder.withIndent('  ').convert(result));
-    section('Finished task "$taskName"');
-
-    if (resultsPath != null) {
-      final Cocoon cocoon = Cocoon();
-      await cocoon.writeTaskResultToFile(
-        builderName: luciBuilder,
-        gitBranch: gitBranch,
-        result: result,
-        resultsPath: resultsPath,
-      );
-    } else if (serviceAccountTokenFile != null) {
-      final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
-
-      /// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
-      await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
-    }
-
-    if (!result.succeeded) {
-      exitCode = 1;
-      if (exitOnFirstTestFailure) {
-        return;
-      }
-    }
   }
 }
 

dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionTest());
+  await task(createGalleryTransitionTest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionE2ETest());
+  await task(createGalleryTransitionE2ETest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createGalleryTransitionHybridTest());
+  await task(createGalleryTransitionHybridTest(args));
 }

dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart

@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
   await task(() async {
-    final TaskResult withoutSemantics = await createGalleryTransitionTest()();
-    final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)();
+    final TaskResult withoutSemantics = await createGalleryTransitionTest(args)();
+    final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)();
     if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) {
       String message = 'Lack of data';
       if (withSemantics.benchmarkScoreKeys.isEmpty) {

dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart

@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 
-Future<void> main() async {
+Future<void> main(List<String> args) async {
   deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(createGalleryTransitionTest());
+  await task(createGalleryTransitionTest(args));
 }

dev/devicelab/bin/tasks/smoke_test_build_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/build_test_task.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+
+/// Smoke test of a successful task.
+Future<void> main(List<String> args) async {
+  deviceOperatingSystem = DeviceOperatingSystem.fake;
+  await task(FakeBuildTestTask(args));
+}
+
+class FakeBuildTestTask extends BuildTestTask {
+  FakeBuildTestTask(List<String> args) : super(args, runFlutterClean: false) {
+    deviceOperatingSystem = DeviceOperatingSystem.fake;
+  }
+
+  @override
+  // In prod, tasks always run some unit of work and the test framework assumes
+  // there will be some work done when managing the isolate. To fake this, add a delay.
+  Future<void> build() => Future<void>.delayed(const Duration(milliseconds: 500));
+
+  @override
+  Future<TaskResult> test() async {
+    await Future<void>.delayed(const Duration(milliseconds: 500));
+    return TaskResult.success(<String, String>{'benchmark': 'data'});
+  }
+}

dev/devicelab/bin/test_runner.dart

@@ -5,10 +5,12 @@
 import 'dart:io';
 
 import 'package:args/command_runner.dart';
+import 'package:flutter_devicelab/command/test.dart';
 import 'package:flutter_devicelab/command/upload_metrics.dart';
 
 final CommandRunner<void> runner =
     CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
+      ..addCommand(TestCommand())
       ..addCommand(UploadMetricsCommand());
 
 Future<void> main(List<String> rawArgs) async {

dev/devicelab/lib/command/test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:args/command_runner.dart';
+
+import 'package:flutter_devicelab/framework/runner.dart';
+
+class TestCommand extends Command<void> {
+  TestCommand() {
+    argParser.addOption('task',
+        abbr: 't',
+        help: 'The name of a task listed under bin/tasks.\n'
+            '   Example: complex_layout__start_up.\n');
+    argParser.addMultiOption('task-args',
+        help: 'The name of a task listed under bin/tasks.\n'
+            'For example, "--task-args build" is passed as "bin/task/task.dart --build"');
+    argParser.addOption(
+      'device-id',
+      abbr: 'd',
+      help: 'Target device id (prefixes are allowed, names are not supported).\n'
+          'The option will be ignored if the test target does not run on a\n'
+          'mobile device. This still respects the device operating system\n'
+          'settings in the test case, and will results in error if no device\n'
+          'with given ID/ID prefix is found.',
+    );
+    argParser.addOption(
+      'git-branch',
+      help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
+          'checkouts run in detached HEAD state, so the branch must be passed.',
+    );
+    argParser.addOption(
+      'local-engine',
+      help: 'Name of a build output within the engine out directory, if you\n'
+          'are building Flutter locally. Use this to select a specific\n'
+          'version of the engine if you have built multiple engine targets.\n'
+          'This path is relative to --local-engine-src-path/out. This option\n'
+          'is required when running an A/B test (see the --ab option).',
+    );
+    argParser.addOption(
+      'local-engine-src-path',
+      help: 'Path to your engine src directory, if you are building Flutter\n'
+          'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
+          'the location based on the value of the --flutter-root option.',
+    );
+    argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.');
+    argParser.addOption('results-file',
+        help: '[Flutter infrastructure] File path for test results. If passed with\n'
+            'task, will write test results to the file.');
+    argParser.addFlag(
+      'silent',
+      negatable: true,
+      defaultsTo: false,
+    );
+  }
+
+  @override
+  String get name => 'test';
+
+  @override
+  String get description => 'Run Flutter DeviceLab test';
+
+  @override
+  Future<void> run() async {
+    final List<String> taskArgsRaw = argResults['task-args'] as List<String>;
+    // Prepend '--' to convert args to options when passed to task
+    final List<String> taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList();
+    print(taskArgs);
+    await runTasks(
+      <String>[argResults['task'] as String],
+      deviceId: argResults['device-id'] as String,
+      gitBranch: argResults['git-branch'] as String,
+      localEngine: argResults['local-engine'] as String,
+      localEngineSrcPath: argResults['local-engine-src-path'] as String,
+      luciBuilder: argResults['luci-builder'] as String,
+      resultsPath: argResults['results-file'] as String,
+      silent: argResults['silent'] as bool,
+      taskArgs: taskArgs,
+    );
+  }
+}

dev/devicelab/lib/framework/framework.dart

@@ -41,8 +41,9 @@ bool _isTaskRegistered = false;
 /// It is OK for a [task] to perform many things. However, only one task can be
 /// registered per Dart VM.
 Future<TaskResult> task(TaskFunction task) async {
-  if (_isTaskRegistered)
+  if (_isTaskRegistered) {
     throw StateError('A task is already registered');
+  }
 
   _isTaskRegistered = true;
 
@@ -59,16 +60,18 @@ Future<TaskResult> task(TaskFunction task) async {
 
 class _TaskRunner {
   _TaskRunner(this.task) {
-    registerExtension('ext.cocoonRunTask',
-        (String method, Map<String, String> parameters) async {
+    registerExtension('ext.cocoonRunTask', (String method, Map<String, String> parameters) async {
       final Duration taskTimeout = parameters.containsKey('timeoutInMinutes')
-        ? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
-        : null;
-      final TaskResult result = await run(taskTimeout);
+          ? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
+          : null;
+      // This is only expected to be passed in unit test runs so they do not
+      // kill the Dart process that is running them.
+      final bool runProcessCleanup = parameters['runProcessCleanup'] != 'false';
+      final bool enableConfig = parameters['enableConfig'] != 'false';
+      final TaskResult result = await run(taskTimeout, runProcessCleanup: runProcessCleanup, enableConfig: enableConfig);
       return ServiceExtensionResponse.result(json.encode(result.toJson()));
     });
-    registerExtension('ext.cocoonRunnerReady',
-        (String method, Map<String, String> parameters) async {
+    registerExtension('ext.cocoonRunnerReady', (String method, Map<String, String> parameters) async {
       return ServiceExtensionResponse.result('"ready"');
     });
   }
@@ -87,59 +90,77 @@ class _TaskRunner {
   /// Signals that this task runner finished running the task.
   Future<TaskResult> get whenDone => _completer.future;
 
-  Future<TaskResult> run(Duration taskTimeout) async {
+  Future<TaskResult> run(Duration taskTimeout, {
+    bool runProcessCleanup = true,
+    bool enableConfig = true,
+  }) async {
     try {
       _taskStarted = true;
       print('Running task with a timeout of $taskTimeout.');
       final String exe = Platform.isWindows ? '.exe' : '';
-      section('Checking running Dart$exe processes');
-      final Set<RunningProcessInfo> beforeRunningDartInstances = await getRunningProcesses(
-        processName: 'dart$exe',
-      ).toSet();
-      final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
-      beforeRunningDartInstances.forEach(print);
-      for (final RunningProcessInfo info in allProcesses) {
-        if (info.commandLine.contains('iproxy')) {
-          print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
+      Set<RunningProcessInfo> beforeRunningDartInstances;
+      if (runProcessCleanup) {
+        section('Checking running Dart$exe processes');
+        beforeRunningDartInstances = await getRunningProcesses(
+          processName: 'dart$exe',
+        ).toSet();
+        final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
+        beforeRunningDartInstances.forEach(print);
+        for (final RunningProcessInfo info in allProcesses) {
+          if (info.commandLine.contains('iproxy')) {
+            print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
+          }
         }
       }
-      print('enabling configs for macOS, Linux, Windows, and Web...');
-      final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
-        'config',
-        '-v',
-        '--enable-macos-desktop',
-        '--enable-windows-desktop',
-        '--enable-linux-desktop',
-        '--enable-web',
-        if (localEngine != null) ...<String>['--local-engine', localEngine],
-      ], canFail: true);
-      if (configResult != 0) {
-        print('Failed to enable configuration, tasks may not run.');
+      if (enableConfig) {
+        print('enabling configs for macOS, Linux, Windows, and Web...');
+        final int configResult = await exec(
+            path.join(flutterDirectory.path, 'bin', 'flutter'),
+            <String>[
+              'config',
+              '-v',
+              '--enable-macos-desktop',
+              '--enable-windows-desktop',
+              '--enable-linux-desktop',
+              '--enable-web',
+              if (localEngine != null) ...<String>['--local-engine', localEngine],
+            ],
+            canFail: true);
+        if (configResult != 0) {
+          print('Failed to enable configuration, tasks may not run.');
+        }
+      } else {
+        section('Skipping flutter config. You should only see this in devicelab unit tests');
       }
 
       Future<TaskResult> futureResult = _performTask();
-      if (taskTimeout != null)
+      if (taskTimeout != null) {
         futureResult = futureResult.timeout(taskTimeout);
+      }
 
       TaskResult result = await futureResult;
 
-      section('Checking running Dart$exe processes after task...');
-      final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
-        processName: 'dart$exe',
-      ).toList();
-      for (final RunningProcessInfo info in afterRunningDartInstances) {
-        if (!beforeRunningDartInstances.contains(info)) {
-          print('$info was leaked by this test.');
-          if (result is TaskResultCheckProcesses) {
-            result = TaskResult.failure('This test leaked dart processes');
-          }
-          final bool killed = await killProcess(info.pid);
-          if (!killed) {
-            print('Failed to kill process ${info.pid}.');
-          } else {
-            print('Killed process id ${info.pid}.');
+      if (runProcessCleanup) {
+        section('Checking running Dart$exe processes after task...');
+        final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
+          processName: 'dart$exe',
+        ).toList();
+        for (final RunningProcessInfo info in afterRunningDartInstances) {
+          if (!beforeRunningDartInstances.contains(info)) {
+            print('$info was leaked by this test.');
+            if (result is TaskResultCheckProcesses) {
+              result = TaskResult.failure('This test leaked dart processes');
+            }
+            final bool killed = await killProcess(info.pid);
+            if (!killed) {
+              print('Failed to kill process ${info.pid}.');
+            } else {
+              print('Killed process id ${info.pid}.');
+            }
           }
         }
+      } else {
+        section('Skipping Dart process cleanup. You should only see this in devicelab unit tests');
       }
       _completer.complete(result);
       return result;
@@ -149,8 +170,10 @@ class _TaskRunner {
       print(stackTrace);
       return TaskResult.failure('Task timed out after $taskTimeout');
     } finally {
-      await checkForRebootRequired();
-      await forceQuitRunningProcesses();
+      if (runProcessCleanup) {
+        await checkForRebootRequired();
+        await forceQuitRunningProcesses();
+      }
       _closeKeepAlivePort();
     }
   }
@@ -188,8 +211,9 @@ class _TaskRunner {
   /// Causes the Dart VM to stay alive until a request to run the task is
   /// received via the VM service protocol.
   void keepVmAliveUntilTaskRunRequested() {
-    if (_taskStarted)
+    if (_taskStarted) {
       throw StateError('Task already started.');
+    }
 
     // Merely creating this port object will cause the VM to stay alive and keep
     // the VM service server running until the port is disposed of.
@@ -218,17 +242,15 @@ class _TaskRunner {
       completer.complete(await task());
     }, onError: (dynamic taskError, Chain taskErrorStack) {
       final String message = 'Task failed: $taskError';
-      stderr
-        ..writeln(message)
-        ..writeln('\nStack trace:')
-        ..writeln(taskErrorStack.terse);
+      stderr..writeln(message)..writeln('\nStack trace:')..writeln(taskErrorStack.terse);
       // IMPORTANT: We're completing the future _successfully_ but with a value
       // that indicates a task failure. This is intentional. At this point we
       // are catching errors coming from arbitrary (and untrustworthy) task
       // code. Our goal is to convert the failure into a readable message.
       // Propagating it further is not useful.
-      if (!completer.isCompleted)
+      if (!completer.isCompleted) {
         completer.complete(TaskResult.failure(message));
+      }
     });
     return completer.future;
   }

dev/devicelab/lib/framework/runner.dart

@@ -6,13 +6,61 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:meta/meta.dart';
 import 'package:vm_service_client/vm_service_client.dart';
 
 import 'package:flutter_devicelab/framework/utils.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 
+import 'cocoon.dart';
 import 'task_result.dart';
 
+Future<void> runTasks(
+  List<String> taskNames, {
+  bool exitOnFirstTestFailure = false,
+  bool silent = false,
+  String deviceId,
+  String gitBranch,
+  String localEngine,
+  String localEngineSrcPath,
+  String luciBuilder,
+  String resultsPath,
+  List<String> taskArgs,
+}) async {
+  for (final String taskName in taskNames) {
+    section('Running task "$taskName"');
+    final TaskResult result = await runTask(
+      taskName,
+      deviceId: deviceId,
+      localEngine: localEngine,
+      localEngineSrcPath: localEngineSrcPath,
+      silent: silent,
+      taskArgs: taskArgs,
+    );
+
+    print('Task result:');
+    print(const JsonEncoder.withIndent('  ').convert(result));
+    section('Finished task "$taskName"');
+
+    if (resultsPath != null) {
+      final Cocoon cocoon = Cocoon();
+      await cocoon.writeTaskResultToFile(
+        builderName: luciBuilder,
+        gitBranch: gitBranch,
+        result: result,
+        resultsPath: resultsPath,
+      );
+    }
+
+    if (!result.succeeded) {
+      exitCode = 1;
+      if (exitOnFirstTestFailure) {
+        return;
+      }
+    }
+  }
+}
+
 /// Runs a task in a separate Dart VM and collects the result using the VM
 /// service protocol.
 ///
@@ -21,17 +69,22 @@ import 'task_result.dart';
 ///
 /// Running the task in [silent] mode will suppress standard output from task
 /// processes and only print standard errors.
+///
+/// [taskArgs] are passed to the task executable for additional configuration.
 Future<TaskResult> runTask(
   String taskName, {
   bool silent = false,
   String localEngine,
   String localEngineSrcPath,
   String deviceId,
+  List<String> taskArgs,
+  @visibleForTesting Map<String, String> isolateParams = const <String, String>{},
 }) async {
   final String taskExecutable = 'bin/tasks/$taskName.dart';
 
-  if (!file(taskExecutable).existsSync())
+  if (!file(taskExecutable).existsSync()) {
     throw 'Executable Dart file not found: $taskExecutable';
+  }
 
   final Process runner = await startProcess(
     dartBin,
@@ -42,10 +95,10 @@ Future<TaskResult> runTask(
       if (localEngine != null) '-DlocalEngine=$localEngine',
       if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
       taskExecutable,
+      ...?taskArgs,
     ],
     environment: <String, String>{
-      if (deviceId != null)
-        DeviceIdEnvName: deviceId,
+      if (deviceId != null) DeviceIdEnvName: deviceId,
     },
   );
 
@@ -63,8 +116,9 @@ Future<TaskResult> runTask(
       .listen((String line) {
     if (!uri.isCompleted) {
       final Uri serviceUri = parseServiceUri(line, prefix: 'Observatory listening on ');
-      if (serviceUri != null)
+      if (serviceUri != null) {
         uri.complete(serviceUri);
+      }
     }
     if (!silent) {
       stdout.writeln('[$taskName] [STDOUT] $line');
@@ -80,13 +134,15 @@ Future<TaskResult> runTask(
 
   try {
     final VMIsolateRef isolate = await _connectToRunnerIsolate(await uri.future);
-    final Map<String, dynamic> taskResultJson = await isolate.invokeExtension('ext.cocoonRunTask') as Map<String, dynamic>;
+    final Map<String, dynamic> taskResultJson =
+        await isolate.invokeExtension('ext.cocoonRunTask', isolateParams) as Map<String, dynamic>;
     final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
     await runner.exitCode;
     return taskResult;
   } finally {
-    if (!runnerFinished)
+    if (!runnerFinished) {
       runner.kill(ProcessSignal.sigkill);
+    }
     await stdoutSub.cancel();
     await stderrSub.cancel();
   }
@@ -98,8 +154,7 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async {
     if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0],
     'ws',
   ];
-  final String url = vmServiceUri.replace(scheme: 'ws', pathSegments:
-      pathSegments).toString();
+  final String url = vmServiceUri.replace(scheme: 'ws', pathSegments: pathSegments).toString();
   final Stopwatch stopwatch = Stopwatch()..start();
 
   while (true) {
@@ -112,8 +167,9 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async {
       final VM vm = await client.getVM();
       final VMIsolateRef isolate = vm.isolates.single;
       final String response = await isolate.invokeExtension('ext.cocoonRunnerReady') as String;
-      if (response != 'ready')
+      if (response != 'ready') {
         throw 'not ready yet';
+      }
       return isolate;
     } catch (error) {
       if (stopwatch.elapsed > const Duration(seconds: 10))

dev/devicelab/lib/framework/task_result.dart

@@ -7,13 +7,20 @@ import 'dart:io';
 
 /// A result of running a single task.
 class TaskResult {
+  TaskResult.empty()
+       : succeeded = true,
+        data = null,
+        detailFiles = null,
+        benchmarkScoreKeys = null,
+        message = 'No tests run';
+
   /// Constructs a successful result.
   TaskResult.success(this.data, {
     this.benchmarkScoreKeys = const <String>[],
     this.detailFiles = const <String>[],
+    this.message = 'success',
   })
-      : succeeded = true,
-        message = 'success' {
+      : succeeded = true {
     const JsonEncoder prettyJson = JsonEncoder.withIndent('  ');
     if (benchmarkScoreKeys != null) {
       for (final String key in benchmarkScoreKeys) {
@@ -49,6 +56,7 @@ class TaskResult {
       return TaskResult.success(json['data'] as Map<String, dynamic>,
         benchmarkScoreKeys: benchmarkScoreKeys,
         detailFiles: detailFiles,
+        message: json['reason'] as String,
       );
     }
 
@@ -106,7 +114,8 @@ class TaskResult {
       json['data'] = data;
       json['detailFiles'] = detailFiles;
       json['benchmarkScoreKeys'] = benchmarkScoreKeys;
-    } else {
+    }
+    if (message != null || !succeeded) {
       json['reason'] = message;
     }
 

dev/devicelab/lib/tasks/build_test_task.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+
+import '../framework/adb.dart';
+import '../framework/task_result.dart';
+import '../framework/utils.dart';
+
+/// [Task] for defining build-test separation.
+///
+/// Using this [Task] allows DeviceLab capacity to only be spent on the [test].
+abstract class BuildTestTask {
+  BuildTestTask(this.args, {this.workingDirectory, this.runFlutterClean = true,}) {
+    final ArgResults argResults = argParser.parse(args);
+    applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
+    buildOnly = argResults[kBuildOnlyFlag] as bool;
+    testOnly = argResults[kTestOnlyFlag] as bool;
+
+  }
+
+  static const String kApplicationBinaryPathOption = 'application-binary-path';
+  static const String kBuildOnlyFlag = 'build';
+  static const String kTestOnlyFlag = 'test';
+
+  final ArgParser argParser = ArgParser()
+    ..addOption(kApplicationBinaryPathOption)
+    ..addFlag(kBuildOnlyFlag)
+    ..addFlag(kTestOnlyFlag);
+
+  /// Args passed from the test runner via "--task-arg".
+  final List<String> args;
+
+  /// If true, skip [test].
+  bool buildOnly = false;
+
+  /// If true, skip [build].
+  bool testOnly = false;
+
+  /// Whether to run `flutter clean` before building the application under test.
+  final bool runFlutterClean;
+
+  /// Path to a built application to use in [test].
+  ///
+  /// If not given, will default to child's expected location.
+  String applicationBinaryPath;
+
+  /// Where the test artifacts are stored, such as performance results.
+  final Directory workingDirectory;
+
+  /// Run Flutter build to create [applicationBinaryPath].
+  Future<void> build() async {
+    await inDirectory<void>(workingDirectory, () async {
+      if (runFlutterClean) {
+        section('FLUTTER CLEAN');
+        await flutter('clean');
+      }
+      section('BUILDING APPLICATION');
+      await flutter('build', options: getBuildArgs(deviceOperatingSystem));
+    });
+
+  }
+
+  /// Run Flutter drive test from [getTestArgs] against the application under test on the device.
+  ///
+  /// This assumes that [applicationBinaryPath] exists.
+  Future<TaskResult> test() async {
+    final Device device = await devices.workingDevice;
+    await device.unlock();
+    await inDirectory<void>(workingDirectory, () async {
+      section('DRIVE START');
+      await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
+    });
+
+    return parseTaskResult();
+  }
+
+  /// Args passed to flutter build to build the application under test.
+  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
+
+  /// Args passed to flutter drive to test the built application.
+  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
+
+  /// Logic to construct [TaskResult] from this test's results.
+  Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
+
+  /// Path to the built application under test.
+  ///
+  /// Tasks can override to support default values. Otherwise, it will default
+  /// to needing to be passed as an argument in the test runner.
+  String getApplicationBinaryPath() => applicationBinaryPath;
+
+  /// Run this task.
+  ///
+  /// Throws [Exception] when unnecessary arguments are passed.
+  Future<TaskResult> call() async {
+    if (buildOnly && testOnly) {
+      throw Exception('Both build and test should not be passed. Pass only one.');
+    }
+
+    if (buildOnly && applicationBinaryPath != null) {
+      throw Exception('Application binary path is only used for tests');
+    }
+
+    if (!testOnly) {
+      build();
+    }
+
+    if (buildOnly) {
+      return TaskResult.empty();
+    }
+
+    return test();
+  }
+}

dev/devicelab/lib/tasks/gallery.dart

@@ -11,13 +11,17 @@ import '../framework/adb.dart';
 import '../framework/framework.dart';
 import '../framework/task_result.dart';
 import '../framework/utils.dart';
+import 'build_test_task.dart';
 
-TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
-  return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
+final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
+
+TaskFunction createGalleryTransitionTest(List<String> args, {bool semanticsEnabled = false}) {
+  return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,);
 }
 
-TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
+TaskFunction createGalleryTransitionE2ETest(List<String> args, {bool semanticsEnabled = false}) {
   return GalleryTransitionTest(
+    args,
     testFile: semanticsEnabled
         ? 'transitions_perf_e2e_with_semantics'
         : 'transitions_perf_e2e',
@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
     transitionDurationFile: null,
     timelineTraceFile: null,
     driverFile: 'transitions_perf_e2e_test',
+    workingDirectory: galleryDirectory,
   );
 }
 
-TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
+TaskFunction createGalleryTransitionHybridTest(List<String> args, {bool semanticsEnabled = false}) {
   return GalleryTransitionTest(
+    args,
     semanticsEnabled: semanticsEnabled,
     driverFile: semanticsEnabled
         ? 'transitions_perf_hybrid_with_semantics_test'
         : 'transitions_perf_hybrid_test',
+    workingDirectory: galleryDirectory,
   );
 }
 
-class GalleryTransitionTest {
-
-  GalleryTransitionTest({
+class GalleryTransitionTest extends BuildTestTask {
+  GalleryTransitionTest(List<String> args, {
     this.semanticsEnabled = false,
     this.testFile = 'transitions_perf',
     this.needFullTimeline = true,
@@ -48,7 +54,8 @@ class GalleryTransitionTest {
     this.timelineTraceFile = 'transitions.timeline',
     this.transitionDurationFile = 'transition_durations.timeline',
     this.driverFile,
-  });
+    Directory workingDirectory,
+  }) : super(args, workingDirectory: workingDirectory);
 
   final bool semanticsEnabled;
   final bool needFullTimeline;
@@ -58,59 +65,48 @@ class GalleryTransitionTest {
   final String transitionDurationFile;
   final String driverFile;
 
-  Future<TaskResult> call() async {
-    final Device device = await devices.workingDevice;
-    await device.unlock();
-    final String deviceId = device.deviceId;
-    final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
-    await inDirectory<void>(galleryDirectory, () async {
-      String applicationBinaryPath;
-      if (deviceOperatingSystem == DeviceOperatingSystem.android) {
-        section('BUILDING APPLICATION');
-        await flutter(
-          'build',
-          options: <String>[
-            'apk',
-            '--no-android-gradle-daemon',
-            '--profile',
-            '-t',
-            'test_driver/$testFile.dart',
-            '--target-platform',
-            'android-arm,android-arm64',
-          ],
-        );
-        applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
+  @override
+  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
+      switch (deviceOperatingSystem) {
+        case DeviceOperatingSystem.android:
+          return <String>[
+              'apk',
+              '--no-android-gradle-daemon',
+              '--profile',
+              '-t',
+              'test_driver/$testFile.dart',
+              '--target-platform',
+              'android-arm,android-arm64',
+            ];
+          default:
+           throw Exception('$deviceOperatingSystem has no build configuration');
       }
+    }
 
-      final String testDriver = driverFile ?? (semanticsEnabled
-          ? '${testFile}_with_semantics_test'
-          : '${testFile}_test');
-      section('DRIVE START');
-      await flutter('drive', options: <String>[
+  @override
+  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
+    final String testDriver = driverFile ?? (semanticsEnabled
+      ? '${testFile}_with_semantics_test'
+      : '${testFile}_test');
+    return <String>[
         '--profile',
         if (needFullTimeline)
           '--trace-startup',
-        if (applicationBinaryPath != null)
-          '--use-application-binary=$applicationBinaryPath'
-        else
-          ...<String>[
-            '-t',
-            'test_driver/$testFile.dart',
-          ],
-        '--driver',
-        'test_driver/$testDriver.dart',
-        '-d',
-        deviceId,
-      ]);
-    });
+        '--use-application-binary=${getApplicationBinaryPath()}',
+        '--driver', 'test_driver/$testDriver.dart',
+        '-d', deviceId,
+      ];
+  }
 
+  @override
+  Future<TaskResult> parseTaskResult() async {
     final Map<String, dynamic> summary = json.decode(
-      file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
+      file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
     ) as Map<String, dynamic>;
 
     if (transitionDurationFile != null) {
       final Map<String, dynamic> original = json.decode(
-        file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
+        file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
       ) as Map<String, dynamic>;
       final Map<String, List<int>> transitions = <String, List<int>>{};
       for (final String key in original.keys) {
@@ -123,9 +119,9 @@ class GalleryTransitionTest {
     return TaskResult.success(summary,
       detailFiles: <String>[
         if (transitionDurationFile != null)
-          '${galleryDirectory.path}/build/$transitionDurationFile.json',
+          '${workingDirectory.path}/build/$transitionDurationFile.json',
         if (timelineTraceFile != null)
-          '${galleryDirectory.path}/build/$timelineTraceFile.json'
+          '${workingDirectory.path}/build/$timelineTraceFile.json'
       ],
       benchmarkScoreKeys: <String>[
         if (transitionDurationFile != null)
@@ -141,6 +137,20 @@ class GalleryTransitionTest {
       ],
     );
   }
+
+  @override
+  String getApplicationBinaryPath() {
+    if (applicationBinaryPath != null) {
+      return applicationBinaryPath;
+    }
+
+    switch (deviceOperatingSystem) {
+      case DeviceOperatingSystem.android:
+        return 'build/app/outputs/flutter-apk/app-profile.apk';
+      default:
+        throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem');
+    }
+  }
 }
 
 int _countMissedTransitions(Map<String, List<int>> transitions) {

dev/devicelab/test/tasks/build_test_task_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/runner.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+
+import '../common.dart';
+
+void main() {
+  final Map<String, String> isolateParams = <String, String>{
+    'enableConfig': 'false',
+    'runProcessCleanup': 'false',
+    'timeoutInMinutes': '1',
+  };
+
+  test('runs build and test when no args are passed', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.data['benchmark'], 'data');
+  });
+
+  test('runs build only when build arg is given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--build'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'No tests run');
+  });
+
+  test('runs test only when test arg is given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--test'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.data['benchmark'], 'data');
+  });
+
+  test('throws exception when build and test arg are given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--build', '--test'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'Task failed: Exception: Both build and test should not be passed. Pass only one.');
+  });
+
+  test('throws exception when build and application binary arg are given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--build', '--application-binary-path=test.apk'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'Task failed: Exception: Application binary path is only used for tests');
+  });
+}

dev/devicelab/test/utils_test.dart

@@ -20,11 +20,10 @@ void main() {
   group('parse service', () {
     const String badOutput = 'No uri here';
     const String sampleOutput = 'An Observatory debugger and profiler on '
-      'Pixel 3 XL is available at: http://127.0.0.1:9090/LpjUpsdEjqI=/';
+        'Pixel 3 XL is available at: http://127.0.0.1:9090/LpjUpsdEjqI=/';
 
     test('uri', () {
-        expect(parseServiceUri(sampleOutput),
-          Uri.parse('http://127.0.0.1:9090/LpjUpsdEjqI=/'));
+      expect(parseServiceUri(sampleOutput), Uri.parse('http://127.0.0.1:9090/LpjUpsdEjqI=/'));
       expect(parseServiceUri(badOutput), null);
     });