Create BuildTestTask (#77307)

dev/devicelab/bin/tasks/smoke_test_build_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/build_test_task.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+
+/// Smoke test of a successful task.
+Future<void> main(List<String> args) async {
+  deviceOperatingSystem = DeviceOperatingSystem.fake;
+  await task(FakeBuildTestTask(args));
+}
+
+class FakeBuildTestTask extends BuildTestTask {
+  FakeBuildTestTask(List<String> args) : super(args, runFlutterClean: false) {
+    deviceOperatingSystem = DeviceOperatingSystem.fake;
+  }
+
+  @override
+  // In prod, tasks always run some unit of work and the test framework assumes
+  // there will be some work done when managing the isolate. To fake this, add a delay.
+  Future<void> build() => Future<void>.delayed(const Duration(milliseconds: 500));
+
+  @override
+  Future<TaskResult> test() async {
+    await Future<void>.delayed(const Duration(milliseconds: 500));
+    return TaskResult.success(<String, String>{'benchmark': 'data'});
+  }
+}

dev/devicelab/lib/framework/framework.dart

@@ -64,7 +64,11 @@ class _TaskRunner {
       final Duration taskTimeout = parameters.containsKey('timeoutInMinutes')
         ? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
         : null;
-      final TaskResult result = await run(taskTimeout);
+      // This is only expected to be passed in unit test runs so they do not
+      // kill the Dart process that is running them and waste time running config.
+      final bool runFlutterConfig = parameters['runFlutterConfig'] != 'false';
+      final bool runProcessCleanup = parameters['runProcessCleanup'] != 'false';
+      final TaskResult result = await run(taskTimeout, runProcessCleanup: runProcessCleanup, runFlutterConfig: runFlutterConfig);
       return ServiceExtensionResponse.result(json.encode(result.toJson()));
     });
     registerExtension('ext.cocoonRunnerReady',
@@ -87,34 +91,47 @@ class _TaskRunner {
   /// Signals that this task runner finished running the task.
   Future<TaskResult> get whenDone => _completer.future;
 
-  Future<TaskResult> run(Duration taskTimeout) async {
+  Future<TaskResult> run(Duration taskTimeout, {
+    bool runFlutterConfig = true,
+    bool runProcessCleanup = true,
+  }) async {
     try {
       _taskStarted = true;
       print('Running task with a timeout of $taskTimeout.');
       final String exe = Platform.isWindows ? '.exe' : '';
-      section('Checking running Dart$exe processes');
-      final Set<RunningProcessInfo> beforeRunningDartInstances = await getRunningProcesses(
-        processName: 'dart$exe',
-      ).toSet();
-      final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
-      beforeRunningDartInstances.forEach(print);
-      for (final RunningProcessInfo info in allProcesses) {
-        if (info.commandLine.contains('iproxy')) {
-          print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
+      Set<RunningProcessInfo> beforeRunningDartInstances;
+      if (runProcessCleanup) {
+        section('Checking running Dart$exe processes');
+        beforeRunningDartInstances = await getRunningProcesses(
+          processName: 'dart$exe',
+        ).toSet();
+        final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
+        beforeRunningDartInstances.forEach(print);
+        for (final RunningProcessInfo info in allProcesses) {
+          if (info.commandLine.contains('iproxy')) {
+            print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
+          }
         }
+      } else {
+        section('Skipping check running Dart$exe processes');
       }
-      print('enabling configs for macOS, Linux, Windows, and Web...');
-      final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
-        'config',
-        '-v',
-        '--enable-macos-desktop',
-        '--enable-windows-desktop',
-        '--enable-linux-desktop',
-        '--enable-web',
-        if (localEngine != null) ...<String>['--local-engine', localEngine],
-      ], canFail: true);
-      if (configResult != 0) {
-        print('Failed to enable configuration, tasks may not run.');
+
+      if (runFlutterConfig) {
+        print('enabling configs for macOS, Linux, Windows, and Web...');
+        final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
+          'config',
+          '-v',
+          '--enable-macos-desktop',
+          '--enable-windows-desktop',
+          '--enable-linux-desktop',
+          '--enable-web',
+          if (localEngine != null) ...<String>['--local-engine', localEngine],
+        ], canFail: true);
+        if (configResult != 0) {
+          print('Failed to enable configuration, tasks may not run.');
+        }
+      } else {
+        print('Skipping enabling configs for macOS, Linux, Windows, and Web');
       }
 
       Future<TaskResult> futureResult = _performTask();
@@ -123,23 +140,27 @@ class _TaskRunner {
 
       TaskResult result = await futureResult;
 
-      section('Checking running Dart$exe processes after task...');
-      final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
-        processName: 'dart$exe',
-      ).toList();
-      for (final RunningProcessInfo info in afterRunningDartInstances) {
-        if (!beforeRunningDartInstances.contains(info)) {
-          print('$info was leaked by this test.');
-          if (result is TaskResultCheckProcesses) {
-            result = TaskResult.failure('This test leaked dart processes');
-          }
-          final bool killed = await killProcess(info.pid);
-          if (!killed) {
-            print('Failed to kill process ${info.pid}.');
-          } else {
-            print('Killed process id ${info.pid}.');
+      if (runProcessCleanup) {
+        section('Checking running Dart$exe processes after task...');
+        final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
+          processName: 'dart$exe',
+        ).toList();
+        for (final RunningProcessInfo info in afterRunningDartInstances) {
+          if (!beforeRunningDartInstances.contains(info)) {
+            print('$info was leaked by this test.');
+            if (result is TaskResultCheckProcesses) {
+              result = TaskResult.failure('This test leaked dart processes');
+            }
+            final bool killed = await killProcess(info.pid);
+            if (!killed) {
+              print('Failed to kill process ${info.pid}.');
+            } else {
+              print('Killed process id ${info.pid}.');
+            }
           }
         }
+      } else {
+        print('Skipping check running Dart$exe processes after task');
       }
       _completer.complete(result);
       return result;

dev/devicelab/lib/framework/runner.dart

@@ -6,6 +6,7 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:meta/meta.dart';
 import 'package:vm_service_client/vm_service_client.dart';
 
 import 'adb.dart';
@@ -76,6 +77,7 @@ Future<TaskResult> runTask(
   String localEngineSrcPath,
   String deviceId,
   List<String> taskArgs,
+  @visibleForTesting Map<String, String> isolateParams,
 }) async {
   final String taskExecutable = 'bin/tasks/$taskName.dart';
 
@@ -130,7 +132,7 @@ Future<TaskResult> runTask(
 
   try {
     final VMIsolateRef isolate = await _connectToRunnerIsolate(await uri.future);
-    final Map<String, dynamic> taskResultJson = await isolate.invokeExtension('ext.cocoonRunTask') as Map<String, dynamic>;
+    final Map<String, dynamic> taskResultJson = await isolate.invokeExtension('ext.cocoonRunTask', isolateParams) as Map<String, dynamic>;
     final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
     await runner.exitCode;
     return taskResult;

dev/devicelab/lib/framework/task_result.dart

@@ -7,13 +7,20 @@ import 'dart:io';
 
 /// A result of running a single task.
 class TaskResult {
+   TaskResult.buildOnly()
+       : succeeded = true,
+        data = null,
+        detailFiles = null,
+        benchmarkScoreKeys = null,
+        message = 'No tests run';
+
   /// Constructs a successful result.
   TaskResult.success(this.data, {
     this.benchmarkScoreKeys = const <String>[],
     this.detailFiles = const <String>[],
+    this.message = 'success',
   })
-      : succeeded = true,
-        message = 'success' {
+      : succeeded = true {
     const JsonEncoder prettyJson = JsonEncoder.withIndent('  ');
     if (benchmarkScoreKeys != null) {
       for (final String key in benchmarkScoreKeys) {
@@ -49,6 +56,7 @@ class TaskResult {
       return TaskResult.success(json['data'] as Map<String, dynamic>,
         benchmarkScoreKeys: benchmarkScoreKeys,
         detailFiles: detailFiles,
+        message: json['reason'] as String,
       );
     }
 
@@ -106,7 +114,9 @@ class TaskResult {
       json['data'] = data;
       json['detailFiles'] = detailFiles;
       json['benchmarkScoreKeys'] = benchmarkScoreKeys;
-    } else {
+    }
+
+    if (message != null || !succeeded) {
       json['reason'] = message;
     }
 

dev/devicelab/lib/tasks/build_test_task.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+
+import '../framework/adb.dart';
+import '../framework/task_result.dart';
+import '../framework/utils.dart';
+
+/// [Task] for defining build-test separation.
+///
+/// Using this [Task] allows DeviceLab capacity to only be spent on the [test].
+abstract class BuildTestTask {
+  BuildTestTask(this.args, {this.workingDirectory, this.runFlutterClean = true,}) {
+    final ArgResults argResults = argParser.parse(args);
+    applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
+    buildOnly = argResults[kBuildOnlyFlag] as bool;
+    testOnly = argResults[kTestOnlyFlag] as bool;
+
+  }
+
+  static const String kApplicationBinaryPathOption = 'application-binary-path';
+  static const String kBuildOnlyFlag = 'build';
+  static const String kTestOnlyFlag = 'test';
+
+  final ArgParser argParser = ArgParser()
+    ..addOption(kApplicationBinaryPathOption)
+    ..addFlag(kBuildOnlyFlag)
+    ..addFlag(kTestOnlyFlag);
+
+  /// Args passed from the test runner via "--task-arg".
+  final List<String> args;
+
+  /// If true, skip [test].
+  bool buildOnly = false;
+
+  /// If true, skip [build].
+  bool testOnly = false;
+
+  /// Whether to run `flutter clean` before building the application under test.
+  final bool runFlutterClean;
+
+  /// Path to a built application to use in [test].
+  ///
+  /// If not given, will default to child's expected location.
+  String applicationBinaryPath;
+
+  /// Where the test artifacts are stored, such as performance results.
+  final Directory workingDirectory;
+
+  /// Run Flutter build to create [applicationBinaryPath].
+  Future<void> build() async {
+    await inDirectory<void>(workingDirectory, () async {
+      if (runFlutterClean) {
+        section('FLUTTER CLEAN');
+        await flutter('clean');
+      }
+      section('BUILDING APPLICATION');
+      await flutter('build', options: getBuildArgs(deviceOperatingSystem));
+    });
+
+  }
+
+  /// Run Flutter drive test from [getTestArgs] against the application under test on the device.
+  ///
+  /// This assumes that [applicationBinaryPath] exists.
+  Future<TaskResult> test() async {
+    final Device device = await devices.workingDevice;
+    await device.unlock();
+    await inDirectory<void>(workingDirectory, () async {
+      section('DRIVE START');
+      await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
+    });
+
+    return parseTaskResult();
+  }
+
+  /// Args passed to flutter build to build the application under test.
+  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
+
+  /// Args passed to flutter drive to test the built application.
+  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
+
+  /// Logic to construct [TaskResult] from this test's results.
+  Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
+
+  /// Path to the built application under test.
+  ///
+  /// Tasks can override to support default values. Otherwise, it will default
+  /// to needing to be passed as an argument in the test runner.
+  String getApplicationBinaryPath() => applicationBinaryPath;
+
+  /// Run this task.
+  ///
+  /// Throws [Exception] when unnecessary arguments are passed.
+  Future<TaskResult> call() async {
+    if (buildOnly && testOnly) {
+      throw Exception('Both build and test should not be passed. Pass only one.');
+    }
+
+    if (buildOnly && applicationBinaryPath != null) {
+      throw Exception('Application binary path is only used for tests');
+    }
+
+    if (!testOnly) {
+      build();
+    }
+
+    if (buildOnly) {
+      return TaskResult.buildOnly();
+    }
+
+    return test();
+  }
+}

dev/devicelab/test/tasks/build_test_task_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/runner.dart';
+import 'package:flutter_devicelab/framework/task_result.dart';
+
+import '../common.dart';
+
+void main() {
+  final Map<String, String> isolateParams = <String, String>{
+    'runFlutterConfig': 'false',
+    'runProcessCleanup': 'false',
+    'timeoutInMinutes': '1',
+  };
+
+  test('runs build and test when no args are passed', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.data['benchmark'], 'data');
+  });
+
+  test('runs build only when build arg is given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--build'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'No tests run');
+  });
+
+  test('runs test only when test arg is given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--test'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.data['benchmark'], 'data');
+  });
+
+  test('throws exception when build and test arg are given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--build', '--test'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'Task failed: Exception: Both build and test should not be passed. Pass only one.');
+  });
+
+  test('throws exception when build and application binary arg are given', () async {
+    final TaskResult result = await runTask(
+      'smoke_test_build_test',
+      taskArgs: <String>['--build', '--application-binary-path=test.apk'],
+      deviceId: 'FAKE_SUCCESS',
+      isolateParams: isolateParams,
+    );
+    expect(result.message, 'Task failed: Exception: Application binary path is only used for tests');
+  });
+}