Unverified Commit 297c7b5c authored by Casey Hillers's avatar Casey Hillers Committed by GitHub

[devicelab] Separate build and test from Flutter gallery tests (#76415)

parent fc35508a
...@@ -9,7 +9,6 @@ import 'package:args/args.dart'; ...@@ -9,7 +9,6 @@ import 'package:args/args.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:flutter_devicelab/framework/ab.dart'; import 'package:flutter_devicelab/framework/ab.dart';
import 'package:flutter_devicelab/framework/cocoon.dart';
import 'package:flutter_devicelab/framework/manifest.dart'; import 'package:flutter_devicelab/framework/manifest.dart';
import 'package:flutter_devicelab/framework/runner.dart'; import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/task_result.dart'; import 'package:flutter_devicelab/framework/task_result.dart';
...@@ -105,46 +104,14 @@ Future<void> main(List<String> rawArgs) async { ...@@ -105,46 +104,14 @@ Future<void> main(List<String> rawArgs) async {
if (args.wasParsed('ab')) { if (args.wasParsed('ab')) {
await _runABTest(); await _runABTest();
} else { } else {
await _runTasks(); await runTasks(_taskNames,
}
}
Future<void> _runTasks() async {
for (final String taskName in _taskNames) {
section('Running task "$taskName"');
final TaskResult result = await runTask(
taskName,
silent: silent, silent: silent,
localEngine: localEngine, exitOnFirstTestFailure: exitOnFirstTestFailure,
localEngineSrcPath: localEngineSrcPath,
deviceId: deviceId, deviceId: deviceId,
gitBranch: gitBranch,
luciBuilder: luciBuilder,
resultsPath: resultsPath,
); );
print('Task result:');
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,
result: result,
resultsPath: resultsPath,
);
} else if (serviceAccountTokenFile != null) {
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
/// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
}
if (!result.succeeded) {
exitCode = 1;
if (exitOnFirstTestFailure) {
return;
}
}
} }
} }
......
...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; ...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android; deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionTest()); await task(createGalleryTransitionTest(args));
} }
...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; ...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android; deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionE2ETest()); await task(createGalleryTransitionE2ETest(args));
} }
...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; ...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios; deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionE2ETest()); await task(createGalleryTransitionE2ETest(args));
} }
...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; ...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios; deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionE2ETest()); await task(createGalleryTransitionE2ETest(args));
} }
...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; ...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android; deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionHybridTest()); await task(createGalleryTransitionHybridTest(args));
} }
...@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart'; ...@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart'; import 'package:flutter_devicelab/framework/task_result.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android; deviceOperatingSystem = DeviceOperatingSystem.android;
await task(() async { await task(() async {
final TaskResult withoutSemantics = await createGalleryTransitionTest()(); final TaskResult withoutSemantics = await createGalleryTransitionTest(args)();
final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)(); final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)();
if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) { if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) {
String message = 'Lack of data'; String message = 'Lack of data';
if (withSemantics.benchmarkScoreKeys.isEmpty) { if (withSemantics.benchmarkScoreKeys.isEmpty) {
......
...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; ...@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async { Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios; deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionTest()); await task(createGalleryTransitionTest(args));
} }
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/build_test_task.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
/// Smoke test of a successful task.
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.fake;
await task(FakeBuildTestTask(args));
}
class FakeBuildTestTask extends BuildTestTask {
FakeBuildTestTask(List<String> args) : super(args, runFlutterClean: false) {
deviceOperatingSystem = DeviceOperatingSystem.fake;
}
@override
// In prod, tasks always run some unit of work and the test framework assumes
// there will be some work done when managing the isolate. To fake this, add a delay.
Future<void> build() => Future<void>.delayed(const Duration(milliseconds: 500));
@override
Future<TaskResult> test() async {
await Future<void>.delayed(const Duration(milliseconds: 500));
return TaskResult.success(<String, String>{'benchmark': 'data'});
}
}
...@@ -5,10 +5,12 @@ ...@@ -5,10 +5,12 @@
import 'dart:io'; import 'dart:io';
import 'package:args/command_runner.dart'; import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/command/test.dart';
import 'package:flutter_devicelab/command/upload_metrics.dart'; import 'package:flutter_devicelab/command/upload_metrics.dart';
final CommandRunner<void> runner = final CommandRunner<void> runner =
CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications') CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
..addCommand(TestCommand())
..addCommand(UploadMetricsCommand()); ..addCommand(UploadMetricsCommand());
Future<void> main(List<String> rawArgs) async { Future<void> main(List<String> rawArgs) async {
......
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/framework/runner.dart';
class TestCommand extends Command<void> {
TestCommand() {
argParser.addOption('task',
abbr: 't',
help: 'The name of a task listed under bin/tasks.\n'
' Example: complex_layout__start_up.\n');
argParser.addMultiOption('task-args',
help: 'The name of a task listed under bin/tasks.\n'
'For example, "--task-args build" is passed as "bin/task/task.dart --build"');
argParser.addOption(
'device-id',
abbr: 'd',
help: 'Target device id (prefixes are allowed, names are not supported).\n'
'The option will be ignored if the test target does not run on a\n'
'mobile device. This still respects the device operating system\n'
'settings in the test case, and will results in error if no device\n'
'with given ID/ID prefix is found.',
);
argParser.addOption(
'git-branch',
help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
);
argParser.addOption(
'local-engine',
help: 'Name of a build output within the engine out directory, if you\n'
'are building Flutter locally. Use this to select a specific\n'
'version of the engine if you have built multiple engine targets.\n'
'This path is relative to --local-engine-src-path/out. This option\n'
'is required when running an A/B test (see the --ab option).',
);
argParser.addOption(
'local-engine-src-path',
help: 'Path to your engine src directory, if you are building Flutter\n'
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
'the location based on the value of the --flutter-root option.',
);
argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.');
argParser.addOption('results-file',
help: '[Flutter infrastructure] File path for test results. If passed with\n'
'task, will write test results to the file.');
argParser.addFlag(
'silent',
negatable: true,
defaultsTo: false,
);
}
@override
String get name => 'test';
@override
String get description => 'Run Flutter DeviceLab test';
@override
Future<void> run() async {
final List<String> taskArgsRaw = argResults['task-args'] as List<String>;
// Prepend '--' to convert args to options when passed to task
final List<String> taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList();
print(taskArgs);
await runTasks(
<String>[argResults['task'] as String],
deviceId: argResults['device-id'] as String,
gitBranch: argResults['git-branch'] as String,
localEngine: argResults['local-engine'] as String,
localEngineSrcPath: argResults['local-engine-src-path'] as String,
luciBuilder: argResults['luci-builder'] as String,
resultsPath: argResults['results-file'] as String,
silent: argResults['silent'] as bool,
taskArgs: taskArgs,
);
}
}
...@@ -41,8 +41,9 @@ bool _isTaskRegistered = false; ...@@ -41,8 +41,9 @@ bool _isTaskRegistered = false;
/// It is OK for a [task] to perform many things. However, only one task can be /// It is OK for a [task] to perform many things. However, only one task can be
/// registered per Dart VM. /// registered per Dart VM.
Future<TaskResult> task(TaskFunction task) async { Future<TaskResult> task(TaskFunction task) async {
if (_isTaskRegistered) if (_isTaskRegistered) {
throw StateError('A task is already registered'); throw StateError('A task is already registered');
}
_isTaskRegistered = true; _isTaskRegistered = true;
...@@ -59,16 +60,18 @@ Future<TaskResult> task(TaskFunction task) async { ...@@ -59,16 +60,18 @@ Future<TaskResult> task(TaskFunction task) async {
class _TaskRunner { class _TaskRunner {
_TaskRunner(this.task) { _TaskRunner(this.task) {
registerExtension('ext.cocoonRunTask', registerExtension('ext.cocoonRunTask', (String method, Map<String, String> parameters) async {
(String method, Map<String, String> parameters) async {
final Duration taskTimeout = parameters.containsKey('timeoutInMinutes') final Duration taskTimeout = parameters.containsKey('timeoutInMinutes')
? Duration(minutes: int.parse(parameters['timeoutInMinutes'])) ? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
: null; : null;
final TaskResult result = await run(taskTimeout); // This is only expected to be passed in unit test runs so they do not
// kill the Dart process that is running them.
final bool runProcessCleanup = parameters['runProcessCleanup'] != 'false';
final bool enableConfig = parameters['enableConfig'] != 'false';
final TaskResult result = await run(taskTimeout, runProcessCleanup: runProcessCleanup, enableConfig: enableConfig);
return ServiceExtensionResponse.result(json.encode(result.toJson())); return ServiceExtensionResponse.result(json.encode(result.toJson()));
}); });
registerExtension('ext.cocoonRunnerReady', registerExtension('ext.cocoonRunnerReady', (String method, Map<String, String> parameters) async {
(String method, Map<String, String> parameters) async {
return ServiceExtensionResponse.result('"ready"'); return ServiceExtensionResponse.result('"ready"');
}); });
} }
...@@ -87,59 +90,77 @@ class _TaskRunner { ...@@ -87,59 +90,77 @@ class _TaskRunner {
/// Signals that this task runner finished running the task. /// Signals that this task runner finished running the task.
Future<TaskResult> get whenDone => _completer.future; Future<TaskResult> get whenDone => _completer.future;
Future<TaskResult> run(Duration taskTimeout) async { Future<TaskResult> run(Duration taskTimeout, {
bool runProcessCleanup = true,
bool enableConfig = true,
}) async {
try { try {
_taskStarted = true; _taskStarted = true;
print('Running task with a timeout of $taskTimeout.'); print('Running task with a timeout of $taskTimeout.');
final String exe = Platform.isWindows ? '.exe' : ''; final String exe = Platform.isWindows ? '.exe' : '';
section('Checking running Dart$exe processes'); Set<RunningProcessInfo> beforeRunningDartInstances;
final Set<RunningProcessInfo> beforeRunningDartInstances = await getRunningProcesses( if (runProcessCleanup) {
processName: 'dart$exe', section('Checking running Dart$exe processes');
).toSet(); beforeRunningDartInstances = await getRunningProcesses(
final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet(); processName: 'dart$exe',
beforeRunningDartInstances.forEach(print); ).toSet();
for (final RunningProcessInfo info in allProcesses) { final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
if (info.commandLine.contains('iproxy')) { beforeRunningDartInstances.forEach(print);
print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} '); for (final RunningProcessInfo info in allProcesses) {
if (info.commandLine.contains('iproxy')) {
print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
}
} }
} }
print('enabling configs for macOS, Linux, Windows, and Web...'); if (enableConfig) {
final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[ print('enabling configs for macOS, Linux, Windows, and Web...');
'config', final int configResult = await exec(
'-v', path.join(flutterDirectory.path, 'bin', 'flutter'),
'--enable-macos-desktop', <String>[
'--enable-windows-desktop', 'config',
'--enable-linux-desktop', '-v',
'--enable-web', '--enable-macos-desktop',
if (localEngine != null) ...<String>['--local-engine', localEngine], '--enable-windows-desktop',
], canFail: true); '--enable-linux-desktop',
if (configResult != 0) { '--enable-web',
print('Failed to enable configuration, tasks may not run.'); if (localEngine != null) ...<String>['--local-engine', localEngine],
],
canFail: true);
if (configResult != 0) {
print('Failed to enable configuration, tasks may not run.');
}
} else {
section('Skipping flutter config. You should only see this in devicelab unit tests');
} }
Future<TaskResult> futureResult = _performTask(); Future<TaskResult> futureResult = _performTask();
if (taskTimeout != null) if (taskTimeout != null) {
futureResult = futureResult.timeout(taskTimeout); futureResult = futureResult.timeout(taskTimeout);
}
TaskResult result = await futureResult; TaskResult result = await futureResult;
section('Checking running Dart$exe processes after task...'); if (runProcessCleanup) {
final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses( section('Checking running Dart$exe processes after task...');
processName: 'dart$exe', final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
).toList(); processName: 'dart$exe',
for (final RunningProcessInfo info in afterRunningDartInstances) { ).toList();
if (!beforeRunningDartInstances.contains(info)) { for (final RunningProcessInfo info in afterRunningDartInstances) {
print('$info was leaked by this test.'); if (!beforeRunningDartInstances.contains(info)) {
if (result is TaskResultCheckProcesses) { print('$info was leaked by this test.');
result = TaskResult.failure('This test leaked dart processes'); if (result is TaskResultCheckProcesses) {
} result = TaskResult.failure('This test leaked dart processes');
final bool killed = await killProcess(info.pid); }
if (!killed) { final bool killed = await killProcess(info.pid);
print('Failed to kill process ${info.pid}.'); if (!killed) {
} else { print('Failed to kill process ${info.pid}.');
print('Killed process id ${info.pid}.'); } else {
print('Killed process id ${info.pid}.');
}
} }
} }
} else {
section('Skipping Dart process cleanup. You should only see this in devicelab unit tests');
} }
_completer.complete(result); _completer.complete(result);
return result; return result;
...@@ -149,8 +170,10 @@ class _TaskRunner { ...@@ -149,8 +170,10 @@ class _TaskRunner {
print(stackTrace); print(stackTrace);
return TaskResult.failure('Task timed out after $taskTimeout'); return TaskResult.failure('Task timed out after $taskTimeout');
} finally { } finally {
await checkForRebootRequired(); if (runProcessCleanup) {
await forceQuitRunningProcesses(); await checkForRebootRequired();
await forceQuitRunningProcesses();
}
_closeKeepAlivePort(); _closeKeepAlivePort();
} }
} }
...@@ -188,8 +211,9 @@ class _TaskRunner { ...@@ -188,8 +211,9 @@ class _TaskRunner {
/// Causes the Dart VM to stay alive until a request to run the task is /// Causes the Dart VM to stay alive until a request to run the task is
/// received via the VM service protocol. /// received via the VM service protocol.
void keepVmAliveUntilTaskRunRequested() { void keepVmAliveUntilTaskRunRequested() {
if (_taskStarted) if (_taskStarted) {
throw StateError('Task already started.'); throw StateError('Task already started.');
}
// Merely creating this port object will cause the VM to stay alive and keep // Merely creating this port object will cause the VM to stay alive and keep
// the VM service server running until the port is disposed of. // the VM service server running until the port is disposed of.
...@@ -218,17 +242,15 @@ class _TaskRunner { ...@@ -218,17 +242,15 @@ class _TaskRunner {
completer.complete(await task()); completer.complete(await task());
}, onError: (dynamic taskError, Chain taskErrorStack) { }, onError: (dynamic taskError, Chain taskErrorStack) {
final String message = 'Task failed: $taskError'; final String message = 'Task failed: $taskError';
stderr stderr..writeln(message)..writeln('\nStack trace:')..writeln(taskErrorStack.terse);
..writeln(message)
..writeln('\nStack trace:')
..writeln(taskErrorStack.terse);
// IMPORTANT: We're completing the future _successfully_ but with a value // IMPORTANT: We're completing the future _successfully_ but with a value
// that indicates a task failure. This is intentional. At this point we // that indicates a task failure. This is intentional. At this point we
// are catching errors coming from arbitrary (and untrustworthy) task // are catching errors coming from arbitrary (and untrustworthy) task
// code. Our goal is to convert the failure into a readable message. // code. Our goal is to convert the failure into a readable message.
// Propagating it further is not useful. // Propagating it further is not useful.
if (!completer.isCompleted) if (!completer.isCompleted) {
completer.complete(TaskResult.failure(message)); completer.complete(TaskResult.failure(message));
}
}); });
return completer.future; return completer.future;
} }
......
...@@ -6,13 +6,61 @@ import 'dart:async'; ...@@ -6,13 +6,61 @@ import 'dart:async';
import 'dart:convert'; import 'dart:convert';
import 'dart:io'; import 'dart:io';
import 'package:meta/meta.dart';
import 'package:vm_service_client/vm_service_client.dart'; import 'package:vm_service_client/vm_service_client.dart';
import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/adb.dart';
import 'cocoon.dart';
import 'task_result.dart'; import 'task_result.dart';
Future<void> runTasks(
List<String> taskNames, {
bool exitOnFirstTestFailure = false,
bool silent = false,
String deviceId,
String gitBranch,
String localEngine,
String localEngineSrcPath,
String luciBuilder,
String resultsPath,
List<String> taskArgs,
}) async {
for (final String taskName in taskNames) {
section('Running task "$taskName"');
final TaskResult result = await runTask(
taskName,
deviceId: deviceId,
localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath,
silent: silent,
taskArgs: taskArgs,
);
print('Task result:');
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,
result: result,
resultsPath: resultsPath,
);
}
if (!result.succeeded) {
exitCode = 1;
if (exitOnFirstTestFailure) {
return;
}
}
}
}
/// Runs a task in a separate Dart VM and collects the result using the VM /// Runs a task in a separate Dart VM and collects the result using the VM
/// service protocol. /// service protocol.
/// ///
...@@ -21,17 +69,22 @@ import 'task_result.dart'; ...@@ -21,17 +69,22 @@ import 'task_result.dart';
/// ///
/// Running the task in [silent] mode will suppress standard output from task /// Running the task in [silent] mode will suppress standard output from task
/// processes and only print standard errors. /// processes and only print standard errors.
///
/// [taskArgs] are passed to the task executable for additional configuration.
Future<TaskResult> runTask( Future<TaskResult> runTask(
String taskName, { String taskName, {
bool silent = false, bool silent = false,
String localEngine, String localEngine,
String localEngineSrcPath, String localEngineSrcPath,
String deviceId, String deviceId,
List<String> taskArgs,
@visibleForTesting Map<String, String> isolateParams = const <String, String>{},
}) async { }) async {
final String taskExecutable = 'bin/tasks/$taskName.dart'; final String taskExecutable = 'bin/tasks/$taskName.dart';
if (!file(taskExecutable).existsSync()) if (!file(taskExecutable).existsSync()) {
throw 'Executable Dart file not found: $taskExecutable'; throw 'Executable Dart file not found: $taskExecutable';
}
final Process runner = await startProcess( final Process runner = await startProcess(
dartBin, dartBin,
...@@ -42,10 +95,10 @@ Future<TaskResult> runTask( ...@@ -42,10 +95,10 @@ Future<TaskResult> runTask(
if (localEngine != null) '-DlocalEngine=$localEngine', if (localEngine != null) '-DlocalEngine=$localEngine',
if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath', if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
taskExecutable, taskExecutable,
...?taskArgs,
], ],
environment: <String, String>{ environment: <String, String>{
if (deviceId != null) if (deviceId != null) DeviceIdEnvName: deviceId,
DeviceIdEnvName: deviceId,
}, },
); );
...@@ -63,8 +116,9 @@ Future<TaskResult> runTask( ...@@ -63,8 +116,9 @@ Future<TaskResult> runTask(
.listen((String line) { .listen((String line) {
if (!uri.isCompleted) { if (!uri.isCompleted) {
final Uri serviceUri = parseServiceUri(line, prefix: 'Observatory listening on '); final Uri serviceUri = parseServiceUri(line, prefix: 'Observatory listening on ');
if (serviceUri != null) if (serviceUri != null) {
uri.complete(serviceUri); uri.complete(serviceUri);
}
} }
if (!silent) { if (!silent) {
stdout.writeln('[$taskName] [STDOUT] $line'); stdout.writeln('[$taskName] [STDOUT] $line');
...@@ -80,13 +134,15 @@ Future<TaskResult> runTask( ...@@ -80,13 +134,15 @@ Future<TaskResult> runTask(
try { try {
final VMIsolateRef isolate = await _connectToRunnerIsolate(await uri.future); final VMIsolateRef isolate = await _connectToRunnerIsolate(await uri.future);
final Map<String, dynamic> taskResultJson = await isolate.invokeExtension('ext.cocoonRunTask') as Map<String, dynamic>; final Map<String, dynamic> taskResultJson =
await isolate.invokeExtension('ext.cocoonRunTask', isolateParams) as Map<String, dynamic>;
final TaskResult taskResult = TaskResult.fromJson(taskResultJson); final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
await runner.exitCode; await runner.exitCode;
return taskResult; return taskResult;
} finally { } finally {
if (!runnerFinished) if (!runnerFinished) {
runner.kill(ProcessSignal.sigkill); runner.kill(ProcessSignal.sigkill);
}
await stdoutSub.cancel(); await stdoutSub.cancel();
await stderrSub.cancel(); await stderrSub.cancel();
} }
...@@ -98,8 +154,7 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async { ...@@ -98,8 +154,7 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async {
if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0], if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0],
'ws', 'ws',
]; ];
final String url = vmServiceUri.replace(scheme: 'ws', pathSegments: final String url = vmServiceUri.replace(scheme: 'ws', pathSegments: pathSegments).toString();
pathSegments).toString();
final Stopwatch stopwatch = Stopwatch()..start(); final Stopwatch stopwatch = Stopwatch()..start();
while (true) { while (true) {
...@@ -112,8 +167,9 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async { ...@@ -112,8 +167,9 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async {
final VM vm = await client.getVM(); final VM vm = await client.getVM();
final VMIsolateRef isolate = vm.isolates.single; final VMIsolateRef isolate = vm.isolates.single;
final String response = await isolate.invokeExtension('ext.cocoonRunnerReady') as String; final String response = await isolate.invokeExtension('ext.cocoonRunnerReady') as String;
if (response != 'ready') if (response != 'ready') {
throw 'not ready yet'; throw 'not ready yet';
}
return isolate; return isolate;
} catch (error) { } catch (error) {
if (stopwatch.elapsed > const Duration(seconds: 10)) if (stopwatch.elapsed > const Duration(seconds: 10))
......
...@@ -7,13 +7,20 @@ import 'dart:io'; ...@@ -7,13 +7,20 @@ import 'dart:io';
/// A result of running a single task. /// A result of running a single task.
class TaskResult { class TaskResult {
TaskResult.empty()
: succeeded = true,
data = null,
detailFiles = null,
benchmarkScoreKeys = null,
message = 'No tests run';
/// Constructs a successful result. /// Constructs a successful result.
TaskResult.success(this.data, { TaskResult.success(this.data, {
this.benchmarkScoreKeys = const <String>[], this.benchmarkScoreKeys = const <String>[],
this.detailFiles = const <String>[], this.detailFiles = const <String>[],
this.message = 'success',
}) })
: succeeded = true, : succeeded = true {
message = 'success' {
const JsonEncoder prettyJson = JsonEncoder.withIndent(' '); const JsonEncoder prettyJson = JsonEncoder.withIndent(' ');
if (benchmarkScoreKeys != null) { if (benchmarkScoreKeys != null) {
for (final String key in benchmarkScoreKeys) { for (final String key in benchmarkScoreKeys) {
...@@ -49,6 +56,7 @@ class TaskResult { ...@@ -49,6 +56,7 @@ class TaskResult {
return TaskResult.success(json['data'] as Map<String, dynamic>, return TaskResult.success(json['data'] as Map<String, dynamic>,
benchmarkScoreKeys: benchmarkScoreKeys, benchmarkScoreKeys: benchmarkScoreKeys,
detailFiles: detailFiles, detailFiles: detailFiles,
message: json['reason'] as String,
); );
} }
...@@ -106,7 +114,8 @@ class TaskResult { ...@@ -106,7 +114,8 @@ class TaskResult {
json['data'] = data; json['data'] = data;
json['detailFiles'] = detailFiles; json['detailFiles'] = detailFiles;
json['benchmarkScoreKeys'] = benchmarkScoreKeys; json['benchmarkScoreKeys'] = benchmarkScoreKeys;
} else { }
if (message != null || !succeeded) {
json['reason'] = message; json['reason'] = message;
} }
......
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:args/args.dart';
import '../framework/adb.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
/// [Task] for defining build-test separation.
///
/// Using this [Task] allows DeviceLab capacity to only be spent on the [test].
abstract class BuildTestTask {
BuildTestTask(this.args, {this.workingDirectory, this.runFlutterClean = true,}) {
final ArgResults argResults = argParser.parse(args);
applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
buildOnly = argResults[kBuildOnlyFlag] as bool;
testOnly = argResults[kTestOnlyFlag] as bool;
}
static const String kApplicationBinaryPathOption = 'application-binary-path';
static const String kBuildOnlyFlag = 'build';
static const String kTestOnlyFlag = 'test';
final ArgParser argParser = ArgParser()
..addOption(kApplicationBinaryPathOption)
..addFlag(kBuildOnlyFlag)
..addFlag(kTestOnlyFlag);
/// Args passed from the test runner via "--task-arg".
final List<String> args;
/// If true, skip [test].
bool buildOnly = false;
/// If true, skip [build].
bool testOnly = false;
/// Whether to run `flutter clean` before building the application under test.
final bool runFlutterClean;
/// Path to a built application to use in [test].
///
/// If not given, will default to child's expected location.
String applicationBinaryPath;
/// Where the test artifacts are stored, such as performance results.
final Directory workingDirectory;
/// Run Flutter build to create [applicationBinaryPath].
Future<void> build() async {
await inDirectory<void>(workingDirectory, () async {
if (runFlutterClean) {
section('FLUTTER CLEAN');
await flutter('clean');
}
section('BUILDING APPLICATION');
await flutter('build', options: getBuildArgs(deviceOperatingSystem));
});
}
/// Run Flutter drive test from [getTestArgs] against the application under test on the device.
///
/// This assumes that [applicationBinaryPath] exists.
Future<TaskResult> test() async {
final Device device = await devices.workingDevice;
await device.unlock();
await inDirectory<void>(workingDirectory, () async {
section('DRIVE START');
await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
});
return parseTaskResult();
}
/// Args passed to flutter build to build the application under test.
List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
/// Args passed to flutter drive to test the built application.
List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
/// Logic to construct [TaskResult] from this test's results.
Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
/// Path to the built application under test.
///
/// Tasks can override to support default values. Otherwise, it will default
/// to needing to be passed as an argument in the test runner.
String getApplicationBinaryPath() => applicationBinaryPath;
/// Run this task.
///
/// Throws [Exception] when unnecessary arguments are passed.
Future<TaskResult> call() async {
if (buildOnly && testOnly) {
throw Exception('Both build and test should not be passed. Pass only one.');
}
if (buildOnly && applicationBinaryPath != null) {
throw Exception('Application binary path is only used for tests');
}
if (!testOnly) {
build();
}
if (buildOnly) {
return TaskResult.empty();
}
return test();
}
}
...@@ -11,13 +11,17 @@ import '../framework/adb.dart'; ...@@ -11,13 +11,17 @@ import '../framework/adb.dart';
import '../framework/framework.dart'; import '../framework/framework.dart';
import '../framework/task_result.dart'; import '../framework/task_result.dart';
import '../framework/utils.dart'; import '../framework/utils.dart';
import 'build_test_task.dart';
TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) { final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
TaskFunction createGalleryTransitionTest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,);
} }
TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) { TaskFunction createGalleryTransitionE2ETest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest( return GalleryTransitionTest(
args,
testFile: semanticsEnabled testFile: semanticsEnabled
? 'transitions_perf_e2e_with_semantics' ? 'transitions_perf_e2e_with_semantics'
: 'transitions_perf_e2e', : 'transitions_perf_e2e',
...@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) { ...@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
transitionDurationFile: null, transitionDurationFile: null,
timelineTraceFile: null, timelineTraceFile: null,
driverFile: 'transitions_perf_e2e_test', driverFile: 'transitions_perf_e2e_test',
workingDirectory: galleryDirectory,
); );
} }
TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) { TaskFunction createGalleryTransitionHybridTest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest( return GalleryTransitionTest(
args,
semanticsEnabled: semanticsEnabled, semanticsEnabled: semanticsEnabled,
driverFile: semanticsEnabled driverFile: semanticsEnabled
? 'transitions_perf_hybrid_with_semantics_test' ? 'transitions_perf_hybrid_with_semantics_test'
: 'transitions_perf_hybrid_test', : 'transitions_perf_hybrid_test',
workingDirectory: galleryDirectory,
); );
} }
class GalleryTransitionTest { class GalleryTransitionTest extends BuildTestTask {
GalleryTransitionTest(List<String> args, {
GalleryTransitionTest({
this.semanticsEnabled = false, this.semanticsEnabled = false,
this.testFile = 'transitions_perf', this.testFile = 'transitions_perf',
this.needFullTimeline = true, this.needFullTimeline = true,
...@@ -48,7 +54,8 @@ class GalleryTransitionTest { ...@@ -48,7 +54,8 @@ class GalleryTransitionTest {
this.timelineTraceFile = 'transitions.timeline', this.timelineTraceFile = 'transitions.timeline',
this.transitionDurationFile = 'transition_durations.timeline', this.transitionDurationFile = 'transition_durations.timeline',
this.driverFile, this.driverFile,
}); Directory workingDirectory,
}) : super(args, workingDirectory: workingDirectory);
final bool semanticsEnabled; final bool semanticsEnabled;
final bool needFullTimeline; final bool needFullTimeline;
...@@ -58,59 +65,48 @@ class GalleryTransitionTest { ...@@ -58,59 +65,48 @@ class GalleryTransitionTest {
final String transitionDurationFile; final String transitionDurationFile;
final String driverFile; final String driverFile;
Future<TaskResult> call() async { @override
final Device device = await devices.workingDevice; List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
await device.unlock(); switch (deviceOperatingSystem) {
final String deviceId = device.deviceId; case DeviceOperatingSystem.android:
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery'); return <String>[
await inDirectory<void>(galleryDirectory, () async { 'apk',
String applicationBinaryPath; '--no-android-gradle-daemon',
if (deviceOperatingSystem == DeviceOperatingSystem.android) { '--profile',
section('BUILDING APPLICATION'); '-t',
await flutter( 'test_driver/$testFile.dart',
'build', '--target-platform',
options: <String>[ 'android-arm,android-arm64',
'apk', ];
'--no-android-gradle-daemon', default:
'--profile', throw Exception('$deviceOperatingSystem has no build configuration');
'-t',
'test_driver/$testFile.dart',
'--target-platform',
'android-arm,android-arm64',
],
);
applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
} }
}
final String testDriver = driverFile ?? (semanticsEnabled @override
? '${testFile}_with_semantics_test' List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
: '${testFile}_test'); final String testDriver = driverFile ?? (semanticsEnabled
section('DRIVE START'); ? '${testFile}_with_semantics_test'
await flutter('drive', options: <String>[ : '${testFile}_test');
return <String>[
'--profile', '--profile',
if (needFullTimeline) if (needFullTimeline)
'--trace-startup', '--trace-startup',
if (applicationBinaryPath != null) '--use-application-binary=${getApplicationBinaryPath()}',
'--use-application-binary=$applicationBinaryPath' '--driver', 'test_driver/$testDriver.dart',
else '-d', deviceId,
...<String>[ ];
'-t', }
'test_driver/$testFile.dart',
],
'--driver',
'test_driver/$testDriver.dart',
'-d',
deviceId,
]);
});
@override
Future<TaskResult> parseTaskResult() async {
final Map<String, dynamic> summary = json.decode( final Map<String, dynamic> summary = json.decode(
file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(), file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
) as Map<String, dynamic>; ) as Map<String, dynamic>;
if (transitionDurationFile != null) { if (transitionDurationFile != null) {
final Map<String, dynamic> original = json.decode( final Map<String, dynamic> original = json.decode(
file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(), file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
) as Map<String, dynamic>; ) as Map<String, dynamic>;
final Map<String, List<int>> transitions = <String, List<int>>{}; final Map<String, List<int>> transitions = <String, List<int>>{};
for (final String key in original.keys) { for (final String key in original.keys) {
...@@ -123,9 +119,9 @@ class GalleryTransitionTest { ...@@ -123,9 +119,9 @@ class GalleryTransitionTest {
return TaskResult.success(summary, return TaskResult.success(summary,
detailFiles: <String>[ detailFiles: <String>[
if (transitionDurationFile != null) if (transitionDurationFile != null)
'${galleryDirectory.path}/build/$transitionDurationFile.json', '${workingDirectory.path}/build/$transitionDurationFile.json',
if (timelineTraceFile != null) if (timelineTraceFile != null)
'${galleryDirectory.path}/build/$timelineTraceFile.json' '${workingDirectory.path}/build/$timelineTraceFile.json'
], ],
benchmarkScoreKeys: <String>[ benchmarkScoreKeys: <String>[
if (transitionDurationFile != null) if (transitionDurationFile != null)
...@@ -141,6 +137,20 @@ class GalleryTransitionTest { ...@@ -141,6 +137,20 @@ class GalleryTransitionTest {
], ],
); );
} }
@override
String getApplicationBinaryPath() {
if (applicationBinaryPath != null) {
return applicationBinaryPath;
}
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
return 'build/app/outputs/flutter-apk/app-profile.apk';
default:
throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem');
}
}
} }
int _countMissedTransitions(Map<String, List<int>> transitions) { int _countMissedTransitions(Map<String, List<int>> transitions) {
......
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import '../common.dart';
void main() {
final Map<String, String> isolateParams = <String, String>{
'enableConfig': 'false',
'runProcessCleanup': 'false',
'timeoutInMinutes': '1',
};
test('runs build and test when no args are passed', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.data['benchmark'], 'data');
});
test('runs build only when build arg is given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--build'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.message, 'No tests run');
});
test('runs test only when test arg is given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--test'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.data['benchmark'], 'data');
});
test('throws exception when build and test arg are given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--build', '--test'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.message, 'Task failed: Exception: Both build and test should not be passed. Pass only one.');
});
test('throws exception when build and application binary arg are given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--build', '--application-binary-path=test.apk'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.message, 'Task failed: Exception: Application binary path is only used for tests');
});
}
...@@ -20,11 +20,10 @@ void main() { ...@@ -20,11 +20,10 @@ void main() {
group('parse service', () { group('parse service', () {
const String badOutput = 'No uri here'; const String badOutput = 'No uri here';
const String sampleOutput = 'An Observatory debugger and profiler on ' const String sampleOutput = 'An Observatory debugger and profiler on '
'Pixel 3 XL is available at: http://127.0.0.1:9090/LpjUpsdEjqI=/'; 'Pixel 3 XL is available at: http://127.0.0.1:9090/LpjUpsdEjqI=/';
test('uri', () { test('uri', () {
expect(parseServiceUri(sampleOutput), expect(parseServiceUri(sampleOutput), Uri.parse('http://127.0.0.1:9090/LpjUpsdEjqI=/'));
Uri.parse('http://127.0.0.1:9090/LpjUpsdEjqI=/'));
expect(parseServiceUri(badOutput), null); expect(parseServiceUri(badOutput), null);
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment