Skip to content

F
Front-End
Unverified Commit 297c7b5c authored by Casey Hillers's avatar Casey Hillers Committed by GitHub
Browse files
Options

[devicelab] Separate build and test from Flutter gallery tests (#76415)

parent fc35508a
Hide whitespace changes
Inline Side-by-side
Showing with 534 additions and 174 deletions
dev/devicelab/bin/run.dart
View file @ 297c7b5c
......@@ -9,7 +9,6 @@ import 'package:args/args.dart';
import 'package:path/path.dart' as path;
import 'package:flutter_devicelab/framework/ab.dart';
import 'package:flutter_devicelab/framework/cocoon.dart';
import 'package:flutter_devicelab/framework/manifest.dart';
import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
......@@ -105,46 +104,14 @@ Future<void> main(List<String> rawArgs) async {
if (args.wasParsed('ab')) {
await _runABTest();
} else {
await _runTasks();
}
}
Future<void> _runTasks() async {
for (final String taskName in _taskNames) {
section('Running task "$taskName"');
final TaskResult result = await runTask(
taskName,
await runTasks(_taskNames,
silent: silent,
localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath,
exitOnFirstTestFailure: exitOnFirstTestFailure,
deviceId: deviceId,
gitBranch: gitBranch,
luciBuilder: luciBuilder,
resultsPath: resultsPath,
);
print('Task result:');
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,
result: result,
resultsPath: resultsPath,
);
} else if (serviceAccountTokenFile != null) {
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
/// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
}
if (!result.succeeded) {
exitCode = 1;
if (exitOnFirstTestFailure) {
return;
}
}
}
}
......
dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart
View file @ 297c7b5c
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionTest());
await task(createGalleryTransitionTest(args));
}
dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart
View file @ 297c7b5c
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionE2ETest());
await task(createGalleryTransitionE2ETest(args));
}
dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart
View file @ 297c7b5c
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionE2ETest());
await task(createGalleryTransitionE2ETest(args));
}
dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart
View file @ 297c7b5c
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionE2ETest());
await task(createGalleryTransitionE2ETest(args));
}
dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart
View file @ 297c7b5c
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createGalleryTransitionHybridTest());
await task(createGalleryTransitionHybridTest(args));
}
dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart
View file @ 297c7b5c
......@@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(() async {
final TaskResult withoutSemantics = await createGalleryTransitionTest()();
final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)();
final TaskResult withoutSemantics = await createGalleryTransitionTest(args)();
final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)();
if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) {
String message = 'Lack of data';
if (withSemantics.benchmarkScoreKeys.isEmpty) {
......
dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart
View file @ 297c7b5c
......@@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(createGalleryTransitionTest());
await task(createGalleryTransitionTest(args));
}
dev/devicelab/bin/tasks/smoke_test_build_test.dart 0 → 100644
View file @ 297c7b5c
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/build_test_task.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
/// Smoke test of a successful task.
Future<void> main(List<String> args) async {
deviceOperatingSystem = DeviceOperatingSystem.fake;
await task(FakeBuildTestTask(args));
}
class FakeBuildTestTask extends BuildTestTask {
FakeBuildTestTask(List<String> args) : super(args, runFlutterClean: false) {
deviceOperatingSystem = DeviceOperatingSystem.fake;
}
@override
// In prod, tasks always run some unit of work and the test framework assumes
// there will be some work done when managing the isolate. To fake this, add a delay.
Future<void> build() => Future<void>.delayed(const Duration(milliseconds: 500));
@override
Future<TaskResult> test() async {
await Future<void>.delayed(const Duration(milliseconds: 500));
return TaskResult.success(<String, String>{'benchmark': 'data'});
}
}
dev/devicelab/bin/test_runner.dart
View file @ 297c7b5c
......@@ -5,10 +5,12 @@
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/command/test.dart';
import 'package:flutter_devicelab/command/upload_metrics.dart';
final CommandRunner<void> runner =
CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
..addCommand(TestCommand())
..addCommand(UploadMetricsCommand());
Future<void> main(List<String> rawArgs) async {
......
dev/devicelab/lib/command/test.dart 0 → 100644
View file @ 297c7b5c
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/framework/runner.dart';
class TestCommand extends Command<void> {
TestCommand() {
argParser.addOption('task',
abbr: 't',
help: 'The name of a task listed under bin/tasks.\n'
' Example: complex_layout__start_up.\n');
argParser.addMultiOption('task-args',
help: 'The name of a task listed under bin/tasks.\n'
'For example, "--task-args build" is passed as "bin/task/task.dart --build"');
argParser.addOption(
'device-id',
abbr: 'd',
help: 'Target device id (prefixes are allowed, names are not supported).\n'
'The option will be ignored if the test target does not run on a\n'
'mobile device. This still respects the device operating system\n'
'settings in the test case, and will results in error if no device\n'
'with given ID/ID prefix is found.',
);
argParser.addOption(
'git-branch',
help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
);
argParser.addOption(
'local-engine',
help: 'Name of a build output within the engine out directory, if you\n'
'are building Flutter locally. Use this to select a specific\n'
'version of the engine if you have built multiple engine targets.\n'
'This path is relative to --local-engine-src-path/out. This option\n'
'is required when running an A/B test (see the --ab option).',
);
argParser.addOption(
'local-engine-src-path',
help: 'Path to your engine src directory, if you are building Flutter\n'
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
'the location based on the value of the --flutter-root option.',
);
argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.');
argParser.addOption('results-file',
help: '[Flutter infrastructure] File path for test results. If passed with\n'
'task, will write test results to the file.');
argParser.addFlag(
'silent',
negatable: true,
defaultsTo: false,
);
}
@override
String get name => 'test';
@override
String get description => 'Run Flutter DeviceLab test';
@override
Future<void> run() async {
final List<String> taskArgsRaw = argResults['task-args'] as List<String>;
// Prepend '--' to convert args to options when passed to task
final List<String> taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList();
print(taskArgs);
await runTasks(
<String>[argResults['task'] as String],
deviceId: argResults['device-id'] as String,
gitBranch: argResults['git-branch'] as String,
localEngine: argResults['local-engine'] as String,
localEngineSrcPath: argResults['local-engine-src-path'] as String,
luciBuilder: argResults['luci-builder'] as String,
resultsPath: argResults['results-file'] as String,
silent: argResults['silent'] as bool,
taskArgs: taskArgs,
);
}
}
dev/devicelab/lib/framework/framework.dart
View file @ 297c7b5c
......@@ -41,8 +41,9 @@ bool _isTaskRegistered = false;
/// It is OK for a [task] to perform many things. However, only one task can be
/// registered per Dart VM.
Future<TaskResult> task(TaskFunction task) async {
if (_isTaskRegistered)
if (_isTaskRegistered) {
throw StateError('A task is already registered');
}
_isTaskRegistered = true;
......@@ -59,16 +60,18 @@ Future<TaskResult> task(TaskFunction task) async {
class _TaskRunner {
_TaskRunner(this.task) {
registerExtension('ext.cocoonRunTask',
(String method, Map<String, String> parameters) async {
registerExtension('ext.cocoonRunTask', (String method, Map<String, String> parameters) async {
final Duration taskTimeout = parameters.containsKey('timeoutInMinutes')
? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
: null;
final TaskResult result = await run(taskTimeout);
? Duration(minutes: int.parse(parameters['timeoutInMinutes']))
: null;
// This is only expected to be passed in unit test runs so they do not
// kill the Dart process that is running them.
final bool runProcessCleanup = parameters['runProcessCleanup'] != 'false';
final bool enableConfig = parameters['enableConfig'] != 'false';
final TaskResult result = await run(taskTimeout, runProcessCleanup: runProcessCleanup, enableConfig: enableConfig);
return ServiceExtensionResponse.result(json.encode(result.toJson()));
});
registerExtension('ext.cocoonRunnerReady',
(String method, Map<String, String> parameters) async {
registerExtension('ext.cocoonRunnerReady', (String method, Map<String, String> parameters) async {
return ServiceExtensionResponse.result('"ready"');
});
}
......@@ -87,59 +90,77 @@ class _TaskRunner {
/// Signals that this task runner finished running the task.
Future<TaskResult> get whenDone => _completer.future;
Future<TaskResult> run(Duration taskTimeout) async {
Future<TaskResult> run(Duration taskTimeout, {
bool runProcessCleanup = true,
bool enableConfig = true,
}) async {
try {
_taskStarted = true;
print('Running task with a timeout of $taskTimeout.');
final String exe = Platform.isWindows ? '.exe' : '';
section('Checking running Dart$exe processes');
final Set<RunningProcessInfo> beforeRunningDartInstances = await getRunningProcesses(
processName: 'dart$exe',
).toSet();
final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
beforeRunningDartInstances.forEach(print);
for (final RunningProcessInfo info in allProcesses) {
if (info.commandLine.contains('iproxy')) {
print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
Set<RunningProcessInfo> beforeRunningDartInstances;
if (runProcessCleanup) {
section('Checking running Dart$exe processes');
beforeRunningDartInstances = await getRunningProcesses(
processName: 'dart$exe',
).toSet();
final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet();
beforeRunningDartInstances.forEach(print);
for (final RunningProcessInfo info in allProcesses) {
if (info.commandLine.contains('iproxy')) {
print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
}
}
}
print('enabling configs for macOS, Linux, Windows, and Web...');
final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
'config',
'-v',
'--enable-macos-desktop',
'--enable-windows-desktop',
'--enable-linux-desktop',
'--enable-web',
if (localEngine != null) ...<String>['--local-engine', localEngine],
], canFail: true);
if (configResult != 0) {
print('Failed to enable configuration, tasks may not run.');
if (enableConfig) {
print('enabling configs for macOS, Linux, Windows, and Web...');
final int configResult = await exec(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>[
'config',
'-v',
'--enable-macos-desktop',
'--enable-windows-desktop',
'--enable-linux-desktop',
'--enable-web',
if (localEngine != null) ...<String>['--local-engine', localEngine],
],
canFail: true);
if (configResult != 0) {
print('Failed to enable configuration, tasks may not run.');
}
} else {
section('Skipping flutter config. You should only see this in devicelab unit tests');
}
Future<TaskResult> futureResult = _performTask();
if (taskTimeout != null)
if (taskTimeout != null) {
futureResult = futureResult.timeout(taskTimeout);
}
TaskResult result = await futureResult;
section('Checking running Dart$exe processes after task...');
final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
processName: 'dart$exe',
).toList();
for (final RunningProcessInfo info in afterRunningDartInstances) {
if (!beforeRunningDartInstances.contains(info)) {
print('$info was leaked by this test.');
if (result is TaskResultCheckProcesses) {
result = TaskResult.failure('This test leaked dart processes');
}
final bool killed = await killProcess(info.pid);
if (!killed) {
print('Failed to kill process ${info.pid}.');
} else {
print('Killed process id ${info.pid}.');
if (runProcessCleanup) {
section('Checking running Dart$exe processes after task...');
final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
processName: 'dart$exe',
).toList();
for (final RunningProcessInfo info in afterRunningDartInstances) {
if (!beforeRunningDartInstances.contains(info)) {
print('$info was leaked by this test.');
if (result is TaskResultCheckProcesses) {
result = TaskResult.failure('This test leaked dart processes');
}
final bool killed = await killProcess(info.pid);
if (!killed) {
print('Failed to kill process ${info.pid}.');
} else {
print('Killed process id ${info.pid}.');
}
}
}
} else {
section('Skipping Dart process cleanup. You should only see this in devicelab unit tests');
}
_completer.complete(result);
return result;
......@@ -149,8 +170,10 @@ class _TaskRunner {
print(stackTrace);
return TaskResult.failure('Task timed out after $taskTimeout');
} finally {
await checkForRebootRequired();
await forceQuitRunningProcesses();
if (runProcessCleanup) {
await checkForRebootRequired();
await forceQuitRunningProcesses();
}
_closeKeepAlivePort();
}
}
......@@ -188,8 +211,9 @@ class _TaskRunner {
/// Causes the Dart VM to stay alive until a request to run the task is
/// received via the VM service protocol.
void keepVmAliveUntilTaskRunRequested() {
if (_taskStarted)
if (_taskStarted) {
throw StateError('Task already started.');
}
// Merely creating this port object will cause the VM to stay alive and keep
// the VM service server running until the port is disposed of.
......@@ -218,17 +242,15 @@ class _TaskRunner {
completer.complete(await task());
}, onError: (dynamic taskError, Chain taskErrorStack) {
final String message = 'Task failed: $taskError';
stderr
..writeln(message)
..writeln('\nStack trace:')
..writeln(taskErrorStack.terse);
stderr..writeln(message)..writeln('\nStack trace:')..writeln(taskErrorStack.terse);
// IMPORTANT: We're completing the future _successfully_ but with a value
// that indicates a task failure. This is intentional. At this point we
// are catching errors coming from arbitrary (and untrustworthy) task
// code. Our goal is to convert the failure into a readable message.
// Propagating it further is not useful.
if (!completer.isCompleted)
if (!completer.isCompleted) {
completer.complete(TaskResult.failure(message));
}
});
return completer.future;
}
......
dev/devicelab/lib/framework/runner.dart
View file @ 297c7b5c
......@@ -6,13 +6,61 @@ import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:meta/meta.dart';
import 'package:vm_service_client/vm_service_client.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'cocoon.dart';
import 'task_result.dart';
Future<void> runTasks(
List<String> taskNames, {
bool exitOnFirstTestFailure = false,
bool silent = false,
String deviceId,
String gitBranch,
String localEngine,
String localEngineSrcPath,
String luciBuilder,
String resultsPath,
List<String> taskArgs,
}) async {
for (final String taskName in taskNames) {
section('Running task "$taskName"');
final TaskResult result = await runTask(
taskName,
deviceId: deviceId,
localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath,
silent: silent,
taskArgs: taskArgs,
);
print('Task result:');
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,
result: result,
resultsPath: resultsPath,
);
}
if (!result.succeeded) {
exitCode = 1;
if (exitOnFirstTestFailure) {
return;
}
}
}
}
/// Runs a task in a separate Dart VM and collects the result using the VM
/// service protocol.
///
......@@ -21,17 +69,22 @@ import 'task_result.dart';
///
/// Running the task in [silent] mode will suppress standard output from task
/// processes and only print standard errors.
///
/// [taskArgs] are passed to the task executable for additional configuration.
Future<TaskResult> runTask(
String taskName, {
bool silent = false,
String localEngine,
String localEngineSrcPath,
String deviceId,
List<String> taskArgs,
@visibleForTesting Map<String, String> isolateParams = const <String, String>{},
}) async {
final String taskExecutable = 'bin/tasks/$taskName.dart';
if (!file(taskExecutable).existsSync())
if (!file(taskExecutable).existsSync()) {
throw 'Executable Dart file not found: $taskExecutable';
}
final Process runner = await startProcess(
dartBin,
......@@ -42,10 +95,10 @@ Future<TaskResult> runTask(
if (localEngine != null) '-DlocalEngine=$localEngine',
if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
taskExecutable,
...?taskArgs,
],
environment: <String, String>{
if (deviceId != null)
DeviceIdEnvName: deviceId,
if (deviceId != null) DeviceIdEnvName: deviceId,
},
);
......@@ -63,8 +116,9 @@ Future<TaskResult> runTask(
.listen((String line) {
if (!uri.isCompleted) {
final Uri serviceUri = parseServiceUri(line, prefix: 'Observatory listening on ');
if (serviceUri != null)
if (serviceUri != null) {
uri.complete(serviceUri);
}
}
if (!silent) {
stdout.writeln('[$taskName] [STDOUT] $line');
......@@ -80,13 +134,15 @@ Future<TaskResult> runTask(
try {
final VMIsolateRef isolate = await _connectToRunnerIsolate(await uri.future);
final Map<String, dynamic> taskResultJson = await isolate.invokeExtension('ext.cocoonRunTask') as Map<String, dynamic>;
final Map<String, dynamic> taskResultJson =
await isolate.invokeExtension('ext.cocoonRunTask', isolateParams) as Map<String, dynamic>;
final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
await runner.exitCode;
return taskResult;
} finally {
if (!runnerFinished)
if (!runnerFinished) {
runner.kill(ProcessSignal.sigkill);
}
await stdoutSub.cancel();
await stderrSub.cancel();
}
......@@ -98,8 +154,7 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async {
if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0],
'ws',
];
final String url = vmServiceUri.replace(scheme: 'ws', pathSegments:
pathSegments).toString();
final String url = vmServiceUri.replace(scheme: 'ws', pathSegments: pathSegments).toString();
final Stopwatch stopwatch = Stopwatch()..start();
while (true) {
......@@ -112,8 +167,9 @@ Future<VMIsolateRef> _connectToRunnerIsolate(Uri vmServiceUri) async {
final VM vm = await client.getVM();
final VMIsolateRef isolate = vm.isolates.single;
final String response = await isolate.invokeExtension('ext.cocoonRunnerReady') as String;
if (response != 'ready')
if (response != 'ready') {
throw 'not ready yet';
}
return isolate;
} catch (error) {
if (stopwatch.elapsed > const Duration(seconds: 10))
......
dev/devicelab/lib/framework/task_result.dart
View file @ 297c7b5c
......@@ -7,13 +7,20 @@ import 'dart:io';
/// A result of running a single task.
class TaskResult {
TaskResult.empty()
: succeeded = true,
data = null,
detailFiles = null,
benchmarkScoreKeys = null,
message = 'No tests run';
/// Constructs a successful result.
TaskResult.success(this.data, {
this.benchmarkScoreKeys = const <String>[],
this.detailFiles = const <String>[],
this.message = 'success',
})
: succeeded = true,
message = 'success' {
: succeeded = true {
const JsonEncoder prettyJson = JsonEncoder.withIndent(' ');
if (benchmarkScoreKeys != null) {
for (final String key in benchmarkScoreKeys) {
......@@ -49,6 +56,7 @@ class TaskResult {
return TaskResult.success(json['data'] as Map<String, dynamic>,
benchmarkScoreKeys: benchmarkScoreKeys,
detailFiles: detailFiles,
message: json['reason'] as String,
);
}
......@@ -106,7 +114,8 @@ class TaskResult {
json['data'] = data;
json['detailFiles'] = detailFiles;
json['benchmarkScoreKeys'] = benchmarkScoreKeys;
} else {
}
if (message != null || !succeeded) {
json['reason'] = message;
}
......
dev/devicelab/lib/tasks/build_test_task.dart 0 → 100644
View file @ 297c7b5c
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:args/args.dart';
import '../framework/adb.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
/// [Task] for defining build-test separation.
///
/// Using this [Task] allows DeviceLab capacity to only be spent on the [test].
abstract class BuildTestTask {
BuildTestTask(this.args, {this.workingDirectory, this.runFlutterClean = true,}) {
final ArgResults argResults = argParser.parse(args);
applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String;
buildOnly = argResults[kBuildOnlyFlag] as bool;
testOnly = argResults[kTestOnlyFlag] as bool;
}
static const String kApplicationBinaryPathOption = 'application-binary-path';
static const String kBuildOnlyFlag = 'build';
static const String kTestOnlyFlag = 'test';
final ArgParser argParser = ArgParser()
..addOption(kApplicationBinaryPathOption)
..addFlag(kBuildOnlyFlag)
..addFlag(kTestOnlyFlag);
/// Args passed from the test runner via "--task-arg".
final List<String> args;
/// If true, skip [test].
bool buildOnly = false;
/// If true, skip [build].
bool testOnly = false;
/// Whether to run `flutter clean` before building the application under test.
final bool runFlutterClean;
/// Path to a built application to use in [test].
///
/// If not given, will default to child's expected location.
String applicationBinaryPath;
/// Where the test artifacts are stored, such as performance results.
final Directory workingDirectory;
/// Run Flutter build to create [applicationBinaryPath].
Future<void> build() async {
await inDirectory<void>(workingDirectory, () async {
if (runFlutterClean) {
section('FLUTTER CLEAN');
await flutter('clean');
}
section('BUILDING APPLICATION');
await flutter('build', options: getBuildArgs(deviceOperatingSystem));
});
}
/// Run Flutter drive test from [getTestArgs] against the application under test on the device.
///
/// This assumes that [applicationBinaryPath] exists.
Future<TaskResult> test() async {
final Device device = await devices.workingDevice;
await device.unlock();
await inDirectory<void>(workingDirectory, () async {
section('DRIVE START');
await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId));
});
return parseTaskResult();
}
/// Args passed to flutter build to build the application under test.
List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented');
/// Args passed to flutter drive to test the built application.
List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented');
/// Logic to construct [TaskResult] from this test's results.
Future<TaskResult> parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented');
/// Path to the built application under test.
///
/// Tasks can override to support default values. Otherwise, it will default
/// to needing to be passed as an argument in the test runner.
String getApplicationBinaryPath() => applicationBinaryPath;
/// Run this task.
///
/// Throws [Exception] when unnecessary arguments are passed.
Future<TaskResult> call() async {
if (buildOnly && testOnly) {
throw Exception('Both build and test should not be passed. Pass only one.');
}
if (buildOnly && applicationBinaryPath != null) {
throw Exception('Application binary path is only used for tests');
}
if (!testOnly) {
build();
}
if (buildOnly) {
return TaskResult.empty();
}
return test();
}
}
dev/devicelab/lib/tasks/gallery.dart
View file @ 297c7b5c
......@@ -11,13 +11,17 @@ import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
import 'build_test_task.dart';
TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
TaskFunction createGalleryTransitionTest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,);
}
TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
TaskFunction createGalleryTransitionE2ETest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(
args,
testFile: semanticsEnabled
? 'transitions_perf_e2e_with_semantics'
: 'transitions_perf_e2e',
......@@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) {
transitionDurationFile: null,
timelineTraceFile: null,
driverFile: 'transitions_perf_e2e_test',
workingDirectory: galleryDirectory,
);
}
TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
TaskFunction createGalleryTransitionHybridTest(List<String> args, {bool semanticsEnabled = false}) {
return GalleryTransitionTest(
args,
semanticsEnabled: semanticsEnabled,
driverFile: semanticsEnabled
? 'transitions_perf_hybrid_with_semantics_test'
: 'transitions_perf_hybrid_test',
workingDirectory: galleryDirectory,
);
}
class GalleryTransitionTest {
GalleryTransitionTest({
class GalleryTransitionTest extends BuildTestTask {
GalleryTransitionTest(List<String> args, {
this.semanticsEnabled = false,
this.testFile = 'transitions_perf',
this.needFullTimeline = true,
......@@ -48,7 +54,8 @@ class GalleryTransitionTest {
this.timelineTraceFile = 'transitions.timeline',
this.transitionDurationFile = 'transition_durations.timeline',
this.driverFile,
});
Directory workingDirectory,
}) : super(args, workingDirectory: workingDirectory);
final bool semanticsEnabled;
final bool needFullTimeline;
......@@ -58,59 +65,48 @@ class GalleryTransitionTest {
final String transitionDurationFile;
final String driverFile;
Future<TaskResult> call() async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
await inDirectory<void>(galleryDirectory, () async {
String applicationBinaryPath;
if (deviceOperatingSystem == DeviceOperatingSystem.android) {
section('BUILDING APPLICATION');
await flutter(
'build',
options: <String>[
'apk',
'--no-android-gradle-daemon',
'--profile',
'-t',
'test_driver/$testFile.dart',
'--target-platform',
'android-arm,android-arm64',
],
);
applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
@override
List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
return <String>[
'apk',
'--no-android-gradle-daemon',
'--profile',
'-t',
'test_driver/$testFile.dart',
'--target-platform',
'android-arm,android-arm64',
];
default:
throw Exception('$deviceOperatingSystem has no build configuration');
}
}
final String testDriver = driverFile ?? (semanticsEnabled
? '${testFile}_with_semantics_test'
: '${testFile}_test');
section('DRIVE START');
await flutter('drive', options: <String>[
@override
List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
final String testDriver = driverFile ?? (semanticsEnabled
? '${testFile}_with_semantics_test'
: '${testFile}_test');
return <String>[
'--profile',
if (needFullTimeline)
'--trace-startup',
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath'
else
...<String>[
'-t',
'test_driver/$testFile.dart',
],
'--driver',
'test_driver/$testDriver.dart',
'-d',
deviceId,
]);
});
'--use-application-binary=${getApplicationBinaryPath()}',
'--driver', 'test_driver/$testDriver.dart',
'-d', deviceId,
];
}
@override
Future<TaskResult> parseTaskResult() async {
final Map<String, dynamic> summary = json.decode(
file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
) as Map<String, dynamic>;
if (transitionDurationFile != null) {
final Map<String, dynamic> original = json.decode(
file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
) as Map<String, dynamic>;
final Map<String, List<int>> transitions = <String, List<int>>{};
for (final String key in original.keys) {
......@@ -123,9 +119,9 @@ class GalleryTransitionTest {
return TaskResult.success(summary,
detailFiles: <String>[
if (transitionDurationFile != null)
'${galleryDirectory.path}/build/$transitionDurationFile.json',
'${workingDirectory.path}/build/$transitionDurationFile.json',
if (timelineTraceFile != null)
'${galleryDirectory.path}/build/$timelineTraceFile.json'
'${workingDirectory.path}/build/$timelineTraceFile.json'
],
benchmarkScoreKeys: <String>[
if (transitionDurationFile != null)
......@@ -141,6 +137,20 @@ class GalleryTransitionTest {
],
);
}
@override
String getApplicationBinaryPath() {
if (applicationBinaryPath != null) {
return applicationBinaryPath;
}
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
return 'build/app/outputs/flutter-apk/app-profile.apk';
default:
throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem');
}
}
}
int _countMissedTransitions(Map<String, List<int>> transitions) {
......
dev/devicelab/test/tasks/build_test_task_test.dart 0 → 100644
View file @ 297c7b5c
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import '../common.dart';
void main() {
final Map<String, String> isolateParams = <String, String>{
'enableConfig': 'false',
'runProcessCleanup': 'false',
'timeoutInMinutes': '1',
};
test('runs build and test when no args are passed', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.data['benchmark'], 'data');
});
test('runs build only when build arg is given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--build'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.message, 'No tests run');
});
test('runs test only when test arg is given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--test'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.data['benchmark'], 'data');
});
test('throws exception when build and test arg are given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--build', '--test'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.message, 'Task failed: Exception: Both build and test should not be passed. Pass only one.');
});
test('throws exception when build and application binary arg are given', () async {
final TaskResult result = await runTask(
'smoke_test_build_test',
taskArgs: <String>['--build', '--application-binary-path=test.apk'],
deviceId: 'FAKE_SUCCESS',
isolateParams: isolateParams,
);
expect(result.message, 'Task failed: Exception: Application binary path is only used for tests');
});
}
dev/devicelab/test/utils_test.dart
View file @ 297c7b5c
......@@ -20,11 +20,10 @@ void main() {
group('parse service', () {
const String badOutput = 'No uri here';
const String sampleOutput = 'An Observatory debugger and profiler on '
'Pixel 3 XL is available at: http://127.0.0.1:9090/LpjUpsdEjqI=/';
'Pixel 3 XL is available at: http://127.0.0.1:9090/LpjUpsdEjqI=/';
test('uri', () {
expect(parseServiceUri(sampleOutput),
Uri.parse('http://127.0.0.1:9090/LpjUpsdEjqI=/'));
expect(parseServiceUri(sampleOutput), Uri.parse('http://127.0.0.1:9090/LpjUpsdEjqI=/'));
expect(parseServiceUri(badOutput), null);
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab