Skip to content

F
Front-End
Unverified Commit 57f097b5 authored by Ian Hickson's avatar Ian Hickson Committed by GitHub
Browse files
Options

Refactor devicelab bin/run.dart and other cleanup (#96320)

parent 1c0eade9
Hide whitespace changes
Inline Side-by-side
Showing with 466 additions and 352 deletions
dev/devicelab/bin/run.dart
View file @ 57f097b5
...@@ -13,102 +13,108 @@ import 'package:flutter_devicelab/framework/task_result.dart'; ...@@ -13,102 +13,108 @@ import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
late ArgResults args; /// Runs tasks.
///
List<String> _taskNames = <String>[]; /// The tasks are chosen depending on the command-line options.
Future<void> main(List<String> rawArgs) async {
/// The device-id to run test on. // This is populated by a callback in the ArgParser.
String? deviceId; final List<String> taskNames = <String>[];
final ArgParser argParser = createArgParser(taskNames);
/// The git branch being tested on. ArgResults args;
String? gitBranch; try {
args = argParser.parse(rawArgs); // populates taskNames as a side-effect
} on FormatException catch (error) {
stderr.writeln('${error.message}\n');
stderr.writeln('Usage:\n');
stderr.writeln(argParser.usage);
exit(1);
}
/// The build of the local engine to use. /// Suppresses standard output, prints only standard error output.
/// final bool silent = (args['silent'] as bool?) ?? false;
/// Required for A/B test mode.
String? localEngine;
/// The path to the engine "src/" directory. /// The build of the local engine to use.
String? localEngineSrcPath; ///
/// Required for A/B test mode.
final String? localEngine = args['local-engine'] as String?;
/// Name of the LUCI builder this test is currently running on. /// The path to the engine "src/" directory.
/// final String? localEngineSrcPath = args['local-engine-src-path'] as String?;
/// This is only passed on CI runs for Cocoon to be able to uniquely identify
/// this test run.
String? luciBuilder;
/// Whether to exit on first test failure. /// The device-id to run test on.
bool exitOnFirstTestFailure = false; final String? deviceId = args['device-id'] as String?;
/// Path to write test results to. /// Whether to exit on first test failure.
String? resultsPath; final bool exitOnFirstTestFailure = (args['exit'] as bool?) ?? false;
/// File containing a service account token. /// Whether to tell tasks to clean up after themselves.
/// final bool terminateStrayDartProcesses = (args['terminate-stray-dart-processes'] as bool?) ?? false;
/// If passed, the test run results will be uploaded to Flutter infrastructure.
String? serviceAccountTokenFile;
/// Suppresses standard output, prints only standard error output. /// The git branch being tested on.
bool silent = false; final String? gitBranch = args['git-branch'] as String?;
/// Runs tasks. /// Name of the LUCI builder this test is currently running on.
/// ///
/// The tasks are chosen depending on the command-line options /// This is only passed on CI runs for Cocoon to be able to uniquely identify
/// (see [_argParser]). /// this test run.
Future<void> main(List<String> rawArgs) async { final String? luciBuilder = args['luci-builder'] as String?;
try {
args = _argParser.parse(rawArgs);
} on FormatException catch (error) {
stderr.writeln('${error.message}\n');
stderr.writeln('Usage:\n');
stderr.writeln(_argParser.usage);
exitCode = 1;
return;
}
deviceId = args['device-id'] as String?; /// Path to write test results to.
exitOnFirstTestFailure = (args['exit'] as bool?) ?? false; final String? resultsPath = args['results-file'] as String?;
gitBranch = args['git-branch'] as String?;
localEngine = args['local-engine'] as String?;
localEngineSrcPath = args['local-engine-src-path'] as String?;
luciBuilder = args['luci-builder'] as String?;
resultsPath = args['results-file'] as String?;
serviceAccountTokenFile = args['service-account-token-file'] as String?;
silent = (args['silent'] as bool?) ?? false;
if (!args.wasParsed('task')) { if (!args.wasParsed('task')) {
if (args.wasParsed('stage') || args.wasParsed('all')) { if (args.wasParsed('stage') || args.wasParsed('all')) {
addTasks( addTasks(
tasks: loadTaskManifest().tasks, tasks: loadTaskManifest().tasks,
args: args, args: args,
taskNames: _taskNames, taskNames: taskNames,
); );
} }
} }
if (args.wasParsed('list')) { if (args.wasParsed('list')) {
for (int i = 0; i < _taskNames.length; i++) { for (int i = 0; i < taskNames.length; i++) {
print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}'); print('${(i + 1).toString().padLeft(3)} - ${taskNames[i]}');
} }
exitCode = 0; exit(0);
return;
} }
if (_taskNames.isEmpty) { if (taskNames.isEmpty) {
stderr.writeln('Failed to find tasks to run based on supplied options.'); stderr.writeln('Failed to find tasks to run based on supplied options.');
exitCode = 1; exit(1);
return;
} }
if (args.wasParsed('ab')) { if (args.wasParsed('ab')) {
await _runABTest(); final int runsPerTest = int.parse(args['ab'] as String);
final String resultsFile = args['ab-result-file'] as String? ?? 'ABresults#.json';
if (taskNames.length > 1) {
stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${taskNames.join(', ')}.\n');
stderr.writeln(argParser.usage);
exit(1);
}
if (localEngine == null) {
stderr.writeln('When running in A/B test mode --local-engine is required.\n');
stderr.writeln(argParser.usage);
exit(1);
}
await _runABTest(
runsPerTest: runsPerTest,
silent: silent,
localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath,
deviceId: deviceId,
resultsFile: resultsFile,
taskName: taskNames.single,
);
} else { } else {
await runTasks(_taskNames, await runTasks(taskNames,
silent: silent, silent: silent,
localEngine: localEngine, localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath, localEngineSrcPath: localEngineSrcPath,
deviceId: deviceId, deviceId: deviceId,
exitOnFirstTestFailure: exitOnFirstTestFailure, exitOnFirstTestFailure: exitOnFirstTestFailure,
terminateStrayDartProcesses: terminateStrayDartProcesses,
gitBranch: gitBranch, gitBranch: gitBranch,
luciBuilder: luciBuilder, luciBuilder: luciBuilder,
resultsPath: resultsPath, resultsPath: resultsPath,
...@@ -116,26 +122,18 @@ Future<void> main(List<String> rawArgs) async { ...@@ -116,26 +122,18 @@ Future<void> main(List<String> rawArgs) async {
} }
} }
Future<void> _runABTest() async { Future<void> _runABTest({
final int runsPerTest = int.parse(args['ab'] as String); required int runsPerTest,
required bool silent,
if (_taskNames.length > 1) { required String localEngine,
stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${_taskNames.join(', ')}.\n'); required String? localEngineSrcPath,
stderr.writeln(_argParser.usage); required String? deviceId,
exit(1); required String resultsFile,
} required String taskName,
}) async {
if (!args.wasParsed('local-engine')) {
stderr.writeln('When running in A/B test mode --local-engine is required.\n');
stderr.writeln(_argParser.usage);
exit(1);
}
final String taskName = _taskNames.single;
print('$taskName A/B test. Will run $runsPerTest times.'); print('$taskName A/B test. Will run $runsPerTest times.');
final ABTest abTest = ABTest(localEngine!, taskName); final ABTest abTest = ABTest(localEngine, taskName);
for (int i = 1; i <= runsPerTest; i++) { for (int i = 1; i <= runsPerTest; i++) {
section('Run #$i'); section('Run #$i');
...@@ -182,7 +180,7 @@ Future<void> _runABTest() async { ...@@ -182,7 +180,7 @@ Future<void> _runABTest() async {
} }
abTest.finalize(); abTest.finalize();
final File jsonFile = _uniqueFile(args['ab-result-file'] as String? ?? 'ABresults#.json'); final File jsonFile = _uniqueFile(resultsFile);
jsonFile.writeAsStringSync(const JsonEncoder.withIndent(' ').convert(abTest.jsonMap)); jsonFile.writeAsStringSync(const JsonEncoder.withIndent(' ').convert(abTest.jsonMap));
if (silent != true) { if (silent != true) {
...@@ -234,142 +232,152 @@ void addTasks({ ...@@ -234,142 +232,152 @@ void addTasks({
} }
} }
/// Command-line options for the `run.dart` command. ArgParser createArgParser(List<String> taskNames) {
final ArgParser _argParser = ArgParser() return ArgParser()
..addMultiOption( ..addMultiOption(
'task', 'task',
abbr: 't', abbr: 't',
help: 'Either:\n' help: 'Either:\n'
' - the name of a task defined in manifest.yaml.\n' ' - the name of a task defined in manifest.yaml.\n'
' Example: complex_layout__start_up.\n' ' Example: complex_layout__start_up.\n'
' - the path to a Dart file corresponding to a task,\n' ' - the path to a Dart file corresponding to a task,\n'
' which resides in bin/tasks.\n' ' which resides in bin/tasks.\n'
' Example: bin/tasks/complex_layout__start_up.dart.\n' ' Example: bin/tasks/complex_layout__start_up.dart.\n'
'\n' '\n'
'This option may be repeated to specify multiple tasks.', 'This option may be repeated to specify multiple tasks.',
callback: (List<String> value) { callback: (List<String> value) {
for (final String nameOrPath in value) { for (final String nameOrPath in value) {
final List<String> fragments = path.split(nameOrPath); final List<String> fragments = path.split(nameOrPath);
final bool isDartFile = fragments.last.endsWith('.dart'); final bool isDartFile = fragments.last.endsWith('.dart');
if (fragments.length == 1 && !isDartFile) { if (fragments.length == 1 && !isDartFile) {
// Not a path // Not a path
_taskNames.add(nameOrPath); taskNames.add(nameOrPath);
} else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) { } else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
// Unsupported executable location // Unsupported executable location
throw FormatException('Invalid value for option -t (--task): $nameOrPath'); throw FormatException('Invalid value for option -t (--task): $nameOrPath');
} else { } else {
_taskNames.add(path.withoutExtension(fragments.last)); taskNames.add(path.withoutExtension(fragments.last));
}
}
},
)
..addOption(
'device-id',
abbr: 'd',
help: 'Target device id (prefixes are allowed, names are not supported).\n'
'The option will be ignored if the test target does not run on a\n'
'mobile device. This still respects the device operating system\n'
'settings in the test case, and will results in error if no device\n'
'with given ID/ID prefix is found.',
)
..addOption(
'ab',
help: 'Runs an A/B test comparing the default engine with the local\n'
'engine build for one task. This option does not support running\n'
'multiple tasks. The value is the number of times to run the task.\n'
'The task is expected to be a benchmark that reports score keys.\n'
'The A/B test collects the metrics collected by the test and\n'
'produces a report containing averages, noise, and the speed-up\n'
'between the two engines. --local-engine is required when running\n'
'an A/B test.',
callback: (String? value) {
if (value != null && int.tryParse(value) == null) {
throw ArgParserException('Option --ab must be a number, but was "$value".');
}
},
)
..addOption(
'ab-result-file',
help: 'The filename in which to place the json encoded results of an A/B test.\n'
'The filename may contain a single # character to be replaced by a sequence\n'
'number if the name already exists.',
)
..addFlag(
'all',
abbr: 'a',
help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
)
..addOption(
'continue-from',
abbr: 'c',
help: 'With --all or --stage, continue from the given test.',
)
..addFlag(
'exit',
defaultsTo: true,
help: 'Exit on the first test failure. Currently flakes are intentionally (though '
'incorrectly) not considered to be failures.',
)
..addOption(
'git-branch',
help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
)
..addOption(
'local-engine',
help: 'Name of a build output within the engine out directory, if you\n'
'are building Flutter locally. Use this to select a specific\n'
'version of the engine if you have built multiple engine targets.\n'
'This path is relative to --local-engine-src-path/out. This option\n'
'is required when running an A/B test (see the --ab option).',
)
..addFlag(
'list',
abbr: 'l',
help: "Don't actually run the tasks, but list out the tasks that would\n"
'have been run, in the order they would have run.',
)
..addOption(
'local-engine-src-path',
help: 'Path to your engine src directory, if you are building Flutter\n'
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
'the location based on the value of the --flutter-root option.',
)
..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
..addFlag(
'match-host-platform',
defaultsTo: true,
help: 'Only run tests that match the host platform (e.g. do not run a\n'
'test with a `required_agent_capabilities` value of "mac/android"\n'
'on a windows host). Each test publishes its '
'`required_agent_capabilities`\nin the `manifest.yaml` file.',
)
..addOption(
'results-file',
help: '[Flutter infrastructure] File path for test results. If passed with\n'
'task, will write test results to the file.'
)
..addOption(
'service-account-token-file',
help: '[Flutter infrastructure] Authentication for uploading results.',
)
..addOption(
'stage',
abbr: 's',
help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
'their stages are read from manifest.yaml.',
)
..addFlag(
'silent',
help: 'Reduce verbosity slightly.',
)
..addFlag(
'terminate-stray-dart-processes',
defaultsTo: true,
help: 'Whether to send a SIGKILL signal to any Dart processes that are still '
'running when a task is completed. If any Dart processes are terminated '
'in this way, the test is considered to have failed.',
)
..addMultiOption(
'test',
hide: true,
callback: (List<String> value) {
if (value.isNotEmpty) {
throw const FormatException(
'Invalid option --test. Did you mean --task (-t)?',
);
} }
} },
}, );
) }
..addOption(
'device-id',
abbr: 'd',
help: 'Target device id (prefixes are allowed, names are not supported).\n'
'The option will be ignored if the test target does not run on a\n'
'mobile device. This still respects the device operating system\n'
'settings in the test case, and will results in error if no device\n'
'with given ID/ID prefix is found.',
)
..addOption(
'ab',
help: 'Runs an A/B test comparing the default engine with the local\n'
'engine build for one task. This option does not support running\n'
'multiple tasks. The value is the number of times to run the task.\n'
'The task is expected to be a benchmark that reports score keys.\n'
'The A/B test collects the metrics collected by the test and\n'
'produces a report containing averages, noise, and the speed-up\n'
'between the two engines. --local-engine is required when running\n'
'an A/B test.',
callback: (String? value) {
if (value != null && int.tryParse(value) == null) {
throw ArgParserException('Option --ab must be a number, but was "$value".');
}
},
)
..addOption(
'ab-result-file',
help: 'The filename in which to place the json encoded results of an A/B test.\n'
'The filename may contain a single # character to be replaced by a sequence\n'
'number if the name already exists.',
)
..addFlag(
'all',
abbr: 'a',
help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
)
..addOption(
'continue-from',
abbr: 'c',
help: 'With --all or --stage, continue from the given test.',
)
..addFlag(
'exit',
defaultsTo: true,
help: 'Exit on the first test failure.',
)
..addOption(
'git-branch',
help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
)
..addOption(
'local-engine',
help: 'Name of a build output within the engine out directory, if you\n'
'are building Flutter locally. Use this to select a specific\n'
'version of the engine if you have built multiple engine targets.\n'
'This path is relative to --local-engine-src-path/out. This option\n'
'is required when running an A/B test (see the --ab option).',
)
..addFlag(
'list',
abbr: 'l',
help: "Don't actually run the tasks, but list out the tasks that would\n"
'have been run, in the order they would have run.',
)
..addOption(
'local-engine-src-path',
help: 'Path to your engine src directory, if you are building Flutter\n'
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
'the location based on the value of the --flutter-root option.',
)
..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
..addFlag(
'match-host-platform',
defaultsTo: true,
help: 'Only run tests that match the host platform (e.g. do not run a\n'
'test with a `required_agent_capabilities` value of "mac/android"\n'
'on a windows host). Each test publishes its '
'`required_agent_capabilities`\nin the `manifest.yaml` file.',
)
..addOption(
'results-file',
help: '[Flutter infrastructure] File path for test results. If passed with\n'
'task, will write test results to the file.'
)
..addOption(
'service-account-token-file',
help: '[Flutter infrastructure] Authentication for uploading results.',
)
..addOption(
'stage',
abbr: 's',
help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
'their stages are read from manifest.yaml.',
)
..addFlag(
'silent',
)
..addMultiOption(
'test',
hide: true,
callback: (List<String> value) {
if (value.isNotEmpty) {
throw const FormatException(
'Invalid option --test. Did you mean --task (-t)?',
);
}
},
);
dev/devicelab/lib/framework/framework.dart
View file @ 57f097b5
...@@ -10,6 +10,7 @@ import 'dart:isolate'; ...@@ -10,6 +10,7 @@ import 'dart:isolate';
import 'package:logging/logging.dart'; import 'package:logging/logging.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:process/process.dart';
import 'package:stack_trace/stack_trace.dart'; import 'package:stack_trace/stack_trace.dart';
import 'devices.dart'; import 'devices.dart';
...@@ -41,35 +42,43 @@ bool _isTaskRegistered = false; ...@@ -41,35 +42,43 @@ bool _isTaskRegistered = false;
/// ///
/// It is OK for a [task] to perform many things. However, only one task can be /// It is OK for a [task] to perform many things. However, only one task can be
/// registered per Dart VM. /// registered per Dart VM.
Future<TaskResult> task(TaskFunction task) async { ///
/// If no `processManager` is provided, a default [LocalProcessManager] is created
/// for the task.
Future<TaskResult> task(TaskFunction task, { ProcessManager? processManager }) async {
if (_isTaskRegistered) if (_isTaskRegistered)
throw StateError('A task is already registered'); throw StateError('A task is already registered');
_isTaskRegistered = true; _isTaskRegistered = true;
processManager ??= const LocalProcessManager();
// TODO(ianh): allow overriding logging. // TODO(ianh): allow overriding logging.
Logger.root.level = Level.ALL; Logger.root.level = Level.ALL;
Logger.root.onRecord.listen((LogRecord rec) { Logger.root.onRecord.listen((LogRecord rec) {
print('${rec.level.name}: ${rec.time}: ${rec.message}'); print('${rec.level.name}: ${rec.time}: ${rec.message}');
}); });
final _TaskRunner runner = _TaskRunner(task); final _TaskRunner runner = _TaskRunner(task, processManager);
runner.keepVmAliveUntilTaskRunRequested(); runner.keepVmAliveUntilTaskRunRequested();
return runner.whenDone; return runner.whenDone;
} }
class _TaskRunner { class _TaskRunner {
_TaskRunner(this.task) { _TaskRunner(this.task, this.processManager) {
registerExtension('ext.cocoonRunTask', registerExtension('ext.cocoonRunTask',
(String method, Map<String, String> parameters) async { (String method, Map<String, String> parameters) async {
final Duration? taskTimeout = parameters.containsKey('timeoutInMinutes') final Duration? taskTimeout = parameters.containsKey('timeoutInMinutes')
? Duration(minutes: int.parse(parameters['timeoutInMinutes']!)) ? Duration(minutes: int.parse(parameters['timeoutInMinutes']!))
: null; : null;
// This is only expected to be passed in unit test runs so they do not final bool runFlutterConfig = parameters['runFlutterConfig'] != 'false'; // used by tests to avoid changing the configuration
// kill the Dart process that is running them and waste time running config.
final bool runFlutterConfig = parameters['runFlutterConfig'] != 'false';
final bool runProcessCleanup = parameters['runProcessCleanup'] != 'false'; final bool runProcessCleanup = parameters['runProcessCleanup'] != 'false';
final TaskResult result = await run(taskTimeout, runProcessCleanup: runProcessCleanup, runFlutterConfig: runFlutterConfig); final String? localEngine = parameters['localEngine'];
final TaskResult result = await run(
taskTimeout,
runProcessCleanup: runProcessCleanup,
runFlutterConfig: runFlutterConfig,
localEngine: localEngine,
);
return ServiceExtensionResponse.result(json.encode(result.toJson())); return ServiceExtensionResponse.result(json.encode(result.toJson()));
}); });
registerExtension('ext.cocoonRunnerReady', registerExtension('ext.cocoonRunnerReady',
...@@ -79,6 +88,7 @@ class _TaskRunner { ...@@ -79,6 +88,7 @@ class _TaskRunner {
} }
final TaskFunction task; final TaskFunction task;
final ProcessManager processManager;
Future<Device?> _getWorkingDeviceIfAvailable() async { Future<Device?> _getWorkingDeviceIfAvailable() async {
try { try {
...@@ -103,6 +113,7 @@ class _TaskRunner { ...@@ -103,6 +113,7 @@ class _TaskRunner {
Future<TaskResult> run(Duration? taskTimeout, { Future<TaskResult> run(Duration? taskTimeout, {
bool runFlutterConfig = true, bool runFlutterConfig = true,
bool runProcessCleanup = true, bool runProcessCleanup = true,
required String? localEngine,
}) async { }) async {
try { try {
_taskStarted = true; _taskStarted = true;
...@@ -113,20 +124,19 @@ class _TaskRunner { ...@@ -113,20 +124,19 @@ class _TaskRunner {
section('Checking running Dart$exe processes'); section('Checking running Dart$exe processes');
beforeRunningDartInstances = await getRunningProcesses( beforeRunningDartInstances = await getRunningProcesses(
processName: 'dart$exe', processName: 'dart$exe',
).toSet(); processManager: processManager,
final Set<RunningProcessInfo> allProcesses = await getRunningProcesses().toSet(); );
final Set<RunningProcessInfo> allProcesses = await getRunningProcesses(processManager: processManager);
beforeRunningDartInstances.forEach(print); beforeRunningDartInstances.forEach(print);
for (final RunningProcessInfo info in allProcesses) { for (final RunningProcessInfo info in allProcesses) {
if (info.commandLine.contains('iproxy')) { if (info.commandLine.contains('iproxy')) {
print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} '); print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ');
} }
} }
} else {
section('Skipping check running Dart$exe processes');
} }
if (runFlutterConfig) { if (runFlutterConfig) {
print('enabling configs for macOS, Linux, Windows, and Web...'); print('Enabling configs for macOS, Linux, Windows, and Web...');
final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[ final int configResult = await exec(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
'config', 'config',
'-v', '-v',
...@@ -134,13 +144,11 @@ class _TaskRunner { ...@@ -134,13 +144,11 @@ class _TaskRunner {
'--enable-windows-desktop', '--enable-windows-desktop',
'--enable-linux-desktop', '--enable-linux-desktop',
'--enable-web', '--enable-web',
if (localEngine != null) ...<String>['--local-engine', localEngine!], if (localEngine != null) ...<String>['--local-engine', localEngine],
], canFail: true); ], canFail: true);
if (configResult != 0) { if (configResult != 0) {
print('Failed to enable configuration, tasks may not run.'); print('Failed to enable configuration, tasks may not run.');
} }
} else {
print('Skipping enabling configs for macOS, Linux, Windows, and Web');
} }
final Device? device = await _getWorkingDeviceIfAvailable(); final Device? device = await _getWorkingDeviceIfAvailable();
...@@ -169,26 +177,24 @@ class _TaskRunner { ...@@ -169,26 +177,24 @@ class _TaskRunner {
} }
if (runProcessCleanup) { if (runProcessCleanup) {
section('Checking running Dart$exe processes after task...'); section('Terminating lingering Dart$exe processes after task...');
final List<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses( final Set<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses(
processName: 'dart$exe', processName: 'dart$exe',
).toList(); processManager: processManager,
);
for (final RunningProcessInfo info in afterRunningDartInstances) { for (final RunningProcessInfo info in afterRunningDartInstances) {
if (!beforeRunningDartInstances.contains(info)) { if (!beforeRunningDartInstances.contains(info)) {
print('$info was leaked by this test.'); print('$info was leaked by this test.');
if (result is TaskResultCheckProcesses) { if (result is TaskResultCheckProcesses) {
result = TaskResult.failure('This test leaked dart processes'); result = TaskResult.failure('This test leaked dart processes');
} }
final bool killed = await killProcess(info.pid); if (await info.terminate(processManager: processManager)) {
if (!killed) {
print('Failed to kill process ${info.pid}.');
} else {
print('Killed process id ${info.pid}.'); print('Killed process id ${info.pid}.');
} else {
print('Failed to kill process ${info.pid}.');
} }
} }
} }
} else {
print('Skipping check running Dart$exe processes after task');
} }
_completer.complete(result); _completer.complete(result);
return result; return result;
......
dev/devicelab/lib/framework/runner.dart
View file @ 57f097b5
...@@ -4,7 +4,6 @@ ...@@ -4,7 +4,6 @@
import 'dart:async'; import 'dart:async';
import 'dart:convert'; import 'dart:convert';
// import 'dart:core' as core;
import 'dart:io'; import 'dart:io';
import 'package:meta/meta.dart'; import 'package:meta/meta.dart';
...@@ -29,6 +28,10 @@ import 'utils.dart'; ...@@ -29,6 +28,10 @@ import 'utils.dart';
Future<void> runTasks( Future<void> runTasks(
List<String> taskNames, { List<String> taskNames, {
bool exitOnFirstTestFailure = false, bool exitOnFirstTestFailure = false,
// terminateStrayDartProcesses defaults to false so that tests don't have to specify it.
// It is set based on the --terminate-stray-dart-processes command line argument in
// normal execution, and that flag defaults to true.
bool terminateStrayDartProcesses = false,
bool silent = false, bool silent = false,
String? deviceId, String? deviceId,
String? gitBranch, String? gitBranch,
...@@ -50,6 +53,7 @@ Future<void> runTasks( ...@@ -50,6 +53,7 @@ Future<void> runTasks(
deviceId: deviceId, deviceId: deviceId,
localEngine: localEngine, localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath, localEngineSrcPath: localEngineSrcPath,
terminateStrayDartProcesses: terminateStrayDartProcesses,
silent: silent, silent: silent,
taskArgs: taskArgs, taskArgs: taskArgs,
resultsPath: resultsPath, resultsPath: resultsPath,
...@@ -58,15 +62,17 @@ Future<void> runTasks( ...@@ -58,15 +62,17 @@ Future<void> runTasks(
isolateParams: isolateParams, isolateParams: isolateParams,
); );
section('Flaky status for "$taskName"');
if (!result.succeeded) { if (!result.succeeded) {
retry++; retry += 1;
} else { } else {
section('Flaky status for "$taskName"');
if (retry > 0) { if (retry > 0) {
print('Total ${retry+1} executions: $retry failures and 1 success'); print('Total ${retry+1} executions: $retry failures and 1 false positive.');
print('flaky: true'); print('flaky: true');
// TODO(ianh): stop ignoring this failure. We should set exitCode=1, and quit
// if exitOnFirstTestFailure is true.
} else { } else {
print('Total ${retry+1} executions: 1 success'); print('Test passed on first attempt.');
print('flaky: false'); print('flaky: false');
} }
break; break;
...@@ -74,7 +80,8 @@ Future<void> runTasks( ...@@ -74,7 +80,8 @@ Future<void> runTasks(
} }
if (!result.succeeded) { if (!result.succeeded) {
print('Total $retry executions: 0 success'); section('Flaky status for "$taskName"');
print('Consistently failed across all $retry executions.');
print('flaky: false'); print('flaky: false');
exitCode = 1; exitCode = 1;
if (exitOnFirstTestFailure) { if (exitOnFirstTestFailure) {
...@@ -92,6 +99,7 @@ Future<TaskResult> rerunTask( ...@@ -92,6 +99,7 @@ Future<TaskResult> rerunTask(
String? deviceId, String? deviceId,
String? localEngine, String? localEngine,
String? localEngineSrcPath, String? localEngineSrcPath,
bool terminateStrayDartProcesses = false,
bool silent = false, bool silent = false,
List<String>? taskArgs, List<String>? taskArgs,
String? resultsPath, String? resultsPath,
...@@ -105,6 +113,7 @@ Future<TaskResult> rerunTask( ...@@ -105,6 +113,7 @@ Future<TaskResult> rerunTask(
deviceId: deviceId, deviceId: deviceId,
localEngine: localEngine, localEngine: localEngine,
localEngineSrcPath: localEngineSrcPath, localEngineSrcPath: localEngineSrcPath,
terminateStrayDartProcesses: terminateStrayDartProcesses,
silent: silent, silent: silent,
taskArgs: taskArgs, taskArgs: taskArgs,
isolateParams: isolateParams, isolateParams: isolateParams,
...@@ -138,11 +147,12 @@ Future<TaskResult> rerunTask( ...@@ -138,11 +147,12 @@ Future<TaskResult> rerunTask(
/// [taskArgs] are passed to the task executable for additional configuration. /// [taskArgs] are passed to the task executable for additional configuration.
Future<TaskResult> runTask( Future<TaskResult> runTask(
String taskName, { String taskName, {
bool terminateStrayDartProcesses = false,
bool silent = false, bool silent = false,
String? localEngine, String? localEngine,
String? localEngineSrcPath, String? localEngineSrcPath,
String? deviceId, String? deviceId,
List<String> ?taskArgs, List<String>? taskArgs,
@visibleForTesting Map<String, String>? isolateParams, @visibleForTesting Map<String, String>? isolateParams,
}) async { }) async {
final String taskExecutable = 'bin/tasks/$taskName.dart'; final String taskExecutable = 'bin/tasks/$taskName.dart';
...@@ -198,17 +208,26 @@ Future<TaskResult> runTask( ...@@ -198,17 +208,26 @@ Future<TaskResult> runTask(
try { try {
final ConnectionResult result = await _connectToRunnerIsolate(await uri.future); final ConnectionResult result = await _connectToRunnerIsolate(await uri.future);
print('[$taskName] Connected to VM server.');
isolateParams = isolateParams == null ? <String, String>{} : Map<String, String>.of(isolateParams);
isolateParams['runProcessCleanup'] = terminateStrayDartProcesses.toString();
final Map<String, dynamic> taskResultJson = (await result.vmService.callServiceExtension( final Map<String, dynamic> taskResultJson = (await result.vmService.callServiceExtension(
'ext.cocoonRunTask', 'ext.cocoonRunTask',
args: isolateParams, args: isolateParams,
isolateId: result.isolate.id, isolateId: result.isolate.id,
)).json!; )).json!;
final TaskResult taskResult = TaskResult.fromJson(taskResultJson); final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
await runner.exitCode; final int exitCode = await runner.exitCode;
print('[$taskName] Process terminated with exit code $exitCode.');
return taskResult; return taskResult;
} catch (error, stack) {
print('[$taskName] Task runner system failed with exception!\n$error\n$stack');
rethrow;
} finally { } finally {
if (!runnerFinished) if (!runnerFinished) {
print('[$taskName] Terminating process...');
runner.kill(ProcessSignal.sigkill); runner.kill(ProcessSignal.sigkill);
}
await stdoutSub.cancel(); await stdoutSub.cancel();
await stderrSub.cancel(); await stderrSub.cancel();
} }
......
dev/devicelab/lib/framework/running_processes.dart
View file @ 57f097b5
...@@ -9,12 +9,12 @@ import 'package:process/process.dart'; ...@@ -9,12 +9,12 @@ import 'package:process/process.dart';
@immutable @immutable
class RunningProcessInfo { class RunningProcessInfo {
const RunningProcessInfo(this.pid, this.creationDate, this.commandLine) const RunningProcessInfo(this.pid, this.commandLine, this.creationDate)
: assert(pid != null), : assert(pid != null),
assert(commandLine != null); assert(commandLine != null);
final int pid;
final String commandLine; final String commandLine;
final String pid;
final DateTime creationDate; final DateTime creationDate;
@override @override
...@@ -25,57 +25,54 @@ class RunningProcessInfo { ...@@ -25,57 +25,54 @@ class RunningProcessInfo {
&& other.creationDate == creationDate; && other.creationDate == creationDate;
} }
Future<bool> terminate({required ProcessManager processManager}) async {
// This returns true when the signal is sent, not when the process goes away.
// See also https://github.com/dart-lang/sdk/issues/40759 (killPid should wait for process to be terminated).
if (Platform.isWindows) {
// TODO(ianh): Move Windows to killPid once we can.
// - killPid on Windows has not-useful return code: https://github.com/dart-lang/sdk/issues/47675
final ProcessResult result = await processManager.run(<String>[
'taskkill.exe',
'/pid',
'$pid',
'/f',
]);
return result.exitCode == 0;
}
return processManager.killPid(pid, ProcessSignal.sigkill);
}
@override @override
int get hashCode => Object.hash(pid, commandLine, creationDate); int get hashCode => Object.hash(pid, commandLine, creationDate);
@override @override
String toString() { String toString() {
return 'RunningProcesses{pid: $pid, commandLine: $commandLine, creationDate: $creationDate}'; return 'RunningProcesses(pid: $pid, commandLine: $commandLine, creationDate: $creationDate)';
}
}
Future<bool> killProcess(String pid, {ProcessManager? processManager}) async {
assert(pid != null, 'Must specify a pid to kill');
processManager ??= const LocalProcessManager();
ProcessResult result;
if (Platform.isWindows) {
result = await processManager.run(<String>[
'taskkill.exe',
'/pid',
pid,
'/f',
]);
} else {
result = await processManager.run(<String>[
'kill',
'-9',
pid,
]);
} }
return result.exitCode == 0;
} }
Stream<RunningProcessInfo> getRunningProcesses({ Future<Set<RunningProcessInfo>> getRunningProcesses({
String? processName, String? processName,
ProcessManager? processManager, required ProcessManager processManager,
}) { }) {
processManager ??= const LocalProcessManager();
if (Platform.isWindows) { if (Platform.isWindows) {
return windowsRunningProcesses(processName); return windowsRunningProcesses(processName, processManager);
} }
return posixRunningProcesses(processName, processManager); return posixRunningProcesses(processName, processManager);
} }
@visibleForTesting @visibleForTesting
Stream<RunningProcessInfo> windowsRunningProcesses(String? processName) async* { Future<Set<RunningProcessInfo>> windowsRunningProcesses(
// PowerShell script to get the command line arguments and create time of String? processName,
// a process. ProcessManager processManager,
) async {
// PowerShell script to get the command line arguments and create time of a process.
// See: https://docs.microsoft.com/en-us/windows/desktop/cimwin32prov/win32-process // See: https://docs.microsoft.com/en-us/windows/desktop/cimwin32prov/win32-process
final String script = processName != null final String script = processName != null
? '"Get-CimInstance Win32_Process -Filter \\"name=\'$processName\'\\" | Select-Object ProcessId,CreationDate,CommandLine | Format-Table -AutoSize | Out-String -Width 4096"' ? '"Get-CimInstance Win32_Process -Filter \\"name=\'$processName\'\\" | Select-Object ProcessId,CreationDate,CommandLine | Format-Table -AutoSize | Out-String -Width 4096"'
: '"Get-CimInstance Win32_Process | Select-Object ProcessId,CreationDate,CommandLine | Format-Table -AutoSize | Out-String -Width 4096"'; : '"Get-CimInstance Win32_Process | Select-Object ProcessId,CreationDate,CommandLine | Format-Table -AutoSize | Out-String -Width 4096"';
// Unfortunately, there doesn't seem to be a good way to get ProcessManager to // TODO(ianh): Unfortunately, there doesn't seem to be a good way to get
// run this. // ProcessManager to run this.
final ProcessResult result = await Process.run( final ProcessResult result = await Process.run(
'powershell -command $script', 'powershell -command $script',
<String>[], <String>[],
...@@ -84,11 +81,9 @@ Stream<RunningProcessInfo> windowsRunningProcesses(String? processName) async* { ...@@ -84,11 +81,9 @@ Stream<RunningProcessInfo> windowsRunningProcesses(String? processName) async* {
print('Could not list processes!'); print('Could not list processes!');
print(result.stderr); print(result.stderr);
print(result.stdout); print(result.stdout);
return; return <RunningProcessInfo>{};
}
for (final RunningProcessInfo info in processPowershellOutput(result.stdout as String)) {
yield info;
} }
return processPowershellOutput(result.stdout as String).toSet();
} }
/// Parses the output of the PowerShell script from [windowsRunningProcesses]. /// Parses the output of the PowerShell script from [windowsRunningProcesses].
...@@ -149,22 +144,22 @@ Iterable<RunningProcessInfo> processPowershellOutput(String output) sync* { ...@@ -149,22 +144,22 @@ Iterable<RunningProcessInfo> processPowershellOutput(String output) sync* {
time = '${hours + 12}${time.substring(2)}'; time = '${hours + 12}${time.substring(2)}';
} }
final String pid = line.substring(0, processIdHeaderSize).trim(); final int pid = int.parse(line.substring(0, processIdHeaderSize).trim());
final DateTime creationDate = DateTime.parse('$year-$month-${day}T$time'); final DateTime creationDate = DateTime.parse('$year-$month-${day}T$time');
final String commandLine = line.substring(commandLineHeaderStart).trim(); final String commandLine = line.substring(commandLineHeaderStart).trim();
yield RunningProcessInfo(pid, creationDate, commandLine); yield RunningProcessInfo(pid, commandLine, creationDate);
} }
} }
@visibleForTesting @visibleForTesting
Stream<RunningProcessInfo> posixRunningProcesses( Future<Set<RunningProcessInfo>> posixRunningProcesses(
String? processName, String? processName,
ProcessManager processManager, ProcessManager processManager,
) async* { ) async {
// Cirrus is missing this in Linux for some reason. // Cirrus is missing this in Linux for some reason.
if (!processManager.canRun('ps')) { if (!processManager.canRun('ps')) {
print('Cannot list processes on this system: `ps` not available.'); print('Cannot list processes on this system: "ps" not available.');
return; return <RunningProcessInfo>{};
} }
final ProcessResult result = await processManager.run(<String>[ final ProcessResult result = await processManager.run(<String>[
'ps', 'ps',
...@@ -175,11 +170,9 @@ Stream<RunningProcessInfo> posixRunningProcesses( ...@@ -175,11 +170,9 @@ Stream<RunningProcessInfo> posixRunningProcesses(
print('Could not list processes!'); print('Could not list processes!');
print(result.stderr); print(result.stderr);
print(result.stdout); print(result.stdout);
return; return <RunningProcessInfo>{};
}
for (final RunningProcessInfo info in processPsOutput(result.stdout as String, processName)) {
yield info;
} }
return processPsOutput(result.stdout as String, processName).toSet();
} }
/// Parses the output of the command in [posixRunningProcesses]. /// Parses the output of the command in [posixRunningProcesses].
...@@ -240,8 +233,8 @@ Iterable<RunningProcessInfo> processPsOutput( ...@@ -240,8 +233,8 @@ Iterable<RunningProcessInfo> processPsOutput(
final DateTime creationDate = DateTime.parse('$year-$month-${day}T$time'); final DateTime creationDate = DateTime.parse('$year-$month-${day}T$time');
line = line.substring(24).trim(); line = line.substring(24).trim();
final int nextSpace = line.indexOf(' '); final int nextSpace = line.indexOf(' ');
final String pid = line.substring(0, nextSpace); final int pid = int.parse(line.substring(0, nextSpace));
final String commandLine = line.substring(nextSpace + 1); final String commandLine = line.substring(nextSpace + 1);
yield RunningProcessInfo(pid, creationDate, commandLine); yield RunningProcessInfo(pid, commandLine, creationDate);
} }
} }
dev/devicelab/lib/framework/utils.dart
View file @ 57f097b5
...@@ -20,14 +20,18 @@ import 'task_result.dart'; ...@@ -20,14 +20,18 @@ import 'task_result.dart';
String cwd = Directory.current.path; String cwd = Directory.current.path;
/// The local engine to use for [flutter] and [evalFlutter], if any. /// The local engine to use for [flutter] and [evalFlutter], if any.
String? get localEngine { ///
/// This is set as an environment variable when running the task, see runTask in runner.dart.
String? get localEngineFromEnv {
const bool isDefined = bool.hasEnvironment('localEngine'); const bool isDefined = bool.hasEnvironment('localEngine');
return isDefined ? const String.fromEnvironment('localEngine') : null; return isDefined ? const String.fromEnvironment('localEngine') : null;
} }
/// The local engine source path to use if a local engine is used for [flutter] /// The local engine source path to use if a local engine is used for [flutter]
/// and [evalFlutter]. /// and [evalFlutter].
String? get localEngineSrcPath { ///
/// This is set as an environment variable when running the task, see runTask in runner.dart.
String? get localEngineSrcPathFromEnv {
const bool isDefined = bool.hasEnvironment('localEngineSrcPath'); const bool isDefined = bool.hasEnvironment('localEngineSrcPath');
return isDefined ? const String.fromEnvironment('localEngineSrcPath') : null; return isDefined ? const String.fromEnvironment('localEngineSrcPath') : null;
} }
...@@ -45,9 +49,9 @@ class ProcessInfo { ...@@ -45,9 +49,9 @@ class ProcessInfo {
@override @override
String toString() { String toString() {
return ''' return '''
command : $command command: $command
started : $startTime started: $startTime
pid : ${process.pid} pid : ${process.pid}
''' '''
.trim(); .trim();
} }
...@@ -275,7 +279,7 @@ Future<Process> startProcess( ...@@ -275,7 +279,7 @@ Future<Process> startProcess(
final Map<String, String> newEnvironment = Map<String, String>.from(environment ?? <String, String>{}); final Map<String, String> newEnvironment = Map<String, String>.from(environment ?? <String, String>{});
newEnvironment['BOT'] = isBot ? 'true' : 'false'; newEnvironment['BOT'] = isBot ? 'true' : 'false';
newEnvironment['LANG'] = 'en_US.UTF-8'; newEnvironment['LANG'] = 'en_US.UTF-8';
print('\nExecuting: $command in $finalWorkingDirectory with environment $newEnvironment'); print('Executing "$command" in "$finalWorkingDirectory" with environment $newEnvironment');
final Process process = await _processManager.start( final Process process = await _processManager.start(
<String>[executable, ...?arguments], <String>[executable, ...?arguments],
environment: newEnvironment, environment: newEnvironment,
...@@ -285,7 +289,6 @@ Future<Process> startProcess( ...@@ -285,7 +289,6 @@ Future<Process> startProcess(
_runningProcesses.add(processInfo); _runningProcesses.add(processInfo);
unawaited(process.exitCode.then<void>((int exitCode) { unawaited(process.exitCode.then<void>((int exitCode) {
print('"$executable" exit code: $exitCode');
_runningProcesses.remove(processInfo); _runningProcesses.remove(processInfo);
})); }));
...@@ -303,7 +306,7 @@ Future<void> forceQuitRunningProcesses() async { ...@@ -303,7 +306,7 @@ Future<void> forceQuitRunningProcesses() async {
for (final ProcessInfo p in _runningProcesses) { for (final ProcessInfo p in _runningProcesses) {
print('Force-quitting process:\n$p'); print('Force-quitting process:\n$p');
if (!p.process.kill()) { if (!p.process.kill()) {
print('Failed to force quit process'); print('Failed to force quit process.');
} }
} }
_runningProcesses.clear(); _runningProcesses.clear();
...@@ -349,7 +352,7 @@ Future<int> _execute( ...@@ -349,7 +352,7 @@ Future<int> _execute(
stderr: stderr, stderr: stderr,
printStdout: printStdout, printStdout: printStdout,
printStderr: printStderr, printStderr: printStderr,
); );
final int exitCode = await process.exitCode; final int exitCode = await process.exitCode;
if (exitCode != 0 && !canFail) if (exitCode != 0 && !canFail)
...@@ -373,23 +376,23 @@ Future<void> forwardStandardStreams( ...@@ -373,23 +376,23 @@ Future<void> forwardStandardStreams(
final Completer<void> stdoutDone = Completer<void>(); final Completer<void> stdoutDone = Completer<void>();
final Completer<void> stderrDone = Completer<void>(); final Completer<void> stderrDone = Completer<void>();
process.stdout process.stdout
.transform<String>(utf8.decoder) .transform<String>(utf8.decoder)
.transform<String>(const LineSplitter()) .transform<String>(const LineSplitter())
.listen((String line) { .listen((String line) {
if (printStdout) { if (printStdout) {
print('stdout: $line'); print('stdout: $line');
} }
output?.writeln(line); output?.writeln(line);
}, onDone: () { stdoutDone.complete(); }); }, onDone: () { stdoutDone.complete(); });
process.stderr process.stderr
.transform<String>(utf8.decoder) .transform<String>(utf8.decoder)
.transform<String>(const LineSplitter()) .transform<String>(const LineSplitter())
.listen((String line) { .listen((String line) {
if (printStderr) { if (printStderr) {
print('stderr: $line'); print('stderr: $line');
} }
stderr?.writeln(line); stderr?.writeln(line);
}, onDone: () { stderrDone.complete(); }); }, onDone: () { stderrDone.complete(); });
return Future.wait<void>(<Future<void>>[ return Future.wait<void>(<Future<void>>[
stdoutDone.future, stdoutDone.future,
...@@ -436,6 +439,8 @@ List<String> flutterCommandArgs(String command, List<String> options) { ...@@ -436,6 +439,8 @@ List<String> flutterCommandArgs(String command, List<String> options) {
'run', 'run',
'screenshot', 'screenshot',
}; };
final String? localEngine = localEngineFromEnv;
final String? localEngineSrcPath = localEngineSrcPathFromEnv;
return <String>[ return <String>[
command, command,
if (deviceOperatingSystem == DeviceOperatingSystem.ios && supportedDeviceTimeoutCommands.contains(command)) if (deviceOperatingSystem == DeviceOperatingSystem.ios && supportedDeviceTimeoutCommands.contains(command))
...@@ -448,8 +453,8 @@ List<String> flutterCommandArgs(String command, List<String> options) { ...@@ -448,8 +453,8 @@ List<String> flutterCommandArgs(String command, List<String> options) {
'--screenshot', '--screenshot',
hostAgent.dumpDirectory!.path, hostAgent.dumpDirectory!.path,
], ],
if (localEngine != null) ...<String>['--local-engine', localEngine!], if (localEngine != null) ...<String>['--local-engine', localEngine],
if (localEngineSrcPath != null) ...<String>['--local-engine-src-path', localEngineSrcPath!], if (localEngineSrcPath != null) ...<String>['--local-engine-src-path', localEngineSrcPath],
...options, ...options,
]; ];
} }
......
dev/devicelab/lib/tasks/perf_tests.dart
View file @ 57f097b5
...@@ -847,12 +847,14 @@ class PerfTest { ...@@ -847,12 +847,14 @@ class PerfTest {
final Device device = await devices.workingDevice; final Device device = await devices.workingDevice;
await device.unlock(); await device.unlock();
final String deviceId = device.deviceId; final String deviceId = device.deviceId;
final String? localEngine = localEngineFromEnv;
final String? localEngineSrcPath = localEngineSrcPathFromEnv;
await flutter('drive', options: <String>[ await flutter('drive', options: <String>[
if (localEngine != null) if (localEngine != null)
...<String>['--local-engine', localEngine!], ...<String>['--local-engine', localEngine],
if (localEngineSrcPath != null) if (localEngineSrcPath != null)
...<String>['--local-engine-src-path', localEngineSrcPath!], ...<String>['--local-engine-src-path', localEngineSrcPath],
'--no-dds', '--no-dds',
'--no-android-gradle-daemon', '--no-android-gradle-daemon',
'-v', '-v',
...@@ -1028,15 +1030,16 @@ class PerfTestWithSkSL extends PerfTest { ...@@ -1028,15 +1030,16 @@ class PerfTestWithSkSL extends PerfTest {
if (File(_vmserviceFileName).existsSync()) { if (File(_vmserviceFileName).existsSync()) {
File(_vmserviceFileName).deleteSync(); File(_vmserviceFileName).deleteSync();
} }
final String? localEngine = localEngineFromEnv;
final String? localEngineSrcPath = localEngineSrcPathFromEnv;
_runProcess = await startProcess( _runProcess = await startProcess(
_flutterPath, _flutterPath,
<String>[ <String>[
'run', 'run',
if (localEngine != null) if (localEngine != null)
...<String>['--local-engine', localEngine!], ...<String>['--local-engine', localEngine],
if (localEngineSrcPath != null) if (localEngineSrcPath != null)
...<String>['--local-engine-src-path', localEngineSrcPath!], ...<String>['--local-engine-src-path', localEngineSrcPath],
'--no-dds', '--no-dds',
if (deviceOperatingSystem == DeviceOperatingSystem.ios) if (deviceOperatingSystem == DeviceOperatingSystem.ios)
...<String>[ ...<String>[
......
dev/devicelab/test/run_test.dart
View file @ 57f097b5
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
import 'dart:io'; import 'dart:io';
import 'package:flutter_devicelab/framework/utils.dart' show rm;
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:process/process.dart'; import 'package:process/process.dart';
...@@ -20,6 +21,7 @@ void main() { ...@@ -20,6 +21,7 @@ void main() {
final ProcessResult scriptProcess = processManager.runSync(<String>[ final ProcessResult scriptProcess = processManager.runSync(<String>[
dart, dart,
'bin/run.dart', 'bin/run.dart',
'--no-terminate-stray-dart-processes',
...otherArgs, ...otherArgs,
for (final String testName in testNames) ...<String>['-t', testName], for (final String testName in testNames) ...<String>['-t', testName],
]); ]);
...@@ -87,9 +89,14 @@ void main() { ...@@ -87,9 +89,14 @@ void main() {
test('runs A/B test', () async { test('runs A/B test', () async {
final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_devicelab_ab_test.');
final File abResultsFile = File(path.join(tempDirectory.path, 'test_results.json'));
expect(abResultsFile.existsSync(), isFalse);
final ProcessResult result = await runScript( final ProcessResult result = await runScript(
<String>['smoke_test_success'], <String>['smoke_test_success'],
<String>['--ab=2', '--local-engine=host_debug_unopt'], <String>['--ab=2', '--local-engine=host_debug_unopt', '--ab-result-file', abResultsFile.path],
); );
expect(result.exitCode, 0); expect(result.exitCode, 0);
...@@ -137,6 +144,9 @@ void main() { ...@@ -137,6 +144,9 @@ void main() {
'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n', 'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
), ),
); );
expect(abResultsFile.existsSync(), isTrue);
rm(tempDirectory, recursive: true);
}); });
test('fails to upload results to Cocoon if flags given', () async { test('fails to upload results to Cocoon if flags given', () async {
......
dev/devicelab/test/runner_test.dart
View file @ 57f097b5
...@@ -11,7 +11,6 @@ import 'common.dart'; ...@@ -11,7 +11,6 @@ import 'common.dart';
void main() { void main() {
final Map<String, String> isolateParams = <String, String>{ final Map<String, String> isolateParams = <String, String>{
'runFlutterConfig': 'false', 'runFlutterConfig': 'false',
'runProcessCleanup': 'false',
'timeoutInMinutes': '1', 'timeoutInMinutes': '1',
}; };
List<String> printLog; List<String> printLog;
...@@ -27,7 +26,7 @@ void main() { ...@@ -27,7 +26,7 @@ void main() {
logs: printLog, logs: printLog,
); );
expect(printLog.length, 2); expect(printLog.length, 2);
expect(printLog[0], 'Total 1 executions: 1 success'); expect(printLog[0], 'Test passed on first attempt.');
expect(printLog[1], 'flaky: false'); expect(printLog[1], 'flaky: false');
}); });
...@@ -40,7 +39,7 @@ void main() { ...@@ -40,7 +39,7 @@ void main() {
logs: printLog, logs: printLog,
); );
expect(printLog.length, 2); expect(printLog.length, 2);
expect(printLog[0], 'Total 3 executions: 0 success'); expect(printLog[0], 'Consistently failed across all 3 executions.');
expect(printLog[1], 'flaky: false'); expect(printLog[1], 'flaky: false');
}); });
}); });
......
dev/devicelab/test/running_processes_test.dart
View file @ 57f097b5
...@@ -2,7 +2,13 @@ ...@@ -2,7 +2,13 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:flutter_devicelab/framework/running_processes.dart'; import 'package:flutter_devicelab/framework/running_processes.dart';
import 'package:process/process.dart';
import 'common.dart'; import 'common.dart';
void main() { void main() {
...@@ -24,19 +30,19 @@ ProcessId CreationDate CommandLine ...@@ -24,19 +30,19 @@ ProcessId CreationDate CommandLine
results, results,
equals(<RunningProcessInfo>[ equals(<RunningProcessInfo>[
RunningProcessInfo( RunningProcessInfo(
'6552', 6552,
DateTime(2019, 7, 3, 17, 0, 27),
r'"C:\tools\dart-sdk\bin\dart.exe" .\bin\agent.dart ci', r'"C:\tools\dart-sdk\bin\dart.exe" .\bin\agent.dart ci',
DateTime(2019, 7, 3, 17, 0, 27),
), ),
RunningProcessInfo( RunningProcessInfo(
'6553', 6553,
DateTime(2019, 7, 3, 22, 0, 27),
r'"C:\tools\dart-sdk1\bin\dart.exe" .\bin\agent.dart ci', r'"C:\tools\dart-sdk1\bin\dart.exe" .\bin\agent.dart ci',
DateTime(2019, 7, 3, 22, 0, 27),
), ),
RunningProcessInfo( RunningProcessInfo(
'6554', 6554,
DateTime(2019, 7, 3, 11, 0, 27),
r'"C:\tools\dart-sdk2\bin\dart.exe" .\bin\agent.dart ci', r'"C:\tools\dart-sdk2\bin\dart.exe" .\bin\agent.dart ci',
DateTime(2019, 7, 3, 11, 0, 27),
), ),
])); ]));
}); });
...@@ -55,15 +61,81 @@ Sat Mar 9 20:13:00 2019 49 /usr/sbin/syslogd ...@@ -55,15 +61,81 @@ Sat Mar 9 20:13:00 2019 49 /usr/sbin/syslogd
results, results,
equals(<RunningProcessInfo>[ equals(<RunningProcessInfo>[
RunningProcessInfo( RunningProcessInfo(
'1', 1,
DateTime(2019, 3, 9, 20, 12, 47),
'/sbin/launchd', '/sbin/launchd',
DateTime(2019, 3, 9, 20, 12, 47),
), ),
RunningProcessInfo( RunningProcessInfo(
'49', 49,
DateTime(2019, 3, 9, 20, 13),
'/usr/sbin/syslogd', '/usr/sbin/syslogd',
DateTime(2019, 3, 9, 20, 13),
), ),
])); ]));
}); });
test('RunningProcessInfo.terminate', () {
final RunningProcessInfo process = RunningProcessInfo(123, 'test', DateTime(456));
final FakeProcessManager fakeProcessManager = FakeProcessManager();
process.terminate(processManager: fakeProcessManager);
if (Platform.isWindows) {
expect(fakeProcessManager.log, <String>['run([taskkill.exe, /pid, 123, /f], null, null, null, null, null, null)']);
} else {
expect(fakeProcessManager.log, <String>['killPid(123, SIGKILL)']);
}
});
}
class FakeProcessManager implements ProcessManager {
final List<String> log = <String>[];
@override
bool canRun(Object? a, { String? workingDirectory }) {
log.add('canRun($a, $workingDirectory)');
return true;
}
@override
bool killPid(int a, [ProcessSignal? b]) {
log.add('killPid($a, $b)');
return true;
}
@override
Future<ProcessResult> run(List<Object> a, {
Map<String, String>? environment,
bool? includeParentEnvironment,
bool? runInShell,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
}) async {
log.add('run($a, $environment, $includeParentEnvironment, $runInShell, $stderrEncoding, $stdoutEncoding, $workingDirectory)');
return ProcessResult(1, 0, 'stdout', 'stderr');
}
@override
ProcessResult runSync(List<Object> a, {
Map<String, String>? environment,
bool? includeParentEnvironment,
bool? runInShell,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
}) {
log.add('runSync($a, $environment, $includeParentEnvironment, $runInShell, $stderrEncoding, $stdoutEncoding, $workingDirectory)');
return ProcessResult(1, 0, 'stdout', 'stderr');
}
@override
Future<Process> start(
List<Object> a, {
Map<String, String>? environment,
bool? includeParentEnvironment,
ProcessStartMode? mode,
bool? runInShell,
String? workingDirectory,
}) {
log.add('start($a, $environment, $includeParentEnvironment, $mode, $runInShell, $workingDirectory)');
return Completer<Process>().future;
}
} }
dev/devicelab/test/tasks/build_test_task_test.dart
View file @ 57f097b5
...@@ -12,7 +12,6 @@ import '../common.dart'; ...@@ -12,7 +12,6 @@ import '../common.dart';
void main() { void main() {
final Map<String, String> isolateParams = <String, String>{ final Map<String, String> isolateParams = <String, String>{
'runFlutterConfig': 'false', 'runFlutterConfig': 'false',
'runProcessCleanup': 'false',
'timeoutInMinutes': '1', 'timeoutInMinutes': '1',
}; };
...@@ -66,7 +65,7 @@ void main() { ...@@ -66,7 +65,7 @@ void main() {
final String capturedPrint = capturedPrintLines.toString(); final String capturedPrint = capturedPrintLines.toString();
expect(capturedPrint, expect(capturedPrint,
contains('with environment {FLUTTER_DEVICELAB_DEVICEID: FAKE_SUCCESS, BOT: true, LANG: en_US.UTF-8}')); contains('with environment {FLUTTER_DEVICELAB_DEVICEID: FAKE_SUCCESS, BOT: true, LANG: en_US.UTF-8}'));
expect(capturedPrint, contains('exit code: 0')); expect(capturedPrint, contains('Process terminated with exit code 0.'));
}); });
test('throws exception when build and test arg are given', () async { test('throws exception when build and test arg are given', () async {
......
dev/devicelab/test/utils_test.dart
View file @ 57f097b5
...@@ -36,8 +36,8 @@ void main() { ...@@ -36,8 +36,8 @@ void main() {
group('engine environment declarations', () { group('engine environment declarations', () {
test('localEngine', () { test('localEngine', () {
expect(localEngine, null); expect(localEngineFromEnv, null);
expect(localEngineSrcPath, null); expect(localEngineSrcPathFromEnv, null);
}); });
}); });
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab