Unverified Commit 5e216d44 authored by Ian Hickson's avatar Ian Hickson Committed by GitHub

Simplify devicelab logic and fix tests (#139122)

- fix https://github.com/flutter/flutter/issues/53707 by having the test not expect a timeout but instead actually look for the retry message
- simplify the `--task` option to only accept task names rather than also accepting paths
- remove some obsolete options that referred to the manifest which no longer seems to exist
parent c5328650
......@@ -66,16 +66,11 @@ To run a test, use option `-t` (`--task`):
```sh
# from the .../flutter/dev/devicelab directory
../../bin/cache/dart-sdk/bin/dart bin/test_runner.dart test -t {NAME_OR_PATH_OF_TEST}
../../bin/cache/dart-sdk/bin/dart bin/test_runner.dart test -t {NAME_OF_TEST}
```
Where `NAME_OR_PATH_OF_TEST` can be either of:
* the _name_ of a task, which is a file's basename in `bin/tasks`. Example:
`complex_layout__start_up`.
* the path to a Dart _file_ corresponding to a task, which resides in
`bin/tasks`. Tip: most shells support path auto-completion using the Tab key.
Example: `bin/tasks/complex_layout__start_up.dart`.
Where `NAME_OR_PATH_OF_TEST` is the name of a task, which is a file's
basename in `bin/tasks`. Example: `complex_layout__start_up`.
To run multiple tests, repeat option `-t` (`--task`) multiple times:
......
......@@ -10,7 +10,6 @@ import 'package:flutter_devicelab/framework/ab.dart';
import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path;
/// Runs tasks.
///
......@@ -235,29 +234,12 @@ ArgParser createArgParser(List<String> taskNames) {
..addMultiOption(
'task',
abbr: 't',
help: 'Either:\n'
' - the name of a task defined in manifest.yaml.\n'
' Example: complex_layout__start_up.\n'
' - the path to a Dart file corresponding to a task,\n'
' which resides in bin/tasks.\n'
' Example: bin/tasks/complex_layout__start_up.dart.\n'
help: 'Name of a Dart file in bin/tasks.\n'
' Example: complex_layout__start_up\n'
'\n'
'This option may be repeated to specify multiple tasks.',
callback: (List<String> value) {
for (final String nameOrPath in value) {
final List<String> fragments = path.split(nameOrPath);
final bool isDartFile = fragments.last.endsWith('.dart');
if (fragments.length == 1 && !isDartFile) {
// Not a path
taskNames.add(nameOrPath);
} else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
// Unsupported executable location
throw FormatException('Invalid value for option -t (--task): $nameOrPath');
} else {
taskNames.add(path.withoutExtension(fragments.last));
}
}
callback: (List<String> tasks) {
taskNames.addAll(tasks);
},
)
..addOption(
......@@ -339,14 +321,6 @@ ArgParser createArgParser(List<String> taskNames) {
'the location based on the value of the --flutter-root option.',
)
..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
..addFlag(
'match-host-platform',
defaultsTo: true,
help: 'Only run tests that match the host platform (e.g. do not run a\n'
'test with a `required_agent_capabilities` value of "mac/android"\n'
'on a windows host). Each test publishes its '
'`required_agent_capabilities`\nin the `manifest.yaml` file.',
)
..addOption(
'results-file',
help: '[Flutter infrastructure] File path for test results. If passed with\n'
......
......@@ -7,5 +7,8 @@
/// By not calling `task()` the VM service extension is not registered and
/// therefore will not accept requests to run tasks. When the runner attempts to
/// connect and run the test it will receive a "method not found" error from the
/// VM service, will likely retry and finally time out.
/// VM service, will likely retry forever.
///
/// The test in ../../test/run_test.dart runs this task until it detects
/// the retry message and then aborts the task.
Future<void> main() async {}
......@@ -171,7 +171,8 @@ Future<TaskResult> runTask(
final String taskExecutable = 'bin/tasks/$taskName.dart';
if (!file(taskExecutable).existsSync()) {
throw 'Executable Dart file not found: $taskExecutable';
print('Executable Dart file not found: $taskExecutable');
exit(1);
}
if (useEmulator) {
......@@ -288,7 +289,13 @@ Future<ConnectionResult> _connectToRunnerIsolate(Uri vmServiceUri) async {
return ConnectionResult(client, isolate);
} catch (error) {
if (stopwatch.elapsed > const Duration(seconds: 10)) {
print('VM service still not ready after ${stopwatch.elapsed}: $error\nContinuing to retry...');
print(
'VM service still not ready. It is possible the target has failed.\n'
'Latest connection error:\n'
' $error\n'
'Continuing to retry...\n',
);
stopwatch.reset();
}
await Future<void>.delayed(const Duration(milliseconds: 50));
}
......
......@@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:flutter_devicelab/framework/utils.dart' show rm;
......@@ -12,45 +13,54 @@ import 'common.dart';
void main() {
const ProcessManager processManager = LocalProcessManager();
final String dart = path.absolute(
path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
group('run.dart script', () {
Future<ProcessResult> runScript(List<String> testNames,
// The tasks here refer to files in ../bin/tasks/*.dart
Future<ProcessResult> runScript(List<String> taskNames,
[List<String> otherArgs = const <String>[]]) async {
final String dart = path.absolute(
path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
final ProcessResult scriptProcess = processManager.runSync(<String>[
dart,
'bin/run.dart',
'--no-terminate-stray-dart-processes',
...otherArgs,
for (final String testName in testNames) ...<String>['-t', testName],
for (final String testName in taskNames) ...<String>['-t', testName],
]);
return scriptProcess;
}
Future<void> expectScriptResult(
List<String> testNames,
List<String> taskNames,
int expectedExitCode,
{String? deviceId}
) async {
final ProcessResult result = await runScript(testNames, <String>[
final ProcessResult result = await runScript(taskNames, <String>[
if (deviceId != null) ...<String>['-d', deviceId],
]);
expect(result.exitCode, expectedExitCode,
expect(
result.exitCode,
expectedExitCode,
reason:
'[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]'
'\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]');
'[ stderr from test process ]\n'
'\n'
'${result.stderr}\n'
'\n'
'[ end of stderr ]\n'
'\n'
'[ stdout from test process ]\n'
'\n'
'${result.stdout}\n'
'\n'
'[ end of stdout ]',
);
}
test('exits with code 0 when succeeds', () async {
await expectScriptResult(<String>['smoke_test_success'], 0);
});
test('accepts file paths', () async {
await expectScriptResult(
<String>['bin/tasks/smoke_test_success.dart'], 0);
});
test('rejects invalid file paths', () async {
await expectScriptResult(<String>['lib/framework/adb.dart'], 1);
});
......@@ -63,9 +73,18 @@ void main() {
await expectScriptResult(<String>['smoke_test_failure'], 1);
});
test('exits with code 1 when fails to connect', () async {
await expectScriptResult(<String>['smoke_test_setup_failure'], 1);
}, skip: true); // https://github.com/flutter/flutter/issues/53707
test('prints a message after a few seconds when failing to connect (this test takes >10s)', () async {
final Process process = await processManager.start(<String>[
dart,
'bin/run.dart',
'--no-terminate-stray-dart-processes',
'-t', 'smoke_test_setup_failure',
]);
await process.stdout.transform(utf8.decoder).where(
(String line) => line.contains('VM service still not ready. It is possible the target has failed')
).first;
expect(process.kill(), isTrue);
});
test('exits with code 1 when results are mixed', () async {
await expectScriptResult(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment