Skip to content

F
Front-End
Unverified Commit ff9082cf authored by Yegor's avatar Yegor Committed by GitHub
Browse files
Options

Refactor command utilities for tests (#68324) 

* refactor command running utilities for tests
Co-authored-by: 's avatarJonah Williams <jonahwilliams@google.com>
parent 03b5825d
Show whitespace changes
Inline Side-by-side
Showing with 158 additions and 99 deletions
dev/bots/run_command.dart
View file @ ff9082cf
...@@ -2,92 +2,113 @@ ...@@ -2,92 +2,113 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:async';
import 'dart:convert'; import 'dart:convert';
import 'dart:core' hide print; import 'dart:core' hide print;
import 'dart:io' hide exit; import 'dart:io' as io;
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'utils.dart'; import 'utils.dart';
// TODO(ianh): These two functions should be refactored into something that avoids all this code duplication. /// Runs the `executable` and returns standard output as a stream of lines.
///
/// The returned stream reaches its end immediately after the command exits.
///
/// If `expectNonZeroExit` is false and the process exits with a non-zero exit
/// code fails the test immediately by exiting the test process with exit code
/// 1.
Stream<String> runAndGetStdout(String executable, List<String> arguments, { Stream<String> runAndGetStdout(String executable, List<String> arguments, {
String workingDirectory, String workingDirectory,
Map<String, String> environment, Map<String, String> environment,
bool expectNonZeroExit = false, bool expectNonZeroExit = false,
int expectedExitCode,
String failureMessage,
bool skip = false,
}) async* { }) async* {
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; final StreamController<String> output = StreamController<String>();
final String relativeWorkingDir = path.relative(workingDirectory); final Future<CommandResult> command = runCommand(
if (skip) { executable,
printProgress('SKIPPING', relativeWorkingDir, commandDescription); arguments,
return;
}
printProgress('RUNNING', relativeWorkingDir, commandDescription);
final Stopwatch time = Stopwatch()..start();
final Process process = await Process.start(executable, arguments,
workingDirectory: workingDirectory, workingDirectory: workingDirectory,
environment: environment, environment: environment,
expectNonZeroExit: expectNonZeroExit,
// Capture the output so it's not printed to the console by default.
outputMode: OutputMode.capture,
outputListener: (String line, io.Process process) {
output.add(line);
},
); );
stderr.addStream(process.stderr); // Close the stream controller after the command is complete. Otherwise,
final Stream<String> lines = process.stdout.transform(utf8.decoder).transform(const LineSplitter()); // the yield* will never finish.
yield* lines; command.whenComplete(output.close);
yield* output.stream;
}
/// Represents a running process launched using [startCommand].
class Command {
Command._(this.process, this._time, this._savedStdout, this._savedStderr);
/// The raw process that was launched for this command.
final io.Process process;
final Stopwatch _time;
final Future<List<List<int>>> _savedStdout;
final Future<List<List<int>>> _savedStderr;
/// Evaluates when the [process] exits.
///
/// Returns the result of running the command.
Future<CommandResult> get onExit async {
final int exitCode = await process.exitCode; final int exitCode = await process.exitCode;
if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) { _time.stop();
exitWithError(<String>[
if (failureMessage != null) // Saved output is null when OutputMode.print is used.
failureMessage final String flattenedStdout = _savedStdout != null ? _flattenToString(await _savedStdout) : null;
else final String flattenedStderr = _savedStderr != null ? _flattenToString(await _savedStderr) : null;
'${bold}ERROR: ${red}Last command exited with $exitCode (expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'}).$reset', return CommandResult._(exitCode, _time.elapsed, flattenedStdout, flattenedStderr);
'${bold}Command: $green$commandDescription$reset',
'${bold}Relative working directory: $cyan$relativeWorkingDir$reset',
]);
} }
print('$clock ELAPSED TIME: ${prettyPrintDuration(time.elapsed)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
} }
/// Runs the `executable` and waits until the process exits. /// The result of running a command using [startCommand] and [runCommand];
/// class CommandResult {
/// If the process exits with a non-zero exit code, exits this process with CommandResult._(this.exitCode, this.elapsedTime, this.flattenedStdout, this.flattenedStderr);
/// exit code 1, unless `expectNonZeroExit` is set to true.
/// The exit code of the process.
final int exitCode;
/// The amount of time it took the process to complete.
final Duration elapsedTime;
/// Standard output decoded as a string using UTF8 decoder.
final String flattenedStdout;
/// Standard error output decoded as a string using UTF8 decoder.
final String flattenedStderr;
}
/// Starts the `executable` and returns a command object representing the
/// running process.
/// ///
/// `outputListener` is called for every line of standard output from the /// `outputListener` is called for every line of standard output from the
/// process, and is given the [Process] object. This can be used to interrupt /// process, and is given the [Process] object. This can be used to interrupt
/// an indefinitely running process, for example, by waiting until the process /// an indefinitely running process, for example, by waiting until the process
/// emits certain output. /// emits certain output.
Future<void> runCommand(String executable, List<String> arguments, { ///
/// `outputMode` controls where the standard output from the command process
/// goes. See [OutputMode].
Future<Command> startCommand(String executable, List<String> arguments, {
String workingDirectory, String workingDirectory,
Map<String, String> environment, Map<String, String> environment,
bool expectNonZeroExit = false,
int expectedExitCode,
String failureMessage,
OutputMode outputMode = OutputMode.print, OutputMode outputMode = OutputMode.print,
CapturedOutput output,
bool skip = false,
bool Function(String) removeLine, bool Function(String) removeLine,
void Function(String, Process) outputListener, void Function(String, io.Process) outputListener,
}) async { }) async {
assert(
(outputMode == OutputMode.capture) == (output != null),
'The output parameter must be non-null with and only with OutputMode.capture',
);
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
final String relativeWorkingDir = path.relative(workingDirectory ?? Directory.current.path); final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path);
if (skip) {
printProgress('SKIPPING', relativeWorkingDir, commandDescription);
return;
}
printProgress('RUNNING', relativeWorkingDir, commandDescription); printProgress('RUNNING', relativeWorkingDir, commandDescription);
final Stopwatch time = Stopwatch()..start(); final Stopwatch time = Stopwatch()..start();
final Process process = await Process.start(executable, arguments, final io.Process process = await io.Process.start(executable, arguments,
workingDirectory: workingDirectory, workingDirectory: workingDirectory,
environment: environment, environment: environment,
); );
...@@ -108,56 +129,103 @@ Future<void> runCommand(String executable, List<String> arguments, { ...@@ -108,56 +129,103 @@ Future<void> runCommand(String executable, List<String> arguments, {
switch (outputMode) { switch (outputMode) {
case OutputMode.print: case OutputMode.print:
await Future.wait<void>(<Future<void>>[ await Future.wait<void>(<Future<void>>[
stdout.addStream(stdoutSource), io.stdout.addStream(stdoutSource),
stderr.addStream(process.stderr), io.stderr.addStream(process.stderr),
]); ]);
break; break;
case OutputMode.capture: case OutputMode.capture:
case OutputMode.discard:
savedStdout = stdoutSource.toList(); savedStdout = stdoutSource.toList();
savedStderr = process.stderr.toList(); savedStderr = process.stderr.toList();
break; break;
} }
final int exitCode = await process.exitCode; return Command._(process, time, savedStdout, savedStderr);
if (output != null) { }
output.stdout = _flattenToString(await savedStdout);
output.stderr = _flattenToString(await savedStderr); /// Runs the `executable` and waits until the process exits.
///
/// If the process exits with a non-zero exit code, exits this process with
/// exit code 1, unless `expectNonZeroExit` is set to true.
///
/// `outputListener` is called for every line of standard output from the
/// process, and is given the [Process] object. This can be used to interrupt
/// an indefinitely running process, for example, by waiting until the process
/// emits certain output.
///
/// Returns the result of the finished process, or null if `skip` is true.
///
/// `outputMode` controls where the standard output from the command process
/// goes. See [OutputMode].
Future<CommandResult> runCommand(String executable, List<String> arguments, {
String workingDirectory,
Map<String, String> environment,
bool expectNonZeroExit = false,
int expectedExitCode,
String failureMessage,
OutputMode outputMode = OutputMode.print,
bool skip = false,
bool Function(String) removeLine,
void Function(String, io.Process) outputListener,
}) async {
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path);
if (skip) {
printProgress('SKIPPING', relativeWorkingDir, commandDescription);
return null;
} }
if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) { final Command command = await startCommand(executable, arguments,
workingDirectory: workingDirectory,
environment: environment,
outputMode: outputMode,
removeLine: removeLine,
outputListener: outputListener,
);
final CommandResult result = await command.onExit;
if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) {
// Print the output when we get unexpected results (unless output was // Print the output when we get unexpected results (unless output was
// printed already). // printed already).
switch (outputMode) { switch (outputMode) {
case OutputMode.print: case OutputMode.print:
break; break;
case OutputMode.capture: case OutputMode.capture:
case OutputMode.discard: io.stdout.writeln(result.flattenedStdout);
stdout.writeln(_flattenToString(await savedStdout)); io.stderr.writeln(result.flattenedStderr);
stderr.writeln(_flattenToString(await savedStderr));
break; break;
} }
exitWithError(<String>[ exitWithError(<String>[
if (failureMessage != null) if (failureMessage != null)
failureMessage failureMessage
else else
'${bold}ERROR: ${red}Last command exited with $exitCode (expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'}).$reset', '${bold}ERROR: ${red}Last command exited with ${result.exitCode} (expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'}).$reset',
'${bold}Command: $green$commandDescription$reset', '${bold}Command: $green$commandDescription$reset',
'${bold}Relative working directory: $cyan$relativeWorkingDir$reset', '${bold}Relative working directory: $cyan$relativeWorkingDir$reset',
]); ]);
} }
print('$clock ELAPSED TIME: ${prettyPrintDuration(time.elapsed)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset'); print('$clock ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
return result;
} }
/// Flattens a nested list of UTF-8 code units into a single string. /// Flattens a nested list of UTF-8 code units into a single string.
String _flattenToString(List<List<int>> chunks) => String _flattenToString(List<List<int>> chunks) =>
utf8.decode(chunks.expand<int>((List<int> ints) => ints).toList()); utf8.decode(chunks.expand<int>((List<int> ints) => ints).toList());
/// Specifies what to do with command output from [runCommand]. /// Specifies what to do with the command output from [runCommand] and [startCommand].
enum OutputMode { print, capture, discard } enum OutputMode {
/// Forwards standard output and standard error streams to the test process'
/// Stores command output from [runCommand] when used with [OutputMode.capture]. /// respective standard streams.
class CapturedOutput { ///
String stdout; /// Use this mode if all you want is print the output of the command to the
String stderr; /// console. The output is no longer available after the process exits.
print,
/// Saves standard output and standard error streams in memory.
///
/// Captured output can be retrieved from the [CommandResult] object.
///
/// Use this mode in tests that need to inspect the output of a command, or
/// when the output should not be printed to console.
capture,
} }
dev/bots/test.dart
View file @ ff9082cf
...@@ -26,7 +26,7 @@ typedef ShardRunner = Future<void> Function(); ...@@ -26,7 +26,7 @@ typedef ShardRunner = Future<void> Function();
/// ///
/// If the output does not match expectations, the function shall return an /// If the output does not match expectations, the function shall return an
/// appropriate error message. /// appropriate error message.
typedef OutputChecker = String Function(CapturedOutput); typedef OutputChecker = String Function(CommandResult);
final String exe = Platform.isWindows ? '.exe' : ''; final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : ''; final String bat = Platform.isWindows ? '.bat' : '';
...@@ -143,9 +143,8 @@ Future<void> _validateEngineHash() async { ...@@ -143,9 +143,8 @@ Future<void> _validateEngineHash() async {
return; return;
} }
final String expectedVersion = File(engineVersionFile).readAsStringSync().trim(); final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
final CapturedOutput flutterTesterOutput = CapturedOutput(); final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
await runCommand(flutterTester, <String>['--help'], output: flutterTesterOutput, outputMode: OutputMode.capture); final String actualVersion = result.flattenedStderr.split('\n').firstWhere((final String line) {
final String actualVersion = flutterTesterOutput.stderr.split('\n').firstWhere((final String line) {
return line.startsWith('Flutter Engine Version:'); return line.startsWith('Flutter Engine Version:');
}); });
if (!actualVersion.contains(expectedVersion)) { if (!actualVersion.contains(expectedVersion)) {
...@@ -190,8 +189,8 @@ Future<void> _runSmokeTests() async { ...@@ -190,8 +189,8 @@ Future<void> _runSmokeTests() async {
script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'), script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
expectFailure: true, expectFailure: true,
printOutput: false, printOutput: false,
outputChecker: (CapturedOutput output) { outputChecker: (CommandResult result) {
return output.stdout.contains('failingPendingTimerTest') return result.flattenedStdout.contains('failingPendingTimerTest')
? null ? null
: 'Failed to find the stack trace for the pending Timer.'; : 'Failed to find the stack trace for the pending Timer.';
} }
...@@ -230,7 +229,7 @@ Future<void> _runSmokeTests() async { ...@@ -230,7 +229,7 @@ Future<void> _runSmokeTests() async {
<String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')], <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'), workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
expectNonZeroExit: true, expectNonZeroExit: true,
outputMode: OutputMode.discard, outputMode: OutputMode.capture,
), ),
], ],
); );
...@@ -287,7 +286,7 @@ Future<void> _runToolCoverage() async { ...@@ -287,7 +286,7 @@ Future<void> _runToolCoverage() async {
'--report-on=lib/' '--report-on=lib/'
], ],
workingDirectory: toolRoot, workingDirectory: toolRoot,
outputMode: OutputMode.discard, outputMode: OutputMode.capture,
); );
} }
...@@ -642,11 +641,11 @@ Future<void> _runFrameworkTests() async { ...@@ -642,11 +641,11 @@ Future<void> _runFrameworkTests() async {
script: path.join('test', 'bindings_test_failure.dart'), script: path.join('test', 'bindings_test_failure.dart'),
expectFailure: true, expectFailure: true,
printOutput: false, printOutput: false,
outputChecker: (CapturedOutput output) { outputChecker: (CommandResult result) {
final Iterable<Match> matches = httpClientWarning.allMatches(output.stdout); final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout);
if (matches == null || matches.isEmpty || matches.length > 1) { if (matches == null || matches.isEmpty || matches.length > 1) {
return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n' return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n'
'stdout:\n${output.stdout}'; 'stdout:\n${result.flattenedStdout}';
} }
return null; return null;
}, },
...@@ -868,9 +867,8 @@ Future<void> _runWebDebugTest(String target, { ...@@ -868,9 +867,8 @@ Future<void> _runWebDebugTest(String target, {
List<String> additionalArguments = const<String>[], List<String> additionalArguments = const<String>[],
}) async { }) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web'); final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final CapturedOutput output = CapturedOutput();
bool success = false; bool success = false;
await runCommand( final CommandResult result = await runCommand(
flutter, flutter,
<String>[ <String>[
'run', 'run',
...@@ -889,7 +887,6 @@ Future<void> _runWebDebugTest(String target, { ...@@ -889,7 +887,6 @@ Future<void> _runWebDebugTest(String target, {
'-t', '-t',
target, target,
], ],
output: output,
outputMode: OutputMode.capture, outputMode: OutputMode.capture,
outputListener: (String line, Process process) { outputListener: (String line, Process process) {
if (line.contains('--- TEST SUCCEEDED ---')) { if (line.contains('--- TEST SUCCEEDED ---')) {
...@@ -908,7 +905,8 @@ Future<void> _runWebDebugTest(String target, { ...@@ -908,7 +905,8 @@ Future<void> _runWebDebugTest(String target, {
if (success) { if (success) {
print('${green}Web stack trace integration test passed.$reset'); print('${green}Web stack trace integration test passed.$reset');
} else { } else {
print(output.stdout); print(result.flattenedStdout);
print(result.flattenedStderr);
print('${red}Web stack trace integration test failed.$reset'); print('${red}Web stack trace integration test failed.$reset');
exit(1); exit(1);
} }
...@@ -1125,29 +1123,22 @@ Future<void> _runFlutterTest(String workingDirectory, { ...@@ -1125,29 +1123,22 @@ Future<void> _runFlutterTest(String workingDirectory, {
args.addAll(tests); args.addAll(tests);
if (!shouldProcessOutput) { if (!shouldProcessOutput) {
OutputMode outputMode = OutputMode.discard; final OutputMode outputMode = outputChecker == null && printOutput
CapturedOutput output; ? OutputMode.print
: OutputMode.capture;
if (outputChecker != null) { final CommandResult result = await runCommand(
outputMode = OutputMode.capture;
output = CapturedOutput();
} else if (printOutput) {
outputMode = OutputMode.print;
}
await runCommand(
flutter, flutter,
args, args,
workingDirectory: workingDirectory, workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure, expectNonZeroExit: expectFailure,
outputMode: outputMode, outputMode: outputMode,
output: output,
skip: skip, skip: skip,
environment: environment, environment: environment,
); );
if (outputChecker != null) { if (outputChecker != null) {
final String message = outputChecker(output); final String message = outputChecker(result);
if (message != null) if (message != null)
exitWithError(<String>[message]); exitWithError(<String>[message]);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab