Unverified Commit c1a2e44c authored by Jonah Williams's avatar Jonah Williams Committed by GitHub

Allow flaky tests to pass or fail and mark web tests as flaky (#34456)

parent f68bc1be
...@@ -85,6 +85,7 @@ Future<void> runCommand(String executable, List<String> arguments, { ...@@ -85,6 +85,7 @@ Future<void> runCommand(String executable, List<String> arguments, {
String failureMessage, String failureMessage,
bool printOutput = true, bool printOutput = true,
bool skip = false, bool skip = false,
bool expectFlaky = false,
Duration timeout = _kLongTimeout, Duration timeout = _kLongTimeout,
}) async { }) async {
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
...@@ -114,9 +115,13 @@ Future<void> runCommand(String executable, List<String> arguments, { ...@@ -114,9 +115,13 @@ Future<void> runCommand(String executable, List<String> arguments, {
final int exitCode = await process.exitCode.timeout(timeout, onTimeout: () { final int exitCode = await process.exitCode.timeout(timeout, onTimeout: () {
stderr.writeln('Process timed out after $timeout'); stderr.writeln('Process timed out after $timeout');
return expectNonZeroExit ? 0 : 1; return (expectNonZeroExit || expectFlaky) ? 0 : 1;
}); });
print('$clock ELAPSED TIME: $bold${elapsedTime(start)}$reset for $commandDescription in $relativeWorkingDir: '); print('$clock ELAPSED TIME: $bold${elapsedTime(start)}$reset for $commandDescription in $relativeWorkingDir: ');
// If the test is flaky we don't care about the actual exit.
if (expectFlaky) {
return;
}
if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) { if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) {
if (failureMessage != null) { if (failureMessage != null) {
print(failureMessage); print(failureMessage);
......
...@@ -341,7 +341,7 @@ Future<void> _runTests() async { ...@@ -341,7 +341,7 @@ Future<void> _runTests() async {
} }
Future<void> _runWebTests() async { Future<void> _runWebTests() async {
await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter'), expectFailure: false, tests: <String>[ await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter'), tests: <String>[
'test/foundation/', 'test/foundation/',
'test/physics/', 'test/physics/',
'test/rendering/', 'test/rendering/',
...@@ -607,7 +607,6 @@ class EvalResult { ...@@ -607,7 +607,6 @@ class EvalResult {
} }
Future<void> _runFlutterWebTest(String workingDirectory, { Future<void> _runFlutterWebTest(String workingDirectory, {
bool expectFailure = false,
bool printOutput = true, bool printOutput = true,
bool skip = false, bool skip = false,
Duration timeout = _kLongTimeout, Duration timeout = _kLongTimeout,
...@@ -627,7 +626,7 @@ Future<void> _runFlutterWebTest(String workingDirectory, { ...@@ -627,7 +626,7 @@ Future<void> _runFlutterWebTest(String workingDirectory, {
flutter, flutter,
args, args,
workingDirectory: workingDirectory, workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure, expectFlaky: true,
timeout: timeout, timeout: timeout,
environment: <String, String>{ environment: <String, String>{
'FLUTTER_WEB': 'true', 'FLUTTER_WEB': 'true',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment