Unverified Commit 03aa059c authored by Ian Hickson's avatar Ian Hickson Committed by GitHub

Do not abort at first error when tests fail. (#108936)

parent 274a9349
This diff is collapsed.
...@@ -149,8 +149,8 @@ Future<Command> startCommand(String executable, List<String> arguments, { ...@@ -149,8 +149,8 @@ Future<Command> startCommand(String executable, List<String> arguments, {
/// Runs the `executable` and waits until the process exits. /// Runs the `executable` and waits until the process exits.
/// ///
/// If the process exits with a non-zero exit code, exits this process with /// If the process exits with a non-zero exit code and `expectNonZeroExit` is
/// exit code 1, unless `expectNonZeroExit` is set to true. /// false, calls foundError (which does not terminate execution!).
/// ///
/// `outputListener` is called for every line of standard output from the /// `outputListener` is called for every line of standard output from the
/// process, and is given the [Process] object. This can be used to interrupt /// process, and is given the [Process] object. This can be used to interrupt
...@@ -195,7 +195,7 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, { ...@@ -195,7 +195,7 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
io.stdout.writeln(result.flattenedStderr); io.stdout.writeln(result.flattenedStderr);
break; break;
} }
exitWithError(<String>[ foundError(<String>[
if (failureMessage != null) if (failureMessage != null)
failureMessage failureMessage
else else
...@@ -203,8 +203,9 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, { ...@@ -203,8 +203,9 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
'${bold}Command: $green$commandDescription$reset', '${bold}Command: $green$commandDescription$reset',
'${bold}Relative working directory: $cyan$relativeWorkingDir$reset', '${bold}Relative working directory: $cyan$relativeWorkingDir$reset',
]); ]);
} else {
print('$clock ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
} }
print('$clock ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
return result; return result;
} }
......
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:async'; import 'dart:core' hide print;
import 'dart:io'; import 'dart:io' hide exit;
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'package:shelf/shelf.dart'; import 'package:shelf/shelf.dart';
...@@ -43,6 +43,10 @@ Future<void> main() async { ...@@ -43,6 +43,10 @@ Future<void> main() async {
await runWebServiceWorkerTestWithCachingResources(headless: false, testType: ServiceWorkerTestType.withFlutterJsShort); await runWebServiceWorkerTestWithCachingResources(headless: false, testType: ServiceWorkerTestType.withFlutterJsShort);
await runWebServiceWorkerTestWithCachingResources(headless: false, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent); await runWebServiceWorkerTestWithCachingResources(headless: false, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent);
await runWebServiceWorkerTestWithBlockedServiceWorkers(headless: false); await runWebServiceWorkerTestWithBlockedServiceWorkers(headless: false);
if (hasError) {
print('One or more tests failed.');
reportErrorsAndExit();
}
} }
Future<void> _setAppVersion(int version) async { Future<void> _setAppVersion(int version) async {
......
This diff is collapsed.
...@@ -12,21 +12,22 @@ import 'common.dart'; ...@@ -12,21 +12,22 @@ import 'common.dart';
typedef AsyncVoidCallback = Future<void> Function(); typedef AsyncVoidCallback = Future<void> Function();
Future<String> capture(AsyncVoidCallback callback, { int exitCode = 0 }) async { Future<String> capture(AsyncVoidCallback callback, { bool shouldHaveErrors = false }) async {
final StringBuffer buffer = StringBuffer(); final StringBuffer buffer = StringBuffer();
final PrintCallback oldPrint = print; final PrintCallback oldPrint = print;
try { try {
print = (Object line) { print = (Object line) {
buffer.writeln(line); buffer.writeln(line);
}; };
try { await callback();
await callback(); expect(
expect(exitCode, 0); hasError,
} on ExitException catch (error) { shouldHaveErrors,
expect(error.exitCode, exitCode); reason: buffer.isEmpty ? '(No output to report.)' : hasError ? 'Unexpected errors:\n$buffer' : 'Unexpected success:\n$buffer',
} );
} finally { } finally {
print = oldPrint; print = oldPrint;
resetErrorStatus();
} }
if (stdout.supportsAnsiEscapes) { if (stdout.supportsAnsiEscapes) {
// Remove ANSI escapes when this test is running on a terminal. // Remove ANSI escapes when this test is running on a terminal.
...@@ -42,7 +43,7 @@ void main() { ...@@ -42,7 +43,7 @@ void main() {
final String dartPath = path.canonicalize(path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', dartName)); final String dartPath = path.canonicalize(path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', dartName));
test('analyze.dart - verifyDeprecations', () async { test('analyze.dart - verifyDeprecations', () async {
final String result = await capture(() => verifyDeprecations(testRootPath, minimumMatches: 2), exitCode: 1); final String result = await capture(() => verifyDeprecations(testRootPath, minimumMatches: 2), shouldHaveErrors: true);
final String lines = <String>[ final String lines = <String>[
'test/analyze-test-input/root/packages/foo/deprecation.dart:12: Deprecation notice does not match required pattern.', 'test/analyze-test-input/root/packages/foo/deprecation.dart:12: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:18: Deprecation notice should be a grammatically correct sentence and start with a capital letter; see style guide: STYLE_GUIDE_URL', 'test/analyze-test-input/root/packages/foo/deprecation.dart:18: Deprecation notice should be a grammatically correct sentence and start with a capital letter; see style guide: STYLE_GUIDE_URL',
...@@ -73,7 +74,7 @@ void main() { ...@@ -73,7 +74,7 @@ void main() {
}); });
test('analyze.dart - verifyGoldenTags', () async { test('analyze.dart - verifyGoldenTags', () async {
final String result = await capture(() => verifyGoldenTags(testRootPath, minimumMatches: 6), exitCode: 1); final String result = await capture(() => verifyGoldenTags(testRootPath, minimumMatches: 6), shouldHaveErrors: true);
const String noTag = "Files containing golden tests must be tagged using @Tags(<String>['reduced-test-set']) " const String noTag = "Files containing golden tests must be tagged using @Tags(<String>['reduced-test-set']) "
'at the top of the file before import statements.'; 'at the top of the file before import statements.';
const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'."; const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'.";
...@@ -117,7 +118,7 @@ void main() { ...@@ -117,7 +118,7 @@ void main() {
}); });
test('analyze.dart - verifyNoMissingLicense', () async { test('analyze.dart - verifyNoMissingLicense', () async {
final String result = await capture(() => verifyNoMissingLicense(testRootPath, checkMinimums: false), exitCode: 1); final String result = await capture(() => verifyNoMissingLicense(testRootPath, checkMinimums: false), shouldHaveErrors: true);
final String file = 'test/analyze-test-input/root/packages/foo/foo.dart' final String file = 'test/analyze-test-input/root/packages/foo/foo.dart'
.replaceAll('/', Platform.isWindows ? r'\' : '/'); .replaceAll('/', Platform.isWindows ? r'\' : '/');
expect(result, expect(result,
...@@ -137,7 +138,7 @@ void main() { ...@@ -137,7 +138,7 @@ void main() {
}); });
test('analyze.dart - verifyNoTrailingSpaces', () async { test('analyze.dart - verifyNoTrailingSpaces', () async {
final String result = await capture(() => verifyNoTrailingSpaces(testRootPath, minimumMatches: 2), exitCode: 1); final String result = await capture(() => verifyNoTrailingSpaces(testRootPath, minimumMatches: 2), shouldHaveErrors: true);
final String lines = <String>[ final String lines = <String>[
'test/analyze-test-input/root/packages/foo/spaces.txt:5: trailing U+0020 space character', 'test/analyze-test-input/root/packages/foo/spaces.txt:5: trailing U+0020 space character',
'test/analyze-test-input/root/packages/foo/spaces.txt:9: trailing blank line', 'test/analyze-test-input/root/packages/foo/spaces.txt:9: trailing blank line',
...@@ -155,7 +156,7 @@ void main() { ...@@ -155,7 +156,7 @@ void main() {
final String result = await capture(() => verifyNoBinaries( final String result = await capture(() => verifyNoBinaries(
testRootPath, testRootPath,
legacyBinaries: <Hash256>{const Hash256(0x39A050CD69434936, 0, 0, 0)}, legacyBinaries: <Hash256>{const Hash256(0x39A050CD69434936, 0, 0, 0)},
), exitCode: Platform.isWindows ? 0 : 1); ), shouldHaveErrors: !Platform.isWindows);
if (!Platform.isWindows) { if (!Platform.isWindows) {
expect(result, expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
...@@ -173,7 +174,7 @@ void main() { ...@@ -173,7 +174,7 @@ void main() {
}); });
test('analyze.dart - verifyInternationalizations - comparison fails', () async { test('analyze.dart - verifyInternationalizations - comparison fails', () async {
final String result = await capture(() => verifyInternationalizations(testRootPath, dartPath), exitCode: 1); final String result = await capture(() => verifyInternationalizations(testRootPath, dartPath), shouldHaveErrors: true);
final String genLocalizationsScript = path.join('dev', 'tools', 'localization', 'bin', 'gen_localizations.dart'); final String genLocalizationsScript = path.join('dev', 'tools', 'localization', 'bin', 'gen_localizations.dart');
expect(result, expect(result,
contains('$dartName $genLocalizationsScript --cupertino')); contains('$dartName $genLocalizationsScript --cupertino'));
...@@ -201,7 +202,7 @@ void main() { ...@@ -201,7 +202,7 @@ void main() {
final String result = await capture(() => verifyNullInitializedDebugExpensiveFields( final String result = await capture(() => verifyNullInitializedDebugExpensiveFields(
testRootPath, testRootPath,
minimumMatches: 1, minimumMatches: 1,
), exitCode: 1); ), shouldHaveErrors: true);
expect(result, contains('L15')); expect(result, contains('L15'));
expect(result, isNot(contains('L12'))); expect(result, isNot(contains('L12')));
......
...@@ -4,9 +4,11 @@ ...@@ -4,9 +4,11 @@
import 'dart:core' as core_internals show print; import 'dart:core' as core_internals show print;
import 'dart:core' hide print; import 'dart:core' hide print;
import 'dart:io' as io_internals show exit; import 'dart:io' as system show exit;
import 'dart:io' hide exit; import 'dart:io' hide exit;
import 'package:meta/meta.dart';
final bool hasColor = stdout.supportsAnsiEscapes; final bool hasColor = stdout.supportsAnsiEscapes;
final String bold = hasColor ? '\x1B[1m' : ''; // used for shard titles final String bold = hasColor ? '\x1B[1m' : ''; // used for shard titles
...@@ -16,36 +18,41 @@ final String yellow = hasColor ? '\x1B[33m' : ''; // used for skips ...@@ -16,36 +18,41 @@ final String yellow = hasColor ? '\x1B[33m' : ''; // used for skips
final String cyan = hasColor ? '\x1B[36m' : ''; // used for paths final String cyan = hasColor ? '\x1B[36m' : ''; // used for paths
final String reverse = hasColor ? '\x1B[7m' : ''; // used for clocks final String reverse = hasColor ? '\x1B[7m' : ''; // used for clocks
final String reset = hasColor ? '\x1B[0m' : ''; final String reset = hasColor ? '\x1B[0m' : '';
final String redLine = '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset';
class ExitException implements Exception { typedef PrintCallback = void Function(Object line);
ExitException(this.exitCode);
final int exitCode; // Allow print() to be overridden, for tests.
PrintCallback print = core_internals.print;
void apply() { bool get hasError => _hasError;
io_internals.exit(exitCode); bool _hasError = false;
}
}
// We actually reimplement exit() so that it uses exceptions rather Iterable<String> get errorMessages => _errorMessages;
// than truly immediately terminating the application, so that we can List<String> _errorMessages = <String>[];
// test the exit code in unit tests (see test/analyze_test.dart).
void exit(int exitCode) {
throw ExitException(exitCode);
}
void exitWithError(List<String> messages) { void foundError(List<String> messages) {
final String redLine = '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset'; assert(messages.isNotEmpty);
print(redLine); print(redLine);
messages.forEach(print); messages.forEach(print);
print(redLine); print(redLine);
exit(1); _errorMessages.addAll(messages);
_hasError = true;
} }
typedef PrintCallback = void Function(Object line); @visibleForTesting
void resetErrorStatus() {
_hasError = false;
_errorMessages.clear();
}
// Allow print() to be overridden, for tests. Never reportErrorsAndExit() {
PrintCallback print = core_internals.print; print(redLine);
print('For your convenience, the error messages reported above are repeated here:');
_errorMessages.forEach(print);
print(redLine);
system.exit(1);
}
String get clock { String get clock {
final DateTime now = DateTime.now(); final DateTime now = DateTime.now();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment