Skip to content

F
Front-End
Unverified Commit 97901da1 authored by Ian Hickson's avatar Ian Hickson Committed by GitHub
Browse files
Options

Cleaner test.dart output. (#109206)

parent d50c5b1b
Show whitespace changes
Inline Side-by-side
Showing with 573 additions and 742 deletions
dev/bots/analyze.dart
View file @ 97901da1
...@@ -44,13 +44,13 @@ Future<void> main(List<String> arguments) async { ...@@ -44,13 +44,13 @@ Future<void> main(List<String> arguments) async {
); );
dart = path.join(dartSdk, 'bin', Platform.isWindows ? 'dart.exe' : 'dart'); dart = path.join(dartSdk, 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
pub = path.join(dartSdk, 'bin', Platform.isWindows ? 'pub.bat' : 'pub'); pub = path.join(dartSdk, 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
print('$clock STARTING ANALYSIS'); printProgress('STARTING ANALYSIS');
await run(arguments); await run(arguments);
if (hasError) { if (hasError) {
print('$clock ${bold}Test failed.$reset'); printProgress('${bold}Test failed.$reset');
reportErrorsAndExit(); reportErrorsAndExit();
} }
print('$clock ${bold}Analysis successful.$reset'); printProgress('${bold}Analysis successful.$reset');
} }
/// Scans [arguments] for an argument of the form `--dart-sdk` or /// Scans [arguments] for an argument of the form `--dart-sdk` or
...@@ -88,85 +88,85 @@ Future<void> run(List<String> arguments) async { ...@@ -88,85 +88,85 @@ Future<void> run(List<String> arguments) async {
foundError(<String>['The analyze.dart script must be run with --enable-asserts.']); foundError(<String>['The analyze.dart script must be run with --enable-asserts.']);
} }
print('$clock No Double.clamp'); printProgress('No Double.clamp');
await verifyNoDoubleClamp(flutterRoot); await verifyNoDoubleClamp(flutterRoot);
print('$clock All tool test files end in _test.dart...'); printProgress('All tool test files end in _test.dart...');
await verifyToolTestsEndInTestDart(flutterRoot); await verifyToolTestsEndInTestDart(flutterRoot);
print('$clock No sync*/async*'); printProgress('No sync*/async*');
await verifyNoSyncAsyncStar(flutterPackages); await verifyNoSyncAsyncStar(flutterPackages);
await verifyNoSyncAsyncStar(flutterExamples, minimumMatches: 200); await verifyNoSyncAsyncStar(flutterExamples, minimumMatches: 200);
print('$clock No runtimeType in toString...'); printProgress('No runtimeType in toString...');
await verifyNoRuntimeTypeInToString(flutterRoot); await verifyNoRuntimeTypeInToString(flutterRoot);
print('$clock Debug mode instead of checked mode...'); printProgress('Debug mode instead of checked mode...');
await verifyNoCheckedMode(flutterRoot); await verifyNoCheckedMode(flutterRoot);
print('$clock Links for creating GitHub issues'); printProgress('Links for creating GitHub issues');
await verifyIssueLinks(flutterRoot); await verifyIssueLinks(flutterRoot);
print('$clock Unexpected binaries...'); printProgress('Unexpected binaries...');
await verifyNoBinaries(flutterRoot); await verifyNoBinaries(flutterRoot);
print('$clock Trailing spaces...'); printProgress('Trailing spaces...');
await verifyNoTrailingSpaces(flutterRoot); // assumes no unexpected binaries, so should be after verifyNoBinaries await verifyNoTrailingSpaces(flutterRoot); // assumes no unexpected binaries, so should be after verifyNoBinaries
print('$clock Deprecations...'); printProgress('Deprecations...');
await verifyDeprecations(flutterRoot); await verifyDeprecations(flutterRoot);
print('$clock Goldens...'); printProgress('Goldens...');
await verifyGoldenTags(flutterPackages); await verifyGoldenTags(flutterPackages);
print('$clock Skip test comments...'); printProgress('Skip test comments...');
await verifySkipTestComments(flutterRoot); await verifySkipTestComments(flutterRoot);
print('$clock Licenses...'); printProgress('Licenses...');
await verifyNoMissingLicense(flutterRoot); await verifyNoMissingLicense(flutterRoot);
print('$clock Test imports...'); printProgress('Test imports...');
await verifyNoTestImports(flutterRoot); await verifyNoTestImports(flutterRoot);
print('$clock Bad imports (framework)...'); printProgress('Bad imports (framework)...');
await verifyNoBadImportsInFlutter(flutterRoot); await verifyNoBadImportsInFlutter(flutterRoot);
print('$clock Bad imports (tools)...'); printProgress('Bad imports (tools)...');
await verifyNoBadImportsInFlutterTools(flutterRoot); await verifyNoBadImportsInFlutterTools(flutterRoot);
print('$clock Internationalization...'); printProgress('Internationalization...');
await verifyInternationalizations(flutterRoot, dart); await verifyInternationalizations(flutterRoot, dart);
print('$clock Integration test timeouts...'); printProgress('Integration test timeouts...');
await verifyIntegrationTestTimeouts(flutterRoot); await verifyIntegrationTestTimeouts(flutterRoot);
print('$clock null initialized debug fields...'); printProgress('null initialized debug fields...');
await verifyNullInitializedDebugExpensiveFields(flutterRoot); await verifyNullInitializedDebugExpensiveFields(flutterRoot);
// Ensure that all package dependencies are in sync. // Ensure that all package dependencies are in sync.
print('$clock Package dependencies...'); printProgress('Package dependencies...');
await runCommand(flutter, <String>['update-packages', '--verify-only'], await runCommand(flutter, <String>['update-packages', '--verify-only'],
workingDirectory: flutterRoot, workingDirectory: flutterRoot,
); );
/// Ensure that no new dependencies have been accidentally /// Ensure that no new dependencies have been accidentally
/// added to core packages. /// added to core packages.
print('$clock Package Allowlist...'); printProgress('Package Allowlist...');
await _checkConsumerDependencies(); await _checkConsumerDependencies();
// Analyze all the Dart code in the repo. // Analyze all the Dart code in the repo.
print('$clock Dart analysis...'); printProgress('Dart analysis...');
await _runFlutterAnalyze(flutterRoot, options: <String>[ await _runFlutterAnalyze(flutterRoot, options: <String>[
'--flutter-repo', '--flutter-repo',
...arguments, ...arguments,
]); ]);
print('$clock Executable allowlist...'); printProgress('Executable allowlist...');
await _checkForNewExecutables(); await _checkForNewExecutables();
// Try with the --watch analyzer, to make sure it returns success also. // Try with the --watch analyzer, to make sure it returns success also.
// The --benchmark argument exits after one run. // The --benchmark argument exits after one run.
print('$clock Dart analysis (with --watch)...'); printProgress('Dart analysis (with --watch)...');
await _runFlutterAnalyze(flutterRoot, options: <String>[ await _runFlutterAnalyze(flutterRoot, options: <String>[
'--flutter-repo', '--flutter-repo',
'--watch', '--watch',
...@@ -175,14 +175,14 @@ Future<void> run(List<String> arguments) async { ...@@ -175,14 +175,14 @@ Future<void> run(List<String> arguments) async {
]); ]);
// Analyze the code in `{@tool snippet}` sections in the repo. // Analyze the code in `{@tool snippet}` sections in the repo.
print('$clock Snippet code...'); printProgress('Snippet code...');
await runCommand(dart, await runCommand(dart,
<String>['--enable-asserts', path.join(flutterRoot, 'dev', 'bots', 'analyze_snippet_code.dart'), '--verbose'], <String>['--enable-asserts', path.join(flutterRoot, 'dev', 'bots', 'analyze_snippet_code.dart'), '--verbose'],
workingDirectory: flutterRoot, workingDirectory: flutterRoot,
); );
// Try analysis against a big version of the gallery; generate into a temporary directory. // Try analysis against a big version of the gallery; generate into a temporary directory.
print('$clock Dart analysis (mega gallery)...'); printProgress('Dart analysis (mega gallery)...');
final Directory outDir = Directory.systemTemp.createTempSync('flutter_mega_gallery.'); final Directory outDir = Directory.systemTemp.createTempSync('flutter_mega_gallery.');
try { try {
await runCommand(dart, await runCommand(dart,
...@@ -548,27 +548,23 @@ String _generateLicense(String prefix) { ...@@ -548,27 +548,23 @@ String _generateLicense(String prefix) {
Future<void> verifyNoMissingLicense(String workingDirectory, { bool checkMinimums = true }) async { Future<void> verifyNoMissingLicense(String workingDirectory, { bool checkMinimums = true }) async {
final int? overrideMinimumMatches = checkMinimums ? null : 0; final int? overrideMinimumMatches = checkMinimums ? null : 0;
int failed = 0; await _verifyNoMissingLicenseForExtension(workingDirectory, 'dart', overrideMinimumMatches ?? 2000, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'dart', overrideMinimumMatches ?? 2000, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'java', overrideMinimumMatches ?? 39, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'java', overrideMinimumMatches ?? 39, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'h', overrideMinimumMatches ?? 30, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'h', overrideMinimumMatches ?? 30, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'm', overrideMinimumMatches ?? 30, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'm', overrideMinimumMatches ?? 30, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'cpp', overrideMinimumMatches ?? 0, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'cpp', overrideMinimumMatches ?? 0, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'swift', overrideMinimumMatches ?? 10, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'swift', overrideMinimumMatches ?? 10, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'gradle', overrideMinimumMatches ?? 80, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'gradle', overrideMinimumMatches ?? 80, _generateLicense('// ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'gn', overrideMinimumMatches ?? 0, _generateLicense('# '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'gn', overrideMinimumMatches ?? 0, _generateLicense('# ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'sh', overrideMinimumMatches ?? 1, _generateLicense('# '), header: r'#!/usr/bin/env bash\n',);
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'sh', overrideMinimumMatches ?? 1, _generateLicense('# '), header: r'#!/usr/bin/env bash\n',); await _verifyNoMissingLicenseForExtension(workingDirectory, 'bat', overrideMinimumMatches ?? 1, _generateLicense('REM '), header: r'@ECHO off\n');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'bat', overrideMinimumMatches ?? 1, _generateLicense('REM '), header: r'@ECHO off\n'); await _verifyNoMissingLicenseForExtension(workingDirectory, 'ps1', overrideMinimumMatches ?? 1, _generateLicense('# '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'ps1', overrideMinimumMatches ?? 1, _generateLicense('# ')); await _verifyNoMissingLicenseForExtension(workingDirectory, 'html', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', trailingBlank: false, header: r'<!DOCTYPE HTML>\n');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'html', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', trailingBlank: false, header: r'<!DOCTYPE HTML>\n'); await _verifyNoMissingLicenseForExtension(workingDirectory, 'xml', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', header: r'(<\?xml version="1.0" encoding="utf-8"\?>\n)?');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'xml', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', header: r'(<\?xml version="1.0" encoding="utf-8"\?>\n)?'); await _verifyNoMissingLicenseForExtension(workingDirectory, 'frag', overrideMinimumMatches ?? 1, _generateLicense('// '), header: r'#version 320 es(\n)+');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'frag', overrideMinimumMatches ?? 1, _generateLicense('// '), header: r'#version 320 es(\n)+');
if (failed > 0) {
foundError(<String>['License check failed.']);
}
} }
Future<int> _verifyNoMissingLicenseForExtension( Future<void> _verifyNoMissingLicenseForExtension(
String workingDirectory, String workingDirectory,
String extension, String extension,
int minimumMatches, int minimumMatches,
...@@ -592,10 +588,8 @@ Future<int> _verifyNoMissingLicenseForExtension( ...@@ -592,10 +588,8 @@ Future<int> _verifyNoMissingLicenseForExtension(
} }
// Fail if any errors // Fail if any errors
if (errors.isNotEmpty) { if (errors.isNotEmpty) {
final String redLine = '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset';
final String fileDoes = errors.length == 1 ? 'file does' : '${errors.length} files do'; final String fileDoes = errors.length == 1 ? 'file does' : '${errors.length} files do';
print(<String>[ foundError(<String>[
redLine,
'${bold}The following $fileDoes not have the right license header for $extension files:$reset', '${bold}The following $fileDoes not have the right license header for $extension files:$reset',
...errors.map<String>((String error) => ' $error'), ...errors.map<String>((String error) => ' $error'),
'The expected license header is:', 'The expected license header is:',
...@@ -603,11 +597,8 @@ Future<int> _verifyNoMissingLicenseForExtension( ...@@ -603,11 +597,8 @@ Future<int> _verifyNoMissingLicenseForExtension(
if (header.isNotEmpty) 'followed by the following license text:', if (header.isNotEmpty) 'followed by the following license text:',
license, license,
if (trailingBlank) '...followed by a blank line.', if (trailingBlank) '...followed by a blank line.',
redLine, ]);
].join('\n'));
return 1;
} }
return 0;
} }
class _Line { class _Line {
...@@ -1650,7 +1641,7 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, { ...@@ -1650,7 +1641,7 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
final String relativeWorkingDir = path.relative(workingDirectory); final String relativeWorkingDir = path.relative(workingDirectory);
if (!runSilently) { if (!runSilently) {
printProgress('RUNNING', relativeWorkingDir, commandDescription); print('RUNNING: cd $cyan$relativeWorkingDir$reset; $green$commandDescription$reset');
} }
final Stopwatch time = Stopwatch()..start(); final Stopwatch time = Stopwatch()..start();
...@@ -1669,12 +1660,12 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, { ...@@ -1669,12 +1660,12 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
); );
if (!runSilently) { if (!runSilently) {
print('$clock ELAPSED TIME: $bold${prettyPrintDuration(time.elapsed)}$reset for $commandDescription in $relativeWorkingDir'); print('ELAPSED TIME: $bold${prettyPrintDuration(time.elapsed)}$reset for $commandDescription in $relativeWorkingDir');
} }
if (exitCode != 0 && !allowNonZeroExit) { if (exitCode != 0 && !allowNonZeroExit) {
stderr.write(result.stderr);
foundError(<String>[ foundError(<String>[
result.stderr,
'${bold}ERROR:$red Last command exited with $exitCode.$reset', '${bold}ERROR:$red Last command exited with $exitCode.$reset',
'${bold}Command:$red $commandDescription$reset', '${bold}Command:$red $commandDescription$reset',
'${bold}Relative working directory:$red $relativeWorkingDir$reset', '${bold}Relative working directory:$red $relativeWorkingDir$reset',
...@@ -1838,9 +1829,8 @@ const Set<String> kExecutableAllowlist = <String>{ ...@@ -1838,9 +1829,8 @@ const Set<String> kExecutableAllowlist = <String>{
Future<void> _checkForNewExecutables() async { Future<void> _checkForNewExecutables() async {
// 0b001001001 // 0b001001001
const int executableBitMask = 0x49; const int executableBitMask = 0x49;
final List<File> files = await _gitFiles(flutterRoot); final List<File> files = await _gitFiles(flutterRoot);
int unexpectedExecutableCount = 0; final List<String> errors = <String>[];
for (final File file in files) { for (final File file in files) {
final String relativePath = path.relative( final String relativePath = path.relative(
file.path, file.path,
...@@ -1849,14 +1839,14 @@ Future<void> _checkForNewExecutables() async { ...@@ -1849,14 +1839,14 @@ Future<void> _checkForNewExecutables() async {
final FileStat stat = file.statSync(); final FileStat stat = file.statSync();
final bool isExecutable = stat.mode & executableBitMask != 0x0; final bool isExecutable = stat.mode & executableBitMask != 0x0;
if (isExecutable && !kExecutableAllowlist.contains(relativePath)) { if (isExecutable && !kExecutableAllowlist.contains(relativePath)) {
unexpectedExecutableCount += 1; errors.add('$relativePath is executable: ${(stat.mode & 0x1FF).toRadixString(2)}');
print('$relativePath is executable: ${(stat.mode & 0x1FF).toRadixString(2)}');
} }
} }
if (unexpectedExecutableCount > 0) { if (errors.isNotEmpty) {
throw Exception( throw Exception(
'found $unexpectedExecutableCount unexpected executable file' '${errors.join('\n')}\n'
'${unexpectedExecutableCount == 1 ? '' : 's'}! If this was intended, you ' 'found ${errors.length} unexpected executable file'
'${errors.length == 1 ? '' : 's'}! If this was intended, you '
'must add this file to kExecutableAllowlist in dev/bots/analyze.dart', 'must add this file to kExecutableAllowlist in dev/bots/analyze.dart',
); );
} }
......
dev/bots/flutter_compact_formatter.dart deleted 100644 → 0
View file @ d50c5b1b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
final Stopwatch _stopwatch = Stopwatch();
/// A wrapper around package:test's JSON reporter.
///
/// This class behaves similarly to the compact reporter, but suppresses all
/// output except for progress until the end of testing. In other words, errors,
/// [print] calls, and skipped test messages will not be printed during the run
/// of the suite.
///
/// It also processes the JSON data into a collection of [TestResult]s for any
/// other post processing needs, e.g. sending data to analytics.
class FlutterCompactFormatter {
FlutterCompactFormatter() {
_stopwatch.start();
}
/// Whether to use color escape codes in writing to stdout.
final bool useColor = stdout.supportsAnsiEscapes;
/// The terminal escape for green text, or the empty string if this is Windows
/// or not outputting to a terminal.
String get _green => useColor ? '\u001b[32m' : '';
/// The terminal escape for red text, or the empty string if this is Windows
/// or not outputting to a terminal.
String get _red => useColor ? '\u001b[31m' : '';
/// The terminal escape for yellow text, or the empty string if this is
/// Windows or not outputting to a terminal.
String get _yellow => useColor ? '\u001b[33m' : '';
/// The terminal escape for gray text, or the empty string if this is
/// Windows or not outputting to a terminal.
String get _gray => useColor ? '\u001b[1;30m' : '';
/// The terminal escape for bold text, or the empty string if this is
/// Windows or not outputting to a terminal.
String get _bold => useColor ? '\u001b[1m' : '';
/// The terminal escape for removing test coloring, or the empty string if
/// this is Windows or not outputting to a terminal.
String get _noColor => useColor ? '\u001b[0m' : '';
/// The terminal escape for clearing the line, or a carriage return if
/// this is Windows or not outputting to a terminal.
String get _clearLine => useColor ? '\x1b[2K\r' : '\r';
final Map<int, TestResult> _tests = <int, TestResult>{};
/// The test results from this run.
Iterable<TestResult> get tests => _tests.values;
/// The number of tests that were started.
int started = 0;
/// The number of test failures.
int failures = 0;
/// The number of skipped tests.
int skips = 0;
/// The number of successful tests.
int successes = 0;
/// Process a single line of JSON output from the JSON test reporter.
///
/// Callers are responsible for splitting multiple lines before calling this
/// method.
TestResult? processRawOutput(String raw) {
assert(raw != null);
// We might be getting messages from Flutter Tool about updating/building.
if (!raw.startsWith('{')) {
print(raw);
return null;
}
final Map<String, dynamic> decoded = json.decode(raw) as Map<String, dynamic>;
final TestResult? originalResult = _tests[decoded['testID']];
switch (decoded['type'] as String) {
case 'done':
stdout.write(_clearLine);
stdout.write('$_bold${_stopwatch.elapsed}$_noColor ');
stdout.writeln(
'$_green+$successes $_yellow~$skips $_red-$failures:$_bold$_gray Done.$_noColor');
break;
case 'testStart':
final Map<String, dynamic> testData = decoded['test'] as Map<String, dynamic>;
if (testData['url'] == null) {
started += 1;
stdout.write(_clearLine);
stdout.write('$_bold${_stopwatch.elapsed}$_noColor ');
stdout.write(
'$_green+$successes $_yellow~$skips $_red-$failures: $_gray${testData['name']}$_noColor');
break;
}
_tests[testData['id'] as int] = TestResult(
id: testData['id'] as int,
name: testData['name'] as String,
line: testData['root_line'] as int? ?? testData['line'] as int,
column: testData['root_column'] as int? ?? testData['column'] as int,
path: testData['root_url'] as String? ?? testData['url'] as String,
startTime: decoded['time'] as int,
);
break;
case 'testDone':
if (originalResult == null) {
break;
}
originalResult.endTime = decoded['time'] as int;
if (decoded['skipped'] == true) {
skips += 1;
originalResult.status = TestStatus.skipped;
} else {
if (decoded['result'] == 'success') {
originalResult.status =TestStatus.succeeded;
successes += 1;
} else {
originalResult.status = TestStatus.failed;
failures += 1;
}
}
break;
case 'error':
final String error = decoded['error'] as String;
final String stackTrace = decoded['stackTrace'] as String;
if (originalResult != null) {
originalResult.errorMessage = error;
originalResult.stackTrace = stackTrace;
} else {
if (error != null) {
stderr.writeln(error);
}
if (stackTrace != null) {
stderr.writeln(stackTrace);
}
}
break;
case 'print':
if (originalResult != null) {
originalResult.messages.add(decoded['message'] as String);
}
break;
case 'group':
case 'allSuites':
case 'start':
case 'suite':
default:
break;
}
return originalResult;
}
/// Print summary of test results.
void finish() {
final List<String> skipped = <String>[];
final List<String> failed = <String>[];
for (final TestResult result in _tests.values) {
switch (result.status) {
case TestStatus.started:
failed.add('${_red}Unexpectedly failed to complete a test!');
failed.add(result.toString() + _noColor);
break;
case TestStatus.skipped:
skipped.add(
'${_yellow}Skipped ${result.name} (${result.pathLineColumn}).$_noColor');
break;
case TestStatus.failed:
failed.addAll(<String>[
'$_bold${_red}Failed ${result.name} (${result.pathLineColumn}):',
result.errorMessage!,
_noColor + _red,
result.stackTrace!,
]);
failed.addAll(result.messages);
failed.add(_noColor);
break;
case TestStatus.succeeded:
break;
}
}
skipped.forEach(print);
failed.forEach(print);
if (failed.isEmpty) {
print('${_green}Completed, $successes test(s) passing ($skips skipped).$_noColor');
} else {
print('$_gray$failures test(s) failed.$_noColor');
}
}
}
/// The state of a test received from the JSON reporter.
enum TestStatus {
/// Test execution has started.
started,
/// Test completed successfully.
succeeded,
/// Test failed.
failed,
/// Test was skipped.
skipped,
}
/// The detailed status of a test run.
class TestResult {
TestResult({
required this.id,
required this.name,
required this.line,
required this.column,
required this.path,
required this.startTime,
this.status = TestStatus.started,
}) : assert(id != null),
assert(name != null),
assert(line != null),
assert(column != null),
assert(path != null),
assert(startTime != null),
assert(status != null),
messages = <String>[];
/// The state of the test.
TestStatus status;
/// The internal ID of the test used by the JSON reporter.
final int id;
/// The name of the test, specified via the `test` method.
final String name;
/// The line number from the original file.
final int line;
/// The column from the original file.
final int column;
/// The path of the original test file.
final String path;
/// A friendly print out of the [path], [line], and [column] of the test.
String get pathLineColumn => '$path:$line:$column';
/// The start time of the test, in milliseconds relative to suite startup.
final int startTime;
/// The stdout of the test.
final List<String> messages;
/// The error message from the test, from an `expect`, an [Exception] or
/// [Error].
String? errorMessage;
/// The stacktrace from a test failure.
String? stackTrace;
/// The time, in milliseconds relative to suite startup, that the test ended.
int? endTime;
/// The total time, in milliseconds, that the test took.
int get totalTime => (endTime ?? _stopwatch.elapsedMilliseconds) - startTime;
@override
String toString() => '{$runtimeType: {$id, $name, ${totalTime}ms, $pathLineColumn}}';
}
dev/bots/run_command.dart
View file @ 97901da1
...@@ -50,23 +50,9 @@ class Command { ...@@ -50,23 +50,9 @@ class Command {
/// The raw process that was launched for this command. /// The raw process that was launched for this command.
final io.Process process; final io.Process process;
final Stopwatch _time; final Stopwatch _time;
final Future<List<List<int>>>? _savedStdout; final Future<String> _savedStdout;
final Future<List<List<int>>>? _savedStderr; final Future<String> _savedStderr;
/// Evaluates when the [process] exits.
///
/// Returns the result of running the command.
Future<CommandResult> get onExit async {
final int exitCode = await process.exitCode;
_time.stop();
// Saved output is null when OutputMode.print is used.
final String? flattenedStdout = _savedStdout != null ? _flattenToString((await _savedStdout)!) : null;
final String? flattenedStderr = _savedStderr != null ? _flattenToString((await _savedStderr)!) : null;
return CommandResult._(exitCode, _time.elapsed, flattenedStdout, flattenedStderr);
}
} }
/// The result of running a command using [startCommand] and [runCommand]; /// The result of running a command using [startCommand] and [runCommand];
...@@ -105,46 +91,50 @@ Future<Command> startCommand(String executable, List<String> arguments, { ...@@ -105,46 +91,50 @@ Future<Command> startCommand(String executable, List<String> arguments, {
}) async { }) async {
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path); final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path);
printProgress('RUNNING', relativeWorkingDir, commandDescription); print('RUNNING: cd $cyan$relativeWorkingDir$reset; $green$commandDescription$reset');
final Stopwatch time = Stopwatch()..start(); final Stopwatch time = Stopwatch()..start();
final io.Process process = await io.Process.start(executable, arguments, final io.Process process = await io.Process.start(executable, arguments,
workingDirectory: workingDirectory, workingDirectory: workingDirectory,
environment: environment, environment: environment,
); );
return Command._(
Future<List<List<int>>> savedStdout = Future<List<List<int>>>.value(<List<int>>[]); process,
Future<List<List<int>>> savedStderr = Future<List<List<int>>>.value(<List<int>>[]); time,
final Stream<List<int>> stdoutSource = process.stdout process.stdout
.transform<String>(const Utf8Decoder()) .transform<String>(const Utf8Decoder())
.transform(const LineSplitter()) .transform(const LineSplitter())
.where((String line) => removeLine == null || !removeLine(line)) .where((String line) => removeLine == null || !removeLine(line))
.map((String line) { .map<String>((String line) {
final String formattedLine = '$line\n'; final String formattedLine = '$line\n';
if (outputListener != null) { if (outputListener != null) {
outputListener(formattedLine, process); outputListener(formattedLine, process);
} }
return formattedLine; switch (outputMode) {
case OutputMode.print:
print(line);
break;
case OutputMode.capture:
break;
}
return line;
}) })
.transform(const Utf8Encoder()); .join('\n'),
process.stderr
.transform<String>(const Utf8Decoder())
.transform(const LineSplitter())
.map<String>((String line) {
switch (outputMode) { switch (outputMode) {
case OutputMode.print: case OutputMode.print:
stdoutSource.listen((List<int> output) { print(line);
io.stdout.add(output);
savedStdout.then((List<List<int>> list) => list.add(output));
});
process.stderr.listen((List<int> output) {
io.stdout.add(output);
savedStdout.then((List<List<int>> list) => list.add(output));
});
break; break;
case OutputMode.capture: case OutputMode.capture:
savedStdout = stdoutSource.toList();
savedStderr = process.stderr.toList();
break; break;
} }
return line;
return Command._(process, time, savedStdout, savedStderr); })
.join('\n'),
);
} }
/// Runs the `executable` and waits until the process exits. /// Runs the `executable` and waits until the process exits.
...@@ -182,7 +172,12 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, { ...@@ -182,7 +172,12 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
outputListener: outputListener, outputListener: outputListener,
); );
final CommandResult result = await command.onExit; final CommandResult result = CommandResult._(
await command.process.exitCode,
command._time.elapsed,
await command._savedStdout,
await command._savedStderr,
);
if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) { if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) {
// Print the output when we get unexpected results (unless output was // Print the output when we get unexpected results (unless output was
...@@ -191,28 +186,24 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, { ...@@ -191,28 +186,24 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
case OutputMode.print: case OutputMode.print:
break; break;
case OutputMode.capture: case OutputMode.capture:
io.stdout.writeln(result.flattenedStdout); print(result.flattenedStdout);
io.stdout.writeln(result.flattenedStderr); print(result.flattenedStderr);
break; break;
} }
foundError(<String>[ foundError(<String>[
if (failureMessage != null) if (failureMessage != null)
failureMessage failureMessage
else else
'${bold}ERROR: ${red}Last command exited with ${result.exitCode} (expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'}).$reset', '$bold${red}Command exited with exit code ${result.exitCode} but expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'} exit code.$reset',
'${bold}Command: $green$commandDescription$reset', '${bold}Command: $green$commandDescription$reset',
'${bold}Relative working directory: $cyan$relativeWorkingDir$reset', '${bold}Relative working directory: $cyan$relativeWorkingDir$reset',
]); ]);
} else { } else {
print('$clock ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset'); print('ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
} }
return result; return result;
} }
/// Flattens a nested list of UTF-8 code units into a single string.
String _flattenToString(List<List<int>> chunks) =>
utf8.decode(chunks.expand<int>((List<int> ints) => ints).toList());
/// Specifies what to do with the command output from [runCommand] and [startCommand]. /// Specifies what to do with the command output from [runCommand] and [startCommand].
enum OutputMode { enum OutputMode {
/// Forwards standard output and standard error streams to the test process' /// Forwards standard output and standard error streams to the test process'
......
dev/bots/service_worker_test.dart
View file @ 97901da1
...@@ -163,8 +163,8 @@ Future<void> runWebServiceWorkerTest({ ...@@ -163,8 +163,8 @@ Future<void> runWebServiceWorkerTest({
Future<void> startAppServer({ Future<void> startAppServer({
required String cacheControl, required String cacheControl,
}) async { }) async {
final int serverPort = await findAvailablePort(); final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePort(); final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
server = await AppServer.start( server = await AppServer.start(
headless: headless, headless: headless,
cacheControl: cacheControl, cacheControl: cacheControl,
...@@ -201,7 +201,7 @@ Future<void> runWebServiceWorkerTest({ ...@@ -201,7 +201,7 @@ Future<void> runWebServiceWorkerTest({
final bool shouldExpectFlutterJs = testType != ServiceWorkerTestType.withoutFlutterJs; final bool shouldExpectFlutterJs = testType != ServiceWorkerTestType.withoutFlutterJs;
print('BEGIN runWebServiceWorkerTest(headless: $headless, testType: $testType)\n'); print('BEGIN runWebServiceWorkerTest(headless: $headless, testType: $testType)');
try { try {
///// /////
...@@ -417,7 +417,7 @@ Future<void> runWebServiceWorkerTest({ ...@@ -417,7 +417,7 @@ Future<void> runWebServiceWorkerTest({
await server?.stop(); await server?.stop();
} }
print('END runWebServiceWorkerTest(headless: $headless, testType: $testType)\n'); print('END runWebServiceWorkerTest(headless: $headless, testType: $testType)');
} }
Future<void> runWebServiceWorkerTestWithCachingResources({ Future<void> runWebServiceWorkerTestWithCachingResources({
...@@ -435,8 +435,8 @@ Future<void> runWebServiceWorkerTestWithCachingResources({ ...@@ -435,8 +435,8 @@ Future<void> runWebServiceWorkerTestWithCachingResources({
Future<void> startAppServer({ Future<void> startAppServer({
required String cacheControl, required String cacheControl,
}) async { }) async {
final int serverPort = await findAvailablePort(); final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePort(); final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
server = await AppServer.start( server = await AppServer.start(
headless: headless, headless: headless,
cacheControl: cacheControl, cacheControl: cacheControl,
...@@ -472,7 +472,7 @@ Future<void> runWebServiceWorkerTestWithCachingResources({ ...@@ -472,7 +472,7 @@ Future<void> runWebServiceWorkerTestWithCachingResources({
final bool shouldExpectFlutterJs = testType != ServiceWorkerTestType.withoutFlutterJs; final bool shouldExpectFlutterJs = testType != ServiceWorkerTestType.withoutFlutterJs;
print('BEGIN runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)\n'); print('BEGIN runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)');
try { try {
////////////////////////////////////////////////////// //////////////////////////////////////////////////////
...@@ -576,7 +576,7 @@ Future<void> runWebServiceWorkerTestWithCachingResources({ ...@@ -576,7 +576,7 @@ Future<void> runWebServiceWorkerTestWithCachingResources({
await server?.stop(); await server?.stop();
} }
print('END runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)\n'); print('END runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)');
} }
Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
...@@ -593,8 +593,8 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({ ...@@ -593,8 +593,8 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
Future<void> startAppServer({ Future<void> startAppServer({
required String cacheControl, required String cacheControl,
}) async { }) async {
final int serverPort = await findAvailablePort(); final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePort(); final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
server = await AppServer.start( server = await AppServer.start(
headless: headless, headless: headless,
cacheControl: cacheControl, cacheControl: cacheControl,
...@@ -628,7 +628,7 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({ ...@@ -628,7 +628,7 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
workingDirectory: _testAppWebDirectory, workingDirectory: _testAppWebDirectory,
); );
print('BEGIN runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)\n'); print('BEGIN runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)');
try { try {
await _rebuildApp(version: 1, testType: ServiceWorkerTestType.blockedServiceWorkers, target: _targetWithBlockedServiceWorkers); await _rebuildApp(version: 1, testType: ServiceWorkerTestType.blockedServiceWorkers, target: _targetWithBlockedServiceWorkers);
...@@ -662,5 +662,5 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({ ...@@ -662,5 +662,5 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
); );
await server?.stop(); await server?.stop();
} }
print('END runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)\n'); print('END runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)');
} }
dev/bots/test.dart
View file @ 97901da1
...@@ -2,9 +2,55 @@ ...@@ -2,9 +2,55 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:async'; // Runs the tests for the flutter/flutter repository.
//
//
// By default, test output is filtered and only errors are shown. (If a
// particular test takes longer than _quietTimeout in utils.dart, the output is
// shown then also, in case something has hung.)
//
// --verbose stops the output cleanup and just outputs everything verbatim.
//
//
// By default, errors are non-fatal; all tests are executed and the output
// ends with a summary of the errors that were detected.
//
// Exit code is 1 if there was an error.
//
// --abort-on-error causes the script to exit immediately when hitting an error.
//
//
// By default, all tests are run. However, the tests support being split by
// shard and subshard. (Inspect the code to see what shards and subshards are
// supported.)
//
// If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
// the shard and sub-shard, by parsing it in the form shard-subshard-platform,
// ignoring the platform.
//
// For local testing you can just set the SHARD and SUBSHARD environment
// variables. For example, to run all the framework tests you can just set
// SHARD=framework_tests. Some shards support named subshards, like
// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of
// two" as in run the first half of the tests).
//
// So for example to run specifically the third subshard of the Web tests you
// would set SHARD=web_tests SUBSHARD=2 (it's zero-based).
//
// By default, where supported, tests within a shard are executed in a random
// order to (eventually) catch inter-test dependencies.
//
// --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
//
//
// All other arguments are treated as arguments to pass to the flutter tool when
// running tests.
import 'dart:convert'; import 'dart:convert';
import 'dart:core' as system show print;
import 'dart:core' hide print; import 'dart:core' hide print;
import 'dart:io' as system show exit;
import 'dart:io' hide exit; import 'dart:io' hide exit;
import 'dart:math' as math; import 'dart:math' as math;
import 'dart:typed_data'; import 'dart:typed_data';
...@@ -15,7 +61,6 @@ import 'package:file/local.dart'; ...@@ -15,7 +61,6 @@ import 'package:file/local.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import 'browser.dart'; import 'browser.dart';
import 'flutter_compact_formatter.dart';
import 'run_command.dart'; import 'run_command.dart';
import 'service_worker_test.dart'; import 'service_worker_test.dart';
import 'utils.dart'; import 'utils.dart';
...@@ -61,8 +106,6 @@ final List<String> flutterTestArgs = <String>[]; ...@@ -61,8 +106,6 @@ final List<String> flutterTestArgs = <String>[];
/// if such flags are provided to `test.dart`. /// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{}; final Map<String,String> localEngineEnv = <String, String>{};
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
const String kShardKey = 'SHARD'; const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD'; const String kSubshardKey = 'SUBSHARD';
...@@ -170,30 +213,32 @@ String get shuffleSeed { ...@@ -170,30 +213,32 @@ String get shuffleSeed {
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt /// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
Future<void> main(List<String> args) async { Future<void> main(List<String> args) async {
print('$clock STARTING ANALYSIS'); try {
flutterTestArgs.addAll(args); printProgress('STARTING ANALYSIS');
final Set<String> removeArgs = <String>{};
for (final String arg in args) { for (final String arg in args) {
if (arg.startsWith('--local-engine=')) { if (arg.startsWith('--local-engine=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length); localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
} flutterTestArgs.add(arg);
if (arg.startsWith('--local-engine-src-path=')) { } else if (arg.startsWith('--local-engine-src-path=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length); localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
} flutterTestArgs.add(arg);
if (arg.startsWith('--test-randomize-ordering-seed=')) { } else if (arg.startsWith('--test-randomize-ordering-seed=')) {
_shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length); _shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
removeArgs.add(arg); } else if (arg.startsWith('--verbose')) {
} print = (Object? message) {
if (arg == '--no-smoke-tests') { system.print(message);
// This flag is deprecated, ignore it. };
removeArgs.add(arg); } else if (arg.startsWith('--abort-on-error')) {
onError = () {
system.exit(1);
};
} else {
flutterTestArgs.add(arg);
} }
} }
flutterTestArgs.removeWhere((String arg) => removeArgs.contains(arg));
if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) { if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}'); printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
} }
print('═' * 80);
await selectShard(<String, ShardRunner>{ await selectShard(<String, ShardRunner>{
'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests, 'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
'build_tests': _runBuildTests, 'build_tests': _runBuildTests,
...@@ -212,13 +257,24 @@ Future<void> main(List<String> args) async { ...@@ -212,13 +257,24 @@ Future<void> main(List<String> args) async {
'web_long_running_tests': _runWebLongRunningTests, 'web_long_running_tests': _runWebLongRunningTests,
'flutter_plugins': _runFlutterPluginsTests, 'flutter_plugins': _runFlutterPluginsTests,
'skp_generator': _runSkpGeneratorTests, 'skp_generator': _runSkpGeneratorTests,
kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script. kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
}); });
} catch (error, stackTrace) {
foundError(<String>[
'UNEXPECTED ERROR!',
error.toString(),
...stackTrace.toString().split('\n'),
'The test.dart script should be corrected to catch this error and call foundError().',
'${yellow}Some tests are likely to have been skipped.$reset',
]);
system.exit(255);
}
if (hasError) { if (hasError) {
print('$clock ${bold}Test failed.$reset'); printProgress('${bold}Test failed.$reset');
reportErrorsAndExit(); reportErrorsAndExit();
} }
print('$clock ${bold}Test successful.$reset'); printProgress('${bold}Test successful.$reset');
system.exit(0);
} }
final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? ''; final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
...@@ -232,22 +288,36 @@ Future<void> _validateEngineHash() async { ...@@ -232,22 +288,36 @@ Future<void> _validateEngineHash() async {
// and then use this script to run Flutter's test suites. // and then use this script to run Flutter's test suites.
// Because the artifacts have been changed, this particular test will return // Because the artifacts have been changed, this particular test will return
// a false positive and should be skipped. // a false positive and should be skipped.
print('${yellow}Skipping Flutter Engine Version Validation for swarming ' print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $_luciBotId.');
'bot $_luciBotId.');
return; return;
} }
final String expectedVersion = File(engineVersionFile).readAsStringSync().trim(); final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture); final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
final String actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) { if (result.flattenedStdout!.isNotEmpty) {
foundError(<String>[
'${red}The stdout of `$flutterTester --help` was not empty:$reset',
...result.flattenedStdout!.split('\n').map((String line) => ' $gray┆$reset $line'),
]);
}
final String actualVersion;
try {
actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
return line.startsWith('Flutter Engine Version:'); return line.startsWith('Flutter Engine Version:');
}); });
} on StateError {
foundError(<String>[
'${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset',
...result.flattenedStderr!.split('\n').map((String line) => ' $gray┆$reset $line'),
]);
return;
}
if (!actualVersion.contains(expectedVersion)) { if (!actualVersion.contains(expectedVersion)) {
foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".']); foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']);
} }
} }
Future<void> _runTestHarnessTests() async { Future<void> _runTestHarnessTests() async {
print('${green}Running test harness tests...$reset'); printProgress('${green}Running test harness tests...$reset');
await _validateEngineHash(); await _validateEngineHash();
...@@ -338,7 +408,7 @@ Future<void> _runTestHarnessTests() async { ...@@ -338,7 +408,7 @@ Future<void> _runTestHarnessTests() async {
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools'); final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
Future<void> _runGeneralToolTests() async { Future<void> _runGeneralToolTests() async {
await _dartRunTest( await _runDartTest(
_toolsPath, _toolsPath,
testPaths: <String>[path.join('test', 'general.shard')], testPaths: <String>[path.join('test', 'general.shard')],
enableFlutterToolAsserts: false, enableFlutterToolAsserts: false,
...@@ -351,7 +421,7 @@ Future<void> _runGeneralToolTests() async { ...@@ -351,7 +421,7 @@ Future<void> _runGeneralToolTests() async {
} }
Future<void> _runCommandsToolTests() async { Future<void> _runCommandsToolTests() async {
await _dartRunTest( await _runDartTest(
_toolsPath, _toolsPath,
forceSingleCore: true, forceSingleCore: true,
testPaths: <String>[path.join('test', 'commands.shard')], testPaths: <String>[path.join('test', 'commands.shard')],
...@@ -359,7 +429,7 @@ Future<void> _runCommandsToolTests() async { ...@@ -359,7 +429,7 @@ Future<void> _runCommandsToolTests() async {
} }
Future<void> _runWebToolTests() async { Future<void> _runWebToolTests() async {
await _dartRunTest( await _runDartTest(
_toolsPath, _toolsPath,
forceSingleCore: true, forceSingleCore: true,
testPaths: <String>[path.join('test', 'web.shard')], testPaths: <String>[path.join('test', 'web.shard')],
...@@ -368,7 +438,7 @@ Future<void> _runWebToolTests() async { ...@@ -368,7 +438,7 @@ Future<void> _runWebToolTests() async {
} }
Future<void> _runToolHostCrossArchTests() { Future<void> _runToolHostCrossArchTests() {
return _dartRunTest( return _runDartTest(
_toolsPath, _toolsPath,
// These are integration tests // These are integration tests
forceSingleCore: true, forceSingleCore: true,
...@@ -382,7 +452,7 @@ Future<void> _runIntegrationToolTests() async { ...@@ -382,7 +452,7 @@ Future<void> _runIntegrationToolTests() async {
.map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath)) .map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
.where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList(); .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();
await _dartRunTest( await _runDartTest(
_toolsPath, _toolsPath,
forceSingleCore: true, forceSingleCore: true,
testPaths: _selectIndexOfTotalSubshard<String>(allTests), testPaths: _selectIndexOfTotalSubshard<String>(allTests),
...@@ -545,7 +615,7 @@ Future<void> _flutterBuildApk(String relativePathToApplication, { ...@@ -545,7 +615,7 @@ Future<void> _flutterBuildApk(String relativePathToApplication, {
bool verifyCaching = false, bool verifyCaching = false,
List<String> additionalArgs = const <String>[], List<String> additionalArgs = const <String>[],
}) async { }) async {
print('${green}Testing APK build$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing APK ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'APK', 'apk', await _flutterBuild(relativePathToApplication, 'APK', 'apk',
release: release, release: release,
verifyCaching: verifyCaching, verifyCaching: verifyCaching,
...@@ -559,7 +629,7 @@ Future<void> _flutterBuildIpa(String relativePathToApplication, { ...@@ -559,7 +629,7 @@ Future<void> _flutterBuildIpa(String relativePathToApplication, {
bool verifyCaching = false, bool verifyCaching = false,
}) async { }) async {
assert(Platform.isMacOS); assert(Platform.isMacOS);
print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing IPA ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'IPA', 'ios', await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
release: release, release: release,
verifyCaching: verifyCaching, verifyCaching: verifyCaching,
...@@ -574,7 +644,7 @@ Future<void> _flutterBuildLinux(String relativePathToApplication, { ...@@ -574,7 +644,7 @@ Future<void> _flutterBuildLinux(String relativePathToApplication, {
}) async { }) async {
assert(Platform.isLinux); assert(Platform.isLinux);
await runCommand(flutter, <String>['config', '--enable-linux-desktop']); await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
print('${green}Testing Linux build$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing Linux ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'Linux', 'linux', await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
release: release, release: release,
verifyCaching: verifyCaching, verifyCaching: verifyCaching,
...@@ -589,7 +659,7 @@ Future<void> _flutterBuildMacOS(String relativePathToApplication, { ...@@ -589,7 +659,7 @@ Future<void> _flutterBuildMacOS(String relativePathToApplication, {
}) async { }) async {
assert(Platform.isMacOS); assert(Platform.isMacOS);
await runCommand(flutter, <String>['config', '--enable-macos-desktop']); await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
print('${green}Testing macOS build$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing macOS ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'macOS', 'macos', await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
release: release, release: release,
verifyCaching: verifyCaching, verifyCaching: verifyCaching,
...@@ -603,7 +673,7 @@ Future<void> _flutterBuildWin32(String relativePathToApplication, { ...@@ -603,7 +673,7 @@ Future<void> _flutterBuildWin32(String relativePathToApplication, {
List<String> additionalArgs = const <String>[], List<String> additionalArgs = const <String>[],
}) async { }) async {
assert(Platform.isWindows); assert(Platform.isWindows);
print('${green}Testing Windows build$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing ${release ? 'release' : 'debug'} Windows build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'Windows', 'windows', await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
release: release, release: release,
verifyCaching: verifyCaching, verifyCaching: verifyCaching,
...@@ -634,7 +704,7 @@ Future<void> _flutterBuild( ...@@ -634,7 +704,7 @@ Future<void> _flutterBuild(
); );
if (verifyCaching) { if (verifyCaching) {
print('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter, await runCommand(flutter,
<String>[ <String>[
'build', 'build',
...@@ -668,7 +738,7 @@ bool _allTargetsCached(File performanceFile) { ...@@ -668,7 +738,7 @@ bool _allTargetsCached(File performanceFile) {
} }
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async { Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...'); printProgress('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter, await runCommand(flutter,
<String>['build', 'web', '-v', '--target=$target'], <String>['build', 'web', '-v', '--target=$target'],
workingDirectory: path.join(flutterRoot, relativePathToApplication), workingDirectory: path.join(flutterRoot, relativePathToApplication),
...@@ -681,12 +751,14 @@ Future<void> _flutterBuildDart2js(String relativePathToApplication, String targe ...@@ -681,12 +751,14 @@ Future<void> _flutterBuildDart2js(String relativePathToApplication, String targe
Future<void> _runAddToAppLifeCycleTests() async { Future<void> _runAddToAppLifeCycleTests() async {
if (Platform.isMacOS) { if (Platform.isMacOS) {
print('${green}Running add-to-app life cycle iOS integration tests$reset...'); printProgress('${green}Running add-to-app life cycle iOS integration tests$reset...');
final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle'); final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
await runCommand('./build_and_test.sh', await runCommand('./build_and_test.sh',
<String>[], <String>[],
workingDirectory: addToAppDir, workingDirectory: addToAppDir,
); );
} else {
printProgress('${yellow}Skipped on this platform (only iOS has add-to-add lifecycle tests at this time).$reset');
} }
} }
...@@ -696,7 +768,7 @@ Future<void> _runFrameworkTests() async { ...@@ -696,7 +768,7 @@ Future<void> _runFrameworkTests() async {
final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation']; final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
Future<void> runWidgets() async { Future<void> runWidgets() async {
print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset'); printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest( await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'), path.join(flutterRoot, 'packages', 'flutter'),
...@@ -733,7 +805,7 @@ Future<void> _runFrameworkTests() async { ...@@ -733,7 +805,7 @@ Future<void> _runFrameworkTests() async {
.where((Directory dir) => dir.path.endsWith('widgets') == false) .where((Directory dir) => dir.path.endsWith('widgets') == false)
.map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator) .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
.toList(); .toList();
print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset'); printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest( await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'), path.join(flutterRoot, 'packages', 'flutter'),
...@@ -774,6 +846,7 @@ Future<void> _runFrameworkTests() async { ...@@ -774,6 +846,7 @@ Future<void> _runFrameworkTests() async {
required Set<String> allowed, required Set<String> allowed,
required Set<String> disallowed, required Set<String> disallowed,
}) async { }) async {
try {
await runCommand( await runCommand(
flutter, flutter,
<String>[ <String>[
...@@ -798,6 +871,12 @@ Future<void> _runFrameworkTests() async { ...@@ -798,6 +871,12 @@ Future<void> _runFrameworkTests() async {
} }
} }
return results; return results;
} catch (error, stackTrace) {
return <String>[
error.toString(),
...stackTrace.toString().trimRight().split('\n'),
];
}
} }
final List<String> results = <String>[]; final List<String> results = <String>[];
...@@ -877,12 +956,12 @@ Future<void> _runFrameworkTests() async { ...@@ -877,12 +956,12 @@ Future<void> _runFrameworkTests() async {
} }
Future<void> runMisc() async { Future<void> runMisc() async {
print('${green}Running package tests$reset for directories other than packages/flutter'); printProgress('${green}Running package tests$reset for directories other than packages/flutter');
await _runTestHarnessTests(); await _runTestHarnessTests();
await runExampleTests(); await runExampleTests();
await _dartRunTest(path.join(flutterRoot, 'dev', 'bots')); await _runDartTest(path.join(flutterRoot, 'dev', 'bots'));
await _dartRunTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209 await _runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
await _dartRunTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true); await _runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
// TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/pull/91127 has landed. // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/pull/91127 has landed.
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false); await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests')); await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
...@@ -942,7 +1021,7 @@ Future<void> _runFrameworkCoverage() async { ...@@ -942,7 +1021,7 @@ Future<void> _runFrameworkCoverage() async {
if (!coverageFile.existsSync()) { if (!coverageFile.existsSync()) {
foundError(<String>[ foundError(<String>[
'${red}Coverage file not found.$reset', '${red}Coverage file not found.$reset',
'Expected to find: $cyan${coverageFile.absolute}$reset', 'Expected to find: $cyan${coverageFile.absolute.path}$reset',
'This file is normally obtained by running `${green}flutter update-packages$reset`.', 'This file is normally obtained by running `${green}flutter update-packages$reset`.',
]); ]);
return; return;
...@@ -954,7 +1033,7 @@ Future<void> _runFrameworkCoverage() async { ...@@ -954,7 +1033,7 @@ Future<void> _runFrameworkCoverage() async {
if (!coverageFile.existsSync()) { if (!coverageFile.existsSync()) {
foundError(<String>[ foundError(<String>[
'${red}Coverage file not found.$reset', '${red}Coverage file not found.$reset',
'Expected to find: $cyan${coverageFile.absolute}$reset', 'Expected to find: $cyan${coverageFile.absolute.path}$reset',
'This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.', 'This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.',
]); ]);
return; return;
...@@ -1172,7 +1251,7 @@ Future<void> _runFlutterDriverWebTest({ ...@@ -1172,7 +1251,7 @@ Future<void> _runFlutterDriverWebTest({
bool expectFailure = false, bool expectFailure = false,
bool silenceBrowserOutput = false, bool silenceBrowserOutput = false,
}) async { }) async {
print('${green}Running integration tests $target in $buildMode mode.$reset'); printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
await runCommand( await runCommand(
flutter, flutter,
<String>[ 'clean' ], <String>[ 'clean' ],
...@@ -1206,7 +1285,6 @@ Future<void> _runFlutterDriverWebTest({ ...@@ -1206,7 +1285,6 @@ Future<void> _runFlutterDriverWebTest({
return false; return false;
}, },
); );
print('${green}Integration test passed.$reset');
} }
// Compiles a sample web app and checks that its JS doesn't contain certain // Compiles a sample web app and checks that its JS doesn't contain certain
...@@ -1287,7 +1365,7 @@ Future<String> getFlutterPluginsVersion({ ...@@ -1287,7 +1365,7 @@ Future<String> getFlutterPluginsVersion({
/// Executes the test suite for the flutter/plugins repo. /// Executes the test suite for the flutter/plugins repo.
Future<void> _runFlutterPluginsTests() async { Future<void> _runFlutterPluginsTests() async {
Future<void> runAnalyze() async { Future<void> runAnalyze() async {
print('${green}Running analysis for flutter/plugins$reset'); printProgress('${green}Running analysis for flutter/plugins$reset');
final Directory checkout = Directory.systemTemp.createTempSync('flutter_plugins.'); final Directory checkout = Directory.systemTemp.createTempSync('flutter_plugins.');
await runCommand( await runCommand(
'git', 'git',
...@@ -1348,7 +1426,7 @@ Future<void> _runFlutterPluginsTests() async { ...@@ -1348,7 +1426,7 @@ Future<void> _runFlutterPluginsTests() async {
/// ///
/// Generated SKPs are ditched, this just verifies that it can run without failure. /// Generated SKPs are ditched, this just verifies that it can run without failure.
Future<void> _runSkpGeneratorTests() async { Future<void> _runSkpGeneratorTests() async {
print('${green}Running skp_generator from flutter/tests$reset'); printProgress('${green}Running skp_generator from flutter/tests$reset');
final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.'); final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.');
await runCommand( await runCommand(
'git', 'git',
...@@ -1388,9 +1466,12 @@ Future<bool> _isChromeDriverRunning() async { ...@@ -1388,9 +1466,12 @@ Future<bool> _isChromeDriverRunning() async {
Future<void> _ensureChromeDriverIsRunning() async { Future<void> _ensureChromeDriverIsRunning() async {
// If we cannot connect to ChromeDriver, assume it is not running. Launch it. // If we cannot connect to ChromeDriver, assume it is not running. Launch it.
if (!await _isChromeDriverRunning()) { if (!await _isChromeDriverRunning()) {
print('Starting chromedriver'); printProgress('Starting chromedriver');
// Assume chromedriver is in the PATH. // Assume chromedriver is in the PATH.
_chromeDriver = await startCommand( _chromeDriver = await startCommand(
// TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
// and it doesn't use most of startCommand's features; we could simplify this a lot by
// inlining the relevant parts of startCommand here.
'chromedriver', 'chromedriver',
<String>['--port=4444'], <String>['--port=4444'],
); );
...@@ -1430,7 +1511,7 @@ Future<void> _stopChromeDriver() async { ...@@ -1430,7 +1511,7 @@ Future<void> _stopChromeDriver() async {
/// The test is written using `package:integration_test` (despite the "e2e" in /// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons). /// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async { Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
print('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset'); printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'); final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
await runCommand( await runCommand(
flutter, flutter,
...@@ -1461,7 +1542,6 @@ Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) ...@@ -1461,7 +1542,6 @@ Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false })
'FLUTTER_WEB': 'true', 'FLUTTER_WEB': 'true',
}, },
); );
print('${green}Integration test passed.$reset');
} }
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async { Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
...@@ -1490,8 +1570,8 @@ Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async { ...@@ -1490,8 +1570,8 @@ Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
); );
// Run the app. // Run the app.
final int serverPort = await findAvailablePort(); final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePort(); final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome( final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html', appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory, appDirectory: appBuildDirectory,
...@@ -1538,8 +1618,8 @@ Future<void> _runWebReleaseTest(String target, { ...@@ -1538,8 +1618,8 @@ Future<void> _runWebReleaseTest(String target, {
); );
// Run the app. // Run the app.
final int serverPort = await findAvailablePort(); final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePort(); final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome( final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html', appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory, appDirectory: appBuildDirectory,
...@@ -1636,7 +1716,7 @@ Future<void> _runFlutterWebTest(String webRenderer, String workingDirectory, Lis ...@@ -1636,7 +1716,7 @@ Future<void> _runFlutterWebTest(String webRenderer, String workingDirectory, Lis
// properly when overriding the local engine (for example, because some platform // properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines). // dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368 // See https://github.com/flutter/flutter/issues/72368
Future<void> _dartRunTest(String workingDirectory, { Future<void> _runDartTest(String workingDirectory, {
List<String>? testPaths, List<String>? testPaths,
bool enableFlutterToolAsserts = true, bool enableFlutterToolAsserts = true,
bool useBuildRunner = false, bool useBuildRunner = false,
...@@ -1671,10 +1751,6 @@ Future<void> _dartRunTest(String workingDirectory, { ...@@ -1671,10 +1751,6 @@ Future<void> _dartRunTest(String workingDirectory, {
'run', 'run',
'test', 'test',
if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed', if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
if (useFlutterTestFormatter)
'-rjson'
else
'-rcompact',
'-j$cpus', '-j$cpus',
if (!hasColor) if (!hasColor)
'--no-color', '--no-color',
...@@ -1702,21 +1778,6 @@ Future<void> _dartRunTest(String workingDirectory, { ...@@ -1702,21 +1778,6 @@ Future<void> _dartRunTest(String workingDirectory, {
// the tool themselves. // the tool themselves.
await runCommand(flutter, <String>['--version'], environment: environment); await runCommand(flutter, <String>['--version'], environment: environment);
} }
if (useFlutterTestFormatter) {
final FlutterCompactFormatter formatter = FlutterCompactFormatter();
Stream<String> testOutput;
try {
testOutput = runAndGetStdout(
dart,
args,
workingDirectory: workingDirectory,
environment: environment,
);
} finally {
formatter.finish();
}
await _processTestOutput(formatter, testOutput);
} else {
await runCommand( await runCommand(
dart, dart,
args, args,
...@@ -1724,7 +1785,6 @@ Future<void> _dartRunTest(String workingDirectory, { ...@@ -1724,7 +1785,6 @@ Future<void> _dartRunTest(String workingDirectory, {
environment: environment, environment: environment,
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null, removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
); );
}
} }
Future<void> _runFlutterTest(String workingDirectory, { Future<void> _runFlutterTest(String workingDirectory, {
...@@ -1741,9 +1801,9 @@ Future<void> _runFlutterTest(String workingDirectory, { ...@@ -1741,9 +1801,9 @@ Future<void> _runFlutterTest(String workingDirectory, {
assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both'); assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
final List<String> tags = <String>[]; final List<String> tags = <String>[];
// Recipe configured reduced test shards will only execute tests with the // Recipe-configured reduced test shards will only execute tests with the
// appropriate tag. // appropriate tag.
if ((Platform.environment['REDUCED_TEST_SET'] ?? 'False') == 'True') { if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
tags.addAll(<String>['-t', 'reduced-test-set']); tags.addAll(<String>['-t', 'reduced-test-set']);
} }
...@@ -1756,11 +1816,6 @@ Future<void> _runFlutterTest(String workingDirectory, { ...@@ -1756,11 +1816,6 @@ Future<void> _runFlutterTest(String workingDirectory, {
...flutterTestArgs, ...flutterTestArgs,
]; ];
final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
if (shouldProcessOutput) {
args.add('--machine');
}
if (script != null) { if (script != null) {
final String fullScriptPath = path.join(workingDirectory, script); final String fullScriptPath = path.join(workingDirectory, script);
if (!FileSystemEntity.isFileSync(fullScriptPath)) { if (!FileSystemEntity.isFileSync(fullScriptPath)) {
...@@ -1778,7 +1833,6 @@ Future<void> _runFlutterTest(String workingDirectory, { ...@@ -1778,7 +1833,6 @@ Future<void> _runFlutterTest(String workingDirectory, {
args.addAll(tests); args.addAll(tests);
if (!shouldProcessOutput) {
final OutputMode outputMode = outputChecker == null && printOutput final OutputMode outputMode = outputChecker == null && printOutput
? OutputMode.print ? OutputMode.print
: OutputMode.capture; : OutputMode.capture;
...@@ -1798,32 +1852,6 @@ Future<void> _runFlutterTest(String workingDirectory, { ...@@ -1798,32 +1852,6 @@ Future<void> _runFlutterTest(String workingDirectory, {
foundError(<String>[message]); foundError(<String>[message]);
} }
} }
return;
}
if (useFlutterTestFormatter) {
final FlutterCompactFormatter formatter = FlutterCompactFormatter();
Stream<String> testOutput;
try {
testOutput = runAndGetStdout(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
environment: environment,
);
} finally {
formatter.finish();
}
await _processTestOutput(formatter, testOutput);
} else {
await runCommand(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
);
}
} }
/// This will force the next run of the Flutter tool (if it uses the provided /// This will force the next run of the Flutter tool (if it uses the provided
...@@ -1843,19 +1871,6 @@ enum CiProviders { ...@@ -1843,19 +1871,6 @@ enum CiProviders {
luci, luci,
} }
Future<void> _processTestOutput(
FlutterCompactFormatter formatter,
Stream<String> testOutput,
) async {
final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
print('Processing...');
});
await testOutput.forEach(formatter.processRawOutput);
heartbeat.cancel();
formatter.finish();
}
CiProviders? get ciProvider { CiProviders? get ciProvider {
if (Platform.environment['CIRRUS_CI'] == 'true') { if (Platform.environment['CIRRUS_CI'] == 'true') {
return CiProviders.cirrus; return CiProviders.cirrus;
...@@ -1874,10 +1889,10 @@ CiProviders? get ciProvider { ...@@ -1874,10 +1889,10 @@ CiProviders? get ciProvider {
Future<String?> verifyVersion(File file) async { Future<String?> verifyVersion(File file) async {
final RegExp pattern = RegExp( final RegExp pattern = RegExp(
r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$'); r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
final String version = await file.readAsString();
if (!file.existsSync()) { if (!file.existsSync()) {
return 'The version logic failed to create the Flutter version file.'; return 'The version logic failed to create the Flutter version file.';
} }
final String version = await file.readAsString();
if (version == '0.0.0-unknown') { if (version == '0.0.0-unknown') {
return 'The version logic failed to determine the Flutter version.'; return 'The version logic failed to determine the Flutter version.';
} }
...@@ -1904,7 +1919,7 @@ List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSub ...@@ -1904,7 +1919,7 @@ List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSub
print('$kSubshardKey environment variable is missing, skipping sharding'); print('$kSubshardKey environment variable is missing, skipping sharding');
return tests; return tests;
} }
print('$bold$subshardKey=$subshardName$reset'); printProgress('$bold$subshardKey=$subshardName$reset');
final RegExp pattern = RegExp(r'^(\d+)_(\d+)$'); final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
final Match? match = pattern.firstMatch(subshardName); final Match? match = pattern.firstMatch(subshardName);
...@@ -1928,7 +1943,7 @@ List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSub ...@@ -1928,7 +1943,7 @@ List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSub
final int start = (index - 1) * testsPerShard; final int start = (index - 1) * testsPerShard;
final int end = math.min(index * testsPerShard, tests.length); final int end = math.min(index * testsPerShard, tests.length);
print('Selecting subshard $index of $total (range ${start + 1}-$end of ${tests.length})'); print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
return tests.sublist(start, end); return tests.sublist(start, end);
} }
...@@ -1939,19 +1954,6 @@ Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async ...@@ -1939,19 +1954,6 @@ Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async
} }
} }
/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
/// the shard and sub-shard (parsing it in the form shard-subshard-platform, ignoring
/// the platform).
///
/// For local testing you can just set the SHARD and SUBSHARD
/// environment variables. For example, to run all the framework tests you can
/// just set SHARD=framework_tests. Some shards support named subshards, like
/// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
/// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of two"
/// as in run the first half of the tests).
///
/// To run specifically the third subshard of
/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0); Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1); Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
...@@ -1966,11 +1968,11 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam ...@@ -1966,11 +1968,11 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam
} }
if (item == null) { if (item == null) {
for (final String currentItem in items.keys) { for (final String currentItem in items.keys) {
print('$bold$key=$currentItem$reset'); printProgress('$bold$key=$currentItem$reset');
await items[currentItem]!(); await items[currentItem]!();
print('');
} }
} else { } else {
printProgress('$bold$key=$item$reset');
if (!items.containsKey(item)) { if (!items.containsKey(item)) {
foundError(<String>[ foundError(<String>[
'${red}Invalid $name: $item$reset', '${red}Invalid $name: $item$reset',
...@@ -1978,7 +1980,6 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam ...@@ -1978,7 +1980,6 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam
]); ]);
return; return;
} }
print('$bold$key=$item$reset');
await items[item]!(); await items[item]!();
} }
} }
dev/bots/test/analyze_test.dart
View file @ 97901da1
...@@ -16,7 +16,7 @@ Future<String> capture(AsyncVoidCallback callback, { bool shouldHaveErrors = fal ...@@ -16,7 +16,7 @@ Future<String> capture(AsyncVoidCallback callback, { bool shouldHaveErrors = fal
final StringBuffer buffer = StringBuffer(); final StringBuffer buffer = StringBuffer();
final PrintCallback oldPrint = print; final PrintCallback oldPrint = print;
try { try {
print = (Object line) { print = (Object? line) {
buffer.writeln(line); buffer.writeln(line);
}; };
await callback(); await callback();
...@@ -45,18 +45,18 @@ void main() { ...@@ -45,18 +45,18 @@ void main() {
test('analyze.dart - verifyDeprecations', () async { test('analyze.dart - verifyDeprecations', () async {
final String result = await capture(() => verifyDeprecations(testRootPath, minimumMatches: 2), shouldHaveErrors: true); final String result = await capture(() => verifyDeprecations(testRootPath, minimumMatches: 2), shouldHaveErrors: true);
final String lines = <String>[ final String lines = <String>[
'test/analyze-test-input/root/packages/foo/deprecation.dart:12: Deprecation notice does not match required pattern.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:12: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:18: Deprecation notice should be a grammatically correct sentence and start with a capital letter; see style guide: STYLE_GUIDE_URL', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:18: Deprecation notice should be a grammatically correct sentence and start with a capital letter; see style guide: STYLE_GUIDE_URL',
'test/analyze-test-input/root/packages/foo/deprecation.dart:25: Deprecation notice should be a grammatically correct sentence and end with a period.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:25: Deprecation notice should be a grammatically correct sentence and end with a period.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:29: Deprecation notice does not match required pattern.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:29: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:32: Deprecation notice does not match required pattern.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:32: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:37: Deprecation notice does not match required pattern.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:37: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:41: Deprecation notice does not match required pattern.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:41: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:48: End of deprecation notice does not match required pattern.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:48: End of deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:51: Unexpected deprecation notice indent.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:51: Unexpected deprecation notice indent.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:70: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:70: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:76: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:76: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:99: Deprecation notice does not match required pattern. You might have used double quotes (") for the string instead of single quotes (\').', '║ test/analyze-test-input/root/packages/foo/deprecation.dart:99: Deprecation notice does not match required pattern. You might have used double quotes (") for the string instead of single quotes (\').',
] ]
.map((String line) { .map((String line) {
return line return line
...@@ -66,55 +66,30 @@ void main() { ...@@ -66,55 +66,30 @@ void main() {
}) })
.join('\n'); .join('\n');
expect(result, expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'$lines\n' '$lines\n'
'See: https://github.com/flutter/flutter/wiki/Tree-hygiene#handling-breaking-changes\n' '║ See: https://github.com/flutter/flutter/wiki/Tree-hygiene#handling-breaking-changes\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╚═══════════════════════════════════════════════════════════════════════════════\n'
); );
}); });
test('analyze.dart - verifyGoldenTags', () async { test('analyze.dart - verifyGoldenTags', () async {
final String result = await capture(() => verifyGoldenTags(testRootPath, minimumMatches: 6), shouldHaveErrors: true); final List<String> result = (await capture(() => verifyGoldenTags(testRootPath, minimumMatches: 6), shouldHaveErrors: true)).split('\n');
const String noTag = "Files containing golden tests must be tagged using @Tags(<String>['reduced-test-set']) " const String noTag = "Files containing golden tests must be tagged using @Tags(<String>['reduced-test-set']) "
'at the top of the file before import statements.'; 'at the top of the file before import statements.';
const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'."; const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'.";
String lines = <String>[ final List<String> lines = <String>[
'test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag', '║ test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
'test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag', '║ test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
] ]
.map((String line) { .map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/'))
return line .toList();
.replaceAll('/', Platform.isWindows ? r'\' : '/'); expect(result.length, 4 + lines.length, reason: 'output had unexpected number of lines:\n${result.join('\n')}');
}) expect(result[0], '╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════');
.join('\n'); expect(result.getRange(1, result.length - 3).toSet(), lines.toSet());
expect(result[result.length - 3], '║ See: https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter');
try { expect(result[result.length - 2], '╚═══════════════════════════════════════════════════════════════════════════════');
expect( expect(result[result.length - 1], ''); // trailing newline
result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'$lines\n'
'See: https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
);
} catch (_) {
// This list of files may come up in one order or the other.
lines = <String>[
'test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
'test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
]
.map((String line) {
return line
.replaceAll('/', Platform.isWindows ? r'\' : '/');
})
.join('\n');
expect(
result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'$lines\n'
'See: https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
);
}
}); });
test('analyze.dart - verifyNoMissingLicense', () async { test('analyze.dart - verifyNoMissingLicense', () async {
...@@ -122,33 +97,30 @@ void main() { ...@@ -122,33 +97,30 @@ void main() {
final String file = 'test/analyze-test-input/root/packages/foo/foo.dart' final String file = 'test/analyze-test-input/root/packages/foo/foo.dart'
.replaceAll('/', Platform.isWindows ? r'\' : '/'); .replaceAll('/', Platform.isWindows ? r'\' : '/');
expect(result, expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'The following file does not have the right license header for dart files:\n' '║ The following file does not have the right license header for dart files:\n'
' $file\n' '║ $file\n'
'The expected license header is:\n' '║ The expected license header is:\n'
'// Copyright 2014 The Flutter Authors. All rights reserved.\n' '║ // Copyright 2014 The Flutter Authors. All rights reserved.\n'
'// Use of this source code is governed by a BSD-style license that can be\n' '║ // Use of this source code is governed by a BSD-style license that can be\n'
'// found in the LICENSE file.\n' '║ // found in the LICENSE file.\n'
'...followed by a blank line.\n' '║ ...followed by a blank line.\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╚═══════════════════════════════════════════════════════════════════════════════\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'License check failed.\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
); );
}); });
test('analyze.dart - verifyNoTrailingSpaces', () async { test('analyze.dart - verifyNoTrailingSpaces', () async {
final String result = await capture(() => verifyNoTrailingSpaces(testRootPath, minimumMatches: 2), shouldHaveErrors: true); final String result = await capture(() => verifyNoTrailingSpaces(testRootPath, minimumMatches: 2), shouldHaveErrors: true);
final String lines = <String>[ final String lines = <String>[
'test/analyze-test-input/root/packages/foo/spaces.txt:5: trailing U+0020 space character', '║ test/analyze-test-input/root/packages/foo/spaces.txt:5: trailing U+0020 space character',
'test/analyze-test-input/root/packages/foo/spaces.txt:9: trailing blank line', '║ test/analyze-test-input/root/packages/foo/spaces.txt:9: trailing blank line',
] ]
.map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/')) .map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/'))
.join('\n'); .join('\n');
expect(result, expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'$lines\n' '$lines\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╚═══════════════════════════════════════════════════════════════════════════════\n'
); );
}); });
...@@ -159,16 +131,16 @@ void main() { ...@@ -159,16 +131,16 @@ void main() {
), shouldHaveErrors: !Platform.isWindows); ), shouldHaveErrors: !Platform.isWindows);
if (!Platform.isWindows) { if (!Platform.isWindows) {
expect(result, expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'test/analyze-test-input/root/packages/foo/serviceaccount.enc:0: file is not valid UTF-8\n' '║ test/analyze-test-input/root/packages/foo/serviceaccount.enc:0: file is not valid UTF-8\n'
'All files in this repository must be UTF-8. In particular, images and other binaries\n' '║ All files in this repository must be UTF-8. In particular, images and other binaries\n'
'must not be checked into this repository. This is because we are very sensitive to the\n' '║ must not be checked into this repository. This is because we are very sensitive to the\n'
'size of the repository as it is distributed to all our developers. If you have a binary\n' '║ size of the repository as it is distributed to all our developers. If you have a binary\n'
'to which you need access, you should consider how to fetch it from another repository;\n' '║ to which you need access, you should consider how to fetch it from another repository;\n'
'for example, the "assets-for-api-docs" repository is used for images in API docs.\n' '║ for example, the "assets-for-api-docs" repository is used for images in API docs.\n'
'To add assets to flutter_tools templates, see the instructions in the wiki:\n' '║ To add assets to flutter_tools templates, see the instructions in the wiki:\n'
'https://github.com/flutter/flutter/wiki/Managing-template-image-assets\n' '║ https://github.com/flutter/flutter/wiki/Managing-template-image-assets\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n' '╚═══════════════════════════════════════════════════════════════════════════════\n'
); );
} }
}); });
......
dev/bots/test/test_test.dart
View file @ 97901da1
...@@ -17,7 +17,17 @@ import 'common.dart'; ...@@ -17,7 +17,17 @@ import 'common.dart';
/// will include the process result's stdio in the failure message. /// will include the process result's stdio in the failure message.
void expectExitCode(ProcessResult result, int expectedExitCode) { void expectExitCode(ProcessResult result, int expectedExitCode) {
if (result.exitCode != expectedExitCode) { if (result.exitCode != expectedExitCode) {
fail('Failure due to exit code ${result.exitCode}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}'); fail(
'Process ${result.pid} exitted with the wrong exit code.\n'
'\n'
'EXPECTED: exit code $expectedExitCode\n'
'ACTUAL: exit code ${result.exitCode}\n'
'\n'
'STDOUT:\n'
'${result.stdout}\n'
'STDERR:\n'
'${result.stderr}'
);
} }
} }
...@@ -96,10 +106,13 @@ void main() { ...@@ -96,10 +106,13 @@ void main() {
group('test.dart script', () { group('test.dart script', () {
const ProcessManager processManager = LocalProcessManager(); const ProcessManager processManager = LocalProcessManager();
Future<ProcessResult> runScript( Future<ProcessResult> runScript([
[Map<String, String>? environment, List<String> otherArgs = const <String>[]]) async { Map<String, String>? environment,
List<String> otherArgs = const <String>[],
]) async {
final String dart = path.absolute( final String dart = path.absolute(
path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')); path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'),
);
final ProcessResult scriptProcess = processManager.runSync(<String>[ final ProcessResult scriptProcess = processManager.runSync(<String>[
dart, dart,
'test.dart', 'test.dart',
...@@ -112,21 +125,21 @@ void main() { ...@@ -112,21 +125,21 @@ void main() {
// When updating this test, try to pick shard numbers that ensure we're checking // When updating this test, try to pick shard numbers that ensure we're checking
// that unequal test distributions don't miss tests. // that unequal test distributions don't miss tests.
ProcessResult result = await runScript( ProcessResult result = await runScript(
<String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '1_3'}, <String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '1_3'},
); );
expectExitCode(result, 0); expectExitCode(result, 0);
expect(result.stdout, contains('Selecting subshard 1 of 3 (range 1-3 of 8)')); expect(result.stdout, contains('Selecting subshard 1 of 3 (tests 1-3 of 8)'));
result = await runScript( result = await runScript(
<String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '3_3'}, <String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '3_3'},
); );
expectExitCode(result, 0); expectExitCode(result, 0);
expect(result.stdout, contains('Selecting subshard 3 of 3 (range 7-8 of 8)')); expect(result.stdout, contains('Selecting subshard 3 of 3 (tests 7-8 of 8)'));
}); });
test('exits with code 1 when SUBSHARD index greater than total', () async { test('exits with code 1 when SUBSHARD index greater than total', () async {
final ProcessResult result = await runScript( final ProcessResult result = await runScript(
<String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '100_99'}, <String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '100_99'},
); );
expectExitCode(result, 1); expectExitCode(result, 1);
expect(result.stdout, contains('Invalid subshard name')); expect(result.stdout, contains('Invalid subshard name'));
......
dev/bots/utils.dart
View file @ 97901da1
...@@ -2,28 +2,82 @@ ...@@ -2,28 +2,82 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
import 'dart:core' as core_internals show print; import 'dart:async';
import 'dart:core' hide print; import 'dart:core' hide print;
import 'dart:io' as system show exit; import 'dart:io' as system show exit;
import 'dart:io' hide exit; import 'dart:io' hide exit;
import 'dart:math' as math;
import 'package:meta/meta.dart'; import 'package:meta/meta.dart';
const Duration _quietTimeout = Duration(minutes: 10); // how long the output should be hidden between calls to printProgress before just being verbose
final bool hasColor = stdout.supportsAnsiEscapes; final bool hasColor = stdout.supportsAnsiEscapes;
final String bold = hasColor ? '\x1B[1m' : ''; // used for shard titles final String bold = hasColor ? '\x1B[1m' : ''; // shard titles
final String red = hasColor ? '\x1B[31m' : ''; // used for errors final String red = hasColor ? '\x1B[31m' : ''; // errors
final String green = hasColor ? '\x1B[32m' : ''; // used for section titles, commands final String green = hasColor ? '\x1B[32m' : ''; // section titles, commands
final String yellow = hasColor ? '\x1B[33m' : ''; // used for skips final String yellow = hasColor ? '\x1B[33m' : ''; // indications that a test was skipped (usually renders orange or brown)
final String cyan = hasColor ? '\x1B[36m' : ''; // used for paths final String cyan = hasColor ? '\x1B[36m' : ''; // paths
final String reverse = hasColor ? '\x1B[7m' : ''; // used for clocks final String reverse = hasColor ? '\x1B[7m' : ''; // clocks
final String gray = hasColor ? '\x1B[30m' : ''; // subtle decorative items (usually renders as dark gray)
final String white = hasColor ? '\x1B[37m' : ''; // last log line (usually renders as light gray)
final String reset = hasColor ? '\x1B[0m' : ''; final String reset = hasColor ? '\x1B[0m' : '';
final String redLine = '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset';
typedef PrintCallback = void Function(Object line); const int kESC = 0x1B;
const int kOpenSquareBracket = 0x5B;
const int kCSIParameterRangeStart = 0x30;
const int kCSIParameterRangeEnd = 0x3F;
const int kCSIIntermediateRangeStart = 0x20;
const int kCSIIntermediateRangeEnd = 0x2F;
const int kCSIFinalRangeStart = 0x40;
const int kCSIFinalRangeEnd = 0x7E;
String get redLine {
if (hasColor) {
return '$red${'━' * stdout.terminalColumns}$reset';
}
return '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━';
}
String get clock {
final DateTime now = DateTime.now();
return '$reverse▌'
'${now.hour.toString().padLeft(2, "0")}:'
'${now.minute.toString().padLeft(2, "0")}:'
'${now.second.toString().padLeft(2, "0")}'
'▐$reset';
}
String prettyPrintDuration(Duration duration) {
String result = '';
final int minutes = duration.inMinutes;
if (minutes > 0) {
result += '${minutes}min ';
}
final int seconds = duration.inSeconds - minutes * 60;
final int milliseconds = duration.inMilliseconds - (seconds * 1000 + minutes * 60 * 1000);
result += '$seconds.${milliseconds.toString().padLeft(3, "0")}s';
return result;
}
typedef PrintCallback = void Function(Object? line);
typedef VoidCallback = void Function();
// Allow print() to be overridden, for tests. // Allow print() to be overridden, for tests.
PrintCallback print = core_internals.print; //
// Files that import this library should not import `print` from dart:core
// and should not use dart:io's `stdout` or `stderr`.
//
// By default this hides log lines between `printProgress` calls unless a
// timeout expires or anything calls `foundError`.
//
// Also used to implement `--verbose` in test.dart.
PrintCallback print = _printQuietly;
// Called by foundError and used to implement `--abort-on-error` in test.dart.
VoidCallback? onError;
bool get hasError => _hasError; bool get hasError => _hasError;
bool _hasError = false; bool _hasError = false;
...@@ -31,22 +85,44 @@ bool _hasError = false; ...@@ -31,22 +85,44 @@ bool _hasError = false;
Iterable<String> get errorMessages => _errorMessages; Iterable<String> get errorMessages => _errorMessages;
List<String> _errorMessages = <String>[]; List<String> _errorMessages = <String>[];
final List<String> _pendingLogs = <String>[];
Timer? _hideTimer; // When this is null, the output is verbose.
void foundError(List<String> messages) { void foundError(List<String> messages) {
assert(messages.isNotEmpty); assert(messages.isNotEmpty);
print(redLine); // Make the error message easy to notice in the logs by
messages.forEach(print); // wrapping it in a red box.
print(redLine); final int width = math.max(15, (hasColor ? stdout.terminalColumns : 80) - 1);
print('$red╔═╡${bold}ERROR$reset$red╞═${"═" * (width - 9)}');
for (final String message in messages.expand((String line) => line.split('\n'))) {
print('$red║$reset $message');
}
print('$red╚${"═" * width}');
// Normally, "print" actually prints to the log. To make the errors visible,
// and to include useful context, print the entire log up to this point, and
// clear it. Subsequent messages will continue to not be logged until there is
// another error.
_pendingLogs.forEach(_printLoudly);
_pendingLogs.clear();
_errorMessages.addAll(messages); _errorMessages.addAll(messages);
_hasError = true; _hasError = true;
if (onError != null) {
onError!();
}
} }
@visibleForTesting @visibleForTesting
void resetErrorStatus() { void resetErrorStatus() {
_hasError = false; _hasError = false;
_errorMessages.clear(); _errorMessages.clear();
_pendingLogs.clear();
_hideTimer?.cancel();
_hideTimer = null;
} }
Never reportErrorsAndExit() { Never reportErrorsAndExit() {
_hideTimer?.cancel();
_hideTimer = null;
print(redLine); print(redLine);
print('For your convenience, the error messages reported above are repeated here:'); print('For your convenience, the error messages reported above are repeated here:');
_errorMessages.forEach(print); _errorMessages.forEach(print);
...@@ -54,35 +130,93 @@ Never reportErrorsAndExit() { ...@@ -54,35 +130,93 @@ Never reportErrorsAndExit() {
system.exit(1); system.exit(1);
} }
String get clock { void printProgress(String message) {
final DateTime now = DateTime.now(); _pendingLogs.clear();
return '$reverse▌' _hideTimer?.cancel();
'${now.hour.toString().padLeft(2, "0")}:' _hideTimer = null;
'${now.minute.toString().padLeft(2, "0")}:' print('$clock $message $reset');
'${now.second.toString().padLeft(2, "0")}' if (hasColor) {
'▐$reset'; // This sets up a timer to switch to verbose mode when the tests take too long,
// so that if a test hangs we can see the logs.
// (This is only supported with a color terminal. When the terminal doesn't
// support colors, the scripts just print everything verbosely, that way in
// CI there's nothing hidden.)
_hideTimer = Timer(_quietTimeout, () {
_hideTimer = null;
_pendingLogs.forEach(_printLoudly);
_pendingLogs.clear();
});
}
} }
String prettyPrintDuration(Duration duration) { final Pattern _lineBreak = RegExp(r'[\r\n]');
String result = '';
final int minutes = duration.inMinutes; void _printQuietly(Object? message) {
if (minutes > 0) { // The point of this function is to avoid printing its output unless the timer
result += '${minutes}min '; // has gone off in which case the function assumes verbose mode is active and
// prints everything. To show that progress is still happening though, rather
// than showing nothing at all, it instead shows the last line of output and
// keeps overwriting it. To do this in color mode, carefully measures the line
// of text ignoring color codes, which is what the parser below does.
if (_hideTimer != null) {
_pendingLogs.add(message.toString());
String line = '$message'.trimRight();
final int start = line.lastIndexOf(_lineBreak) + 1;
int index = start;
int length = 0;
while (index < line.length && length < stdout.terminalColumns) {
if (line.codeUnitAt(index) == kESC) { // 0x1B
index += 1;
if (index < line.length && line.codeUnitAt(index) == kOpenSquareBracket) { // 0x5B, [
// That was the start of a CSI sequence.
index += 1;
while (index < line.length && line.codeUnitAt(index) >= kCSIParameterRangeStart
&& line.codeUnitAt(index) <= kCSIParameterRangeEnd) { // 0x30..0x3F
index += 1; // ...parameter bytes...
}
while (index < line.length && line.codeUnitAt(index) >= kCSIIntermediateRangeStart
&& line.codeUnitAt(index) <= kCSIIntermediateRangeEnd) { // 0x20..0x2F
index += 1; // ...intermediate bytes...
}
if (index < line.length && line.codeUnitAt(index) >= kCSIFinalRangeStart
&& line.codeUnitAt(index) <= kCSIFinalRangeEnd) { // 0x40..0x7E
index += 1; // ...final byte.
}
}
} else {
index += 1;
length += 1;
}
}
line = line.substring(start, index);
if (line.isNotEmpty) {
stdout.write('\r\x1B[2K$white$line$reset');
}
} else {
_printLoudly('$message');
} }
final int seconds = duration.inSeconds - minutes * 60;
final int milliseconds = duration.inMilliseconds - (seconds * 1000 + minutes * 60 * 1000);
result += '$seconds.${milliseconds.toString().padLeft(3, "0")}s';
return result;
} }
void printProgress(String action, String workingDir, String command) { void _printLoudly(String message) {
print('$clock $action: cd $cyan$workingDir$reset; $green$command$reset'); if (hasColor) {
// Overwrite the last line written by _printQuietly.
stdout.writeln('\r\x1B[2K$reset${message.trimRight()}');
} else {
stdout.writeln(message);
}
} }
// THE FOLLOWING CODE IS A VIOLATION OF OUR STYLE GUIDE
// BECAUSE IT INTRODUCES A VERY FLAKY RACE CONDITION
// https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#never-check-if-a-port-is-available-before-using-it-never-add-timeouts-and-other-race-conditions
// DO NOT USE THE FOLLOWING FUNCTIONS
// DO NOT WRITE CODE LIKE THE FOLLOWING FUNCTIONS
// https://github.com/flutter/flutter/issues/109474
int _portCounter = 8080; int _portCounter = 8080;
/// Finds the next available local port. /// Finds the next available local port.
Future<int> findAvailablePort() async { Future<int> findAvailablePortAndPossiblyCauseFlakyTests() async {
while (!await _isPortAvailable(_portCounter)) { while (!await _isPortAvailable(_portCounter)) {
_portCounter += 1; _portCounter += 1;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab