Unverified Commit 54fa2554 authored by Kenzie Davisson's avatar Kenzie Davisson Committed by GitHub

Add support for `setup` steps in flutter customer tests (#135726)

This PR adds support for adding `setup` steps that must be run before the `update` steps in Flutter customer tests.

```
setup=./generate_my_mocks.sh
update=.
test= ...
```

This PR also cleans up the `CustomerTest` class in the spirit of yak-shaving and making the code more robust.
parent 6c4965e3
......@@ -4,6 +4,7 @@
import 'dart:io';
import 'package:collection/collection.dart';
import 'package:meta/meta.dart';
@immutable
......@@ -12,58 +13,50 @@ class CustomerTest {
final String errorPrefix = 'Could not parse: ${testFile.path}\n';
final List<String> contacts = <String>[];
final List<String> fetch = <String>[];
final List<String> setup = <String>[];
final List<Directory> update = <Directory>[];
final List<String> test = <String>[];
int? iterations;
bool hasTests = false;
for (final String line in testFile.readAsLinesSync().map((String line) => line.trim())) {
if (line.isEmpty) {
// blank line
} else if (line.startsWith('#')) {
// comment
} else if (line.startsWith('contact=')) {
contacts.add(line.substring(8));
} else if (line.startsWith('fetch=')) {
fetch.add(line.substring(6));
} else if (line.startsWith('update=')) {
update.add(Directory(line.substring(7)));
} else if (line.startsWith('iterations=')) {
if (line.isEmpty || line.startsWith('#')) {
// Blank line or comment.
continue;
}
final bool isUnknownDirective = _TestDirective.values.firstWhereOrNull((_TestDirective d) => line.startsWith(d.name)) == null;
if (isUnknownDirective) {
throw FormatException('${errorPrefix}Unexpected directive:\n$line');
}
_maybeAddTestConfig(line, directive: _TestDirective.contact, directiveValues: contacts);
_maybeAddTestConfig(line, directive: _TestDirective.fetch, directiveValues: fetch);
_maybeAddTestConfig(line, directive: _TestDirective.setup, directiveValues: setup, platformAgnostic: false);
final String updatePrefix = _directive(_TestDirective.update);
if (line.startsWith(updatePrefix)) {
update.add(Directory(line.substring(updatePrefix.length)));
}
final String iterationsPrefix = _directive(_TestDirective.iterations);
if (line.startsWith(iterationsPrefix)) {
if (iterations != null) {
throw const FormatException('Cannot specify "iterations" directive multiple times.');
throw FormatException('Cannot specify "${_TestDirective.iterations.name}" directive multiple times.');
}
iterations = int.parse(line.substring(11));
iterations = int.parse(line.substring(iterationsPrefix.length));
if (iterations < 1) {
throw const FormatException('The "iterations" directive must have a positive integer value.');
throw FormatException('The "${_TestDirective.iterations.name}" directive must have a positive integer value.');
}
} else if (line.startsWith('test=')) {
hasTests = true;
test.add(line.substring(5));
} else if (line.startsWith('test.windows=')) {
hasTests = true;
if (Platform.isWindows) {
test.add(line.substring(13));
}
} else if (line.startsWith('test.macos=')) {
hasTests = true;
if (Platform.isMacOS) {
test.add(line.substring(11));
}
} else if (line.startsWith('test.linux=')) {
hasTests = true;
if (Platform.isLinux) {
test.add(line.substring(11));
}
} else if (line.startsWith('test.posix=')) {
}
if (line.startsWith(_directive(_TestDirective.test)) || line.startsWith('${_TestDirective.test.name}.')) {
hasTests = true;
if (Platform.isLinux || Platform.isMacOS) {
test.add(line.substring(11));
}
} else {
throw FormatException('${errorPrefix}Unexpected directive:\n$line');
}
_maybeAddTestConfig(line, directive: _TestDirective.test, directiveValues: test, platformAgnostic: false);
}
if (contacts.isEmpty) {
throw FormatException('${errorPrefix}No contacts specified. At least one contact e-mail address must be specified.');
throw FormatException('${errorPrefix}No "${_TestDirective.contact.name}" directives specified. At least one contact e-mail address must be specified.');
}
for (final String email in contacts) {
if (!email.contains(_email) || email.endsWith('@example.com')) {
......@@ -71,33 +64,41 @@ class CustomerTest {
}
}
if (fetch.isEmpty) {
throw FormatException('${errorPrefix}No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
throw FormatException('${errorPrefix}No "${_TestDirective.fetch.name}" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
}
if (fetch.length < 2) {
throw FormatException('${errorPrefix}Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
throw FormatException('${errorPrefix}Only one "${_TestDirective.fetch.name}" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
}
if (!fetch[0].contains(_fetch1)) {
throw FormatException('${errorPrefix}First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
throw FormatException('${errorPrefix}First "${_TestDirective.fetch.name}" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
}
if (!fetch[1].contains(_fetch2)) {
throw FormatException('${errorPrefix}Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH").');
throw FormatException('${errorPrefix}Second "${_TestDirective.fetch.name}" directive does not match expected pattern (expected "git -C tests checkout HASH").');
}
if (update.isEmpty) {
throw FormatException('${errorPrefix}No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
throw FormatException('${errorPrefix}No "${_TestDirective.update.name}" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
}
if (!hasTests) {
throw FormatException('${errorPrefix}No "test" directives specified. At least one command must be specified to run tests.');
throw FormatException('${errorPrefix}No "${_TestDirective.test.name}" directives specified. At least one command must be specified to run tests.');
}
return CustomerTest._(
List<String>.unmodifiable(contacts),
List<String>.unmodifiable(fetch),
List<String>.unmodifiable(setup),
List<Directory>.unmodifiable(update),
List<String>.unmodifiable(test),
iterations,
);
}
const CustomerTest._(this.contacts, this.fetch, this.update, this.tests, this.iterations);
const CustomerTest._(
this.contacts,
this.fetch,
this.setup,
this.update,
this.tests,
this.iterations,
);
// (e-mail regexp from HTML standard)
static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$");
......@@ -106,7 +107,60 @@ class CustomerTest {
final List<String> contacts;
final List<String> fetch;
final List<String> setup;
final List<Directory> update;
final List<String> tests;
final int? iterations;
static void _maybeAddTestConfig(
String line, {
required _TestDirective directive,
required List<String> directiveValues,
bool platformAgnostic = true,
}) {
final List<_PlatformType> platforms = platformAgnostic
? <_PlatformType>[_PlatformType.all]
: _PlatformType.values;
for (final _PlatformType platform in platforms) {
final String directiveName = _directive(directive, platform: platform);
if (line.startsWith(directiveName) && platform.conditionMet) {
directiveValues.add(line.substring(directiveName.length));
}
}
}
static String _directive(
_TestDirective directive, {
_PlatformType platform = _PlatformType.all,
}) {
return switch (platform) {
_PlatformType.all => '${directive.name}=',
_ => '${directive.name}.${platform.name}=',
};
}
}
enum _PlatformType {
all,
windows,
macos,
linux,
posix;
bool get conditionMet => switch (this) {
_PlatformType.all => true,
_PlatformType.windows => Platform.isWindows,
_PlatformType.macos => Platform.isMacOS,
_PlatformType.linux => Platform.isLinux,
_PlatformType.posix => Platform.isLinux || Platform.isMacOS,
};
}
enum _TestDirective {
contact,
fetch,
setup,
update,
test,
iterations,
}
......@@ -108,6 +108,21 @@ Future<bool> runTests({
}
if (success) {
final Directory customerRepo = Directory(path.join(checkout.path, 'tests'));
for (final String setupCommand in instructions.setup) {
if (verbose) {
print('Running setup command: $setupCommand');
}
success = await shell(
setupCommand,
customerRepo,
verbose: verbose,
failedCallback: printHeader,
);
if (!success) {
failure('Setup command failed: $setupCommand');
break;
}
}
for (final Directory updateDirectory in instructions.update) {
final Directory resolvedUpdateDirectory = Directory(path.join(customerRepo.path, updateDirectory.path));
if (verbose) {
......
......@@ -16,6 +16,11 @@ void main() {
contact=abc@gmail.com
fetch=git clone https://github.com/flutter/cocoon.git tests
fetch=git -C tests checkout abc123
setup=flutter --version
setup.windows=flutter doctor
setup.posix=flutter -h
setup.linux=flutter analyze -h
setup.macos=flutter build -h
update=.
# Runs flutter analyze, flutter test, and builds web platform
test.posix=./test_utilities/bin/flutter_test_runner.sh app_flutter
......@@ -30,7 +35,11 @@ test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
test.fetch,
containsAllInOrder(
<String>['git clone https://github.com/flutter/cocoon.git tests', 'git -C tests checkout abc123']));
expect(test.setup.first, 'flutter --version');
if (Platform.isLinux || Platform.isMacOS) {
expect(test.setup.length, 3);
expect(test.setup[1], 'flutter -h');
expect(test.setup[2], Platform.isLinux ? 'flutter analyze -h' : 'flutter build -h');
expect(
test.tests,
containsAllInOrder(<String>[
......@@ -39,6 +48,8 @@ test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
]),
);
} else if (Platform.isWindows) {
expect(test.setup.length, 2);
expect(test.setup[1], 'flutter doctor');
expect(test.tests, containsAllInOrder(<String>['.\test_utilities\bin\flutter_test_runner.bat repo_dashboard']));
}
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment