[customer_testing] Refactor runner to be testable (#76831)

dev/customer_testing/lib/customer_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:meta/meta.dart';
+
+@immutable
+class CustomerTest {
+  factory CustomerTest(File testFile) {
+    final String errorPrefix = 'Could not parse: ${testFile.path}\n';
+    final List<String> contacts = <String>[];
+    final List<String> fetch = <String>[];
+    final List<Directory> update = <Directory>[];
+    final List<String> test = <String>[];
+    bool hasTests = false;
+    for (final String line in testFile.readAsLinesSync().map((String line) => line.trim())) {
+      if (line.isEmpty) {
+        // blank line
+      } else if (line.startsWith('#')) {
+        // comment
+      } else if (line.startsWith('contact=')) {
+        contacts.add(line.substring(8));
+      } else if (line.startsWith('fetch=')) {
+        fetch.add(line.substring(6));
+      } else if (line.startsWith('update=')) {
+        update.add(Directory(line.substring(7)));
+      } else if (line.startsWith('test=')) {
+        hasTests = true;
+        test.add(line.substring(5));
+      } else if (line.startsWith('test.windows=')) {
+        hasTests = true;
+        if (Platform.isWindows)
+          test.add(line.substring(13));
+      } else if (line.startsWith('test.macos=')) {
+        hasTests = true;
+        if (Platform.isMacOS)
+          test.add(line.substring(11));
+      } else if (line.startsWith('test.linux=')) {
+        hasTests = true;
+        if (Platform.isLinux)
+          test.add(line.substring(11));
+      } else if (line.startsWith('test.posix=')) {
+        hasTests = true;
+        if (Platform.isLinux || Platform.isMacOS)
+          test.add(line.substring(11));
+      } else {
+        throw FormatException('${errorPrefix}Unexpected directive:\n$line');
+      }
+    }
+    if (contacts.isEmpty)
+      throw FormatException('${errorPrefix}No contacts specified. At least one contact e-mail address must be specified.');
+    for (final String email in contacts) {
+      if (!email.contains(_email) || email.endsWith('@example.com'))
+        throw FormatException('${errorPrefix}The following e-mail address appears to be an invalid e-mail address: $email');
+    }
+    if (fetch.isEmpty)
+      throw FormatException('${errorPrefix}No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
+    if (fetch.length < 2)
+      throw FormatException('${errorPrefix}Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
+    if (!fetch[0].contains(_fetch1))
+      throw FormatException('${errorPrefix}First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
+    if (!fetch[1].contains(_fetch2))
+      throw FormatException('${errorPrefix}Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH").');
+    if (update.isEmpty)
+      throw FormatException('${errorPrefix}No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
+    if (!hasTests)
+      throw FormatException('${errorPrefix}No "test" directives specified. At least one command must be specified to run tests.');
+    return CustomerTest._(
+      List<String>.unmodifiable(contacts),
+      List<String>.unmodifiable(fetch),
+      List<Directory>.unmodifiable(update),
+      List<String>.unmodifiable(test),
+    );
+  }
+
+  const CustomerTest._(this.contacts, this.fetch, this.update, this.tests);
+
+  // (e-mail regexp from HTML standard)
+  static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$");
+  static final RegExp _fetch1 = RegExp(r'^git(?: -c core.longPaths=true)? clone https://github.com/[-a-zA-Z0-9]+/[-_a-zA-Z0-9]+.git tests$');
+  static final RegExp _fetch2 = RegExp(r'^git(?: -c core.longPaths=true)? -C tests checkout [0-9a-f]+$');
+
+  final List<String> contacts;
+  final List<String> fetch;
+  final List<Directory> update;
+  final List<String> tests;
+}

dev/customer_testing/lib/runner.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import 'customer_test.dart';
+
+Future<bool> runTests({
+  int repeat = 1,
+  bool skipOnFetchFailure = false,
+  bool verbose = false,
+  int numberShards = 1,
+  int shardIndex = 0,
+  List<File> files,
+}) async {
+  if (verbose)
+    print('Starting run_tests.dart...');
+
+  // Best attempt at evenly splitting tests among the shards
+  final List<File> shardedFiles = <File>[];
+  for (int i = shardIndex; i < files.length; i += numberShards) {
+    shardedFiles.add(files[i]);
+  }
+
+  int testCount = 0;
+  int failures = 0;
+
+  if (verbose) {
+    final String s = files.length == 1 ? '' : 's';
+    final String ss = shardedFiles.length == 1 ? '' : 's';
+    print('${files.length} file$s specified. ${shardedFiles.length} test$ss in shard #$shardIndex.');
+    print('');
+  }
+
+  if (verbose) {
+    print('Tests in this shard:');
+    for (final File file in shardedFiles)
+      print(file.path);
+  }
+  print('');
+
+  for (final File file in shardedFiles) {
+    if (verbose)
+      print('Processing ${file.path}...');
+    CustomerTest instructions;
+    try {
+      instructions = CustomerTest(file);
+    } on FormatException catch (error) {
+      print('ERROR: ${error.message}');
+      print('');
+      failures += 1;
+      continue;
+    } on FileSystemException catch (error) {
+      print('ERROR: ${error.message}');
+      print('  ${file.path}');
+      print('');
+      failures += 1;
+      continue;
+    }
+
+    final Directory checkout = Directory.systemTemp.createTempSync('flutter_customer_testing.${path.basenameWithoutExtension(file.path)}.');
+    if (verbose)
+      print('Created temporary directory: ${checkout.path}');
+    try {
+      bool success;
+      bool showContacts = false;
+      for (final String fetchCommand in instructions.fetch) {
+        success = await shell(fetchCommand, checkout, verbose: verbose, silentFailure: skipOnFetchFailure);
+        if (!success) {
+          if (skipOnFetchFailure) {
+            if (verbose) {
+              print('Skipping (fetch failed).');
+            } else {
+              print('Skipping ${file.path} (fetch failed).');
+            }
+          } else {
+            print('ERROR: Failed to fetch repository.');
+            failures += 1;
+            showContacts = true;
+          }
+          break;
+        }
+      }
+      assert(success != null);
+      if (success) {
+        if (verbose)
+          print('Running tests...');
+        final Directory tests = Directory(path.join(checkout.path, 'tests'));
+        // TODO(ianh): Once we have a way to update source code, run that command in each directory of instructions.update
+        for (int iteration = 0; iteration < repeat; iteration += 1) {
+          if (verbose && repeat > 1)
+            print('Round ${iteration + 1} of $repeat.');
+          for (final String testCommand in instructions.tests) {
+            testCount += 1;
+            success = await shell(testCommand, tests, verbose: verbose);
+            if (!success) {
+              print('ERROR: One or more tests from ${path.basenameWithoutExtension(file.path)} failed.');
+              failures += 1;
+              showContacts = true;
+              break;
+            }
+          }
+        }
+        if (verbose && success)
+          print('Tests finished.');
+      }
+      if (showContacts) {
+        final String s = instructions.contacts.length == 1 ? '' : 's';
+        print('Contact$s: ${instructions.contacts.join(", ")}');
+      }
+    } finally {
+      if (verbose)
+        print('Deleting temporary directory...');
+      try {
+        checkout.deleteSync(recursive: true);
+      } on FileSystemException {
+        print('Failed to delete "${checkout.path}".');
+      }
+    }
+    if (verbose)
+      print('');
+  }
+  if (failures > 0) {
+    final String s = failures == 1 ? '' : 's';
+    print('$failures failure$s.');
+    return false;
+  }
+  print('$testCount tests all passed!');
+  return true;
+}
+
+final RegExp _spaces = RegExp(r' +');
+
+Future<bool> shell(String command, Directory directory, { bool verbose = false, bool silentFailure = false }) async {
+  if (verbose)
+    print('>> $command');
+  Process process;
+  if (Platform.isWindows) {
+    process = await Process.start('CMD.EXE', <String>['/S', '/C', command], workingDirectory: directory.path);
+  } else {
+    final List<String> segments = command.trim().split(_spaces);
+    process = await Process.start(segments.first, segments.skip(1).toList(), workingDirectory: directory.path);
+  }
+  final List<String> output = <String>[];
+  utf8.decoder.bind(process.stdout).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
+  utf8.decoder.bind(process.stderr).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
+  final bool success = await process.exitCode == 0;
+  if (success || silentFailure)
+    return success;
+  if (!verbose) {
+    print('>> $command');
+    output.forEach(printLog);
+  }
+  return success;
+}
+
+void printLog(String line) {
+  print('| $line'.trimRight());
+}

dev/customer_testing/pubspec.yaml

@@ -19,4 +19,42 @@ dependencies:
   string_scanner: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   term_glyph: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 5fe8
+dev_dependencies:
+  test: 1.16.5
+
+  _fe_analyzer_shared: 14.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  analyzer: 0.41.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  boolean_selector: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  cli_util: 0.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  convert: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  coverage: 0.15.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  crypto: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  http_multi_server: 2.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  http_parser: 3.1.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  io: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  js: 0.6.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  logging: 0.11.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  matcher: 0.12.10 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  mime: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  node_preamble: 1.4.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  package_config: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pool: 1.5.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf: 0.7.9 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf_packages_handler: 2.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf_static: 0.2.9+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf_web_socket: 0.2.4+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_map_stack_trace: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_maps: 0.10.10 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  stack_trace: 1.10.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  stream_channel: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  test_api: 0.2.19 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  test_core: 0.3.15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  typed_data: 1.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  vm_service: 6.0.1-nullsafety.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  watcher: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  web_socket_channel: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  webkit_inspection_protocol: 0.7.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  yaml: 3.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+
+# PUBSPEC CHECKSUM: 86cb

dev/customer_testing/run_tests.dart

@@ -2,15 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
 import 'dart:io';
 
 import 'package:args/args.dart';
 import 'package:file/local.dart';
 import 'package:glob/glob.dart';
-import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
 
+import 'lib/runner.dart';
+
 Future<void> main(List<String> arguments) async {
   exit(await run(arguments) ? 0 : 1);
 }
@@ -106,9 +106,6 @@ Future<bool> run(List<String> arguments) async {
     return help;
   }
 
-  if (verbose)
-    print('Starting run_tests.dart...');
-
   if (files.length < shardIndex)
     print('Warning: There are more shards than tests. Some shards will not run any tests.');
 
@@ -117,226 +114,12 @@ Future<bool> run(List<String> arguments) async {
     return help;
   }
 
-  // Best attempt at evenly splitting tests among the shards
-  final List<File> shardedFiles = <File>[];
-  for (int i = shardIndex; i < files.length; i += numberShards) {
-    shardedFiles.add(files[i]);
-  }
-
-  int testCount = 0;
-  int failures = 0;
-
-  if (verbose) {
-    final String s = files.length == 1 ? '' : 's';
-    final String ss = shardedFiles.length == 1 ? '' : 's';
-    print('${files.length} file$s specified. ${shardedFiles.length} test$ss in shard #$shardIndex.');
-    print('');
-  }
-
-  if (verbose) {
-    print('Tests in this shard:');
-    for (final File file in shardedFiles)
-      print(file.path);
-  }
-  print('');
-
-  for (final File file in shardedFiles) {
-    if (verbose)
-      print('Processing ${file.path}...');
-    TestFile instructions;
-    try {
-      instructions = TestFile(file);
-    } on FormatException catch (error) {
-      print('ERROR: ${error.message}');
-      print('');
-      failures += 1;
-      continue;
-    } on FileSystemException catch (error) {
-      print('ERROR: ${error.message}');
-      print('  ${file.path}');
-      print('');
-      failures += 1;
-      continue;
-    }
-
-    final Directory checkout = Directory.systemTemp.createTempSync('flutter_customer_testing.${path.basenameWithoutExtension(file.path)}.');
-    if (verbose)
-      print('Created temporary directory: ${checkout.path}');
-    try {
-      bool success;
-      bool showContacts = false;
-      for (final String fetchCommand in instructions.fetch) {
-        success = await shell(fetchCommand, checkout, verbose: verbose, silentFailure: skipOnFetchFailure);
-        if (!success) {
-          if (skipOnFetchFailure) {
-            if (verbose) {
-              print('Skipping (fetch failed).');
-            } else {
-              print('Skipping ${file.path} (fetch failed).');
-            }
-          } else {
-            print('ERROR: Failed to fetch repository.');
-            failures += 1;
-            showContacts = true;
-          }
-          break;
-        }
-      }
-      assert(success != null);
-      if (success) {
-        if (verbose)
-          print('Running tests...');
-        final Directory tests = Directory(path.join(checkout.path, 'tests'));
-        // TODO(ianh): Once we have a way to update source code, run that command in each directory of instructions.update
-        for (int iteration = 0; iteration < repeat; iteration += 1) {
-          if (verbose && repeat > 1)
-            print('Round ${iteration + 1} of $repeat.');
-          for (final String testCommand in instructions.tests) {
-            testCount += 1;
-            success = await shell(testCommand, tests, verbose: verbose);
-            if (!success) {
-              print('ERROR: One or more tests from ${path.basenameWithoutExtension(file.path)} failed.');
-              failures += 1;
-              showContacts = true;
-              break;
-            }
-          }
-        }
-        if (verbose && success)
-          print('Tests finished.');
-      }
-      if (showContacts) {
-        final String s = instructions.contacts.length == 1 ? '' : 's';
-        print('Contact$s: ${instructions.contacts.join(", ")}');
-      }
-    } finally {
-      if (verbose)
-        print('Deleting temporary directory...');
-      try {
-        checkout.deleteSync(recursive: true);
-      } on FileSystemException {
-        print('Failed to delete "${checkout.path}".');
-      }
-    }
-    if (verbose)
-      print('');
-  }
-  if (failures > 0) {
-    final String s = failures == 1 ? '' : 's';
-    print('$failures failure$s.');
-    return false;
-  }
-  print('$testCount tests all passed!');
-  return true;
-}
-
-@immutable
-class TestFile {
-  factory TestFile(File file) {
-    final String errorPrefix = 'Could not parse: ${file.path}\n';
-    final List<String> contacts = <String>[];
-    final List<String> fetch = <String>[];
-    final List<Directory> update = <Directory>[];
-    final List<String> test = <String>[];
-    bool hasTests = false;
-    for (final String line in file.readAsLinesSync().map((String line) => line.trim())) {
-      if (line.isEmpty) {
-        // blank line
-      } else if (line.startsWith('#')) {
-        // comment
-      } else if (line.startsWith('contact=')) {
-        contacts.add(line.substring(8));
-      } else if (line.startsWith('fetch=')) {
-        fetch.add(line.substring(6));
-      } else if (line.startsWith('update=')) {
-        update.add(Directory(line.substring(7)));
-      } else if (line.startsWith('test=')) {
-        hasTests = true;
-        test.add(line.substring(5));
-      } else if (line.startsWith('test.windows=')) {
-        hasTests = true;
-        if (Platform.isWindows)
-          test.add(line.substring(13));
-      } else if (line.startsWith('test.macos=')) {
-        hasTests = true;
-        if (Platform.isMacOS)
-          test.add(line.substring(11));
-      } else if (line.startsWith('test.linux=')) {
-        hasTests = true;
-        if (Platform.isLinux)
-          test.add(line.substring(11));
-      } else if (line.startsWith('test.posix=')) {
-        hasTests = true;
-        if (Platform.isLinux || Platform.isMacOS)
-          test.add(line.substring(11));
-      } else {
-        throw FormatException('${errorPrefix}Unexpected directive:\n$line');
-      }
-    }
-    if (contacts.isEmpty)
-      throw FormatException('${errorPrefix}No contacts specified. At least one contact e-mail address must be specified.');
-    for (final String email in contacts) {
-      if (!email.contains(_email) || email.endsWith('@example.com'))
-        throw FormatException('${errorPrefix}The following e-mail address appears to be an invalid e-mail address: $email');
-    }
-    if (fetch.isEmpty)
-      throw FormatException('${errorPrefix}No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
-    if (fetch.length < 2)
-      throw FormatException('${errorPrefix}Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
-    if (!fetch[0].contains(_fetch1))
-      throw FormatException('${errorPrefix}First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
-    if (!fetch[1].contains(_fetch2))
-      throw FormatException('${errorPrefix}Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH").');
-    if (update.isEmpty)
-      throw FormatException('${errorPrefix}No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
-    if (!hasTests)
-      throw FormatException('${errorPrefix}No "test" directives specified. At least one command must be specified to run tests.');
-    return TestFile._(
-      List<String>.unmodifiable(contacts),
-      List<String>.unmodifiable(fetch),
-      List<Directory>.unmodifiable(update),
-      List<String>.unmodifiable(test),
-    );
-  }
-
-  const TestFile._(this.contacts, this.fetch, this.update, this.tests);
-
-  // (e-mail regexp from HTML standard)
-  static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$");
-  static final RegExp _fetch1 = RegExp(r'^git(?: -c core.longPaths=true)? clone https://github.com/[-a-zA-Z0-9]+/[-_a-zA-Z0-9]+.git tests$');
-  static final RegExp _fetch2 = RegExp(r'^git(?: -c core.longPaths=true)? -C tests checkout [0-9a-f]+$');
-
-  final List<String> contacts;
-  final List<String> fetch;
-  final List<Directory> update;
-  final List<String> tests;
-}
-
-final RegExp _spaces = RegExp(r' +');
-
-Future<bool> shell(String command, Directory directory, { bool verbose = false, bool silentFailure = false }) async {
-  if (verbose)
-    print('>> $command');
-  Process process;
-  if (Platform.isWindows) {
-    process = await Process.start('CMD.EXE', <String>['/S', '/C', command], workingDirectory: directory.path);
-  } else {
-    final List<String> segments = command.trim().split(_spaces);
-    process = await Process.start(segments.first, segments.skip(1).toList(), workingDirectory: directory.path);
-  }
-  final List<String> output = <String>[];
-  utf8.decoder.bind(process.stdout).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
-  utf8.decoder.bind(process.stderr).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
-  final bool success = await process.exitCode == 0;
-  if (success || silentFailure)
-    return success;
-  if (!verbose) {
-    print('>> $command');
-    output.forEach(printLog);
-  }
-  return success;
-}
-
-void printLog(String line) {
-  print('| $line'.trimRight());
+  return runTests(
+    repeat: repeat,
+    skipOnFetchFailure: skipOnFetchFailure,
+    verbose: verbose,
+    numberShards: numberShards,
+    shardIndex: shardIndex,
+    files: files,
+  );
 }

dev/customer_testing/test/common.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
+import 'package:test/test.dart' as test_package show TypeMatcher;
+
+export 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
+
+// Defines a 'package:test' shim.
+// TODO(ianh): Remove this file once https://github.com/dart-lang/matcher/issues/98 is fixed
+
+/// A matcher that compares the type of the actual value to the type argument T.
+test_package.TypeMatcher<T> isInstanceOf<T>() => isA<T>();

dev/customer_testing/test/customer_test_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:file/file.dart';
+import 'package:file/memory.dart';
+
+import 'package:customer_testing/customer_test.dart';
+
+import 'common.dart';
+
+void main() {
+  test('constructs expected model', () async {
+    const String registryContent = '''
+contact=abc@gmail.com
+fetch=git clone https://github.com/flutter/cocoon.git tests
+fetch=git -C tests checkout abc123
+update=.
+# Runs flutter analyze, flutter test, and builds web platform
+test.posix=./test_utilities/bin/flutter_test_runner.sh app_flutter
+test.posix=./test_utilities/bin/flutter_test_runner.sh repo_dashboard
+test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
+    ''';
+    final File registryFile = MemoryFileSystem().file('flutter_cocoon.test')..writeAsStringSync(registryContent);
+
+    final CustomerTest test = CustomerTest(registryFile);
+    expect(test.contacts, containsAll(<String>['abc@gmail.com']));
+    expect(
+        test.fetch,
+        containsAllInOrder(
+            <String>['git clone https://github.com/flutter/cocoon.git tests', 'git -C tests checkout abc123']));
+    if (Platform.isLinux || Platform.isMacOS) {
+      expect(
+          test.tests,
+          containsAllInOrder(<String>[
+            './test_utilities/bin/flutter_test_runner.sh app_flutter',
+            './test_utilities/bin/flutter_test_runner.sh repo_dashboard'
+          ]));
+    } else if (Platform.isWindows) {
+      expect(test.tests, containsAllInOrder(<String>['.\test_utilities\bin\flutter_test_runner.bat repo_dashboard']));
+    }
+  });
+
+  test('throws exception when unknown field is passed', () async {
+    const String registryContent = '''
+contact=abc@gmail.com
+update=.
+fetch=git clone https://github.com/flutter/cocoon.git tests
+fetch=git -C tests checkout abc123
+test.posix=./test_utilities/bin/flutter_test_runner.sh app_flutter
+test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
+unknownfield=super not cool
+    ''';
+    final File registryFile = MemoryFileSystem().file('abc.test')..writeAsStringSync(registryContent);
+
+    expect(() => CustomerTest(registryFile), throwsFormatException);
+  });
+
+  test('throws exception when no tests given', () async {
+    const String registryContent = '''
+contact=abc@gmail.com
+update=.
+fetch=git clone https://github.com/flutter/cocoon.git tests
+''';
+    final File registryFile = MemoryFileSystem().file('abc.test')..writeAsStringSync(registryContent);
+
+    expect(() => CustomerTest(registryFile), throwsFormatException);
+  });
+
+  test('throws exception when only one fetch instruction given', () async {
+    const String registryContent = '''
+contact=abc@gmail.com
+update=.
+fetch=git clone https://github.com/flutter/cocoon.git tests
+test.posix=./test_utilities/bin/flutter_test_runner.sh app_flutter
+test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
+    ''';
+    final File registryFile = MemoryFileSystem().file('abc.test')..writeAsStringSync(registryContent);
+
+    expect(() => CustomerTest(registryFile), throwsFormatException);
+  });
+
+  test('throws exception when no contacts given', () async {
+    const String registryContent = '''
+update=.
+fetch=git clone https://github.com/flutter/cocoon.git tests
+test.posix=./test_utilities/bin/flutter_test_runner.sh app_flutter
+test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
+    ''';
+    final File registryFile = MemoryFileSystem().file('abc.test')..writeAsStringSync(registryContent);
+
+    expect(() => CustomerTest(registryFile), throwsFormatException);
+  });
+}

dev/devicelab/pubspec.yaml

@@ -35,7 +35,6 @@ dependencies:
   web_socket_channel: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
 dev_dependencies:
-  # See packages/flutter_test/pubspec.yaml for why we're pinning this version.
   test: 1.16.5
 
   _fe_analyzer_shared: 14.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"