flutter test: add --machine flag (#10520)

Currently this just prints the observatory URL as a JSON event.
Refactored the code to make this fit in.

examples/hello_world/hello_world.iml

@@ -7,6 +7,7 @@
       <excludeFolder url="file://$MODULE_DIR$/.pub" />
       <excludeFolder url="file://$MODULE_DIR$/build" />
       <excludeFolder url="file://$MODULE_DIR$/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/packages" />
     </content>
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="library" name="Dart Packages" level="project" />

packages/flutter_tools/bin/fuchsia_tester.dart

@@ -88,7 +88,6 @@ Future<Null> run(List<String> args) async {
     }
     loader.installHook(
       shellPath: shellPath,
-      debuggerMode: false,
     );
 
     PackageMap.globalPackagesPath =

packages/flutter_tools/flutter_tools.iml

@@ -7,6 +7,12 @@
       <excludeFolder url="file://$MODULE_DIR$/bin/packages" />
       <excludeFolder url="file://$MODULE_DIR$/build" />
       <excludeFolder url="file://$MODULE_DIR$/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/android/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/base/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/commands/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/data/dart_dependencies_test/asci_casing/.pub" />
+      <excludeFolder url="file://$MODULE_DIR$/test/data/dart_dependencies_test/asci_casing/build" />
+      <excludeFolder url="file://$MODULE_DIR$/test/data/dart_dependencies_test/asci_casing/packages" />
       <excludeFolder url="file://$MODULE_DIR$/test/data/dart_dependencies_test/bad_package/.pub" />
       <excludeFolder url="file://$MODULE_DIR$/test/data/dart_dependencies_test/bad_package/build" />
       <excludeFolder url="file://$MODULE_DIR$/test/data/dart_dependencies_test/bad_package/packages" />
@@ -30,16 +36,10 @@
       <excludeFolder url="file://$MODULE_DIR$/test/data/intellij/plugins/Dart/packages" />
       <excludeFolder url="file://$MODULE_DIR$/test/data/intellij/plugins/packages" />
       <excludeFolder url="file://$MODULE_DIR$/test/data/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/ios/packages" />
       <excludeFolder url="file://$MODULE_DIR$/test/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/replay/osx/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/replay/osx/simulator_application_binary/file/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/replay/osx/simulator_application_binary/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/replay/osx/simulator_application_binary/platform/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/replay/osx/simulator_application_binary/process/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/replay/osx/simulator_application_binary/vmservice/packages" />
       <excludeFolder url="file://$MODULE_DIR$/test/replay/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/src/base/packages" />
-      <excludeFolder url="file://$MODULE_DIR$/test/src/ios/packages" />
+      <excludeFolder url="file://$MODULE_DIR$/test/runner/packages" />
       <excludeFolder url="file://$MODULE_DIR$/test/src/packages" />
       <excludeFolder url="file://$MODULE_DIR$/tool/packages" />
     </content>

packages/flutter_tools/lib/executable.dart

@@ -83,7 +83,7 @@ Future<Null> main(List<String> args) async {
     new RunCommand(verboseHelp: verboseHelp),
     new ScreenshotCommand(),
     new StopCommand(),
-    new TestCommand(),
+    new TestCommand(verboseHelp: verboseHelp),
     new TraceCommand(),
     new UpdatePackagesCommand(hidden: !verboseHelp),
     new UpgradeCommand(),

packages/flutter_tools/lib/src/commands/test.dart

@@ -4,9 +4,6 @@
 
 import 'dart:async';
 
-import 'package:test/src/executable.dart' as test; // ignore: implementation_imports
-
-import '../artifacts.dart';
 import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/io.dart';
@@ -14,16 +11,16 @@ import '../base/logger.dart';
 import '../base/os.dart';
 import '../base/platform.dart';
 import '../base/process_manager.dart';
-import '../base/terminal.dart';
 import '../cache.dart';
-import '../dart/package_map.dart';
 import '../globals.dart';
 import '../runner/flutter_command.dart';
 import '../test/coverage_collector.dart';
-import '../test/flutter_platform.dart' as loader;
+import '../test/event_printer.dart';
+import '../test/runner.dart';
+import '../test/watcher.dart';
 
 class TestCommand extends FlutterCommand {
-  TestCommand() {
+  TestCommand({ bool verboseHelp: false }) {
     usesPubOption();
     argParser.addFlag('start-paused',
         defaultsTo: false,
@@ -53,6 +50,11 @@ class TestCommand extends FlutterCommand {
       defaultsTo: 'coverage/lcov.info',
       help: 'Where to store coverage information (if coverage is enabled).'
     );
+    argParser.addFlag('machine',
+        hide: !verboseHelp,
+        negatable: false,
+        help: 'Handle machine structured JSON command input\n'
+            'and provide output and progress in machine friendly format.');
     commandValidator = () {
       if (!fs.isFileSync('pubspec.yaml')) {
         throwToolExit(
@@ -70,37 +72,12 @@ class TestCommand extends FlutterCommand {
   @override
   String get description => 'Run Flutter unit tests for the current project.';
 
-  Iterable<String> _findTests(Directory directory) {
-    return directory.listSync(recursive: true, followLinks: false)
-                    .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart') &&
-                      fs.isFileSync(entity.path))
-                    .map((FileSystemEntity entity) => fs.path.absolute(entity.path));
-  }
-
   Directory get _currentPackageTestDir {
     // We don't scan the entire package, only the test/ subdirectory, so that
     // files with names like like "hit_test.dart" don't get run.
     return fs.directory('test');
   }
 
-  Future<int> _runTests(List<String> testArgs, Directory testDirectory) async {
-    final Directory currentDirectory = fs.currentDirectory;
-    try {
-      if (testDirectory != null) {
-        printTrace('switching to directory $testDirectory to run tests');
-        PackageMap.globalPackagesPath = fs.path.normalize(fs.path.absolute(PackageMap.globalPackagesPath));
-        fs.currentDirectory = testDirectory;
-      }
-      printTrace('running test package with arguments: $testArgs');
-      await test.main(testArgs);
-      // test.main() sets dart:io's exitCode global.
-      printTrace('test package returned with exit code $exitCode');
-      return exitCode;
-    } finally {
-      fs.currentDirectory = currentDirectory;
-    }
-  }
-
   Future<bool> _collectCoverageData(CoverageCollector collector, { bool mergeCoverageData: false }) async {
     final Status status = logger.startProgress('Collecting coverage information...');
     final String coverageData = await collector.finalizeCoverage(
@@ -161,7 +138,7 @@ class TestCommand extends FlutterCommand {
   }
 
   @override
-  Future<Null> runCommand() async {
+  Future<FlutterCommandResult> runCommand() async {
     if (platform.isWindows) {
       throwToolExit(
           'The test command is currently not supported on Windows: '
@@ -169,57 +146,57 @@ class TestCommand extends FlutterCommand {
       );
     }
 
-    final List<String> testArgs = <String>[];
-
     commandValidator();
 
-    if (!terminal.supportsColor)
-      testArgs.addAll(<String>['--no-color', '-rexpanded']);
+    Iterable<String> files = argResults.rest.map<String>((String testPath) => fs.path.absolute(testPath)).toList();
 
-    CoverageCollector collector;
-    if (argResults['coverage'] || argResults['merge-coverage']) {
-      collector = new CoverageCollector();
-      testArgs.add('--concurrency=1');
+    final bool startPaused = argResults['start-paused'];
+    if (startPaused && files.length != 1) {
+      throwToolExit(
+          'When using --start-paused, you must specify a single test file to run.',
+          exitCode: 1);
     }
 
-    testArgs.add('--');
-
-    Directory testDir;
-    Iterable<String> files = argResults.rest.map<String>((String testPath) => fs.path.absolute(testPath)).toList();
-    if (argResults['start-paused']) {
-      if (files.length != 1)
-        throwToolExit('When using --start-paused, you must specify a single test file to run.', exitCode: 1);
-    } else if (files.isEmpty) {
-      testDir = _currentPackageTestDir;
-      if (!testDir.existsSync())
-        throwToolExit('Test directory "${testDir.path}" not found.');
-      files = _findTests(testDir);
+    Directory workDir;
+    if (files.isEmpty) {
+      workDir = _currentPackageTestDir;
+      if (!workDir.existsSync())
+        throwToolExit('Test directory "${workDir.path}" not found.');
+      files = _findTests(workDir);
       if (files.isEmpty) {
         throwToolExit(
-          'Test directory "${testDir.path}" does not appear to contain any test files.\n'
-          'Test files must be in that directory and end with the pattern "_test.dart".'
+            'Test directory "${workDir.path}" does not appear to contain any test files.\n'
+                'Test files must be in that directory and end with the pattern "_test.dart".'
         );
       }
     }
-    testArgs.addAll(files);
-
-    final InternetAddressType serverType = argResults['ipv6']
-        ? InternetAddressType.IP_V6
-        : InternetAddressType.IP_V4;
-
-    final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester);
-    if (!fs.isFileSync(shellPath))
-      throwToolExit('Cannot find Flutter shell at $shellPath');
-    loader.installHook(
-      shellPath: shellPath,
-      collector: collector,
-      debuggerMode: argResults['start-paused'],
-      serverType: serverType,
-    );
+
+    CoverageCollector collector;
+    if (argResults['coverage'] || argResults['merge-coverage']) {
+      collector = new CoverageCollector();
+    }
+
+    final bool wantEvents = argResults['machine'];
+    if (collector != null && wantEvents) {
+      throwToolExit(
+          "The test command doesn't support --machine and coverage together");
+    }
+
+    TestWatcher watcher;
+    if (collector != null) {
+      watcher = collector;
+    } else if (wantEvents) {
+      watcher = new EventPrinter();
+    }
 
     Cache.releaseLockEarly();
 
-    final int result = await _runTests(testArgs, testDir);
+    final int result = await runTests(files,
+        workDir: workDir,
+        watcher: watcher,
+        enableObservatory: collector != null || startPaused,
+        startPaused: startPaused,
+        ipv6: argResults['ipv6']);
 
     if (collector != null) {
       if (!await _collectCoverageData(collector, mergeCoverageData: argResults['merge-coverage']))
@@ -228,5 +205,13 @@ class TestCommand extends FlutterCommand {
 
     if (result != 0)
       throwToolExit(null);
+    return const FlutterCommandResult(ExitStatus.success);
   }
 }
+
+Iterable<String> _findTests(Directory directory) {
+  return directory.listSync(recursive: true, followLinks: false)
+      .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart') &&
+      fs.isFileSync(entity.path))
+      .map((FileSystemEntity entity) => fs.path.absolute(entity.path));
+}

packages/flutter_tools/lib/src/test/coverage_collector.dart

@@ -11,10 +11,18 @@ import '../base/io.dart';
 import '../dart/package_map.dart';
 import '../globals.dart';
 
+import 'watcher.dart';
+
 /// A class that's used to collect coverage data during tests.
-class CoverageCollector {
+class CoverageCollector extends TestWatcher {
   Map<String, dynamic> _globalHitmap;
 
+  @override
+  Future<Null> onFinishedTests(ProcessEvent event) async {
+    printTrace('test ${event.childIndex}: collecting coverage');
+    await collectCoverage(event.process, event.observatoryUri);
+  }
+
   void _addHitmap(Map<String, dynamic> hitmap) {
     if (_globalHitmap == null)
       _globalHitmap = hitmap;

packages/flutter_tools/lib/src/test/event_printer.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert' show JSON;
+
+import '../base/io.dart' show stdout;
+import 'watcher.dart';
+
+/// Prints JSON events when running a test in --machine mode.
+class EventPrinter extends TestWatcher {
+  EventPrinter({StringSink out}) : this._out = out == null ? stdout: out;
+
+  final StringSink _out;
+
+  @override
+  void onStartedProcess(ProcessEvent event) {
+    _sendEvent("test.startedProcess",
+        <String, dynamic>{"observatoryUri": event.observatoryUri.toString()});
+  }
+
+  void _sendEvent(String name, [dynamic params]) {
+    final Map<String, dynamic> map = <String, dynamic>{ 'event': name};
+    if (params != null) {
+      map['params'] = params;
+    }
+    _send(map);
+  }
+
+  void _send(Map<String, dynamic> command) {
+    final String encoded = JSON.encode(command, toEncodable: _jsonEncodeObject);
+    _out.writeln('\n[$encoded]');
+  }
+
+  dynamic _jsonEncodeObject(dynamic object) {
+    if (object is Uri) {
+      return object.toString();
+    }
+    return object;
+  }
+}

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -19,6 +19,7 @@ import '../base/process_manager.dart';
 import '../dart/package_map.dart';
 import '../globals.dart';
 import 'coverage_collector.dart';
+import 'watcher.dart';
 
 /// The timeout we give the test process to connect to the test harness
 /// once the process has entered its main method.
@@ -53,17 +54,22 @@ final Map<InternetAddressType, InternetAddress> _kHosts = <InternetAddressType,
 void installHook({
   @required String shellPath,
   CoverageCollector collector,
-  bool debuggerMode: false,
+  TestWatcher watcher,
+  bool enableObservatory: false,
+  bool startPaused: false,
   int observatoryPort,
   int diagnosticPort,
   InternetAddressType serverType: InternetAddressType.IP_V4,
 }) {
+  if (startPaused || observatoryPort != null || diagnosticPort != null)
+    assert(enableObservatory);
   hack.registerPlatformPlugin(
     <TestPlatform>[TestPlatform.vm],
     () => new _FlutterPlatform(
       shellPath: shellPath,
-      collector: collector,
-      debuggerMode: debuggerMode,
+      watcher: watcher,
+      enableObservatory: enableObservatory,
+      startPaused: startPaused,
       explicitObservatoryPort: observatoryPort,
       explicitDiagnosticPort: diagnosticPort,
       host: _kHosts[serverType],
@@ -78,8 +84,9 @@ typedef Future<Null> _Finalizer();
 class _FlutterPlatform extends PlatformPlugin {
   _FlutterPlatform({
     @required this.shellPath,
-    this.collector,
-    this.debuggerMode,
+    this.watcher,
+    this.enableObservatory,
+    this.startPaused,
     this.explicitObservatoryPort,
     this.explicitDiagnosticPort,
     this.host,
@@ -88,8 +95,9 @@ class _FlutterPlatform extends PlatformPlugin {
   }
 
   final String shellPath;
-  final CoverageCollector collector;
-  final bool debuggerMode;
+  final TestWatcher watcher;
+  final bool enableObservatory;
+  final bool startPaused;
   final int explicitObservatoryPort;
   final int explicitDiagnosticPort;
   final InternetAddress host;
@@ -105,7 +113,7 @@ class _FlutterPlatform extends PlatformPlugin {
 
   @override
   StreamChannel<dynamic> loadChannel(String testPath, TestPlatform platform) {
-    if (explicitObservatoryPort != null || explicitDiagnosticPort != null || debuggerMode) {
+    if (enableObservatory || explicitDiagnosticPort != null) {
       if (_testCount > 0)
         throwToolExit('installHook() was called with an observatory port, a diagnostic port, both, or debugger mode enabled, but then more than one test suite was run.');
     }
@@ -190,8 +198,8 @@ class _FlutterPlatform extends PlatformPlugin {
         shellPath,
         listenerFile.path,
         packages: PackageMap.globalPackagesPath,
-        enableObservatory: collector != null || debuggerMode,
-        startPaused: debuggerMode,
+        enableObservatory: enableObservatory,
+        startPaused: startPaused,
         observatoryPort: explicitObservatoryPort,
         diagnosticPort: explicitDiagnosticPort,
       );
@@ -216,19 +224,23 @@ class _FlutterPlatform extends PlatformPlugin {
       // Pipe stdout and stderr from the subprocess to our printStatus console.
       // We also keep track of what observatory port the engine used, if any.
       Uri processObservatoryUri;
+
       _pipeStandardStreamsToConsole(
         process,
         reportObservatoryUri: (Uri detectedUri) {
           assert(processObservatoryUri == null);
           assert(explicitObservatoryPort == null ||
                  explicitObservatoryPort == detectedUri.port);
-          if (debuggerMode) {
+          if (startPaused) {
             printStatus('The test process has been started.');
             printStatus('You can now connect to it using observatory. To connect, load the following Web site in your browser:');
             printStatus('  $detectedUri');
             printStatus('You should first set appropriate breakpoints, then resume the test in the debugger.');
           } else {
-            printTrace('test $ourTestCount: using observatory uri $detectedUri from pid ${process.pid} to collect coverage');
+            printTrace('test $ourTestCount: using observatory uri $detectedUri from pid ${process.pid}');
+          }
+          if (watcher != null) {
+            watcher.onStartedProcess(new ProcessEvent(ourTestCount, process, detectedUri));
           }
           processObservatoryUri = detectedUri;
         },
@@ -341,9 +353,9 @@ class _FlutterPlatform extends PlatformPlugin {
           break;
       }
 
-      if (subprocessActive && collector != null) {
-        printTrace('test $ourTestCount: collecting coverage');
-        await collector.collectCoverage(process, processObservatoryUri);
+      if (subprocessActive && watcher != null) {
+        await watcher.onFinishedTests(
+            new ProcessEvent(ourTestCount, process, processObservatoryUri));
       }
     } catch (error, stack) {
       printTrace('test $ourTestCount: error caught during test; ${controllerSinkClosed ? "reporting to console" : "sending to test framework"}');

packages/flutter_tools/lib/src/test/runner.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+// ignore: implementation_imports
+import 'package:test/src/executable.dart' as test;
+
+import '../artifacts.dart';
+import '../base/common.dart';
+import '../base/file_system.dart';
+import '../base/io.dart';
+import '../base/terminal.dart';
+import '../dart/package_map.dart';
+import '../globals.dart';
+import '../test/flutter_platform.dart' as loader;
+import 'watcher.dart';
+
+/// Runs tests using package:test and the Flutter engine.
+Future<int> runTests(
+    List<String> testFiles, {
+    Directory workDir,
+    bool enableObservatory: false,
+    bool startPaused: false,
+    bool ipv6: false,
+    TestWatcher watcher,
+    }) async {
+  // Compute the command-line arguments for package:test.
+  final List<String> testArgs = <String>[];
+  if (!terminal.supportsColor)
+    testArgs.addAll(<String>['--no-color', '-rexpanded']);
+
+  if (enableObservatory) {
+    testArgs.add('--concurrency=1');
+  }
+
+  testArgs.add('--');
+  testArgs.addAll(testFiles);
+
+  // Configure package:test to use the Flutter engine for child processes.
+  final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester);
+  if (!fs.isFileSync(shellPath))
+    throwToolExit('Cannot find Flutter shell at $shellPath');
+
+  final InternetAddressType serverType =
+      ipv6 ? InternetAddressType.IP_V6 : InternetAddressType.IP_V4;
+
+  loader.installHook(
+    shellPath: shellPath,
+    watcher: watcher,
+    enableObservatory: enableObservatory,
+    startPaused: startPaused,
+    serverType: serverType,
+  );
+
+  // Set the package path used for child processes.
+  // TODO(skybrian): why is this global? Move to installHook?
+  PackageMap.globalPackagesPath =
+      fs.path.normalize(fs.path.absolute(PackageMap.globalPackagesPath));
+
+  // Call package:test's main method in the appropriate directory.
+  final Directory saved = fs.currentDirectory;
+  try {
+    if (workDir != null) {
+      printTrace('switching to directory $workDir to run tests');
+      fs.currentDirectory = workDir;
+    }
+
+    printTrace('running test package with arguments: $testArgs');
+    await test.main(testArgs);
+
+    // test.main() sets dart:io's exitCode global.
+    // TODO(skybrian): restore previous value?
+    printTrace('test package returned with exit code $exitCode');
+
+    return exitCode;
+  } finally {
+    fs.currentDirectory = saved;
+  }
+}

packages/flutter_tools/lib/src/test/watcher.dart

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import '../base/io.dart' show Process;
+
+/// Callbacks for reporting progress while running tests.
+class TestWatcher {
+
+  /// Called after a child process starts.
+  ///
+  /// If startPaused was true, the caller needs to resume in Observatory to
+  /// start running the tests.
+  void onStartedProcess(ProcessEvent event) {}
+
+  /// Called after the tests finish but before the process exits.
+  ///
+  /// The child process won't exit until this method completes.
+  /// Not called if the process died.
+  Future<Null> onFinishedTests(ProcessEvent event) async {}
+}
+
+/// Describes a child process started during testing.
+class ProcessEvent {
+  ProcessEvent(this.childIndex, this.process, this.observatoryUri);
+
+  /// The index assigned when the child process was launched.
+  ///
+  /// Indexes are assigned consecutively starting from zero.
+  /// When debugging, there should only be one child process so this will
+  /// always be zero.
+  final int childIndex;
+
+  final Process process;
+
+  /// The observatory Uri or null if not debugging.
+  final Uri observatoryUri;
+}