Try to fix the test framework better than before

The previous attempt to port the 'test' framework to the new framework
wasn't super-successful. This does a better job, hopefully.

packages/flutter_tools/lib/src/test/loader.dart

@@ -11,7 +11,6 @@ import 'package:sky_tools/src/test/json_socket.dart';
 import 'package:sky_tools/src/test/remote_test.dart';
 import 'package:stack_trace/stack_trace.dart';
 import 'package:test/src/backend/group.dart';
-import 'package:test/src/backend/group_entry.dart';
 import 'package:test/src/backend/metadata.dart';
 import 'package:test/src/backend/test_platform.dart';
 import 'package:test/src/runner/configuration.dart';
@@ -30,6 +29,12 @@ final String _kSkyShell = Platform.environment['SKY_SHELL'];
 const String _kHost = '127.0.0.1';
 const String _kPath = '/runner';
 
+// Right now a bunch of our tests crash or assert after the tests have finished running.
+// Mostly this is just because the test puts the framework in an inconsistent state with 
+// a scheduled microtask that verifies that state. Eventually we should fix all these
+// problems but for now we'll just paper over them.
+const bool kExpectAllTestsToCloseCleanly = false;
+
 class _ServerInfo {
   final String url;
   final Future<WebSocket> socket;
@@ -84,10 +89,12 @@ void main() {
 }
 ''');
 
-  Completer<Iterable<GroupEntry>> completer = new Completer<Iterable<GroupEntry>>();
+  Completer<Iterable<RemoteTest>> completer = new Completer<Iterable<RemoteTest>>();
 
-  Process process = await _startProcess(listenerFile.path,
-      packageRoot: p.absolute(config.packageRoot));
+  Process process = await _startProcess(
+    listenerFile.path,
+    packageRoot: p.absolute(config.packageRoot)
+  );
 
   Future cleanupTempDirectory() async {
     if (tempDir == null)
@@ -98,12 +105,43 @@ void main() {
   }
 
   process.exitCode.then((int exitCode) async {
-    info.server.close(force: true);
-    await cleanupTempDirectory();
-    if (!completer.isCompleted) {
-      String error = await process.stderr.transform(UTF8.decoder).first;
-      completer.completeError(
-        new LoadException(path, error), new Trace.current());
+    try {
+      info.server.close(force: true);
+      await cleanupTempDirectory();
+      String output = '';
+      if (exitCode < 0) {
+        // Abnormal termination (high bit of signed 8-bit exitCode is set)
+        switch (exitCode) {
+          case -0x0f: // ProcessSignal.SIGTERM
+            break; // we probably killed it ourselves
+          case -0x0b: // ProcessSignal.SIGSEGV
+            output += 'Segmentation fault in subprocess for: $path\n';
+            break;
+          default:
+            output += 'Unexpected exit code $exitCode from subprocess for: $path\n';
+        }
+      }
+      String stdout = await process.stdout.transform(UTF8.decoder).join('\n');
+      String stderr = await process.stderr.transform(UTF8.decoder).join('\n');
+      if (stdout != '')
+         output += '\nstdout:\n$stdout';
+      if (stderr != '')
+         output += '\nstderr:\n$stderr';
+      if (!completer.isCompleted) {
+        if (output == '')
+          output = 'No output.';
+        completer.completeError(
+          new LoadException(path, output),
+          new Trace.current()
+        );
+      } else {
+        if (kExpectAllTestsToCloseCleanly && output != '')
+          print('Unexpected failure after test claimed to pass:\n$output');
+      }
+    } catch (e) {
+      // Throwing inside this block causes all kinds of hard-to-debug issues
+      // like stack overflows and hangs. So catch everything just in case.
+      print("exception while handling subprocess termination: $e");
     }
   });
 
@@ -116,12 +154,12 @@ void main() {
     if (response["type"] == "print") {
       print(response["line"]);
     } else if (response["type"] == "loadException") {
-      process.kill();
+      process.kill(ProcessSignal.SIGTERM);
       completer.completeError(
           new LoadException(path, response["message"]),
           new Trace.current());
     } else if (response["type"] == "error") {
-      process.kill();
+      process.kill(ProcessSignal.SIGTERM);
       AsyncError asyncError = RemoteException.deserialize(response["error"]);
       completer.completeError(
           new LoadException(path, asyncError.error),
@@ -136,7 +174,7 @@ void main() {
     }
   });
 
-  Iterable<GroupEntry> entries = await completer.future;
+  Iterable<RemoteTest> entries = await completer.future;
 
   return new RunnerSuite(
     const VMEnvironment(),
@@ -144,6 +182,6 @@ void main() {
     path: path,
     platform: TestPlatform.vm,
     os: currentOS,
-    onClose: process.kill
+    onClose: () { process.kill(ProcessSignal.SIGTERM); }
   );
 }

packages/flutter_tools/lib/src/test/remote_listener.dart

@@ -28,9 +28,11 @@ final OperatingSystem currentOS = (() {
 typedef AsyncFunction();
 
 class RemoteListener {
+  RemoteListener._(this._suite, this._socket);
+
   final Suite _suite;
   final WebSocket _socket;
-  LiveTest _liveTest;
+  final Set<LiveTest> _liveTests = new Set<LiveTest>();
 
   static Future start(String server, Metadata metadata, Function getMain()) async {
     WebSocket socket = await WebSocket.connect(server);
@@ -93,8 +95,6 @@ class RemoteListener {
     socket.add(JSON.encode({"type": "loadException", "message": message}));
   }
 
-  RemoteListener._(this._suite, this._socket);
-
   void _send(data) {
     _socket.add(JSON.encode(data));
   }
@@ -119,13 +119,13 @@ class RemoteListener {
   void _handleCommand(String data) {
     var message = JSON.decode(data);
     if (message['command'] == 'run') {
-      assert(_liveTest == null);
       // TODO(ianh): entries[] might return a Group instead of a Test. We don't
       // currently support nested groups.
       Test test = _suite.group.entries[message['index']];
-      _liveTest = test.load(_suite);
+      LiveTest liveTest = test.load(_suite);
+      _liveTests.add(liveTest);
 
-      _liveTest.onStateChange.listen((state) {
+      liveTest.onStateChange.listen((state) {
         _send({
           "type": "state-change",
           "status": state.status.name,
@@ -133,25 +133,30 @@ class RemoteListener {
         });
       });
 
-      _liveTest.onError.listen((asyncError) {
+      liveTest.onError.listen((asyncError) {
         _send({
           "type": "error",
           "error": RemoteException.serialize(
-              asyncError.error, asyncError.stackTrace)
+            asyncError.error,
+            asyncError.stackTrace
+          )
         });
       });
 
-      _liveTest.onPrint.listen((line) {
+      liveTest.onPrint.listen((line) {
         _send({"type": "print", "line": line});
       });
 
-      _liveTest.run().then((_) {
+      liveTest.run().then((_) {
         _send({"type": "complete"});
-        _liveTest = null;
+        _liveTests.remove(liveTest);
       });
     } else if (message['command'] == 'close') {
-      _liveTest.close();
-      _liveTest = null;
+      if (_liveTests.isNotEmpty)
+        print('closing with ${_liveTests.length} live tests');
+      for (LiveTest liveTest in _liveTests)
+        liveTest.close();
+      _liveTests.clear();
     } else {
       print('remote_listener.dart: ignoring command "${message["command"]}" from test harness');
     }

packages/flutter_tools/lib/src/test/remote_test.dart

@@ -17,14 +17,13 @@ import 'package:test/src/util/remote_exception.dart';
 import 'package:sky_tools/src/test/json_socket.dart';
 
 class RemoteTest extends Test {
+  RemoteTest(this.name, this.metadata, this._socket, this._index);
+
   final String name;
   final Metadata metadata;
-
   final JSONSocket _socket;
   final int _index;
 
-  RemoteTest(this.name, this.metadata, this._socket, this._index);
-
   LiveTest load(Suite suite) {
     LiveTestController controller;
     StreamSubscription subscription;
@@ -71,7 +70,13 @@ class RemoteTest extends Test {
 
   // TODO(ianh): Implement this if we need it.
   Test forPlatform(TestPlatform platform, {OperatingSystem os}) {
-    assert(false);
-    return this;
-  }  
+    if (!metadata.testOn.evaluate(platform, os: os))
+      return null;
+    return new RemoteTest(
+      name,
+      metadata.forPlatform(platform, os: os),
+      _socket,
+      _index
+    );
+  }
 }