Check exit code for test subprocess (#7269)

dev/automated_tests/test_smoke_test/README.md

-This directory is used by ///flutter/travis/test.sh to verify that
+This directory is used by //flutter/dev/bots/test.sh to verify that
 `flutter test` actually correctly fails when a test fails.

dev/automated_tests/test_smoke_test/crash1_test.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as system;
+
+import 'package:flutter_test/flutter_test.dart';
+
+// this is a test to make sure our tests consider engine crashes to be failures
+// see //flutter/dev/bots/test.sh
+
+void main() {
+  test('test smoke test -- this test should fail', () async {
+    system.Process.killPid(system.pid, system.ProcessSignal.SIGSEGV);
+  });
+}
\ No newline at end of file

dev/automated_tests/test_smoke_test/crash2_test.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as system;
+
+// this is a test to make sure our tests consider engine crashes to be failures
+// see //flutter/dev/bots/test.sh
+
+void main() {
+  system.Process.killPid(system.pid, system.ProcessSignal.SIGSEGV);
+}
\ No newline at end of file

dev/automated_tests/test_smoke_test/fail_test.dart

@@ -5,7 +5,7 @@
 import 'package:flutter_test/flutter_test.dart';
 
 // this is a test to make sure our tests actually catch failures
-// see ///flutter/travis/test.sh
+// see //flutter/dev/bots/test.sh
 
 void main() {
   test('test smoke test -- this test SHOULD FAIL', () async {

dev/automated_tests/test_smoke_test/missing_import_test.broken_dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// this is a test to make sure our tests consider syntax errors to be failures
+// see //flutter/dev/bots/test.sh
+
+void main() {
+  fail(); // inspired by https://github.com/flutter/flutter/issues/2698
+}

dev/automated_tests/test_smoke_test/pass_test.dart

@@ -5,7 +5,7 @@
 import 'package:flutter_test/flutter_test.dart';
 
 // this is a test to make sure our tests actually catch failures
-// see ///flutter/travis/test.sh
+// see //flutter/dev/bots/test.sh
 
 void main() {
   test('test smoke test -- this test should pass', () async {

dev/automated_tests/test_smoke_test/syntax_error_test.broken_dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// this is a test to make sure our tests consider syntax errors to be failures
+// see //flutter/dev/bots/test.sh
+
+The challenge: demand satisfaction
+If they apologize, no need for further action.

dev/bots/test.sh

@@ -22,6 +22,10 @@ flutter analyze --flutter-repo
 # verify that the tests actually return failure on failure and success on success
 (cd dev/automated_tests; ! flutter test test_smoke_test/fail_test.dart > /dev/null)
 (cd dev/automated_tests; flutter test test_smoke_test/pass_test.dart > /dev/null)
+(cd dev/automated_tests; ! flutter test test_smoke_test/crash1_test.dart > /dev/null)
+(cd dev/automated_tests; ! flutter test test_smoke_test/crash2_test.dart > /dev/null)
+(cd dev/automated_tests; ! flutter test test_smoke_test/syntax_error_test.broken_dart > /dev/null)
+(cd dev/automated_tests; ! flutter test test_smoke_test/missing_import_test.broken_dart > /dev/null)
 (cd packages/flutter_driver; ! flutter drive --use-existing-app -t test_driver/failure.dart >/dev/null 2>&1)
 
 COVERAGE_FLAG=

packages/flutter_tools/lib/src/commands/test.dart

@@ -6,7 +6,7 @@ import 'dart:async';
 import 'dart:io';
 
 import 'package:path/path.dart' as path;
-import 'package:test/src/executable.dart' as executable; // ignore: implementation_imports
+import 'package:test/src/executable.dart' as test; // ignore: implementation_imports
 
 import '../base/common.dart';
 import '../base/logger.dart';
@@ -77,9 +77,9 @@ class TestCommand extends FlutterCommand {
         Directory.current = testDirectory;
       }
       printTrace('running test package with arguments: $testArgs');
-      await executable.main(testArgs);
+      await test.main(testArgs);
+      // test.main() sets dart:io's exitCode global.
       printTrace('test package returned with exit code $exitCode');
-
       return exitCode;
     } finally {
       Directory.current = currentDirectory;
@@ -164,10 +164,10 @@ class TestCommand extends FlutterCommand {
     if (argResults['coverage'])
       testArgs.insert(0, '--concurrency=1');
 
-    loader.installHook();
-    loader.shellPath = tools.getHostToolPath(HostTool.SkyShell);
-    if (!FileSystemEntity.isFileSync(loader.shellPath))
-      throwToolExit('Cannot find Flutter shell at ${loader.shellPath}');
+    final String shellPath = tools.getHostToolPath(HostTool.SkyShell) ?? Platform.environment['SKY_SHELL'];
+    if (!FileSystemEntity.isFileSync(shellPath))
+      throwToolExit('Cannot find Flutter shell at $shellPath');
+    loader.installHook(shellPath: shellPath);
 
     Cache.releaseLockEarly();
 

packages/flutter_tools/lib/src/test/coverage_collector.dart

@@ -20,14 +20,14 @@ class CoverageCollector {
   void collectCoverage({
     String host,
     int port,
-    Process processToKill
+    Process processToKill,
   }) {
     if (enabled) {
       assert(_jobs != null);
       _jobs.add(_startJob(
         host: host,
         port: port,
-        processToKill: processToKill
+        processToKill: processToKill,
       ));
     } else {
       processToKill.kill();
@@ -37,7 +37,7 @@ class CoverageCollector {
   Future<Null> _startJob({
     String host,
     int port,
-    Process processToKill
+    Process processToKill,
   }) async {
     int pid = processToKill.pid;
     printTrace('collecting coverage data from pid $pid on port $port');