flutter_platform.dart 38.9 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';

7
import 'package:meta/meta.dart';
8
import 'package:stream_channel/stream_channel.dart';
9
import 'package:vm_service/vm_service.dart' as vm_service;
10

11 12 13 14 15
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports
import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports
import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports
import 'package:test_core/src/runner/environment.dart'; // ignore: implementation_imports
16

17
import '../base/common.dart';
18
import '../base/file_system.dart';
19
import '../base/io.dart';
20
import '../build_info.dart';
21
import '../compile.dart';
22
import '../convert.dart';
23
import '../dart/package_map.dart';
24
import '../globals.dart' as globals;
25
import '../project.dart';
26
import '../test/test_wrapper.dart';
27
import '../vmservice.dart';
28
import 'test_compiler.dart';
29
import 'test_config.dart';
30
import 'watcher.dart';
31

32 33
/// The timeout we give the test process to connect to the test harness
/// once the process has entered its main method.
34 35 36 37 38
///
/// We time out test execution because we expect some tests to hang and we want
/// to know which test hung, rather than have the entire test harness just do
/// nothing for a few hours until the user (or CI environment) gets bored.
const Duration _kTestStartupTimeout = Duration(minutes: 5);
39 40 41 42

/// The timeout we give the test process to start executing Dart code. When the
/// CPU is under severe load, this can take a while, but it's not indicative of
/// any problem with Flutter, so we give it a large timeout.
43 44
///
/// See comment under [_kTestStartupTimeout] regarding timeouts.
45
const Duration _kTestProcessTimeout = Duration(minutes: 5);
46 47 48 49 50 51 52 53 54 55

/// Message logged by the test process to signal that its main method has begun
/// execution.
///
/// The test harness responds by starting the [_kTestStartupTimeout] countdown.
/// The CPU may be throttled, which can cause a long delay in between when the
/// process is spawned and when dart code execution begins; we don't want to
/// hold that against the test.
const String _kStartTimeoutTimerMessage = 'sky_shell test process has entered main method';

56 57
/// The address at which our WebSocket server resides and at which the sky_shell
/// processes will host the Observatory server.
58
final Map<InternetAddressType, InternetAddress> _kHosts = <InternetAddressType, InternetAddress>{
59 60
  InternetAddressType.IPv4: InternetAddress.loopbackIPv4,
  InternetAddressType.IPv6: InternetAddress.loopbackIPv6,
61
};
62

63 64
typedef PlatformPluginRegistration = void Function(FlutterPlatform platform);

65 66
/// Configure the `test` package to work with Flutter.
///
67
/// On systems where each [FlutterPlatform] is only used to run one test suite
68
/// (that is, one Dart file with a `*_test.dart` file name and a single `void
69
/// main()`), you can set an observatory port explicitly.
70
FlutterPlatform installHook({
71
  TestWrapper testWrapper = const TestWrapper(),
72
  @required String shellPath,
73
  TestWatcher watcher,
74 75 76
  bool enableObservatory = false,
  bool machine = false,
  bool startPaused = false,
77
  bool disableServiceAuthCodes = false,
78
  int port = 0,
79
  String precompiledDillPath,
80
  Map<String, String> precompiledDillFiles,
81
  @required BuildMode buildMode,
82 83
  bool trackWidgetCreation = false,
  bool updateGoldens = false,
84
  bool buildTestAssets = false,
85
  int observatoryPort,
86
  InternetAddressType serverType = InternetAddressType.IPv4,
87
  Uri projectRootDirectory,
88
  FlutterProject flutterProject,
89
  String icudtlPath,
90
  PlatformPluginRegistration platformPluginRegistration,
91
  @required List<String> dartExperiments,
92
}) {
93
  assert(testWrapper != null);
94
  assert(enableObservatory || (!startPaused && observatoryPort == null));
95 96 97

  // registerPlatformPlugin can be injected for testing since it's not very mock-friendly.
  platformPluginRegistration ??= (FlutterPlatform platform) {
98
    testWrapper.registerPlatformPlugin(
99
      <Runtime>[Runtime.vm],
100
      () {
101
        return platform;
102
      },
103 104 105 106 107 108 109 110 111 112 113 114 115 116
    );
  };
  final FlutterPlatform platform = FlutterPlatform(
    shellPath: shellPath,
    watcher: watcher,
    machine: machine,
    enableObservatory: enableObservatory,
    startPaused: startPaused,
    disableServiceAuthCodes: disableServiceAuthCodes,
    explicitObservatoryPort: observatoryPort,
    host: _kHosts[serverType],
    port: port,
    precompiledDillPath: precompiledDillPath,
    precompiledDillFiles: precompiledDillFiles,
117
    buildMode: buildMode,
118 119 120 121 122 123
    trackWidgetCreation: trackWidgetCreation,
    updateGoldens: updateGoldens,
    buildTestAssets: buildTestAssets,
    projectRootDirectory: projectRootDirectory,
    flutterProject: flutterProject,
    icudtlPath: icudtlPath,
124
    dartExperiments: dartExperiments,
125
  );
126 127
  platformPluginRegistration(platform);
  return platform;
128 129
}

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
/// Generates the bootstrap entry point script that will be used to launch an
/// individual test file.
///
/// The [testUrl] argument specifies the path to the test file that is being
/// launched.
///
/// The [host] argument specifies the address at which the test harness is
/// running.
///
/// If [testConfigFile] is specified, it must follow the conventions of test
/// configuration files as outlined in the [flutter_test] library. By default,
/// the test file will be launched directly.
///
/// The [updateGoldens] argument will set the [autoUpdateGoldens] global
/// variable in the [flutter_test] package before invoking the test.
145 146
// NOTE: this API is used by the fuchsia source tree, do not add new
// required or position parameters.
147 148 149 150
String generateTestBootstrap({
  @required Uri testUrl,
  @required InternetAddress host,
  File testConfigFile,
151
  bool updateGoldens = false,
152
  bool nullSafety = false,
153 154 155 156 157
}) {
  assert(testUrl != null);
  assert(host != null);
  assert(updateGoldens != null);

158
  final String websocketUrl = host.type == InternetAddressType.IPv4
159 160 161 162
      ? 'ws://${host.address}'
      : 'ws://[${host.address}]';
  final String encodedWebsocketUrl = Uri.encodeComponent(websocketUrl);

163
  final StringBuffer buffer = StringBuffer();
164
  buffer.write('''
165
import 'dart:async';
166 167
import 'dart:convert';  // ignore: dart_convert_import
import 'dart:io';  // ignore: dart_io_import
168
import 'dart:isolate';
169 170

import 'package:flutter_test/flutter_test.dart';
171
import 'package:test_api/src/remote_listener.dart';
172
import 'package:stream_channel/stream_channel.dart';
173
import 'package:stack_trace/stack_trace.dart';
174 175

import '$testUrl' as test;
176
''');
177 178
  if (testConfigFile != null) {
    buffer.write('''
179
import '${Uri.file(testConfigFile.path)}' as test_config;
180
''');
181
  }
182 183 184 185
  // This type is sensitive to the non-nullable experiment.
  final String beforeLoadTypedef = nullSafety
    ? 'Future<dynamic> Function()?'
    : 'Future<dynamic> Function()';
186 187
  buffer.write('''

188 189
/// Returns a serialized test suite.
StreamChannel<dynamic> serializeSuite(Function getMain(),
190
    {bool hidePrints = true, $beforeLoadTypedef beforeLoad}) {
191 192 193 194 195 196 197 198 199 200 201 202 203 204
  return RemoteListener.start(getMain,
      hidePrints: hidePrints, beforeLoad: beforeLoad);
}

/// Capture any top-level errors (mostly lazy syntax errors, since other are
/// caught below) and report them to the parent isolate.
void catchIsolateErrors() {
  final ReceivePort errorPort = ReceivePort();
  // Treat errors non-fatal because otherwise they'll be double-printed.
  Isolate.current.setErrorsFatal(false);
  Isolate.current.addErrorListener(errorPort.sendPort);
  errorPort.listen((dynamic message) {
    // Masquerade as an IsolateSpawnException because that's what this would
    // be if the error had been detected statically.
205 206 207 208
    final IsolateSpawnException error = IsolateSpawnException(
        message[0] as String);
    final Trace stackTrace = message[1] == null ?
        Trace(const <Frame>[]) : Trace.parse(message[1] as String);
209 210 211 212 213
    Zone.current.handleUncaughtError(error, stackTrace);
  });
}


214 215
void main() {
  print('$_kStartTimeoutTimerMessage');
216
  String serverPort = Platform.environment['SERVER_PORT'] ?? '';
217
  String server = Uri.decodeComponent('$encodedWebsocketUrl:\$serverPort');
218
  StreamChannel<dynamic> channel = serializeSuite(() {
219 220 221
    catchIsolateErrors();
    goldenFileComparator = new LocalFileComparator(Uri.parse('$testUrl'));
    autoUpdateGoldenFiles = $updateGoldens;
222
''');
223 224 225 226 227 228 229 230 231 232 233 234 235
  if (testConfigFile != null) {
    buffer.write('''
    return () => test_config.main(test.main);
''');
  } else {
    buffer.write('''
    return test.main;
''');
  }
  buffer.write('''
  });
  WebSocket.connect(server).then((WebSocket socket) {
    socket.map((dynamic x) {
236
      return json.decode(x as String);
237 238 239 240
    }).pipe(channel.sink);
    socket.addStream(channel.stream.map(json.encode));
  });
}
241
''');
242 243 244
  return buffer.toString();
}

245
enum InitialResult { crashed, timedOut, connected }
246

247
enum TestResult { crashed, harnessBailed, testBailed }
248

249
typedef Finalizer = Future<void> Function();
250

251 252 253
/// The flutter test platform used to integrate with package:test.
class FlutterPlatform extends PlatformPlugin {
  FlutterPlatform({
254 255 256 257 258
    @required this.shellPath,
    this.watcher,
    this.enableObservatory,
    this.machine,
    this.startPaused,
259
    this.disableServiceAuthCodes,
260 261 262 263
    this.explicitObservatoryPort,
    this.host,
    this.port,
    this.precompiledDillPath,
264
    this.precompiledDillFiles,
265
    @required this.buildMode,
266
    this.trackWidgetCreation,
267
    this.updateGoldens,
268
    this.buildTestAssets,
269
    this.projectRootDirectory,
270
    this.flutterProject,
271
    this.icudtlPath,
272
    @required this.dartExperiments,
273 274
  }) : assert(shellPath != null);

275
  final String shellPath;
276 277
  final TestWatcher watcher;
  final bool enableObservatory;
278
  final bool machine;
279
  final bool startPaused;
280
  final bool disableServiceAuthCodes;
281
  final int explicitObservatoryPort;
282
  final InternetAddress host;
283
  final int port;
284
  final String precompiledDillPath;
285
  final Map<String, String> precompiledDillFiles;
286
  final BuildMode buildMode;
287
  final bool trackWidgetCreation;
288
  final bool updateGoldens;
289
  final bool buildTestAssets;
290
  final Uri projectRootDirectory;
291
  final FlutterProject flutterProject;
292
  final String icudtlPath;
293
  final List<String> dartExperiments;
294

295
  Directory fontsDirectory;
296 297 298

  /// The test compiler produces dill files for each test main.
  ///
299
  /// To speed up compilation, each compile is initialized from an existing
300 301
  /// dill file from previous runs, if possible.
  TestCompiler compiler;
302

303 304 305 306 307 308 309
  // Each time loadChannel() is called, we spin up a local WebSocket server,
  // then spin up the engine in a subprocess. We pass the engine a Dart file
  // that connects to our WebSocket server, then we proxy JSON messages from
  // the test harness to the engine and back again. If at any time the engine
  // crashes, we inject an error into that stream. When the process closes,
  // we clean everything up.

310 311
  int _testCount = 0;

312 313 314 315 316 317 318 319 320
  @override
  Future<RunnerSuite> load(
    String path,
    SuitePlatform platform,
    SuiteConfiguration suiteConfig,
    Object message,
  ) async {
    // loadChannel may throw an exception. That's fine; it will cause the
    // LoadSuite to emit an error, which will be presented to the user.
321 322 323 324 325
    // Except for the Declarer error, which is a specific test incompatibility
    // error we need to catch.
    try {
      final StreamChannel<dynamic> channel = loadChannel(path, platform);
      final RunnerSuiteController controller = deserializeSuite(path, platform,
326
        suiteConfig, const PluginEnvironment(), channel, message);
327
      return await controller.suite;
328
    } on Exception catch (err) {
329
      /// Rethrow a less confusing error if it is a test incompatibility.
330
      if (err.toString().contains("type 'Declarer' is not a subtype of type 'Declarer'")) {
331 332 333
        throw UnsupportedError('Package incompatibility between flutter and test packages:\n'
          '  * flutter is incompatible with test <1.4.0.\n'
          '  * flutter is incompatible with mockito <4.0.0\n'
334
          "To fix this error, update test to at least '^1.4.0' and mockito to at least '^4.0.0'\n"
335 336 337 338 339
        );
      }
      // Guess it was a different error.
      rethrow;
    }
340 341
  }

342
  @override
343
  StreamChannel<dynamic> loadChannel(String path, SuitePlatform platform) {
344 345
    if (_testCount > 0) {
      // Fail if there will be a port conflict.
346
      if (explicitObservatoryPort != null) {
347
        throwToolExit('installHook() was called with an observatory port or debugger mode enabled, but then more than one test suite was run.');
348
      }
349
      // Fail if we're passing in a precompiled entry-point.
350
      if (precompiledDillPath != null) {
351
        throwToolExit('installHook() was called with a precompiled test entry-point, but then more than one test suite was run.');
352
      }
353
    }
354
    final int ourTestCount = _testCount;
355
    _testCount += 1;
356 357 358 359
    final StreamController<dynamic> localController = StreamController<dynamic>();
    final StreamController<dynamic> remoteController = StreamController<dynamic>();
    final Completer<_AsyncError> testCompleteCompleter = Completer<_AsyncError>();
    final _FlutterPlatformStreamSinkWrapper<dynamic> remoteSink = _FlutterPlatformStreamSinkWrapper<dynamic>(
360 361 362
      remoteController.sink,
      testCompleteCompleter.future,
    );
363
    final StreamChannel<dynamic> localChannel = StreamChannel<dynamic>.withGuarantees(
364 365 366
      remoteController.stream,
      localController.sink,
    );
367
    final StreamChannel<dynamic> remoteChannel = StreamChannel<dynamic>.withGuarantees(
368 369 370
      localController.stream,
      remoteSink,
    );
371
    testCompleteCompleter.complete(_startTest(path, localChannel, ourTestCount));
372
    return remoteChannel;
373
  }
374

375 376 377 378 379 380 381 382 383
  Future<String> _compileExpressionService(
    String isolateId,
    String expression,
    List<String> definitions,
    List<String> typeDefinitions,
    String libraryUri,
    String klass,
    bool isStatic,
  ) async {
384 385 386 387 388 389 390
    if (compiler == null || compiler.compiler == null) {
      throw 'Compiler is not set up properly to compile $expression';
    }
    final CompilerOutput compilerOutput =
      await compiler.compiler.compileExpression(expression, definitions,
        typeDefinitions, libraryUri, klass, isStatic);
    if (compilerOutput != null && compilerOutput.outputFilename != null) {
391
      return base64.encode(globals.fs.file(compilerOutput.outputFilename).readAsBytesSync());
392 393 394 395
    }
    throw 'Failed to compile $expression';
  }

396 397 398 399 400 401 402
  /// Binds an [HttpServer] serving from `host` on `port`.
  ///
  /// Only intended to be overridden in tests for [FlutterPlatform].
  @protected
  @visibleForTesting
  Future<HttpServer> bind(InternetAddress host, int port) => HttpServer.bind(host, port);

403
  Future<_AsyncError> _startTest(
404 405
    String testPath,
    StreamChannel<dynamic> controller,
406 407
    int ourTestCount,
  ) async {
408
    globals.printTrace('test $ourTestCount: starting test $testPath');
409

410
    _AsyncError outOfBandError; // error that we couldn't send to the harness that we need to send via our future
411

412
    final List<Finalizer> finalizers = <Finalizer>[]; // Will be run in reverse order.
413 414 415
    bool subprocessActive = false;
    bool controllerSinkClosed = false;
    try {
416
      // Callback can't throw since it's just setting a variable.
417
      unawaited(controller.sink.done.whenComplete(() {
418
        controllerSinkClosed = true;
419
      }));
420 421

      // Prepare our WebSocket server to talk to the engine subproces.
422
      final HttpServer server = await bind(host, port);
423
      finalizers.add(() async {
424
        globals.printTrace('test $ourTestCount: shutting down test harness socket server');
425
        await server.close(force: true);
426
      });
427
      final Completer<WebSocket> webSocket = Completer<WebSocket>();
428 429
      server.listen(
        (HttpRequest request) {
430 431 432
          if (!webSocket.isCompleted) {
            webSocket.complete(WebSocketTransformer.upgrade(request));
          }
433
        },
434
        onError: (dynamic error, StackTrace stack) {
435
          // If you reach here, it's unlikely we're going to be able to really handle this well.
436
          globals.printTrace('test $ourTestCount: test harness socket server experienced an unexpected error: $error');
437 438 439 440
          if (!controllerSinkClosed) {
            controller.sink.addError(error, stack);
            controller.sink.close();
          } else {
441
            globals.printError('unexpected error from test harness socket server: $error');
442 443 444 445
          }
        },
        cancelOnError: true,
      );
446

447
      globals.printTrace('test $ourTestCount: starting shell process');
448

449
      // If a kernel file is given, then use that to launch the test.
450 451 452 453 454 455 456 457
      // If mapping is provided, look kernel file from mapping.
      // If all fails, create a "listener" dart that invokes actual test.
      String mainDart;
      if (precompiledDillPath != null) {
        mainDart = precompiledDillPath;
      } else if (precompiledDillFiles != null) {
        mainDart = precompiledDillFiles[testPath];
      }
458
      mainDart ??= _createListenerDart(finalizers, ourTestCount, testPath, server);
459

460
      if (precompiledDillPath == null && precompiledDillFiles == null) {
461
        // Lazily instantiate compiler so it is built only if it is actually used.
462
        compiler ??= TestCompiler(buildMode, trackWidgetCreation, flutterProject, dartExperiments);
463
        mainDart = await compiler.compile(globals.fs.file(mainDart).uri);
464 465

        if (mainDart == null) {
466
          controller.sink.addError(_getErrorMessage('Compilation failed', testPath, shellPath));
467
          return null;
468
        }
469
      }
470

471
      final Process process = await _startProcess(
472
        shellPath,
473
        mainDart,
474
        packages: globalPackagesPath,
475 476
        enableObservatory: enableObservatory,
        startPaused: startPaused,
477
        disableServiceAuthCodes: disableServiceAuthCodes,
478
        observatoryPort: explicitObservatoryPort,
479
        serverPort: server.port,
480 481 482
      );
      subprocessActive = true;
      finalizers.add(() async {
483
        if (subprocessActive) {
484
          globals.printTrace('test $ourTestCount: ensuring end-of-process for shell');
485
          process.kill();
486 487
          final int exitCode = await process.exitCode;
          subprocessActive = false;
488 489
          if (!controllerSinkClosed && exitCode != -15) {
            // ProcessSignal.SIGTERM
490 491
            // We expect SIGTERM (15) because we tried to terminate it.
            // It's negative because signals are returned as negative exit codes.
492 493 494 495
            final String message = _getErrorMessage(
                _getExitCodeMessage(exitCode, 'after tests finished'),
                testPath,
                shellPath);
496 497
            controller.sink.addError(message);
          }
498 499
        }
      });
500

501 502
      final Completer<void> timeout = Completer<void>();
      final Completer<void> gotProcessObservatoryUri = Completer<void>();
503
      if (!enableObservatory) {
504
        gotProcessObservatoryUri.complete();
505
      }
506

507
      // Pipe stdout and stderr from the subprocess to our printStatus console.
508
      // We also keep track of what observatory port the engine used, if any.
509
      Uri processObservatoryUri;
510 511
      _pipeStandardStreamsToConsole(
        process,
512 513
        reportObservatoryUri: (Uri detectedUri) {
          assert(processObservatoryUri == null);
514
          assert(explicitObservatoryPort == null ||
515
              explicitObservatoryPort == detectedUri.port);
516
          if (startPaused && !machine) {
517 518 519 520
            globals.printStatus('The test process has been started.');
            globals.printStatus('You can now connect to it using observatory. To connect, load the following Web site in your browser:');
            globals.printStatus('  $detectedUri');
            globals.printStatus('You should first set appropriate breakpoints, then resume the test in the debugger.');
521
          } else {
522
            globals.printTrace('test $ourTestCount: using observatory uri $detectedUri from pid ${process.pid}');
523
          }
524
          processObservatoryUri = detectedUri;
525
          {
526
            globals.printTrace('Connecting to service protocol: $processObservatoryUri');
527
            final Future<vm_service.VmService> localVmService = connectToVmService(processObservatoryUri,
528
              compileExpression: _compileExpressionService);
529
            localVmService.then((vm_service.VmService vmservice) {
530
              globals.printTrace('Successfully connected to service protocol: $processObservatoryUri');
531 532
            });
          }
533
          gotProcessObservatoryUri.complete();
534 535
          watcher?.handleStartedProcess(
              ProcessEvent(ourTestCount, process, processObservatoryUri));
536 537
        },
        startTimeoutTimer: () {
538
          Future<InitialResult>.delayed(_kTestStartupTimeout)
539
              .then<void>((_) => timeout.complete());
540 541
        },
      );
542

543 544 545 546
      // At this point, three things can happen next:
      // The engine could crash, in which case process.exitCode will complete.
      // The engine could connect to us, in which case webSocket.future will complete.
      // The local test harness could get bored of us.
547
      globals.printTrace('test $ourTestCount: awaiting initial result for pid ${process.pid}');
548 549 550 551 552 553 554
      final InitialResult initialResult = await Future.any<InitialResult>(<Future<InitialResult>>[
        process.exitCode.then<InitialResult>((int exitCode) => InitialResult.crashed),
        timeout.future.then<InitialResult>((void value) => InitialResult.timedOut),
        Future<InitialResult>.delayed(_kTestProcessTimeout, () => InitialResult.timedOut),
        gotProcessObservatoryUri.future.then<InitialResult>((void value) {
          return webSocket.future.then<InitialResult>(
            (WebSocket webSocket) => InitialResult.connected,
555 556
          );
        }),
557 558 559
      ]);

      switch (initialResult) {
560
        case InitialResult.crashed:
561
          globals.printTrace('test $ourTestCount: process with pid ${process.pid} crashed before connecting to test harness');
562
          final int exitCode = await process.exitCode;
563
          subprocessActive = false;
564 565 566 567 568
          final String message = _getErrorMessage(
              _getExitCodeMessage(
                  exitCode, 'before connecting to test harness'),
              testPath,
              shellPath);
569
          controller.sink.addError(message);
570
          // Awaited for with 'sink.done' below.
571
          unawaited(controller.sink.close());
572
          globals.printTrace('test $ourTestCount: waiting for controller sink to close');
573
          await controller.sink.done;
574
          await watcher?.handleTestCrashed(ProcessEvent(ourTestCount, process));
575
          break;
576
        case InitialResult.timedOut:
577 578 579
          // Could happen either if the process takes a long time starting
          // (_kTestProcessTimeout), or if once Dart code starts running, it takes a
          // long time to open the WebSocket connection (_kTestStartupTimeout).
580
          globals.printTrace('test $ourTestCount: timed out waiting for process with pid ${process.pid} to connect to test harness');
581
          final String message = _getErrorMessage('Test never connected to test harness.', testPath, shellPath);
582
          controller.sink.addError(message);
583
          // Awaited for with 'sink.done' below.
584
          unawaited(controller.sink.close());
585
          globals.printTrace('test $ourTestCount: waiting for controller sink to close');
586
          await controller.sink.done;
587 588
          await watcher
              ?.handleTestTimedOut(ProcessEvent(ourTestCount, process));
589
          break;
590
        case InitialResult.connected:
591
          globals.printTrace('test $ourTestCount: process with pid ${process.pid} connected to test harness');
592
          final WebSocket testSocket = await webSocket.future;
593

594
          final Completer<void> harnessDone = Completer<void>();
595 596 597 598 599
          final StreamSubscription<dynamic> harnessToTest =
              controller.stream.listen(
            (dynamic event) {
              testSocket.add(json.encode(event));
            },
600
            onDone: harnessDone.complete,
601
            onError: (dynamic error, StackTrace stack) {
602
              // If you reach here, it's unlikely we're going to be able to really handle this well.
603
              globals.printError('test harness controller stream experienced an unexpected error\ntest: $testPath\nerror: $error');
604 605 606 607
              if (!controllerSinkClosed) {
                controller.sink.addError(error, stack);
                controller.sink.close();
              } else {
608
                globals.printError('unexpected error from test harness controller stream: $error');
609 610 611
              }
            },
            cancelOnError: true,
612 613
          );

614
          final Completer<void> testDone = Completer<void>();
615
          final StreamSubscription<dynamic> testToHarness = testSocket.listen(
616
            (dynamic encodedEvent) {
617 618
              assert(encodedEvent is String); // we shouldn't ever get binary messages
              controller.sink.add(json.decode(encodedEvent as String));
619
            },
620
            onDone: testDone.complete,
621
            onError: (dynamic error, StackTrace stack) {
622
              // If you reach here, it's unlikely we're going to be able to really handle this well.
623
              globals.printError('test socket stream experienced an unexpected error\ntest: $testPath\nerror: $error');
624 625 626 627
              if (!controllerSinkClosed) {
                controller.sink.addError(error, stack);
                controller.sink.close();
              } else {
628
                globals.printError('unexpected error from test socket stream: $error');
629 630 631
              }
            },
            cancelOnError: true,
632 633
          );

634
          globals.printTrace('test $ourTestCount: awaiting test result for pid ${process.pid}');
635 636 637
          final TestResult testResult = await Future.any<TestResult>(<Future<TestResult>>[
            process.exitCode.then<TestResult>((int exitCode) {
              return TestResult.crashed;
638
            }),
639 640
            harnessDone.future.then<TestResult>((void value) {
              return TestResult.harnessBailed;
641
            }),
642 643
            testDone.future.then<TestResult>((void value) {
              return TestResult.testBailed;
644
            }),
645 646
          ]);

647
          await Future.wait<void>(<Future<void>>[
648 649 650
            harnessToTest.cancel(),
            testToHarness.cancel(),
          ]);
651 652

          switch (testResult) {
653
            case TestResult.crashed:
654
              globals.printTrace('test $ourTestCount: process with pid ${process.pid} crashed');
655
              final int exitCode = await process.exitCode;
656
              subprocessActive = false;
657 658 659 660 661
              final String message = _getErrorMessage(
                  _getExitCodeMessage(
                      exitCode, 'before test harness closed its WebSocket'),
                  testPath,
                  shellPath);
662
              controller.sink.addError(message);
663
              // Awaited for with 'sink.done' below.
664
              unawaited(controller.sink.close());
665
              globals.printTrace('test $ourTestCount: waiting for controller sink to close');
666 667
              await controller.sink.done;
              break;
668 669 670
            case TestResult.harnessBailed:
            case TestResult.testBailed:
              if (testResult == TestResult.harnessBailed) {
671
                globals.printTrace('test $ourTestCount: process with pid ${process.pid} no longer needed by test harness');
672
              } else {
673
                assert(testResult == TestResult.testBailed);
674
                globals.printTrace('test $ourTestCount: process with pid ${process.pid} no longer needs test harness');
675
              }
676 677
              await watcher?.handleFinishedTest(
                  ProcessEvent(ourTestCount, process, processObservatoryUri));
678 679 680 681
              break;
          }
          break;
      }
682
    } on Exception catch (error, stack) {
683
      globals.printTrace('test $ourTestCount: error caught during test; ${controllerSinkClosed ? "reporting to console" : "sending to test framework"}');
684
      if (!controllerSinkClosed) {
685
        controller.sink.addError(error, stack);
686
      } else {
687
        globals.printError('unhandled error during test:\n$testPath\n$error\n$stack');
688
        outOfBandError ??= _AsyncError(error, stack);
689 690
      }
    } finally {
691
      globals.printTrace('test $ourTestCount: cleaning up...');
692
      // Finalizers are treated like a stack; run them in reverse order.
693
      for (final Finalizer finalizer in finalizers.reversed) {
694 695
        try {
          await finalizer();
696
        } on Exception catch (error, stack) {
697
          globals.printTrace('test $ourTestCount: error while cleaning up; ${controllerSinkClosed ? "reporting to console" : "sending to test framework"}');
698 699 700
          if (!controllerSinkClosed) {
            controller.sink.addError(error, stack);
          } else {
701
            globals.printError('unhandled error during finalization of test:\n$testPath\n$error\n$stack');
702
            outOfBandError ??= _AsyncError(error, stack);
703 704 705
          }
        }
      }
706
      if (!controllerSinkClosed) {
707
        // Waiting below with await.
708
        unawaited(controller.sink.close());
709
        globals.printTrace('test $ourTestCount: waiting for controller sink to close');
710 711 712 713 714
        await controller.sink.done;
      }
    }
    assert(!subprocessActive);
    assert(controllerSinkClosed);
715
    if (outOfBandError != null) {
716
      globals.printTrace('test $ourTestCount: finished with out-of-band failure');
717
    } else {
718
      globals.printTrace('test $ourTestCount: finished');
719
    }
720
    return outOfBandError;
721 722
  }

723
  String _createListenerDart(
724
    List<Finalizer> finalizers,
725 726 727 728
    int ourTestCount,
    String testPath,
    HttpServer server,
  ) {
729
    // Prepare a temporary directory to store the Dart file that will talk to us.
730
    final Directory tempDir = globals.fs.systemTempDirectory.createTempSync('flutter_test_listener.');
731
    finalizers.add(() async {
732
      globals.printTrace('test $ourTestCount: deleting temporary directory');
733
      tempDir.deleteSync(recursive: true);
734 735 736
    });

    // Prepare the Dart file that will talk to us and start the test.
737
    final File listenerFile = globals.fs.file('${tempDir.path}/listener.dart');
738 739
    listenerFile.createSync();
    listenerFile.writeAsStringSync(_generateTestMain(
740
      testUrl: globals.fs.path.toUri(globals.fs.path.absolute(testPath)),
741 742 743 744
    ));
    return listenerFile.path;
  }

745
  String _generateTestMain({
746
    Uri testUrl,
747
  }) {
748
    assert(testUrl.scheme == 'file');
749 750
    return generateTestBootstrap(
      testUrl: testUrl,
751
      testConfigFile: findTestConfigFile(globals.fs.file(testUrl)),
752 753
      host: host,
      updateGoldens: updateGoldens,
754
      nullSafety: dartExperiments.contains('non-nullable'),
755
    );
756 757 758 759
  }

  File _cachedFontConfig;

760 761 762
  @override
  Future<dynamic> close() async {
    if (compiler != null) {
763
      await compiler.dispose();
764 765
      compiler = null;
    }
766
    if (fontsDirectory != null) {
767
      globals.printTrace('Deleting ${fontsDirectory.path}...');
768 769 770
      fontsDirectory.deleteSync(recursive: true);
      fontsDirectory = null;
    }
771 772
  }

773 774 775
  /// Returns a Fontconfig config file that limits font fallback to the
  /// artifact cache directory.
  File get _fontConfigFile {
776
    if (_cachedFontConfig != null) {
777
      return _cachedFontConfig;
778
    }
779

780
    final StringBuffer sb = StringBuffer();
781
    sb.writeln('<fontconfig>');
782
    sb.writeln('  <dir>${globals.cache.getCacheArtifacts().path}</dir>');
783 784 785
    sb.writeln('  <cachedir>/var/cache/fontconfig</cachedir>');
    sb.writeln('</fontconfig>');

786
    if (fontsDirectory == null) {
787 788
      fontsDirectory = globals.fs.systemTempDirectory.createTempSync('flutter_test_fonts.');
      globals.printTrace('Using this directory for fonts configuration: ${fontsDirectory.path}');
789 790
    }

791
    _cachedFontConfig = globals.fs.file('${fontsDirectory.path}/fonts.conf');
792 793 794 795 796
    _cachedFontConfig.createSync();
    _cachedFontConfig.writeAsStringSync(sb.toString());
    return _cachedFontConfig;
  }

797 798 799 800
  Future<Process> _startProcess(
    String executable,
    String testPath, {
    String packages,
801 802
    bool enableObservatory = false,
    bool startPaused = false,
803
    bool disableServiceAuthCodes = false,
804
    int observatoryPort,
805
    int serverPort,
806
  }) {
807
    assert(executable != null); // Please provide the path to the shell in the SKY_SHELL environment variable.
808
    assert(!startPaused || enableObservatory);
809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
    final List<String> command = <String>[
      executable,
      if (enableObservatory) ...<String>[
        // Some systems drive the _FlutterPlatform class in an unusual way, where
        // only one test file is processed at a time, and the operating
        // environment hands out specific ports ahead of time in a cooperative
        // manner, where we're only allowed to open ports that were given to us in
        // advance like this. For those esoteric systems, we have this feature
        // whereby you can create _FlutterPlatform with a pair of ports.
        //
        // I mention this only so that you won't be tempted, as I was, to apply
        // the obvious simplification to this code and remove this entire feature.
        if (observatoryPort != null) '--observatory-port=$observatoryPort',
        if (startPaused) '--start-paused',
        if (disableServiceAuthCodes) '--disable-service-auth-codes',
      ]
      else
        '--disable-observatory',
      if (host.type == InternetAddressType.IPv6) '--ipv6',
      if (icudtlPath != null) '--icu-data-file-path=$icudtlPath',
829 830
      '--enable-checked-mode',
      '--verify-entry-points',
831 832
      '--enable-software-rendering',
      '--skia-deterministic-rendering',
833 834
      '--enable-dart-profiling',
      '--non-interactive',
835
      '--use-test-fonts',
836 837
      '--packages=$packages',
      testPath,
838 839
    ];

840
    globals.printTrace(command.join(' '));
841 842 843 844 845
    // If the FLUTTER_TEST environment variable has been set, then pass it on
    // for package:flutter_test to handle the value.
    //
    // If FLUTTER_TEST has not been set, assume from this context that this
    // call was invoked by the command 'flutter test'.
846 847
    final String flutterTest = globals.platform.environment.containsKey('FLUTTER_TEST')
        ? globals.platform.environment['FLUTTER_TEST']
848
        : 'true';
849
    final Map<String, String> environment = <String, String>{
850
      'FLUTTER_TEST': flutterTest,
851
      'FONTCONFIG_FILE': _fontConfigFile.path,
852
      'SERVER_PORT': serverPort.toString(),
853
      'APP_NAME': flutterProject?.manifest?.appName ?? '',
854
      if (buildTestAssets)
855
        'UNIT_TEST_ASSETS': globals.fs.path.join(flutterProject?.directory?.path ?? '', 'build', 'unit_test_assets'),
856
    };
857
    return globals.processManager.start(command, environment: environment);
858
  }
859

860 861 862
  void _pipeStandardStreamsToConsole(
    Process process, {
    void startTimeoutTimer(),
863
    void reportObservatoryUri(Uri uri),
864
  }) {
865
    const String observatoryString = 'Observatory listening on ';
866
    for (final Stream<List<int>> stream in <Stream<List<int>>>[
867
      process.stderr,
868
      process.stdout,
869 870 871 872 873 874 875 876 877 878
    ]) {
      stream
          .transform<String>(utf8.decoder)
          .transform<String>(const LineSplitter())
          .listen(
        (String line) {
          if (line == _kStartTimeoutTimerMessage) {
            if (startTimeoutTimer != null) {
              startTimeoutTimer();
            }
879
          } else if (line.startsWith("error: Unable to read Dart source 'package:test/")) {
880 881
            globals.printTrace('Shell: $line');
            globals.printError('\n\nFailed to load test harness. Are you missing a dependency on flutter_test?\n');
882
          } else if (line.startsWith(observatoryString)) {
883
            globals.printTrace('Shell: $line');
884 885 886 887
            try {
              final Uri uri = Uri.parse(line.substring(observatoryString.length));
              if (reportObservatoryUri != null) {
                reportObservatoryUri(uri);
888
              }
889
            } on Exception catch (error) {
890
              globals.printError('Could not parse shell observatory port message: $error');
891
            }
892
          } else if (line != null) {
893
            globals.printStatus('Shell: $line');
894 895 896
          }
        },
        onError: (dynamic error) {
897
          globals. printError('shell console stream for process pid ${process.pid} experienced an unexpected error: $error');
898 899 900
        },
        cancelOnError: true,
      );
901
    }
902
  }
903

904 905 906
  String _getErrorMessage(String what, String testPath, String shellPath) {
    return '$what\nTest: $testPath\nShell: $shellPath\n\n';
  }
907

908 909
  String _getExitCodeMessage(int exitCode, String when) {
    switch (exitCode) {
910 911
      case 1:
        return 'Shell subprocess cleanly reported an error $when. Check the logs above for an error message.';
912 913
      case 0:
        return 'Shell subprocess ended cleanly $when. Did main() call exit()?';
914
      case -0x0f: // ProcessSignal.SIGTERM
915
        return 'Shell subprocess crashed with SIGTERM ($exitCode) $when.';
916
      case -0x0b: // ProcessSignal.SIGSEGV
917
        return 'Shell subprocess crashed with segmentation fault $when.';
918
      case -0x06: // ProcessSignal.SIGABRT
919
        return 'Shell subprocess crashed with SIGABRT ($exitCode) $when.';
920
      case -0x02: // ProcessSignal.SIGINT
921
        return 'Shell subprocess terminated by ^C (SIGINT, $exitCode) $when.';
922 923
      default:
        return 'Shell subprocess crashed with unexpected exit code $exitCode $when.';
924 925 926
    }
  }
}
927

928 929 930 931 932 933 934 935 936 937 938 939 940
// The [_shellProcessClosed] future can't have errors thrown on it because it
// crosses zones (it's fed in a zone created by the test package, but listened
// to by a parent zone, the same zone that calls [close] below).
//
// This is because Dart won't let errors that were fed into a Future in one zone
// propagate to listeners in another zone. (Specifically, the zone in which the
// future was completed with the error, and the zone in which the listener was
// registered, are what matters.)
//
// Because of this, the [_shellProcessClosed] future takes an [_AsyncError]
// object as a result. If it's null, it's as if it had completed correctly; if
// it's non-null, it contains the error and stack trace of the actual error, as
// if it had completed with that error.
941 942
class _FlutterPlatformStreamSinkWrapper<S> implements StreamSink<S> {
  _FlutterPlatformStreamSinkWrapper(this._parent, this._shellProcessClosed);
943

944
  final StreamSink<S> _parent;
945
  final Future<_AsyncError> _shellProcessClosed;
946 947

  @override
948
  Future<void> get done => _done.future;
949
  final Completer<void> _done = Completer<void>();
950 951 952

  @override
  Future<dynamic> close() {
953
    Future.wait<dynamic>(<Future<dynamic>>[
954 955
      _parent.close(),
      _shellProcessClosed,
956
    ]).then<void>(
957 958 959
      (List<dynamic> futureResults) {
        assert(futureResults.length == 2);
        assert(futureResults.first == null);
960 961 962
        final dynamic lastResult = futureResults.last;
        if (lastResult is _AsyncError) {
          _done.completeError(lastResult.error, lastResult.stack);
963
        } else {
964
          assert(lastResult == null);
965 966
          _done.complete();
        }
967
      },
968
      onError: _done.completeError,
969 970 971 972 973 974 975
    );
    return done;
  }

  @override
  void add(S event) => _parent.add(event);
  @override
976
  void addError(dynamic errorEvent, [ StackTrace stackTrace ]) => _parent.addError(errorEvent, stackTrace);
977 978 979
  @override
  Future<dynamic> addStream(Stream<S> stream) => _parent.addStream(stream);
}
980 981 982 983 984 985

@immutable
class _AsyncError {
  const _AsyncError(this.error, this.stack);
  final dynamic error;
  final StackTrace stack;
986
}