process.dart 17.5 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';

7
import 'package:meta/meta.dart';
8
import 'package:process/process.dart';
9

10
import '../convert.dart';
11
import 'io.dart';
12
import 'logger.dart';
13

14
typedef StringConverter = String? Function(String string);
15 16

/// A function that will be run before the VM exits.
17
typedef ShutdownHook = FutureOr<void> Function();
Devon Carew's avatar
Devon Carew committed
18

19
// TODO(ianh): We have way too many ways to run subprocesses in this project.
20 21 22 23
// Convert most of these into one or more lightweight wrappers around the
// [ProcessManager] API using named parameters for the various options.
// See [here](https://github.com/flutter/flutter/pull/14535#discussion_r167041161)
// for more details.
24

25
abstract class ShutdownHooks {
26
  factory ShutdownHooks() => _DefaultShutdownHooks();
27 28 29

  /// Registers a [ShutdownHook] to be executed before the VM exits.
  void addShutdownHook(
30 31
    ShutdownHook shutdownHook
  );
32

33 34 35
  @visibleForTesting
  List<ShutdownHook> get registeredHooks;

36 37 38 39 40 41 42
  /// Runs all registered shutdown hooks and returns a future that completes when
  /// all such hooks have finished.
  ///
  /// Shutdown hooks will be run in groups by their [ShutdownStage]. All shutdown
  /// hooks within a given stage will be started in parallel and will be
  /// guaranteed to run to completion before shutdown hooks in the next stage are
  /// started.
43 44 45 46
  ///
  /// This class is constructed before the [Logger], so it cannot be direct
  /// injected in the constructor.
  Future<void> runShutdownHooks(Logger logger);
47 48
}

49
class _DefaultShutdownHooks implements ShutdownHooks {
50
  _DefaultShutdownHooks();
51

52 53
  @override
  final List<ShutdownHook> registeredHooks = <ShutdownHook>[];
54 55 56 57 58

  bool _shutdownHooksRunning = false;

  @override
  void addShutdownHook(
59 60
    ShutdownHook shutdownHook
  ) {
61
    assert(!_shutdownHooksRunning);
62
    registeredHooks.add(shutdownHook);
63 64 65
  }

  @override
66 67 68 69
  Future<void> runShutdownHooks(Logger logger) async {
    logger.printTrace(
      'Running ${registeredHooks.length} shutdown hook${registeredHooks.length == 1 ? '' : 's'}',
    );
70 71
    _shutdownHooksRunning = true;
    try {
72
      final List<Future<dynamic>> futures = <Future<dynamic>>[];
73
      for (final ShutdownHook shutdownHook in registeredHooks) {
74 75 76
        final FutureOr<dynamic> result = shutdownHook();
        if (result is Future<dynamic>) {
          futures.add(result);
77 78
        }
      }
79
      await Future.wait<dynamic>(futures);
80 81
    } finally {
      _shutdownHooksRunning = false;
82
    }
83
    logger.printTrace('Shutdown hooks complete');
84
  }
85 86
}

87
class ProcessExit implements Exception {
88
  ProcessExit(this.exitCode, {this.immediate = false});
89

90
  final bool immediate;
91 92
  final int exitCode;

Hixie's avatar
Hixie committed
93
  String get message => 'ProcessExit: $exitCode';
94 95

  @override
96 97
  String toString() => message;
}
98 99

class RunResult {
100
  RunResult(this.processResult, this._command)
101
    : assert(_command.isNotEmpty);
102 103 104

  final ProcessResult processResult;

105 106
  final List<String> _command;

107
  int get exitCode => processResult.exitCode;
108 109
  String get stdout => processResult.stdout as String;
  String get stderr => processResult.stderr as String;
110 111 112

  @override
  String toString() {
113
    final StringBuffer out = StringBuffer();
114 115
    if (stdout.isNotEmpty) {
      out.writeln(stdout);
116
    }
117 118
    if (stderr.isNotEmpty) {
      out.writeln(stderr);
119
    }
120 121
    return out.toString().trimRight();
  }
122

123 124
  /// Throws a [ProcessException] with the given `message`.
  void throwException(String message) {
125
    throw ProcessException(
126 127 128 129 130 131
      _command.first,
      _command.skip(1).toList(),
      message,
      exitCode,
    );
  }
132
}
133 134 135 136

typedef RunResultChecker = bool Function(int);

abstract class ProcessUtils {
137
  factory ProcessUtils({
138 139
    required ProcessManager processManager,
    required Logger logger,
140 141 142 143
  }) => _DefaultProcessUtils(
    processManager: processManager,
    logger: logger,
  );
144 145 146 147 148 149

  /// Spawns a child process to run the command [cmd].
  ///
  /// When [throwOnError] is `true`, if the child process finishes with a non-zero
  /// exit code, a [ProcessException] is thrown.
  ///
150
  /// If [throwOnError] is `true`, and [allowedFailures] is supplied,
151
  /// a [ProcessException] is only thrown on a non-zero exit code if
152
  /// [allowedFailures] returns false when passed the exit code.
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
  ///
  /// When [workingDirectory] is set, it is the working directory of the child
  /// process.
  ///
  /// When [allowReentrantFlutter] is set to `true`, the child process is
  /// permitted to call the Flutter tool. By default it is not.
  ///
  /// When [environment] is supplied, it is used as the environment for the child
  /// process.
  ///
  /// When [timeout] is supplied, [runAsync] will kill the child process and
  /// throw a [ProcessException] when it doesn't finish in time.
  ///
  /// If [timeout] is supplied, the command will be retried [timeoutRetries] times
  /// if it times out.
  Future<RunResult> run(
    List<String> cmd, {
    bool throwOnError = false,
171 172
    RunResultChecker? allowedFailures,
    String? workingDirectory,
173
    bool allowReentrantFlutter = false,
174 175
    Map<String, String>? environment,
    Duration? timeout,
176 177 178 179 180 181 182
    int timeoutRetries = 0,
  });

  /// Run the command and block waiting for its result.
  RunResult runSync(
    List<String> cmd, {
    bool throwOnError = false,
183
    bool verboseExceptions = false,
184
    RunResultChecker? allowedFailures,
185
    bool hideStdout = false,
186 187
    String? workingDirectory,
    Map<String, String>? environment,
188
    bool allowReentrantFlutter = false,
189
    Encoding encoding = systemEncoding,
190 191 192 193 194 195
  });

  /// This runs the command in the background from the specified working
  /// directory. Completes when the process has been started.
  Future<Process> start(
    List<String> cmd, {
196
    String? workingDirectory,
197
    bool allowReentrantFlutter = false,
198
    Map<String, String>? environment,
199
    ProcessStartMode mode = ProcessStartMode.normal,
200 201 202 203 204 205 206 207 208 209
  });

  /// This runs the command and streams stdout/stderr from the child process to
  /// this process' stdout/stderr. Completes with the process's exit code.
  ///
  /// If [filter] is null, no lines are removed.
  ///
  /// If [filter] is non-null, all lines that do not match it are removed. If
  /// [mapFunction] is present, all lines that match [filter] are also forwarded
  /// to [mapFunction] for further processing.
210 211 212
  ///
  /// If [stdoutErrorMatcher] is non-null, matching lines from stdout will be
  /// treated as errors, just as if they had been logged to stderr instead.
213 214
  Future<int> stream(
    List<String> cmd, {
215
    String? workingDirectory,
216 217 218
    bool allowReentrantFlutter = false,
    String prefix = '',
    bool trace = false,
219 220 221 222
    RegExp? filter,
    RegExp? stdoutErrorMatcher,
    StringConverter? mapFunction,
    Map<String, String>? environment,
223 224 225 226
  });

  bool exitsHappySync(
    List<String> cli, {
227
    Map<String, String>? environment,
228 229 230 231
  });

  Future<bool> exitsHappy(
    List<String> cli, {
232
    Map<String, String>? environment,
233 234 235 236
  });
}

class _DefaultProcessUtils implements ProcessUtils {
237
  _DefaultProcessUtils({
238 239
    required ProcessManager processManager,
    required Logger logger,
240 241 242 243 244 245 246
  }) : _processManager = processManager,
      _logger = logger;

  final ProcessManager _processManager;

  final Logger _logger;

247 248 249 250
  @override
  Future<RunResult> run(
    List<String> cmd, {
    bool throwOnError = false,
251 252
    RunResultChecker? allowedFailures,
    String? workingDirectory,
253
    bool allowReentrantFlutter = false,
254 255
    Map<String, String>? environment,
    Duration? timeout,
256 257
    int timeoutRetries = 0,
  }) async {
258
    if (cmd.isEmpty) {
259 260 261 262 263 264 265 266
      throw ArgumentError('cmd must be a non-empty list');
    }
    if (timeoutRetries < 0) {
      throw ArgumentError('timeoutRetries must be non-negative');
    }
    _traceCommand(cmd, workingDirectory: workingDirectory);

    // When there is no timeout, there's no need to kill a running process, so
267
    // we can just use _processManager.run().
268
    if (timeout == null) {
269
      final ProcessResult results = await _processManager.run(
270 271 272 273 274
        cmd,
        workingDirectory: workingDirectory,
        environment: _environment(allowReentrantFlutter, environment),
      );
      final RunResult runResult = RunResult(results, cmd);
275
      _logger.printTrace(runResult.toString());
276
      if (throwOnError && runResult.exitCode != 0 &&
277
          (allowedFailures == null || !allowedFailures(runResult.exitCode))) {
278 279 280 281 282 283
        runResult.throwException('Process exited abnormally:\n$runResult');
      }
      return runResult;
    }

    // When there is a timeout, we have to kill the running process, so we have
284
    // to use _processManager.start() through _runCommand() above.
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
    while (true) {
      assert(timeoutRetries >= 0);
      timeoutRetries = timeoutRetries - 1;

      final Process process = await start(
          cmd,
          workingDirectory: workingDirectory,
          allowReentrantFlutter: allowReentrantFlutter,
          environment: environment,
      );

      final StringBuffer stdoutBuffer = StringBuffer();
      final StringBuffer stderrBuffer = StringBuffer();
      final Future<void> stdoutFuture = process.stdout
          .transform<String>(const Utf8Decoder(reportErrors: false))
          .listen(stdoutBuffer.write)
301
          .asFuture<void>();
302 303 304
      final Future<void> stderrFuture = process.stderr
          .transform<String>(const Utf8Decoder(reportErrors: false))
          .listen(stderrBuffer.write)
305
          .asFuture<void>();
306

307 308
      int? exitCode;
      exitCode = await process.exitCode.then<int?>((int x) => x).timeout(timeout, onTimeout: () {
309
        // The process timed out. Kill it.
310
        _processManager.killPid(process.pid);
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
        return null;
      });

      String stdoutString;
      String stderrString;
      try {
        Future<void> stdioFuture =
            Future.wait<void>(<Future<void>>[stdoutFuture, stderrFuture]);
        if (exitCode == null) {
          // If we had to kill the process for a timeout, only wait a short time
          // for the stdio streams to drain in case killing the process didn't
          // work.
          stdioFuture = stdioFuture.timeout(const Duration(seconds: 1));
        }
        await stdioFuture;
326
      } on Exception {
327 328 329 330 331 332 333 334 335 336 337 338
        // Ignore errors on the process' stdout and stderr streams. Just capture
        // whatever we got, and use the exit code
      }
      stdoutString = stdoutBuffer.toString();
      stderrString = stderrBuffer.toString();

      final ProcessResult result = ProcessResult(
          process.pid, exitCode ?? -1, stdoutString, stderrString);
      final RunResult runResult = RunResult(result, cmd);

      // If the process did not timeout. We are done.
      if (exitCode != null) {
339
        _logger.printTrace(runResult.toString());
340
        if (throwOnError && runResult.exitCode != 0 &&
341
            (allowedFailures == null || !allowedFailures(exitCode))) {
342 343 344 345 346 347 348 349 350 351 352
          runResult.throwException('Process exited abnormally:\n$runResult');
        }
        return runResult;
      }

      // If we are out of timeoutRetries, throw a ProcessException.
      if (timeoutRetries < 0) {
        runResult.throwException('Process timed out:\n$runResult');
      }

      // Log the timeout with a trace message in verbose mode.
353 354 355 356
      _logger.printTrace(
        'Process "${cmd[0]}" timed out. $timeoutRetries attempts left:\n'
        '$runResult',
      );
357 358 359 360 361 362 363 364 365
    }

    // Unreachable.
  }

  @override
  RunResult runSync(
    List<String> cmd, {
    bool throwOnError = false,
366
    bool verboseExceptions = false,
367
    RunResultChecker? allowedFailures,
368
    bool hideStdout = false,
369 370
    String? workingDirectory,
    Map<String, String>? environment,
371
    bool allowReentrantFlutter = false,
372
    Encoding encoding = systemEncoding,
373 374
  }) {
    _traceCommand(cmd, workingDirectory: workingDirectory);
375
    final ProcessResult results = _processManager.runSync(
376 377 378
      cmd,
      workingDirectory: workingDirectory,
      environment: _environment(allowReentrantFlutter, environment),
379 380
      stderrEncoding: encoding,
      stdoutEncoding: encoding,
381 382 383
    );
    final RunResult runResult = RunResult(results, cmd);

384
    _logger.printTrace('Exit code ${runResult.exitCode} from: ${cmd.join(' ')}');
385 386

    bool failedExitCode = runResult.exitCode != 0;
387 388
    if (allowedFailures != null && failedExitCode) {
      failedExitCode = !allowedFailures(runResult.exitCode);
389 390 391 392
    }

    if (runResult.stdout.isNotEmpty && !hideStdout) {
      if (failedExitCode && throwOnError) {
393
        _logger.printStatus(runResult.stdout.trim());
394
      } else {
395
        _logger.printTrace(runResult.stdout.trim());
396 397 398 399 400
      }
    }

    if (runResult.stderr.isNotEmpty) {
      if (failedExitCode && throwOnError) {
401
        _logger.printError(runResult.stderr.trim());
402
      } else {
403
        _logger.printTrace(runResult.stderr.trim());
404 405 406 407
      }
    }

    if (failedExitCode && throwOnError) {
408 409 410 411 412 413
      String message = 'The command failed';
      if (verboseExceptions) {
        message = 'The command failed\nStdout:\n${runResult.stdout}\n'
            'Stderr:\n${runResult.stderr}';
      }
      runResult.throwException(message);
414 415 416 417 418 419 420 421
    }

    return runResult;
  }

  @override
  Future<Process> start(
    List<String> cmd, {
422
    String? workingDirectory,
423
    bool allowReentrantFlutter = false,
424
    Map<String, String>? environment,
425
    ProcessStartMode mode = ProcessStartMode.normal,
426 427
  }) {
    _traceCommand(cmd, workingDirectory: workingDirectory);
428
    return _processManager.start(
429 430 431
      cmd,
      workingDirectory: workingDirectory,
      environment: _environment(allowReentrantFlutter, environment),
432
      mode: mode,
433 434 435 436 437 438
    );
  }

  @override
  Future<int> stream(
    List<String> cmd, {
439
    String? workingDirectory,
440 441 442
    bool allowReentrantFlutter = false,
    String prefix = '',
    bool trace = false,
443 444 445 446
    RegExp? filter,
    RegExp? stdoutErrorMatcher,
    StringConverter? mapFunction,
    Map<String, String>? environment,
447 448 449 450 451 452 453 454 455 456 457 458
  }) async {
    final Process process = await start(
      cmd,
      workingDirectory: workingDirectory,
      allowReentrantFlutter: allowReentrantFlutter,
      environment: environment,
    );
    final StreamSubscription<String> stdoutSubscription = process.stdout
      .transform<String>(utf8.decoder)
      .transform<String>(const LineSplitter())
      .where((String line) => filter == null || filter.hasMatch(line))
      .listen((String line) {
459
        String? mappedLine = line;
460
        if (mapFunction != null) {
461
          mappedLine = mapFunction(line);
462
        }
463 464
        if (mappedLine != null) {
          final String message = '$prefix$mappedLine';
465
          if (stdoutErrorMatcher?.hasMatch(mappedLine) ?? false) {
466 467
            _logger.printError(message, wrap: false);
          } else if (trace) {
468
            _logger.printTrace(message);
469
          } else {
470
            _logger.printStatus(message, wrap: false);
471
          }
472 473 474 475 476 477 478
        }
      });
    final StreamSubscription<String> stderrSubscription = process.stderr
      .transform<String>(utf8.decoder)
      .transform<String>(const LineSplitter())
      .where((String line) => filter == null || filter.hasMatch(line))
      .listen((String line) {
479
        String? mappedLine = line;
480
        if (mapFunction != null) {
481
          mappedLine = mapFunction(line);
482
        }
483 484
        if (mappedLine != null) {
          _logger.printError('$prefix$mappedLine', wrap: false);
485
        }
486 487 488 489
      });

    // Wait for stdout to be fully processed
    // because process.exitCode may complete first causing flaky tests.
490
    await Future.wait<void>(<Future<void>>[
491 492 493 494
      stdoutSubscription.asFuture<void>(),
      stderrSubscription.asFuture<void>(),
    ]);

495 496 497 498 499 500 501
    // The streams as futures have already completed, so waiting for the
    // potentially async stream cancellation to complete likely has no benefit.
    // Further, some Stream implementations commonly used in tests don't
    // complete the Future returned here, which causes tests using
    // mocks/FakeAsync to fail when these Futures are awaited.
    unawaited(stdoutSubscription.cancel());
    unawaited(stderrSubscription.cancel());
502

503
    return process.exitCode;
504 505 506 507 508
  }

  @override
  bool exitsHappySync(
    List<String> cli, {
509
    Map<String, String>? environment,
510 511
  }) {
    _traceCommand(cli);
512 513 514 515 516
    if (!_processManager.canRun(cli.first)) {
      _logger.printTrace('$cli either does not exist or is not executable.');
      return false;
    }

517
    try {
518
      return _processManager.runSync(cli, environment: environment).exitCode == 0;
519
    } on Exception catch (error) {
520
      _logger.printTrace('$cli failed with $error');
521 522 523 524 525 526 527
      return false;
    }
  }

  @override
  Future<bool> exitsHappy(
    List<String> cli, {
528
    Map<String, String>? environment,
529 530
  }) async {
    _traceCommand(cli);
531 532 533 534 535
    if (!_processManager.canRun(cli.first)) {
      _logger.printTrace('$cli either does not exist or is not executable.');
      return false;
    }

536
    try {
537
      return (await _processManager.run(cli, environment: environment)).exitCode == 0;
538
    } on Exception catch (error) {
539
      _logger.printTrace('$cli failed with $error');
540 541 542 543
      return false;
    }
  }

544 545
  Map<String, String>? _environment(bool allowReentrantFlutter, [
    Map<String, String>? environment,
546 547
  ]) {
    if (allowReentrantFlutter) {
548
      if (environment == null) {
549
        environment = <String, String>{'FLUTTER_ALREADY_LOCKED': 'true'};
550
      } else {
551
        environment['FLUTTER_ALREADY_LOCKED'] = 'true';
552
      }
553 554 555 556 557
    }

    return environment;
  }

558
  void _traceCommand(List<String> args, { String? workingDirectory }) {
559 560
    final String argsText = args.join(' ');
    if (workingDirectory == null) {
561
      _logger.printTrace('executing: $argsText');
562
    } else {
563
      _logger.printTrace('executing: [$workingDirectory/] $argsText');
564 565 566
    }
  }
}