pub.dart 19.8 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'package:package_config/package_config.dart';
6
import 'package:process/process.dart';
7

8
import '../base/bot_detector.dart';
9
import '../base/common.dart';
10
import '../base/context.dart';
11
import '../base/file_system.dart';
12
import '../base/io.dart' as io;
13
import '../base/logger.dart';
14
import '../base/platform.dart';
15
import '../base/process.dart';
16
import '../cache.dart';
17
import '../convert.dart';
18
import '../dart/package_map.dart';
19
import '../reporting/reporting.dart';
20

21
/// The [Pub] instance.
22
Pub get pub => context.get<Pub>()!;
23

24 25 26 27 28 29
/// The console environment key used by the pub tool.
const String _kPubEnvironmentKey = 'PUB_ENVIRONMENT';

/// The console environment key used by the pub tool to find the cache directory.
const String _kPubCacheEnvironmentKey = 'PUB_CACHE';

30 31 32 33
/// The UNAVAILABLE exit code returned by the pub tool.
/// (see https://github.com/dart-lang/pub/blob/master/lib/src/exit_codes.dart)
const int _kPubExitCodeUnavailable = 69;

34 35
typedef MessageFilter = String Function(String message);

36 37 38 39 40 41
/// Represents Flutter-specific data that is added to the `PUB_ENVIRONMENT`
/// environment variable and allows understanding the type of requests made to
/// the package site on Flutter's behalf.
// DO NOT update without contacting kevmoo.
// We have server-side tooling that assumes the values are consistent.
class PubContext {
42
  PubContext._(this._values) {
43
    for (final String item in _values) {
44 45
      if (!_validContext.hasMatch(item)) {
        throw ArgumentError.value(
46
          _values, 'value', 'Must match RegExp ${_validContext.pattern}');
47 48 49 50 51 52
      }
    }
  }

  static PubContext getVerifyContext(String commandName) =>
      PubContext._(<String>['verify', commandName.replaceAll('-', '_')]);
53

54 55 56 57 58 59
  static final PubContext create = PubContext._(<String>['create']);
  static final PubContext createPackage = PubContext._(<String>['create_pkg']);
  static final PubContext createPlugin = PubContext._(<String>['create_plugin']);
  static final PubContext interactive = PubContext._(<String>['interactive']);
  static final PubContext pubGet = PubContext._(<String>['get']);
  static final PubContext pubUpgrade = PubContext._(<String>['upgrade']);
60
  static final PubContext pubForward = PubContext._(<String>['forward']);
61 62 63
  static final PubContext runTest = PubContext._(<String>['run_test']);
  static final PubContext flutterTests = PubContext._(<String>['flutter_tests']);
  static final PubContext updatePackages = PubContext._(<String>['update_packages']);
64 65 66

  final List<String> _values;

67
  static final RegExp _validContext = RegExp('[a-z][a-z_]*[a-z]');
68 69 70

  @override
  String toString() => 'PubContext: ${_values.join(':')}';
71 72 73 74

  String toAnalyticsString()  {
    return _values.map((String s) => s.replaceAll('_', '-')).toList().join('-');
  }
75 76
}

77 78 79
/// A handle for interacting with the pub tool.
abstract class Pub {
  /// Create a default [Pub] instance.
80
  factory Pub({
81 82 83 84 85 86
    required FileSystem fileSystem,
    required Logger logger,
    required ProcessManager processManager,
    required Platform platform,
    required BotDetector botDetector,
    required Usage usage,
87
  }) = _DefaultPub;
88 89 90 91 92

  /// Runs `pub get`.
  ///
  /// [context] provides extra information to package server requests to
  /// understand usage.
93 94 95 96
  ///
  /// If [shouldSkipThirdPartyGenerator] is true, the overall pub get will be
  /// skipped if the package config file has a "generator" other than "pub".
  /// Defaults to true.
97
  Future<void> get({
98
    required PubContext context,
99 100 101 102
    String directory,
    bool skipIfAbsent = false,
    bool upgrade = false,
    bool offline = false,
103
    bool generateSyntheticPackage = false,
104
    String flutterRootOverride,
105
    bool checkUpToDate = false,
106
    bool shouldSkipThirdPartyGenerator = true,
107
    bool printProgress = true,
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
  });

  /// Runs pub in 'batch' mode.
  ///
  /// forwarding complete lines written by pub to its stdout/stderr streams to
  /// the corresponding stream of this process, optionally applying filtering.
  /// The pub process will not receive anything on its stdin stream.
  ///
  /// The `--trace` argument is passed to `pub` (by mutating the provided
  /// `arguments` list) when `showTraceForErrors` is true, and when `showTraceForErrors`
  /// is null/unset, and `isRunningOnBot` is true.
  ///
  /// [context] provides extra information to package server requests to
  /// understand usage.
  Future<void> batch(
    List<String> arguments, {
124
    required PubContext context,
125 126 127
    String directory,
    MessageFilter filter,
    String failureMessage = 'pub failed',
128
    required bool retry,
129 130
    bool showTraceForErrors,
  });
131 132 133 134 135 136 137

  /// Runs pub in 'interactive' mode.
  ///
  /// directly piping the stdin stream of this process to that of pub, and the
  /// stdout/stderr stream of pub to the corresponding streams of this process.
  Future<void> interactively(
    List<String> arguments, {
138
    String directory,
139
    required io.Stdio stdio,
140 141
    bool touchesPackageConfig = false,
    bool generateSyntheticPackage = false,
142
  });
143 144 145
}

class _DefaultPub implements Pub {
146
  _DefaultPub({
147 148 149 150 151 152
    required FileSystem fileSystem,
    required Logger logger,
    required ProcessManager processManager,
    required Platform platform,
    required BotDetector botDetector,
    required Usage usage,
153
  }) : _fileSystem = fileSystem,
154 155 156 157 158 159 160
       _logger = logger,
       _platform = platform,
       _botDetector = botDetector,
       _usage = usage,
       _processUtils = ProcessUtils(
         logger: logger,
         processManager: processManager,
161 162
       ),
       _processManager = processManager;
163 164 165 166 167 168 169

  final FileSystem _fileSystem;
  final Logger _logger;
  final ProcessUtils _processUtils;
  final Platform _platform;
  final BotDetector _botDetector;
  final Usage _usage;
170
  final ProcessManager _processManager;
171 172 173

  @override
  Future<void> get({
174 175
    required PubContext context,
    String? directory,
176 177 178
    bool skipIfAbsent = false,
    bool upgrade = false,
    bool offline = false,
179
    bool generateSyntheticPackage = false,
180
    String? flutterRootOverride,
181
    bool checkUpToDate = false,
182
    bool shouldSkipThirdPartyGenerator = true,
183
    bool printProgress = true,
184
  }) async {
185 186 187
    directory ??= _fileSystem.currentDirectory.path;
    final File packageConfigFile = _fileSystem.file(
      _fileSystem.path.join(directory, '.dart_tool', 'package_config.json'));
188 189
    final Directory generatedDirectory = _fileSystem.directory(
      _fileSystem.path.join(directory, '.dart_tool', 'flutter_gen'));
190 191 192
    final File lastVersion = _fileSystem.file(
      _fileSystem.path.join(directory, '.dart_tool', 'version'));
    final File currentVersion = _fileSystem.file(
193
      _fileSystem.path.join(Cache.flutterRoot!, 'version'));
194 195 196 197 198 199
    final File pubspecYaml = _fileSystem.file(
      _fileSystem.path.join(directory, 'pubspec.yaml'));
    final File pubLockFile = _fileSystem.file(
      _fileSystem.path.join(directory, 'pubspec.lock')
    );

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
    if (shouldSkipThirdPartyGenerator && packageConfigFile.existsSync()) {
      Map<String, Object?> packageConfigMap;
      try {
        packageConfigMap = jsonDecode(
          packageConfigFile.readAsStringSync(),
        ) as Map<String, Object?>;
      } on FormatException {
        packageConfigMap = <String, Object?>{};
      }

      final bool isPackageConfigGeneratedByThirdParty =
          packageConfigMap.containsKey('generator') &&
          packageConfigMap['generator'] != 'pub';

      if (isPackageConfigGeneratedByThirdParty) {
        _logger.printTrace('Skipping pub get: generated by third-party.');
        return;
      }
    }

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
    // If the pubspec.yaml is older than the package config file and the last
    // flutter version used is the same as the current version skip pub get.
    // This will incorrectly skip pub on the master branch if dependencies
    // are being added/removed from the flutter framework packages, but this
    // can be worked around by manually running pub.
    if (checkUpToDate &&
        packageConfigFile.existsSync() &&
        pubLockFile.existsSync() &&
        pubspecYaml.lastModifiedSync().isBefore(pubLockFile.lastModifiedSync()) &&
        pubspecYaml.lastModifiedSync().isBefore(packageConfigFile.lastModifiedSync()) &&
        lastVersion.existsSync() &&
        lastVersion.readAsStringSync() == currentVersion.readAsStringSync()) {
      _logger.printTrace('Skipping pub get: version match.');
      return;
    }
235

236
    final String command = upgrade ? 'upgrade' : 'get';
237
    final Status? status = printProgress ? _logger.startProgress(
238
      'Running "flutter pub $command" in ${_fileSystem.path.basename(directory)}...',
239
    ) : null;
240 241 242
    final bool verbose = _logger.isVerbose;
    final List<String> args = <String>[
      if (verbose)
243 244 245
        '--verbose'
      else
        '--verbosity=warning',
246 247 248 249 250 251 252 253 254 255 256 257 258
      ...<String>[
        command,
        '--no-precompile',
      ],
      if (offline)
        '--offline',
    ];
    try {
      await batch(
        args,
        context: context,
        directory: directory,
        failureMessage: 'pub $command failed',
259
        retry: !offline,
260
        flutterRootOverride: flutterRootOverride,
261
      );
262
      status?.stop();
263 264
    // The exception is rethrown, so don't catch only Exceptions.
    } catch (exception) { // ignore: avoid_catches_without_on_clauses
265
      status?.cancel();
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
      if (exception is io.ProcessException) {
        final StringBuffer buffer = StringBuffer(exception.message);
        buffer.writeln('Working directory: "$directory"');
        final Map<String, String> env = await _createPubEnvironment(context, flutterRootOverride);
        if (env.entries.isNotEmpty) {
          buffer.writeln('pub env: {');
          for (final MapEntry<String, String> entry in env.entries) {
            buffer.writeln('  "${entry.key}": "${entry.value}",');
          }
          buffer.writeln('}');
        }

        throw io.ProcessException(
          exception.executable,
          exception.arguments,
          buffer.toString(),
          exception.errorCode,
        );
      }
285
      rethrow;
286
    }
287

288
    if (!packageConfigFile.existsSync()) {
289
      throwToolExit('$directory: pub did not create .dart_tools/package_config.json file.');
290
    }
291
    lastVersion.writeAsStringSync(currentVersion.readAsStringSync());
292 293 294 295 296
    await _updatePackageConfig(
      packageConfigFile,
      generatedDirectory,
      generateSyntheticPackage,
    );
297
  }
298

299 300 301
  @override
  Future<void> batch(
    List<String> arguments, {
302 303 304
    required PubContext context,
    String? directory,
    MessageFilter? filter,
305
    String failureMessage = 'pub failed',
306 307 308
    required bool retry,
    bool? showTraceForErrors,
    String? flutterRootOverride,
309
  }) async {
310
    showTraceForErrors ??= await _botDetector.isRunningOnBot;
311

312
    String lastPubMessage = 'no message';
313 314
    bool versionSolvingFailed = false;
    String filterWrapper(String line) {
315
      lastPubMessage = line;
316 317 318 319 320 321 322
      if (line.contains('version solving failed')) {
        versionSolvingFailed = true;
      }
      if (filter == null) {
        return line;
      }
      return filter(line);
323
    }
324 325 326

    if (showTraceForErrors) {
      arguments.insert(0, '--trace');
327
    }
328 329 330
    int attempts = 0;
    int duration = 1;
    int code;
331
    while (true) {
332
      attempts += 1;
333
      code = await _processUtils.stream(
334 335
        _pubCommand(arguments),
        workingDirectory: directory,
336
        mapFunction: filterWrapper, // may set versionSolvingFailed, lastPubMessage
337
        environment: await _createPubEnvironment(context, flutterRootOverride),
338
      );
339 340 341
      String? message;
      if (retry) {
        if (code == _kPubExitCodeUnavailable) {
342
          message = 'server unavailable';
343 344 345 346
        }
      }
      if (message == null) {
        break;
347 348
      }
      versionSolvingFailed = false;
349 350 351 352
      _logger.printStatus(
        '$failureMessage ($message) -- attempting retry $attempts in $duration '
        'second${ duration == 1 ? "" : "s"}...',
      );
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
      await Future<void>.delayed(Duration(seconds: duration));
      if (duration < 64) {
        duration *= 2;
      }
    }
    assert(code != null);

    String result = 'success';
    if (versionSolvingFailed) {
      result = 'version-solving-failed';
    } else if (code != 0) {
      result = 'failure';
    }
    PubResultEvent(
      context: context.toAnalyticsString(),
      result: result,
369
      usage: _usage,
370
    ).send();
371

372
    if (code != 0) {
373
      throwToolExit('$failureMessage ($code; $lastPubMessage)', exitCode: code);
374
    }
375
  }
376 377 378 379

  @override
  Future<void> interactively(
    List<String> arguments, {
380 381
    String? directory,
    required io.Stdio stdio,
382 383
    bool touchesPackageConfig = false,
    bool generateSyntheticPackage = false,
384
  }) async {
385
    // Fully resolved pub or pub.bat is calculated based on current platform.
386 387 388 389 390
    final io.Process process = await _processUtils.start(
      _pubCommand(arguments),
      workingDirectory: directory,
      environment: await _createPubEnvironment(PubContext.interactive),
    );
391

392
    // Pipe the Flutter tool stdin to the pub stdin.
393
    unawaited(process.stdin.addStream(stdio.stdin)
394 395 396
      // If pub exits unexpectedly with an error, that will be reported below
      // by the tool exit after the exit code check.
      .catchError((dynamic err, StackTrace stack) {
397 398
        _logger.printTrace('Echoing stdin to the pub subprocess failed:');
        _logger.printTrace('$err\n$stack');
399 400 401 402 403 404
      }
    ));

    // Pipe the pub stdout and stderr to the tool stdout and stderr.
    try {
      await Future.wait<dynamic>(<Future<dynamic>>[
405 406
        stdio.addStdoutStream(process.stdout),
        stdio.addStderrStream(process.stderr),
407
      ]);
408
    } on Exception catch (err, stack) {
409 410
      _logger.printTrace('Echoing stdout or stderr from the pub subprocess failed:');
      _logger.printTrace('$err\n$stack');
411
    }
412 413 414 415 416 417

    // Wait for pub to exit.
    final int code = await process.exitCode;
    if (code != 0) {
      throwToolExit('pub finished with exit code $code', exitCode: code);
    }
418 419

    if (touchesPackageConfig) {
420
      final String targetDirectory = directory ?? _fileSystem.currentDirectory.path;
421
      final File packageConfigFile = _fileSystem.file(
422
        _fileSystem.path.join(targetDirectory, '.dart_tool', 'package_config.json'));
423
      final Directory generatedDirectory = _fileSystem.directory(
424
        _fileSystem.path.join(targetDirectory, '.dart_tool', 'flutter_gen'));
425
      final File lastVersion = _fileSystem.file(
426
        _fileSystem.path.join(targetDirectory, '.dart_tool', 'version'));
427
      final File currentVersion = _fileSystem.file(
428
        _fileSystem.path.join(Cache.flutterRoot!, 'version'));
429 430 431 432 433 434 435
        lastVersion.writeAsStringSync(currentVersion.readAsStringSync());
      await _updatePackageConfig(
        packageConfigFile,
        generatedDirectory,
        generateSyntheticPackage,
      );
    }
436
  }
437

438 439
  /// The command used for running pub.
  List<String> _pubCommand(List<String> arguments) {
440
    // TODO(zanderso): refactor to use artifacts.
441
    final String sdkPath = _fileSystem.path.joinAll(<String>[
442
      Cache.flutterRoot!,
443 444 445 446
      'bin',
      'cache',
      'dart-sdk',
      'bin',
447
      'dart',
448
    ]);
449 450 451 452 453 454 455
    if (!_processManager.canRun(sdkPath)) {
      throwToolExit(
        'Your Flutter SDK download may be corrupt or missing permissions to run. '
        'Try re-downloading the Flutter SDK into a directory that has read/write '
        'permissions for the current user.'
      );
    }
456
    return <String>[sdkPath, '__deprecated_pub', ...arguments];
457
  }
458

459 460 461 462 463 464 465 466 467
  // Returns the environment value that should be used when running pub.
  //
  // Includes any existing environment variable, if one exists.
  //
  // [context] provides extra information to package server requests to
  // understand usage.
  Future<String> _getPubEnvironmentValue(PubContext pubContext) async {
    // DO NOT update this function without contacting kevmoo.
    // We have server-side tooling that assumes the values are consistent.
468
    final String? existing = _platform.environment[_kPubEnvironmentKey];
469 470 471 472 473 474 475
    final List<String> values = <String>[
      if (existing != null && existing.isNotEmpty) existing,
      if (await _botDetector.isRunningOnBot) 'flutter_bot',
      'flutter_cli',
      ...pubContext._values,
    ];
    return values.join(':');
476 477
  }

478
  String? _getRootPubCacheIfAvailable() {
479 480 481
    if (_platform.environment.containsKey(_kPubCacheEnvironmentKey)) {
      return _platform.environment[_kPubCacheEnvironmentKey];
    }
482

483
    final String cachePath = _fileSystem.path.join(Cache.flutterRoot!, '.pub-cache');
484 485 486 487
    if (_fileSystem.directory(cachePath).existsSync()) {
      _logger.printTrace('Using $cachePath for the pub cache.');
      return cachePath;
    }
488

489
    // Use pub's default location by returning null.
490
    return null;
491
  }
492 493 494 495 496

  /// The full environment used when running pub.
  ///
  /// [context] provides extra information to package server requests to
  /// understand usage.
497
  Future<Map<String, String>> _createPubEnvironment(PubContext context, [ String? flutterRootOverride ]) async {
498
    final Map<String, String> environment = <String, String>{
499
      'FLUTTER_ROOT': flutterRootOverride ?? Cache.flutterRoot!,
500 501
      _kPubEnvironmentKey: await _getPubEnvironmentValue(context),
    };
502
    final String? pubCache = _getRootPubCacheIfAvailable();
503 504 505 506
    if (pubCache != null) {
      environment[_kPubCacheEnvironmentKey] = pubCache;
    }
    return environment;
507
  }
508

509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535
  /// Update the package configuration file.
  ///
  /// Creates a corresponding `package_config_subset` file that is used by the build
  /// system to avoid rebuilds caused by an updated pub timestamp.
  ///
  /// if [generateSyntheticPackage] is true then insert flutter_gen synthetic
  /// package into the package configuration. This is used by the l10n localization
  /// tooling to insert a new reference into the package_config file, allowing the import
  /// of a package URI that is not specified in the pubspec.yaml
  ///
  /// For more information, see:
  ///   * [generateLocalizations], `in lib/src/localizations/gen_l10n.dart`
  Future<void> _updatePackageConfig(
    File packageConfigFile,
    Directory generatedDirectory,
    bool generateSyntheticPackage,
  ) async {
    final PackageConfig packageConfig = await loadPackageConfigWithLogging(packageConfigFile, logger: _logger);

    packageConfigFile.parent
      .childFile('package_config_subset')
      .writeAsStringSync(_computePackageConfigSubset(
        packageConfig,
        _fileSystem,
      ));

    if (!generateSyntheticPackage) {
536 537 538 539 540
      return;
    }
    if (packageConfig.packages.any((Package package) => package.name == 'flutter_gen')) {
      return;
    }
541 542 543 544 545 546 547 548 549

    // TODO(jonahwillams): Using raw json manipulation here because
    // savePackageConfig always writes to local io, and it changes absolute
    // paths to relative on round trip.
    // See: https://github.com/dart-lang/package_config/issues/99,
    // and: https://github.com/dart-lang/package_config/issues/100.

    // Because [loadPackageConfigWithLogging] succeeded [packageConfigFile]
    // we can rely on the file to exist and be correctly formatted.
550 551
    final Map<String, dynamic> jsonContents =
        json.decode(packageConfigFile.readAsStringSync()) as Map<String, dynamic>;
552

553
    (jsonContents['packages'] as List<dynamic>).add(<String, dynamic>{
554 555 556 557 558 559
      'name': 'flutter_gen',
      'rootUri': 'flutter_gen',
      'languageVersion': '2.12',
    });

    packageConfigFile.writeAsStringSync(json.encode(jsonContents));
560
  }
561 562 563 564 565 566 567 568 569 570 571 572 573 574

  // Subset the package config file to only the parts that are relevant for
  // rerunning the dart compiler.
  String _computePackageConfigSubset(PackageConfig packageConfig, FileSystem fileSystem) {
    final StringBuffer buffer = StringBuffer();
    for (final Package package in packageConfig.packages) {
      buffer.writeln(package.name);
      buffer.writeln(package.languageVersion);
      buffer.writeln(package.root);
      buffer.writeln(package.packageUriRoot);
    }
    buffer.writeln(packageConfig.version);
    return buffer.toString();
  }
575
}