update_packages.dart 60.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
// @dart = 2.8

7
import 'dart:collection';
8

9 10
import 'package:meta/meta.dart';

11
import '../base/common.dart';
12
import '../base/context.dart';
13
import '../base/file_system.dart';
14 15 16
import '../base/logger.dart';
import '../base/net.dart';
import '../cache.dart';
17
import '../dart/pub.dart';
18
import '../globals_null_migrated.dart' as globals;
19 20
import '../runner/flutter_command.dart';

21 22
/// Map from package name to package version, used to artificially pin a pub
/// package version in cases when upgrading to the latest breaks Flutter.
23
const Map<String, String> _kManuallyPinnedDependencies = <String, String>{
Ian Hickson's avatar
Ian Hickson committed
24 25
  // Add pinned packages here. Please leave a comment explaining why.
  // PACKAGES WITH INCOMPATIBLE LATER VERSIONS
26 27 28
  // Dart analyzer does not catch renamed or deleted files.
  // Therefore, we control the version of flutter_gallery_assets so that
  // existing tests do not fail when the package has a new version.
29
  'flutter_gallery_assets': '^1.0.1',
30
  'flutter_template_images': '4.0.0', // Must always exactly match flutter_tools template.
Ian Hickson's avatar
Ian Hickson committed
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
  // DART TEAM OWNED NNBD DEPS
  'archive': '">=3.0.0-nullsafety.0"',
  'async': '">=2.5.0-nullsafety.3"',
  'boolean_selector': '">=2.1.0-nullsafety.3"',
  'characters': '">=1.1.0-nullsafety.5"',
  'charcode': '">=1.2.0-nullsafety.3"',
  'clock': '">=1.1.0-nullsafety.3"',
  'collection': '">=1.15.0-nullsafety.5"',
  'fake_async': '">=1.2.0-nullsafety.3"',
  'intl': '">=0.17.0-nullsafety.2"',
  'js': '">=0.6.3-nullsafety.3"',
  'matcher': '">=0.12.10-nullsafety.3"',
  'meta': '">=1.3.0-nullsafety.6"',
  'path': '">=1.8.0-nullsafety.3"',
  'pedantic': '">=1.10.0-nullsafety.3"',
  'petitparser': '">=4.0.0-nullsafety.1"',
  'pool': '">=1.5.0-nullsafety.3"',
  'source_map_stack_trace': '">=2.1.0-nullsafety.4"',
  'source_maps': '">=0.10.10-nullsafety.3"',
  'source_span': '">=1.8.0-nullsafety.4"',
  'stack_trace': '">=1.10.0-nullsafety.6"',
  'stream_channel': '">=2.1.0-nullsafety.3"',
  'string_scanner': '">=1.1.0-nullsafety.3"',
  'term_glyph': '">=1.2.0-nullsafety.3"',
  'test': '">=1.16.0-nullsafety.16"',
  'test_api': '">=0.2.19-nullsafety.6"',
  'test_core': '">=0.3.12-nullsafety.15"',
  'typed_data': '">=1.3.0-nullsafety.5"',
  'vector_math': '">=2.1.0-nullsafety.5"',
60
  'vm_service': '">=6.0.1-nullsafety.1"',
Ian Hickson's avatar
Ian Hickson committed
61 62 63 64 65
  'xml': '">=5.0.0-nullsafety.1"',
  // FLUTTER TEAM OWNED NNBD DEPS
  'connectivity': '">=3.0.0-nullsafety.1"',
  'device_info': '">=2.0.0-nullsafety.1"',
  'file': '">=6.0.0-nullsafety.4"',
66
  'path_provider': '">=2.0.0-nullsafety.1"',
Ian Hickson's avatar
Ian Hickson committed
67 68 69 70
  'platform': '">=3.0.0-nullsafety.4"',
  'process': '">=4.0.0-nullsafety.4"',
  'process_runner': '">=4.0.0-nullsafety.5"',
  'url_launcher': '">=6.0.0-nullsafety.1"',
71 72 73 74
  // This is pinned to avoid the performance regression from a reverted feature
  // from https://github.com/dart-lang/shelf/issues/189 . This can be removed
  // when a new major version of shelf is published.
  'shelf': '1.1.4',
75 76
  // Latest version does not resolve on our CI.
  'video_player': '2.1.1',
77 78
};

79
class UpdatePackagesCommand extends FlutterCommand {
80
  UpdatePackagesCommand() {
81 82 83 84 85 86
    argParser
      ..addFlag(
        'force-upgrade',
        help: 'Attempt to update all the dependencies to their latest versions.\n'
              'This will actually modify the pubspec.yaml files in your checkout.',
        defaultsTo: false,
87
        negatable: false,
88 89 90 91
      )
      ..addFlag(
        'paths',
        help: 'Finds paths in the dependency chain leading from package specified '
92
              'in "--from" to package specified in "--to".',
93
        defaultsTo: false,
94
        negatable: false,
95 96 97
      )
      ..addOption(
        'from',
98
        help: 'Used with "--dependency-path". Specifies the package to begin '
99 100 101 102
              'searching dependency path from.',
      )
      ..addOption(
        'to',
103 104
        help: 'Used with "--dependency-path". Specifies the package that the '
              'sought-after dependency path leads to.',
105 106 107 108 109 110
      )
      ..addFlag(
        'transitive-closure',
        help: 'Prints the dependency graph that is the transitive closure of '
              'packages the Flutter SDK depends on.',
        defaultsTo: false,
111
        negatable: false,
112
      )
113 114
      ..addFlag(
        'consumer-only',
115
        help: 'Only prints the dependency graph that is the transitive closure '
116 117
              'that a consumer of the Flutter SDK will observe (when combined '
              'with transitive-closure).',
118 119 120
        defaultsTo: false,
        negatable: false,
      )
121 122
      ..addFlag(
        'verify-only',
123
        help: 'Verifies the package checksum without changing or updating deps.',
124
        defaultsTo: false,
125
        negatable: false,
126 127 128
      )
      ..addFlag(
        'offline',
129
        help: 'Use cached packages instead of accessing the network.',
130 131
        defaultsTo: false,
        negatable: false,
132 133 134 135 136 137
      )
      ..addFlag(
        'crash',
        help: 'For Flutter CLI testing only, forces this command to throw an unhandled exception.',
        defaultsTo: false,
        negatable: false,
138
      );
139 140
  }

141
  @override
142
  final String name = 'update-packages';
143 144

  @override
145 146
  final String description = 'Update the packages inside the Flutter repo.';

147 148 149
  @override
  final List<String> aliases = <String>['upgrade-packages'];

150
  @override
151
  final bool hidden = true;
152

153 154 155 156

  // Lazy-initialize the net utilities with values from the context.
  Net _cachedNet;
  Net get _net => _cachedNet ??= Net(
157
    httpClientFactory: context.get<HttpClientFactory>(),
158 159 160 161
    logger: globals.logger,
    platform: globals.platform,
  );

162
  Future<void> _downloadCoverageData() async {
163
    final Status status = globals.logger.startProgress(
164 165
      'Downloading lcov data for package:flutter...',
    );
166
    final String urlBase = globals.platform.environment['FLUTTER_STORAGE_BASE_URL'] ?? 'https://storage.googleapis.com';
167
    final Uri coverageUri = Uri.parse('$urlBase/flutter_infra_release/flutter/coverage/lcov.info');
168 169 170 171 172
    final List<int> data = await _net.fetchUrl(coverageUri);
    final String coverageDir = globals.fs.path.join(
      Cache.flutterRoot,
      'packages/flutter/coverage',
    );
173
    globals.fs.file(globals.fs.path.join(coverageDir, 'lcov.base.info'))
174 175
      ..createSync(recursive: true)
      ..writeAsBytesSync(data, flush: true);
176
    globals.fs.file(globals.fs.path.join(coverageDir, 'lcov.info'))
177 178
      ..createSync(recursive: true)
      ..writeAsBytesSync(data, flush: true);
Devon Carew's avatar
Devon Carew committed
179
    status.stop();
180 181
  }

182
  @override
183
  Future<FlutterCommandResult> runCommand() async {
184 185
    final List<Directory> packages = runner.getRepoPackages();

186 187 188 189 190
    final bool upgrade = boolArg('force-upgrade');
    final bool isPrintPaths = boolArg('paths');
    final bool isPrintTransitiveClosure = boolArg('transitive-closure');
    final bool isVerifyOnly = boolArg('verify-only');
    final bool isConsumerOnly = boolArg('consumer-only');
191
    final bool offline = boolArg('offline');
192 193 194 195 196
    final bool crash = boolArg('crash');

    if (crash) {
      throw StateError('test crash please ignore.');
    }
197 198 199 200 201 202

    if (upgrade && offline) {
      throwToolExit(
          '--force-upgrade cannot be used with the --offline flag'
      );
    }
203

204
    // "consumer" packages are those that constitute our public API (e.g. flutter, flutter_test, flutter_driver, flutter_localizations, integration_test).
205 206 207 208
    if (isConsumerOnly) {
      if (!isPrintTransitiveClosure) {
        throwToolExit(
          '--consumer-only can only be used with the --transitive-closure flag'
209 210
        );
      }
211 212 213 214 215 216 217
      // Only retain flutter, flutter_test, flutter_driver, and flutter_localizations.
      const List<String> consumerPackages = <String>['flutter', 'flutter_test', 'flutter_driver', 'flutter_localizations', 'integration_test'];
      // ensure we only get flutter/packages
      packages.retainWhere((Directory directory) {
        return consumerPackages.any((String package) {
          return directory.path.endsWith('packages${globals.fs.path.separator}$package');
        });
218 219
      });
    }
220 221

    if (isVerifyOnly) {
222
      bool needsUpdate = false;
223
      globals.printStatus('Verifying pubspecs...');
224
      for (final Directory directory in packages) {
225 226
        PubspecYaml pubspec;
        try {
227
          pubspec = PubspecYaml(directory);
228 229 230
        } on String catch (message) {
          throwToolExit(message);
        }
231
        globals.printTrace('Reading pubspec.yaml from ${directory.path}');
232
        if (pubspec.checksum.value == null) {
233 234
          // If the checksum is invalid or missing, we can just ask them run to run
          // upgrade again to compute it.
235
          globals.printError(
236
            'Warning: pubspec in ${directory.path} has out of date dependencies. '
237
            'Please run "flutter update-packages --force-upgrade" to update them correctly.'
238
          );
239
          needsUpdate = true;
240 241
        }
        // all dependencies in the pubspec sorted lexically.
242
        final Map<String, String> checksumDependencies = <String, String>{};
243
        for (final PubspecLine data in pubspec.inputData) {
244
          if (data is PubspecDependency && data.kind == DependencyKind.normal) {
245
            checksumDependencies[data.name] = data.version;
246
          }
247
        }
248
        final String checksum = _computeChecksum(checksumDependencies.keys, (String name) => checksumDependencies[name]);
249 250 251
        if (checksum != pubspec.checksum.value) {
          // If the checksum doesn't match, they may have added or removed some dependencies.
          // we need to run update-packages to recapture the transitive deps.
252
          globals.printError(
253 254 255
            'Warning: pubspec in ${directory.path} has updated or new dependencies. '
            'Please run "flutter update-packages --force-upgrade" to update them correctly '
            '(checksum ${pubspec.checksum.value} != $checksum).'
256
          );
257 258 259
          needsUpdate = true;
        } else {
          // everything is correct in the pubspec.
260
          globals.printTrace('pubspec in ${directory.path} is up to date!');
261
        }
262
      }
263 264 265 266 267 268 269
      if (needsUpdate) {
        throwToolExit(
          'Warning: one or more pubspecs have invalid dependencies. '
          'Please run "flutter update-packages --force-upgrade" to update them correctly.',
          exitCode: 1,
        );
      }
270
      globals.printStatus('All pubspecs were up to date.');
271
      return FlutterCommandResult.success();
272 273
    }

274
    if (upgrade || isPrintPaths || isPrintTransitiveClosure) {
275
      globals.printStatus('Upgrading packages...');
276 277 278 279 280 281 282 283
      // This feature attempts to collect all the packages used across all the
      // pubspec.yamls in the repo (including via transitive dependencies), and
      // find the latest version of each that can be used while keeping each
      // such package fixed at a single version across all the pubspec.yamls.
      //
      // First, collect up the explicit dependencies:
      final List<PubspecYaml> pubspecs = <PubspecYaml>[];
      final Map<String, PubspecDependency> dependencies = <String, PubspecDependency>{};
284
      final Set<String> specialDependencies = <String>{};
285
      for (final Directory directory in packages) { // these are all the directories with pubspec.yamls we care about
286
        globals.printTrace('Reading pubspec.yaml from: ${directory.path}');
287 288
        PubspecYaml pubspec;
        try {
289
          pubspec = PubspecYaml(directory); // this parses the pubspec.yaml
290 291 292
        } on String catch (message) {
          throwToolExit(message);
        }
293
        pubspecs.add(pubspec); // remember it for later
294
        for (final PubspecDependency dependency in pubspec.allDependencies) { // this is all the explicit dependencies
295 296 297 298 299 300 301 302 303 304 305 306 307
          if (dependencies.containsKey(dependency.name)) {
            // If we've seen the dependency before, make sure that we are
            // importing it the same way. There's several ways to import a
            // dependency. Hosted (from pub via version number), by path (e.g.
            // pointing at the version of a package we get from the Dart SDK
            // that we download with Flutter), by SDK (e.g. the "flutter"
            // package is explicitly from "sdk: flutter").
            //
            // This makes sure that we don't import a package in two different
            // ways, e.g. by saying "sdk: flutter" in one pubspec.yaml and
            // saying "path: ../../..." in another.
            final PubspecDependency previous = dependencies[dependency.name];
            if (dependency.kind != previous.kind || dependency.lockTarget != previous.lockTarget) {
308 309 310 311 312
              throwToolExit(
                'Inconsistent requirements around ${dependency.name}; '
                'saw ${dependency.kind} (${dependency.lockTarget}) in "${dependency.sourcePath}" '
                'and ${previous.kind} (${previous.lockTarget}) in "${previous.sourcePath}".'
              );
313 314 315 316 317 318 319 320
            }
          }
          // Remember this dependency by name so we can look it up again.
          dependencies[dependency.name] = dependency;
          // Normal dependencies are those we get from pub. The others we
          // already implicitly pin since we pull down one version of the
          // Flutter and Dart SDKs, so we track which those are here so that we
          // can omit them from our list of pinned dependencies later.
321
          if (dependency.kind != DependencyKind.normal) {
322
            specialDependencies.add(dependency.name);
323
          }
324 325 326 327 328 329 330
        }
      }

      // Now that we have all the dependencies we explicitly care about, we are
      // going to create a fake package and then run "pub upgrade" on it. The
      // pub tool will attempt to bring these dependencies up to the most recent
      // possible versions while honoring all their constraints.
331
      final PubDependencyTree tree = PubDependencyTree(); // object to collect results
332
      final Directory tempDir = globals.fs.systemTempDirectory.createTempSync('flutter_update_packages.');
333
      try {
334
        final File fakePackage = _pubspecFor(tempDir);
335 336
        fakePackage.createSync();
        fakePackage.writeAsStringSync(_generateFakePubspec(dependencies.values));
337
        // Create a synthetic flutter SDK so that transitive flutter SDK
338
        // constraints are not affected by this upgrade.
339 340 341
        Directory temporaryFlutterSdk;
        if (upgrade) {
          temporaryFlutterSdk = createTemporaryFlutterSdk(
342
            globals.logger,
343 344 345 346 347
            globals.fs,
            globals.fs.directory(Cache.flutterRoot),
            pubspecs,
          );
        }
348 349

        // Next we run "pub upgrade" on this generated package:
350
        await pub.get(
351 352 353
          context: PubContext.updatePackages,
          directory: tempDir.path,
          upgrade: true,
354
          offline: offline,
355 356 357
          flutterRootOverride: upgrade
            ? temporaryFlutterSdk.path
            : null,
358
          generateSyntheticPackage: false,
359
        );
360 361
        // Cleanup the temporary SDK
        try {
362
          temporaryFlutterSdk?.deleteSync(recursive: true);
363
        } on FileSystemException {
364
          // Failed to delete temporary SDK.
365 366
        }

367 368 369 370 371
        // Then we run "pub deps --style=compact" on the result. We pipe all the
        // output to tree.fill(), which parses it so that it can create a graph
        // of all the dependencies so that we can figure out the transitive
        // dependencies later. It also remembers which version was selected for
        // each package.
372
        await pub.batch(
373
          <String>['deps', '--style=compact'],
374
          context: PubContext.updatePackages,
375
          directory: tempDir.path,
376 377 378 379
          filter: tree.fill,
          retry: false, // errors here are usually fatal since we're not hitting the network
        );
      } finally {
380
        tempDir.deleteSync(recursive: true);
381 382
      }

383 384 385
      // The transitive dependency tree for the fake package does not contain
      // dependencies between Flutter SDK packages and pub packages. We add them
      // here.
386
      for (final PubspecYaml pubspec in pubspecs) {
387
        final String package = pubspec.name;
388 389 390 391
        specialDependencies.add(package);
        tree._versions[package] = pubspec.version;
        assert(!tree._dependencyTree.containsKey(package));
        tree._dependencyTree[package] = <String>{};
392
        for (final PubspecDependency dependency in pubspec.dependencies) {
393 394 395 396 397 398 399 400
          if (dependency.kind == DependencyKind.normal) {
            tree._dependencyTree[package].add(dependency.name);
          }
        }
      }

      if (isPrintTransitiveClosure) {
        tree._dependencyTree.forEach((String from, Set<String> to) {
401
          globals.printStatus('$from -> $to');
402
        });
403
        return FlutterCommandResult.success();
404 405 406
      }

      if (isPrintPaths) {
407
        showDependencyPaths(from: stringArg('from'), to: stringArg('to'), tree: tree);
408
        return FlutterCommandResult.success();
409 410
      }

411 412 413 414 415 416 417 418
      // Now that we have collected all the data, we can apply our dependency
      // versions to each pubspec.yaml that we collected. This mutates the
      // pubspec.yaml files.
      //
      // The specialDependencies argument is the set of package names to not pin
      // to specific versions because they are explicitly pinned by their
      // constraints. Here we list the names we earlier established we didn't
      // need to pin because they come from the Dart or Flutter SDKs.
419
      for (final PubspecYaml pubspec in pubspecs) {
420
        pubspec.apply(tree, specialDependencies);
421
      }
422 423 424 425 426 427 428

      // Now that the pubspec.yamls are updated, we run "pub get" on each one so
      // that the various packages are ready to use. This is what "flutter
      // update-packages" does without --force-upgrade, so we can just fall into
      // the regular code path.
    }

429
    final Stopwatch timer = Stopwatch()..start();
430
    int count = 0;
431

432
    for (final Directory dir in packages) {
433 434 435 436
      await pub.get(
        context: PubContext.updatePackages,
        directory: dir.path,
        offline: offline,
437
        generateSyntheticPackage: false,
438
      );
439
      count += 1;
440
    }
441

442
    await _downloadCoverageData();
443

444
    final double seconds = timer.elapsedMilliseconds / 1000.0;
445
    globals.printStatus("\nRan 'pub' $count time${count == 1 ? "" : "s"} and fetched coverage data in ${seconds.toStringAsFixed(1)}s.");
446

447
    return FlutterCommandResult.success();
448
  }
449 450 451 452 453 454

  void showDependencyPaths({
    @required String from,
    @required String to,
    @required PubDependencyTree tree,
  }) {
455
    if (!tree.contains(from)) {
456
      throwToolExit('Package $from not found in the dependency tree.');
457 458
    }
    if (!tree.contains(to)) {
459
      throwToolExit('Package $to not found in the dependency tree.');
460
    }
461

462
    final Queue<_DependencyLink> traversalQueue = Queue<_DependencyLink>();
463
    final Set<String> visited = <String>{};
464 465
    final List<_DependencyLink> paths = <_DependencyLink>[];

466
    traversalQueue.addFirst(_DependencyLink(from: null, to: from));
467
    while (traversalQueue.isNotEmpty) {
468
      final _DependencyLink link = traversalQueue.removeLast();
469
      if (link.to == to) {
470
        paths.add(link);
471 472
      }
      if (link.from != null) {
473
        visited.add(link.from.to);
474
      }
475
      for (final String dependency in tree._dependencyTree[link.to]) {
476
        if (!visited.contains(dependency)) {
477
          traversalQueue.addFirst(_DependencyLink(from: link, to: dependency));
478 479 480 481 482
        }
      }
    }

    for (_DependencyLink path in paths) {
483
      final StringBuffer buf = StringBuffer();
484
      while (path != null) {
485
        buf.write(path.to);
486
        path = path.from;
487
        if (path != null) {
488
          buf.write(' <- ');
489
        }
490
      }
491
      globals.printStatus(buf.toString(), wrap: false);
492 493 494
    }

    if (paths.isEmpty) {
495
      globals.printStatus('No paths found from $from to $to');
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
    }
  }
}

class _DependencyLink {
  _DependencyLink({
    @required this.from,
    @required this.to,
  });

  final _DependencyLink from;
  final String to;

  @override
  String toString() => '${from?.to} -> $to';
511
}
512

513 514 515 516 517 518
/// The various sections of a pubspec.yaml file.
///
/// We care about the "dependencies", "dev_dependencies", and
/// "dependency_overrides" sections, as well as the "name" and "version" fields
/// in the pubspec header bucketed into [header]. The others are all bucketed
/// into [other].
519
enum Section { header, dependencies, devDependencies, dependencyOverrides, builders, other }
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538

/// The various kinds of dependencies we know and care about.
enum DependencyKind {
  // Dependencies that will be path or sdk dependencies but
  // for which we haven't yet parsed the data.
  unknown,

  // Regular dependencies with a specified version range.
  normal,

  // Dependency that uses an explicit path, e.g. into the Dart SDK.
  path,

  // Dependency defined as coming from an SDK (typically "sdk: flutter").
  sdk,

  // A dependency that was "normal", but for which we later found a "path" or
  // "sdk" dependency in the dependency_overrides section.
  overridden,
539

540
  // A dependency that uses git.
541
  git,
542 543 544 545 546
}

/// This is the string we output next to each of our autogenerated transitive
/// dependencies so that we can ignore them the next time we parse the
/// pubspec.yaml file.
547 548 549 550 551
const String kTransitiveMagicString= '# THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"';


/// This is the string output before a checksum of the packages used.
const String kDependencyChecksum = '# PUBSPEC CHECKSUM: ';
552 553 554 555 556 557 558 559

/// This class represents a pubspec.yaml file for the purposes of upgrading the
/// dependencies as done by this file.
class PubspecYaml {
  /// You create one of these by providing a directory, from which we obtain the
  /// pubspec.yaml and parse it into a line-by-line form.
  factory PubspecYaml(Directory directory) {
    final File file = _pubspecFor(directory);
560
    return _parse(file, file.readAsLinesSync());
561 562
  }

563
  PubspecYaml._(this.file, this.name, this.version, this.inputData, this.checksum);
564 565 566

  final File file; // The actual pubspec.yaml file.

567 568 569 570 571 572
  /// The package name.
  final String name;

  /// The package version.
  final String version;

573 574
  final List<PubspecLine> inputData; // Each line of the pubspec.yaml file, parsed(ish).

575
  /// The package checksum.
576
  ///
577 578 579 580
  /// If this was not found in the pubspec, a synthetic checksum is created
  /// with a value of `-1`.
  final PubspecChecksum checksum;

581 582 583 584 585
  /// This parses each line of a pubspec.yaml file (a list of lines) into
  /// slightly more structured data (in the form of a list of PubspecLine
  /// objects). We don't just use a YAML parser because we care about comments
  /// and also because we can just define the style of pubspec.yaml files we care
  /// about (since they're all under our control).
586 587 588 589
  static PubspecYaml _parse(File file, List<String> lines) {
    final String filename = file.path;
    String packageName;
    String packageVersion;
590
    PubspecChecksum checksum; // the checksum value used to verify that dependencies haven't changed.
591 592 593 594 595 596 597 598 599 600 601 602 603 604
    final List<PubspecLine> result = <PubspecLine>[]; // The output buffer.
    Section section = Section.other; // Which section we're currently reading from.
    bool seenMain = false; // Whether we've seen the "dependencies:" section.
    bool seenDev = false; // Whether we've seen the "dev_dependencies:" section.
    // The masterDependencies map is used to keep track of the objects
    // representing actual dependencies we've seen so far in this file so that
    // if we see dependency overrides we can update the actual dependency so it
    // knows that it's not really a dependency.
    final Map<String, PubspecDependency> masterDependencies = <String, PubspecDependency>{};
    // The "special" dependencies (the ones that use git: or path: or sdk: or
    // whatnot) have the style of having extra data after the line that declares
    // the dependency. So we track what is the "current" (or "last") dependency
    // that we are dealing with using this variable.
    PubspecDependency lastDependency;
605 606
    for (int index = 0; index < lines.length; index += 1) {
      String line = lines[index];
607 608 609 610 611 612
      if (lastDependency == null) {
        // First we look to see if we're transitioning to a new top-level section.
        // The PubspecHeader.parse static method can recognize those headers.
        final PubspecHeader header = PubspecHeader.parse(line); // See if it's a header.
        if (header != null) { // It is!
          section = header.section; // The parser determined what kind of section it is.
613
          if (section == Section.header) {
614
            if (header.name == 'name') {
615
              packageName = header.value;
616
            } else if (header.name == 'version') {
617
              packageVersion = header.value;
618
            }
619
          } else if (section == Section.dependencies) {
620 621
            // If we're entering the "dependencies" section, we want to make sure that
            // it's the first section (of those we care about) that we've seen so far.
622
            if (seenMain) {
623
              throw 'Two dependencies sections found in $filename. There should only be one.';
624
            }
625 626 627 628 629 630 631 632 633
            if (seenDev) {
              throw 'The dependencies section was after the dev_dependencies section in $filename. '
                    'To enable one-pass processing, the dependencies section must come before the '
                    'dev_dependencies section.';
            }
            seenMain = true;
          } else if (section == Section.devDependencies) {
            // Similarly, if we're entering the dev_dependencies section, we should verify
            // that we've not seen one already.
634
            if (seenDev) {
635
              throw 'Two dev_dependencies sections found in $filename. There should only be one.';
636
            }
637 638 639
            seenDev = true;
          }
          result.add(header);
640 641 642 643 644
        } else if (section == Section.builders) {
          // Do nothing.
          // This line isn't a section header, and we're not in a section we care about.
          // We just stick the line into the output unmodified.
          result.add(PubspecLine(line));
645
        } else if (section == Section.other) {
646 647 648 649 650 651 652
          if (line.contains(kDependencyChecksum)) {
            // This is the pubspec checksum. After computing it, we remove it from the output data
            // since it will be recomputed later.
            checksum = PubspecChecksum.parse(line);
          } else {
            // This line isn't a section header, and we're not in a section we care about.
            // We just stick the line into the output unmodified.
653
            result.add(PubspecLine(line));
654
          }
655 656 657 658
        } else {
          // We're in a section we care about. Try to parse out the dependency:
          final PubspecDependency dependency = PubspecDependency.parse(line, filename: filename);
          if (dependency != null) { // We got one!
659 660
            // Track whether or not this a dev dependency.
            dependency.isDevDependency = seenDev;
661 662 663 664 665 666 667 668 669 670 671 672 673 674
            result.add(dependency);
            if (dependency.kind == DependencyKind.unknown) {
              // If we didn't get a version number, then we need to be ready to
              // read the next line as part of this dependency, so keep track of
              // this dependency object.
              lastDependency = dependency;
            }
            if (section != Section.dependencyOverrides) {
              // If we're not in the overrides section, then just remember the
              // dependency, in case it comes up again later in the overrides
              // section.
              //
              // First, make sure it's a unique dependency. Listing dependencies
              // twice doesn't make sense.
675
              if (masterDependencies.containsKey(dependency.name)) {
676
                throw '$filename contains two dependencies on ${dependency.name}.';
677
              }
678 679 680 681 682 683 684 685 686
              masterDependencies[dependency.name] = dependency;
            } else {
              // If we _are_ in the overrides section, then go tell the version
              // we saw earlier (if any -- there might not be, we might be
              // overriding a transitive dependency) that we have overridden it,
              // so that later when we output the dependencies we can leave
              // the line unmodified.
              masterDependencies[dependency.name]?.markOverridden(dependency);
            }
687 688 689 690
          } else if (line.contains(kDependencyChecksum)) {
            // This is the pubspec checksum. After computing it, we remove it from the output data
            // since it will be recomputed later.
            checksum = PubspecChecksum.parse(line);
691 692 693 694
          } else {
            // We're in a section we care about but got a line we didn't
            // recognize. Maybe it's a comment or a blank line or something.
            // Just pass it through.
695
            result.add(PubspecLine(line));
696 697 698 699 700 701 702 703
          }
        }
      } else {
        // If we're here it means the last line was a dependency that needed
        // extra information to be parsed from the next line.
        //
        // Try to parse the line by giving it to the last PubspecDependency
        // object we created. If parseLock fails to recognize the line, it will
704 705
        // throw. If it does recognize the line and needs the following lines in
        // its lockLine, it'll return false.
706 707 708 709 710
        // Otherwise it returns true.
        //
        // If it returns true, then it will have updated itself internally to
        // store the information from this line.
        if (!lastDependency.parseLock(line, filename, lockIsOverride: section == Section.dependencyOverrides)) {
711 712 713 714 715 716 717
          // Ok we're dealing with some "git:" dependency. Consume lines until
          // we are out of the git dependency, and stuff them into the lock
          // line.
          lastDependency._lockLine = line;
          lastDependency._lockIsOverride = section == Section.dependencyOverrides;
          do {
            index += 1;
718 719 720
            if (index == lines.length) {
              throw StateError('Invalid pubspec.yaml: a "git" dependency section terminated early.');
            }
721 722 723
            line = lines[index];
            lastDependency._lockLine += '\n$line';
          } while (line.startsWith('   '));
724 725 726 727 728 729
        }
        // We're done with this special dependency, so reset back to null so
        // we'll go in the top section next time instead.
        lastDependency = null;
      }
    }
730
    return PubspecYaml._(file, packageName, packageVersion, result, checksum ?? PubspecChecksum(null, ''));
731 732
  }

733
  /// This returns all the explicit dependencies that this pubspec.yaml lists under dependencies.
734 735 736 737 738 739
  Iterable<PubspecDependency> get dependencies sync* {
    // It works by iterating over the parsed data from _parse above, collecting
    // all the dependencies that were found, ignoring any that are flagged as as
    // overridden by subsequent entries in the same file and any that have the
    // magic comment flagging them as auto-generated transitive dependencies
    // that we added in a previous run.
740
    for (final PubspecLine data in inputData) {
741
      if (data is PubspecDependency && data.kind != DependencyKind.overridden && !data.isTransitive && !data.isDevDependency) {
742
        yield data;
743
      }
744 745 746 747 748
    }
  }

  /// This returns all regular dependencies and all dev dependencies.
  Iterable<PubspecDependency> get allDependencies sync* {
749
    for (final PubspecLine data in inputData) {
750
      if (data is PubspecDependency && data.kind != DependencyKind.overridden && !data.isTransitive) {
751
        yield data;
752
      }
753 754 755 756 757 758 759 760 761 762
    }
  }

  /// Take a dependency graph with explicit version numbers, and apply them to
  /// the pubspec.yaml, ignoring any that we know are special dependencies (those
  /// that depend on the Flutter or Dart SDK directly and are thus automatically
  /// pinned).
  void apply(PubDependencyTree versions, Set<String> specialDependencies) {
    assert(versions != null);
    final List<String> output = <String>[]; // the string data to output to the file, line by line
763 764
    final Set<String> directDependencies = <String>{}; // packages this pubspec directly depends on (i.e. not transitive)
    final Set<String> devDependencies = <String>{};
765
    Section section = Section.other; // the section we're currently handling
766 767 768 769 770

    // the line number where we're going to insert the transitive dependencies.
    int endOfDirectDependencies;
    // The line number where we're going to insert the transitive dev dependencies.
    int endOfDevDependencies;
771 772 773 774 775 776 777
    // Walk the pre-parsed input file, outputting it unmodified except for
    // updating version numbers, removing the old transitive dependencies lines,
    // and adding our new transitive dependencies lines. We also do a little
    // cleanup, removing trailing spaces, removing double-blank lines, leading
    // blank lines, and trailing blank lines, and ensuring the file ends with a
    // newline. This cleanup lets us be a little more aggressive while building
    // the output.
778
    for (final PubspecLine data in inputData) {
779 780 781 782 783 784
      if (data is PubspecHeader) {
        // This line was a header of some sort.
        //
        // If we're leaving one of the sections in which we can list transitive
        // dependencies, then remember this as the current last known valid
        // place to insert our transitive dependencies.
785
        if (section == Section.dependencies) {
786
          endOfDirectDependencies = output.length;
787 788
        }
        if (section == Section.devDependencies) {
789
          endOfDevDependencies = output.length;
790
        }
791 792 793 794 795 796 797
        section = data.section; // track which section we're now in.
        output.add(data.line); // insert the header into the output
      } else if (data is PubspecDependency) {
        // This was a dependency of some sort.
        // How we handle this depends on the section.
        switch (section) {
          case Section.devDependencies:
798
          case Section.dependencies:
799 800 801 802
            // For the dependencies and dev_dependencies sections, we reinsert
            // the dependency if it wasn't one of our autogenerated transitive
            // dependency lines.
            if (!data.isTransitive) {
803
              // Assert that we haven't seen it in this file already.
804
              assert(!directDependencies.contains(data.name) && !devDependencies.contains(data.name));
805 806 807 808 809 810 811
              if (data.kind == DependencyKind.normal) {
                // This is a regular dependency, so we need to update the
                // version number.
                //
                // We output data that matches the format that
                // PubspecDependency.parse can handle. The data.suffix is any
                // previously-specified trailing comment.
812
                assert(versions.contains(data.name));
813 814 815 816 817 818
                output.add('  ${data.name}: ${versions.versionFor(data.name)}${data.suffix}');
              } else {
                // If it wasn't a regular dependency, then we output the line
                // unmodified. If there was an additional line (e.g. an "sdk:
                // flutter" line) then we output that too.
                output.add(data.line);
819
                if (data.lockLine != null) {
820
                  output.add(data.lockLine);
821
                }
822 823 824
              }
              // Remember that we've dealt with this dependency so we don't
              // mention it again when doing the transitive dependencies.
825 826 827 828
              if (section == Section.dependencies) {
                directDependencies.add(data.name);
              } else {
                devDependencies.add(data.name);
829
              }
830 831 832
            }
            // Since we're in one of the places where we can list dependencies,
            // remember this as the current last known valid place to insert our
833
            // transitive dev dependencies. If the section is for regular dependencies,
834
            // then also remember the line for the end of direct dependencies.
835 836 837 838
            if (section == Section.dependencies) {
              endOfDirectDependencies = output.length;
            }
            endOfDevDependencies = output.length;
839 840 841 842
            break;
          default:
            // In other sections, pass everything through in its original form.
            output.add(data.line);
843
            if (data.lockLine != null) {
844
              output.add(data.lockLine);
845
            }
846 847 848 849 850 851 852 853
            break;
        }
      } else {
        // Not a header, not a dependency, just pass that through unmodified.
        output.add(data.line);
      }
    }

854 855 856 857
    // If there are no dependencies or dev_dependencies sections, these will be
    // null. We have such files in our tests, so account for them here.
    endOfDirectDependencies ??= output.length;
    endOfDevDependencies ??= output.length;
858 859 860 861 862 863 864

    // Now include all the transitive dependencies and transitive dev dependencies.
    // The blocks of text to insert for each dependency section.
    final List<String> transitiveDependencyOutput = <String>[];
    final List<String> transitiveDevDependencyOutput = <String>[];

    // Which dependencies we need to handle for the transitive and dev dependency sections.
865 866
    final Set<String> transitiveDependencies = <String>{};
    final Set<String> transitiveDevDependencies = <String>{};
867

868
    // Merge the lists of dependencies we've seen in this file from dependencies, dev dependencies,
869
    // and the dependencies we know this file mentions that are already pinned
870
    // (and which didn't get special processing above).
871 872 873 874 875
    final Set<String> implied = <String>{
      ...directDependencies,
      ...specialDependencies,
      ...devDependencies,
    };
876

877 878
    // Create a new set to hold the list of packages we've already processed, so
    // that we don't redundantly process them multiple times.
879
    final Set<String> done = <String>{};
880
    for (final String package in directDependencies) {
881
      transitiveDependencies.addAll(versions.getTransitiveDependenciesFor(package, seen: done, exclude: implied));
882
    }
883
    for (final String package in devDependencies) {
884
      transitiveDevDependencies.addAll(versions.getTransitiveDependenciesFor(package, seen: done, exclude: implied));
885
    }
886

887
    // Sort each dependency block lexically so that we don't get noisy diffs when upgrading.
888
    final List<String> transitiveDependenciesAsList = transitiveDependencies.toList()..sort();
889 890
    final List<String> transitiveDevDependenciesAsList = transitiveDevDependencies.toList()..sort();

891 892 893 894
    String computeTransitiveDependencyLineFor(String package) {
      return '  $package: ${versions.versionFor(package)} $kTransitiveMagicString';
    }

895
    // Add a line for each transitive dependency and transitive dev dependency using our magic string to recognize them later.
896
    for (final String package in transitiveDependenciesAsList) {
897
      transitiveDependencyOutput.add(computeTransitiveDependencyLineFor(package));
898
    }
899
    for (final String package in transitiveDevDependenciesAsList) {
900
      transitiveDevDependencyOutput.add(computeTransitiveDependencyLineFor(package));
901
    }
902 903

    // Build a sorted list of all dependencies for the checksum.
904 905 906 907 908 909
    final Set<String> checksumDependencies = <String>{
      ...directDependencies,
      ...devDependencies,
      ...transitiveDependenciesAsList,
      ...transitiveDevDependenciesAsList,
    }..removeAll(specialDependencies);
910 911 912 913 914 915 916 917 918 919

    // Add a blank line before and after each section to keep the resulting output clean.
    transitiveDependencyOutput
      ..insert(0, '')
      ..add('');
    transitiveDevDependencyOutput
      ..insert(0, '')
      ..add('');

    // Compute a new checksum from all sorted dependencies and their version and convert to a hex string.
920
    final String checksumString = _computeChecksum(checksumDependencies, versions.versionFor);
921 922 923 924 925 926 927 928 929

    // Insert the block of transitive dependency declarations into the output after [endOfDirectDependencies],
    // and the blocks of transitive dev dependency declarations into the output after [lastPossiblePlace]. Finally,
    // insert the [checksumString] at the very end.
    output
      ..insertAll(endOfDevDependencies, transitiveDevDependencyOutput)
      ..insertAll(endOfDirectDependencies, transitiveDependencyOutput)
      ..add('')
      ..add('$kDependencyChecksum$checksumString');
930

931
    // Remove trailing lines.
932
    while (output.last.isEmpty) {
933
      output.removeLast();
934
    }
935 936 937

    // Output the result to the pubspec.yaml file, skipping leading and
    // duplicate blank lines and removing trailing spaces.
938
    final StringBuffer contents = StringBuffer();
939 940 941 942
    bool hadBlankLine = true;
    for (String line in output) {
      line = line.trimRight();
      if (line == '') {
943
        if (!hadBlankLine) {
944
          contents.writeln('');
945
        }
946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
        hadBlankLine = true;
      } else {
        contents.writeln(line);
        hadBlankLine = false;
      }
    }
    file.writeAsStringSync(contents.toString());
  }
}

/// This is the base class for the objects that represent lines in the
/// pubspec.yaml files.
class PubspecLine {
  PubspecLine(this.line);

  /// The raw line as we saw it in the original file. This is used so that we can
  /// output the same line unmodified for the majority of lines.
  final String line;
}

966 967 968 969
/// A checksum of the non autogenerated dependencies.
class PubspecChecksum extends PubspecLine {
  PubspecChecksum(this.value, String line) : super(line);

970
  /// The checksum value, computed using [hashValues] over the direct, dev,
971
  /// and special dependencies sorted lexically.
972
  ///
973 974
  /// If the line cannot be parsed, [value] will be null.
  final String value;
975 976

  /// Parses a [PubspecChecksum] from a line.
977
  ///
978 979
  /// The returned PubspecChecksum will have a null [value] if no checksum could
  /// be found on this line. This is a value that [_computeChecksum] cannot return.
980
  static PubspecChecksum parse(String line) {
981
    final List<String> tokens = line.split(kDependencyChecksum);
982
    if (tokens.length != 2) {
983
      return PubspecChecksum(null, line);
984
    }
985
    return PubspecChecksum(tokens.last.trim(), line);
986
  }
987 988
}

989 990
/// A header, e.g. "dependencies:".
class PubspecHeader extends PubspecLine {
991 992 993
  PubspecHeader(String line, this.section, { this.name, this.value }) : super(line);

  /// The section of the pubspec where the parse [line] appears.
994 995
  final Section section;

996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
  /// The name in the pubspec line providing a name/value pair, such as "name"
  /// and "version".
  ///
  /// Example:
  ///
  /// The value of this field extracted from the following line is "version".
  ///
  /// ```
  /// version: 0.16.5
  /// ```
  final String name;

  /// The value in the pubspec line providing a name/value pair, such as "name"
  /// and "version".
  ///
  /// Example:
  ///
  /// The value of this field extracted from the following line is "0.16.5".
  ///
  /// ```
  /// version: 0.16.5
  /// ```
  final String value;

1020 1021 1022 1023 1024 1025 1026
  static PubspecHeader parse(String line) {
    // We recognize any line that:
    //  * doesn't start with a space (i.e. is aligned on the left edge)
    //  * ignoring trailing spaces and comments, ends with a colon
    //  * has contents before the colon
    // We also try to recognize which of the kinds of Sections it is
    // by comparing those contents against known strings.
1027
    if (line.startsWith(' ')) {
1028
      return null;
1029
    }
1030
    final String strippedLine = _stripComments(line);
1031
    if (!strippedLine.contains(':') || strippedLine.length <= 1) {
1032
      return null;
1033
    }
1034 1035 1036
    final List<String> parts = strippedLine.split(':');
    final String sectionName = parts.first;
    final String value = parts.last.trim();
1037 1038
    switch (sectionName) {
      case 'dependencies':
1039
        return PubspecHeader(line, Section.dependencies);
1040
      case 'dev_dependencies':
1041
        return PubspecHeader(line, Section.devDependencies);
1042
      case 'dependency_overrides':
1043
        return PubspecHeader(line, Section.dependencyOverrides);
1044 1045
      case 'builders':
        return PubspecHeader(line, Section.builders);
1046 1047
      case 'name':
      case 'version':
1048
        return PubspecHeader(line, Section.header, name: sectionName, value: value);
1049
      default:
1050
        return PubspecHeader(line, Section.other);
1051 1052 1053 1054 1055 1056 1057
    }
  }

  /// Returns the input after removing trailing spaces and anything after the
  /// first "#".
  static String _stripComments(String line) {
    final int hashIndex = line.indexOf('#');
1058
    if (hashIndex < 0) {
1059
      return line.trimRight();
1060
    }
1061 1062 1063 1064 1065 1066
    return line.substring(0, hashIndex).trimRight();
  }
}

/// A dependency, as represented by a line (or two) from a pubspec.yaml file.
class PubspecDependency extends PubspecLine {
1067 1068 1069 1070
  PubspecDependency(
    String line,
    this.name,
    this.suffix, {
1071 1072
    @required this.isTransitive,
    DependencyKind kind,
1073
    this.version,
1074
    this.sourcePath,
1075 1076
  }) : _kind = kind,
       super(line);
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086

  static PubspecDependency parse(String line, { @required String filename }) {
    // We recognize any line that:
    //  * starts with exactly two spaces, no more or less
    //  * has some content, then a colon
    //
    // If we recognize the line, then we look to see if there's anything after
    // the colon, ignoring comments. If there is, then this is a normal
    // dependency, otherwise it's an unknown one.
    //
1087 1088 1089
    // We also try and save the version string, if any. This is used to verify
    // the checksum of package deps.
    //
1090 1091 1092 1093 1094 1095
    // We also look at the trailing comment, if any, to see if it is the magic
    // string that identifies the line as a transitive dependency that we
    // previously pinned, so we can ignore it.
    //
    // We remember the trailing comment, if any, so that we can reconstruct the
    // line later. We forget the specified version range, if any.
1096
    if (line.length < 4 || line.startsWith('   ') || !line.startsWith('  ')) {
1097
      return null;
1098
    }
1099 1100
    final int colonIndex = line.indexOf(':');
    final int hashIndex = line.indexOf('#');
1101
    if (colonIndex < 3) { // two spaces at 0 and 1, a character at 2
1102
      return null;
1103 1104
    }
    if (hashIndex >= 0 && hashIndex < colonIndex) {
1105
      return null;
1106
    }
1107 1108 1109 1110 1111 1112
    final String package = line.substring(2, colonIndex).trimRight();
    assert(package.isNotEmpty);
    assert(line.startsWith('  $package'));
    String suffix = '';
    bool isTransitive = false;
    String stripped;
1113
    String version = '';
1114 1115 1116 1117 1118
    if (hashIndex >= 0) {
      assert(hashIndex > colonIndex);
      final String trailingComment = line.substring(hashIndex, line.length);
      assert(line.endsWith(trailingComment));
      isTransitive = trailingComment == kTransitiveMagicString;
1119
      suffix = ' $trailingComment';
1120 1121 1122 1123
      stripped = line.substring(colonIndex + 1, hashIndex).trimRight();
    } else {
      stripped = line.substring(colonIndex + 1, line.length).trimRight();
    }
1124 1125 1126
    if (colonIndex != -1) {
      version = line.substring(colonIndex + 1, hashIndex != -1 ? hashIndex : line.length).trim();
    }
1127
    return PubspecDependency(line, package, suffix, isTransitive: isTransitive, version: version, kind: stripped.isEmpty ? DependencyKind.unknown : DependencyKind.normal, sourcePath: filename);
1128 1129 1130 1131
  }

  final String name; // the package name
  final String suffix; // any trailing comment we found
1132
  final String version; // the version string if found, or blank.
1133 1134
  final bool isTransitive; // whether the suffix matched kTransitiveMagicString
  final String sourcePath; // the filename of the pubspec.yaml file, for error messages
1135
  bool isDevDependency; // Whether this dependency is under the `dev dependencies` section.
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155

  DependencyKind get kind => _kind;
  DependencyKind _kind = DependencyKind.normal;

  /// If we're a path or sdk dependency, the path or sdk in question.
  String get lockTarget => _lockTarget;
  String _lockTarget;

  /// If we were a two-line dependency, the second line (see the inherited [line]
  /// for the first).
  String get lockLine => _lockLine;
  String _lockLine;

  /// If we're a path or sdk dependency, whether we were found in a
  /// dependencies/dev_dependencies section, or a dependency_overrides section.
  /// We track this so that we can put ourselves in the right section when
  /// generating the fake pubspec.yaml.
  bool get lockIsOverride => _lockIsOverride;
  bool _lockIsOverride;

1156 1157 1158
  static const String _pathPrefix = '    path: ';
  static const String _sdkPrefix = '    sdk: ';
  static const String _gitPrefix = '    git:';
1159

1160 1161 1162 1163 1164 1165 1166 1167
  /// Whether the dependency points to a package in the Flutter SDK.
  ///
  /// There are two ways one can point to a Flutter package:
  ///
  /// - Using a "sdk: flutter" dependency.
  /// - Using a "path" dependency that points somewhere in the Flutter
  ///   repository other than the "bin" directory.
  bool get pointsToSdk {
1168
    if (_kind == DependencyKind.sdk) {
1169
      return true;
1170
    }
1171 1172

    if (_kind == DependencyKind.path &&
1173 1174
        !globals.fs.path.isWithin(globals.fs.path.join(Cache.flutterRoot, 'bin'), _lockTarget) &&
        globals.fs.path.isWithin(Cache.flutterRoot, _lockTarget)) {
1175
      return true;
1176
    }
1177 1178 1179 1180

    return false;
  }

1181 1182 1183
  /// If parse decided we were a two-line dependency, this is called to parse the second line.
  /// We throw if we couldn't parse this line.
  /// We return true if we parsed it and stored the line in lockLine.
1184
  /// We return false if we parsed it and it's a git dependency that needs the next few lines.
1185 1186 1187
  bool parseLock(String line, String pubspecPath, { @required bool lockIsOverride }) {
    assert(lockIsOverride != null);
    assert(kind == DependencyKind.unknown);
1188
    if (line.startsWith(_pathPrefix)) {
1189
      // We're a path dependency; remember the (absolute) path.
1190 1191
      _lockTarget = globals.fs.path.canonicalize(
          globals.fs.path.absolute(globals.fs.path.dirname(pubspecPath), line.substring(_pathPrefix.length, line.length))
1192
      );
1193
      _kind = DependencyKind.path;
1194
    } else if (line.startsWith(_sdkPrefix)) {
1195
      // We're an SDK dependency.
1196
      _lockTarget = line.substring(_sdkPrefix.length, line.length);
1197
      _kind = DependencyKind.sdk;
1198
    } else if (line.startsWith(_gitPrefix)) {
1199 1200
      // We're a git: dependency. We'll have to get the next few lines.
      _kind = DependencyKind.git;
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
      return false;
    } else {
      throw 'Could not parse additional details for dependency $name; line was: "$line"';
    }
    _lockIsOverride = lockIsOverride;
    _lockLine = line;
    return true;
  }

  void markOverridden(PubspecDependency sibling) {
    // This is called when we find a dependency is mentioned a second time,
    // first in dependencies/dev_dependencies, and then in dependency_overrides.
    // It is called on the one found in dependencies/dev_dependencies, so that
    // we'll later know to report our version as "any" in the fake pubspec.yaml
    // and unmodified in the official pubspec.yamls.
    assert(sibling.name == name);
    assert(sibling.sourcePath == sourcePath);
    assert(sibling.kind != DependencyKind.normal);
    _kind = DependencyKind.overridden;
  }

  /// This generates the entry for this dependency for the pubspec.yaml for the
  /// fake package that we'll use to get the version numbers figured out.
  void describeForFakePubspec(StringBuffer dependencies, StringBuffer overrides) {
    switch (kind) {
      case DependencyKind.unknown:
      case DependencyKind.overridden:
        assert(kind != DependencyKind.unknown);
        break;
      case DependencyKind.normal:
1231
        if (!_kManuallyPinnedDependencies.containsKey(name)) {
1232
          dependencies.writeln('  $name: any');
1233
        }
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
        break;
      case DependencyKind.path:
        if (_lockIsOverride) {
          dependencies.writeln('  $name: any');
          overrides.writeln('  $name:');
          overrides.writeln('    path: $lockTarget');
        } else {
          dependencies.writeln('  $name:');
          dependencies.writeln('    path: $lockTarget');
        }
        break;
      case DependencyKind.sdk:
        if (_lockIsOverride) {
          dependencies.writeln('  $name: any');
          overrides.writeln('  $name:');
          overrides.writeln('    sdk: $lockTarget');
        } else {
          dependencies.writeln('  $name:');
          dependencies.writeln('    sdk: $lockTarget');
        }
        break;
1255 1256 1257 1258 1259 1260 1261 1262 1263
      case DependencyKind.git:
        if (_lockIsOverride) {
          dependencies.writeln('  $name: any');
          overrides.writeln('  $name:');
          overrides.writeln(lockLine);
        } else {
          dependencies.writeln('  $name:');
          dependencies.writeln(lockLine);
        }
1264 1265 1266 1267 1268 1269
    }
  }
}

/// Generates the File object for the pubspec.yaml file of a given Directory.
File _pubspecFor(Directory directory) {
1270 1271
  return directory.fileSystem.file(
    directory.fileSystem.path.join(directory.path, 'pubspec.yaml'));
1272 1273 1274 1275 1276
}

/// Generates the source of a fake pubspec.yaml file given a list of
/// dependencies.
String _generateFakePubspec(Iterable<PubspecDependency> dependencies) {
1277 1278
  final StringBuffer result = StringBuffer();
  final StringBuffer overrides = StringBuffer();
1279
  result.writeln('name: flutter_update_packages');
1280 1281
  result.writeln('environment:');
  result.writeln("  sdk: '>=2.10.0 <3.0.0'");
1282 1283
  result.writeln('dependencies:');
  overrides.writeln('dependency_overrides:');
1284
  if (_kManuallyPinnedDependencies.isNotEmpty) {
1285
    globals.printStatus('WARNING: the following packages use hard-coded version constraints:');
1286
    final Set<String> allTransitive = <String>{
1287
      for (final PubspecDependency dependency in dependencies)
1288
        dependency.name,
1289
    };
1290
    for (final String package in _kManuallyPinnedDependencies.keys) {
1291 1292
      // Don't add pinned dependency if it is not in the set of all transitive dependencies.
      if (!allTransitive.contains(package)) {
1293
        globals.printStatus('Skipping $package because it was not transitive');
1294 1295
        continue;
      }
1296 1297
      final String version = _kManuallyPinnedDependencies[package];
      result.writeln('  $package: $version');
1298
      globals.printStatus('  - $package: $version');
1299 1300
    }
  }
1301
  for (final PubspecDependency dependency in dependencies) {
1302
    if (!dependency.pointsToSdk) {
1303
      dependency.describeForFakePubspec(result, overrides);
1304 1305
    }
  }
1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
  result.write(overrides.toString());
  return result.toString();
}

/// This object tracks the output of a call to "pub deps --style=compact".
///
/// It ends up holding the full graph of dependencies, and the version number for
/// each one.
class PubDependencyTree {
  final Map<String, String> _versions = <String, String>{};
  final Map<String, Set<String>> _dependencyTree = <String, Set<String>>{};

  /// Handles the output from "pub deps --style=compact".
  ///
  /// That output is of this form:
  ///
  /// ```
  /// package_name 0.0.0
  ///
  /// dependencies:
  /// - analyzer 0.31.0-alpha.0 [watcher args package_config collection]
  /// - archive 1.0.31 [crypto args path]
  /// - args 0.13.7
  /// - cli_util 0.1.2+1 [path]
  ///
  /// dev dependencies:
  /// - async 1.13.3 [collection]
  /// - barback 0.15.2+11 [stack_trace source_span pool async collection path]
  ///
  /// dependency overrides:
  /// - analyzer 0.31.0-alpha.0 [watcher args package_config collection]
  /// ```
  ///
  /// We ignore all the lines that don't start with a hyphen. For each other
  /// line, we ignore any line that mentions a package we've already seen (this
  /// happens when the overrides section mentions something that was in the
  /// dependencies section). We ignore if something is a dependency or
  /// dev_dependency (pub won't use different versions for those two).
  ///
1345
  /// We then parse out the package name, version number, and sub-dependencies for
1346 1347 1348 1349 1350 1351
  /// each entry, and store than in our _versions and _dependencyTree fields
  /// above.
  String fill(String message) {
    if (message.startsWith('- ')) {
      final int space2 = message.indexOf(' ', 2);
      int space3 = message.indexOf(' ', space2 + 1);
1352
      if (space3 < 0) {
1353
        space3 = message.length;
1354
      }
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
      final String package = message.substring(2, space2);
      if (!contains(package)) {
        // Some packages get listed in the dependency overrides section too.
        // We just ignore those. The data is the same either way.
        final String version = message.substring(space2 + 1, space3);
        List<String> dependencies;
        if (space3 < message.length) {
          assert(message[space3 + 1] == '[');
          assert(message[message.length - 1] == ']');
          final String allDependencies = message.substring(space3 + 2, message.length - 1);
          dependencies = allDependencies.split(' ');
        } else {
          dependencies = const <String>[];
        }
        _versions[package] = version;
1370
        _dependencyTree[package] = Set<String>.of(dependencies);
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
      }
    }
    return null;
  }

  /// Whether we know about this package.
  bool contains(String package) {
    return _versions.containsKey(package);
  }

  /// The transitive closure of all the dependencies for the given package,
1382
  /// excluding any listed in `seen`.
1383 1384
  Iterable<String> getTransitiveDependenciesFor(
    String package, {
1385 1386 1387 1388 1389
    @required Set<String> seen,
    @required Set<String> exclude,
  }) sync* {
    assert(seen != null);
    assert(exclude != null);
1390 1391 1392 1393 1394
    if (!_dependencyTree.containsKey(package)) {
      // We have no transitive dependencies extracted for flutter_sdk packages
      // because they were omitted from pubspec.yaml used for 'pub upgrade' run.
      return;
    }
1395
    for (final String dependency in _dependencyTree[package]) {
1396
      if (!seen.contains(dependency)) {
1397
        if (!exclude.contains(dependency)) {
1398
          yield dependency;
1399
        }
1400
        seen.add(dependency);
1401
        yield* getTransitiveDependenciesFor(dependency, seen: seen, exclude: exclude);
1402 1403 1404 1405 1406 1407 1408 1409 1410
      }
    }
  }

  /// The version that a particular package ended up with.
  String versionFor(String package) {
    return _versions[package];
  }
}
1411

1412
// Produces a 16-bit checksum from the codePoints of the package name and
1413
// version strings using Fletcher's algorithm.
1414
String _computeChecksum(Iterable<String> names, String Function(String name) getVersion) {
1415 1416
  int lowerCheck = 0;
  int upperCheck = 0;
1417
  final List<String> sortedNames = names.toList()..sort();
1418
  for (final String name in sortedNames) {
1419 1420
    final String version = getVersion(name);
    assert(version != '');
1421
    if (version == null) {
1422
      continue;
1423
    }
1424
    final String value = '$name: $version';
1425
    // Each code unit is 16 bits.
1426
    for (final int codeUnit in value.codeUnits) {
1427 1428 1429 1430 1431 1432 1433 1434
      final int upper = codeUnit >> 8;
      final int lower = codeUnit & 0xFF;
      lowerCheck = (lowerCheck + upper) % 255;
      upperCheck = (upperCheck + lowerCheck) % 255;
      lowerCheck = (lowerCheck + lower) % 255;
      upperCheck = (upperCheck + lowerCheck) % 255;
    }
  }
1435
  return ((upperCheck << 8) | lowerCheck).toRadixString(16).padLeft(4, '0');
1436
}
1437 1438 1439

/// Create a synthetic Flutter SDK so that pub version solving does not get
/// stuck on the old versions.
1440 1441 1442 1443 1444 1445 1446
@visibleForTesting
Directory createTemporaryFlutterSdk(
  Logger logger,
  FileSystem fileSystem,
  Directory realFlutter,
  List<PubspecYaml> pubspecs,
) {
1447 1448 1449 1450 1451 1452 1453
  final Set<String> currentPackages = <String>{};
  for (final FileSystemEntity entity in realFlutter.childDirectory('packages').listSync()) {
    // Verify that a pubspec.yaml exists to ensure this isn't a left over directory.
    if (entity is Directory && entity.childFile('pubspec.yaml').existsSync()) {
      currentPackages.add(fileSystem.path.basename(entity.path));
    }
  }
1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474

  final Map<String, PubspecYaml> pubspecsByName = <String, PubspecYaml>{};
  for (final PubspecYaml pubspec in pubspecs) {
    pubspecsByName[pubspec.name] = pubspec;
  }

  final Directory directory = fileSystem.systemTempDirectory
    .createTempSync('flutter_upgrade_sdk.')
    ..createSync();
  // Fill in version info.
  realFlutter.childFile('version')
    .copySync(directory.childFile('version').path);

  // Directory structure should mirror the current Flutter SDK
  final Directory packages = directory.childDirectory('packages');
  for (final String flutterPackage in currentPackages) {
    final File pubspecFile = packages
      .childDirectory(flutterPackage)
      .childFile('pubspec.yaml')
      ..createSync(recursive: true);
    final PubspecYaml pubspecYaml = pubspecsByName[flutterPackage];
1475 1476 1477 1478 1479 1480
    if (pubspecYaml == null) {
      logger.printError(
        "Unexpected package '$flutterPackage' found in packages directory",
      );
      continue;
    }
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
    final StringBuffer output = StringBuffer('name: $flutterPackage\n');

    // Fill in SDK dependency constraint.
    output.write('''
environment:
  sdk: ">=2.7.0 <3.0.0"
''');

    output.writeln('dependencies:');
    for (final PubspecDependency dependency in pubspecYaml.dependencies) {
      if (dependency.isTransitive || dependency.isDevDependency) {
        continue;
      }
      if (dependency.kind == DependencyKind.sdk) {
        output.writeln('  ${dependency.name}:\n    sdk: flutter');
        continue;
      }
      output.writeln('  ${dependency.name}: any');
    }
    pubspecFile.writeAsStringSync(output.toString());
  }

  // Create the sky engine pubspec.yaml
  directory
    .childDirectory('bin')
    .childDirectory('cache')
    .childDirectory('pkg')
    .childDirectory('sky_engine')
    .childFile('pubspec.yaml')
    ..createSync(recursive: true)
    ..writeAsStringSync('''
name: sky_engine
version: 0.0.99
description: Dart SDK extensions for dart:ui
homepage: http://flutter.io
# sky_engine requires sdk_ext support in the analyzer which was added in 1.11.x
environment:
  sdk: '>=1.11.0 <3.0.0'
''');

  return directory;
}