prepare_package.dart 31.9 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:convert';
7
import 'dart:io' hide Platform;
8
import 'dart:typed_data';
9 10

import 'package:args/args.dart';
11 12
import 'package:crypto/crypto.dart';
import 'package:crypto/src/digest_sink.dart';
13
import 'package:http/http.dart' as http;
14
import 'package:path/path.dart' as path;
15
import 'package:platform/platform.dart' show LocalPlatform, Platform;
16
import 'package:process/process.dart';
17

18 19
const String gobMirror =
    'https://flutter.googlesource.com/mirrors/flutter';
20
const String githubRepo = 'https://github.com/flutter/flutter.git';
21
const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
22
    '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
23
const String releaseFolder = '/releases';
24 25 26
const String gsBase = 'gs://flutter_infra_release';
const String gsReleaseFolder = '$gsBase$releaseFolder';
const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
27
const int shortCacheSeconds = 60;
28 29
const String frameworkVersionTag = 'frameworkVersionFromGit';
const String dartVersionTag = 'dartSdkVersion';
30
const String dartTargetArchTag = 'dartTargetArch';
31

32
/// Exception class for when a process fails to run, so we can catch
33
/// it and provide something more readable than a stack trace.
34
class PreparePackageException implements Exception {
35
  PreparePackageException(this.message, [this.result]);
36

37
  final String message;
38
  final ProcessResult? result;
39
  int get exitCode => result?.exitCode ?? -1;
40 41

  @override
42 43 44 45 46
  String toString() {
    String output = runtimeType.toString();
    if (message != null) {
      output += ': $message';
    }
47
    final String stderr = result?.stderr as String? ?? '';
48 49 50 51 52
    if (stderr.isNotEmpty) {
      output += ':\n$stderr';
    }
    return output;
  }
53 54
}

55 56 57
enum Branch {
  beta,
  stable;
58 59
}

60 61 62 63 64
/// A helper class for classes that want to run a process, optionally have the
/// stderr and stdout reported as the process runs, and capture the stdout
/// properly without dropping any.
class ProcessRunner {
  ProcessRunner({
65
    ProcessManager? processManager,
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
    this.subprocessOutput = true,
    this.defaultWorkingDirectory,
    this.platform = const LocalPlatform(),
  }) : processManager = processManager ?? const LocalProcessManager() {
    environment = Map<String, String>.from(platform.environment);
  }

  /// The platform to use for a starting environment.
  final Platform platform;

  /// Set [subprocessOutput] to show output as processes run. Stdout from the
  /// process will be printed to stdout, and stderr printed to stderr.
  final bool subprocessOutput;

  /// Set the [processManager] in order to inject a test instance to perform
  /// testing.
  final ProcessManager processManager;

  /// Sets the default directory used when `workingDirectory` is not specified
  /// to [runProcess].
86
  final Directory? defaultWorkingDirectory;
87 88

  /// The environment to run processes with.
89
  late Map<String, String> environment;
90 91 92 93 94 95 96 97 98

  /// Run the command and arguments in `commandLine` as a sub-process from
  /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
  /// [Directory.current] if [defaultWorkingDirectory] is not set.
  ///
  /// Set `failOk` if [runProcess] should not throw an exception when the
  /// command completes with a non-zero exit code.
  Future<String> runProcess(
    List<String> commandLine, {
99
    Directory? workingDirectory,
100 101 102 103 104 105 106 107 108
    bool failOk = false,
  }) async {
    workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
    if (subprocessOutput) {
      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
    }
    final List<int> output = <int>[];
    final Completer<void> stdoutComplete = Completer<void>();
    final Completer<void> stderrComplete = Completer<void>();
109
    late Process process;
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
    Future<int> allComplete() async {
      await stderrComplete.future;
      await stdoutComplete.future;
      return process.exitCode;
    }

    try {
      process = await processManager.start(
        commandLine,
        workingDirectory: workingDirectory.absolute.path,
        environment: environment,
      );
      process.stdout.listen(
        (List<int> event) {
          output.addAll(event);
          if (subprocessOutput) {
            stdout.add(event);
          }
        },
        onDone: () async => stdoutComplete.complete(),
      );
      if (subprocessOutput) {
        process.stderr.listen(
          (List<int> event) {
            stderr.add(event);
          },
          onDone: () async => stderrComplete.complete(),
        );
      } else {
        stderrComplete.complete();
      }
    } on ProcessException catch (e) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
143
          'failed with:\n$e';
144 145 146
      throw PreparePackageException(message);
    } on ArgumentError catch (e) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
147
          'failed with:\n$e';
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
      throw PreparePackageException(message);
    }

    final int exitCode = await allComplete();
    if (exitCode != 0 && !failOk) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
      throw PreparePackageException(
        message,
        ProcessResult(0, exitCode, null, 'returned $exitCode'),
      );
    }
    return utf8.decoder.convert(output).trim();
  }
}

163
typedef HttpReader = Future<Uint8List> Function(Uri url, {Map<String, String> headers});
164

165 166
/// Creates a pre-populated Flutter archive from a git repo.
class ArchiveCreator {
167
  /// [tempDir] is the directory to use for creating the archive. The script
168 169 170
  /// will place several GiB of data there, so it should have available space.
  ///
  /// The processManager argument is used to inject a mock of [ProcessManager] for
171
  /// testing purposes.
172 173 174
  ///
  /// If subprocessOutput is true, then output from processes invoked during
  /// archive creation is echoed to stderr and stdout.
175 176 177 178 179 180
  factory ArchiveCreator(
    Directory tempDir,
    Directory outputDir,
    String revision,
    Branch branch, {
    bool strict = true,
181
    ProcessManager? processManager,
182
    bool subprocessOutput = true,
183
    Platform platform = const LocalPlatform(),
184
    HttpReader? httpReader,
185 186 187 188 189 190 191 192 193 194
  }) {
    final Directory flutterRoot = Directory(path.join(tempDir.path, 'flutter'));
    final ProcessRunner processRunner = ProcessRunner(
      processManager: processManager,
      subprocessOutput: subprocessOutput,
      platform: platform,
    )..environment['PUB_CACHE'] = path.join(
      flutterRoot.absolute.path, '.pub-cache',
    );
    final String flutterExecutable = path.join(
195
      flutterRoot.absolute.path,
196
      'bin',
197
      'flutter',
198
    );
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
    final String dartExecutable = path.join(
      flutterRoot.absolute.path,
      'bin',
      'cache',
      'dart-sdk',
      'bin',
      'dart',
    );

    return ArchiveCreator._(
      tempDir: tempDir,
      platform: platform,
      flutterRoot: flutterRoot,
      outputDir: outputDir,
      revision: revision,
      branch: branch,
      strict: strict,
      processRunner: processRunner,
      httpReader: httpReader ?? http.readBytes,
      flutterExecutable: flutterExecutable,
      dartExecutable: dartExecutable,
    );
221 222
  }

223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
  ArchiveCreator._({
    required this.tempDir,
    required this.platform,
    required this.flutterRoot,
    required this.outputDir,
    required this.revision,
    required this.branch,
    required this.strict,
    required ProcessRunner processRunner,
    required this.httpReader,
    required String flutterExecutable,
    required String dartExecutable,
  }) :
    assert(revision.length == 40),
    _processRunner = processRunner,
    _flutter = flutterExecutable,
    _dart = dartExecutable;

241 242
  /// The platform to use for the environment and determining which
  /// platform we're running on.
243
  final Platform platform;
244

245
  /// The branch to build the archive for. The branch must contain [revision].
246
  final Branch branch;
247 248 249 250 251

  /// The git revision hash to build the archive for. This revision has
  /// to be available in the [branch], although it doesn't have to be
  /// at HEAD, since we clone the branch and then reset to this revision
  /// to create the archive.
252
  final String revision;
253 254

  /// The flutter root directory in the [tempDir].
255
  final Directory flutterRoot;
256 257

  /// The temporary directory used to build the archive in.
258
  final Directory tempDir;
259 260

  /// The directory to write the output file to.
261
  final Directory outputDir;
262

263 264 265 266 267
  /// True if the creator should be strict about checking requirements or not.
  ///
  /// In strict mode, will insist that the [revision] be a tagged revision.
  final bool strict;

268
  final Uri _minGitUri = Uri.parse(mingitForWindowsUrl);
269 270
  final ProcessRunner _processRunner;

271 272 273 274 275
  /// Used to tell the [ArchiveCreator] which function to use for reading
  /// bytes from a URL. Used in tests to inject a fake reader. Defaults to
  /// [http.readBytes].
  final HttpReader httpReader;

276
  final Map<String, String> _version = <String, String>{};
277
  late String _flutter;
278 279 280 281 282 283 284
  late String _dart;

  late final Future<String> _dartArch = (() async {
    // Parse 'arch' out of a string like '... "os_arch"\n'.
    return (await _runDart(<String>['--version']))
        .trim().split(' ').last.replaceAll('"', '').split('_')[1];
  })();
285

286 287
  /// Returns a default archive name when given a Git revision.
  /// Used when an output filename is not given.
288
  Future<String> get _archiveName async {
289
    final String os = platform.operatingSystem.toLowerCase();
290 291
    // Include the intended host archetecture in the file name for non-x64.
    final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_';
292 293 294 295 296 297 298 299
    // We don't use .tar.xz on Mac because although it can unpack them
    // on the command line (with tar), the "Archive Utility" that runs
    // when you double-click on them just does some crazy behavior (it
    // converts it to a compressed cpio archive, and when you double
    // click on that, it converts it back to .tar.xz, without ever
    // unpacking it!) So, we use .zip for Mac, and the files are about
    // 220MB larger than they need to be. :-(
    final String suffix = platform.isLinux ? 'tar.xz' : 'zip';
300 301
    final String package = '${os}_$arch${_version[frameworkVersionTag]}';
    return 'flutter_$package-${branch.name}.$suffix';
302 303 304 305
  }

  /// Checks out the flutter repo and prepares it for other operations.
  ///
306 307 308
  /// Returns the version for this release as obtained from the git tags, and
  /// the dart version as obtained from `flutter --version`.
  Future<Map<String, String>> initializeRepo() async {
309
    await _checkoutFlutter();
310 311 312
    if (_version.isEmpty) {
      _version.addAll(await _getVersion());
    }
313
    return _version;
314 315
  }

316
  /// Performs all of the steps needed to create an archive.
317
  Future<File> createArchive() async {
318
    assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive');
319 320 321 322
    final File outputFile = File(path.join(
      outputDir.absolute.path,
      await _archiveName,
    ));
323 324
    await _installMinGitIfNeeded();
    await _populateCaches();
325
    await _validate();
326 327
    await _archiveFiles(outputFile);
    return outputFile;
328 329
  }

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
  /// Validates the integrity of the release package.
  ///
  /// Currently only checks that macOS binaries are codesigned. Will throw a
  /// [PreparePackageException] if the test failes.
  Future<void> _validate() async {
    // Only validate in strict mode, which means `--publish`
    if (!strict || !platform.isMacOS) {
      return;
    }
    // Validate that the dart binary is codesigned
    try {
      // TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701
      await _processRunner.runProcess(
        <String>[
          'codesign',
          '-vvvv',
          '--check-notarization',
347
          _dart,
348 349 350 351 352
        ],
        workingDirectory: flutterRoot,
      );
    } on PreparePackageException catch (e) {
      throw PreparePackageException(
353
        'The binary $_dart was not codesigned!\n${e.message}',
354 355 356 357
      );
    }
  }

358 359
  /// Returns the version map of this release, according the to tags in the
  /// repo and the output of `flutter --version --machine`.
360 361 362 363 364
  ///
  /// This looks for the tag attached to [revision] and, if it doesn't find one,
  /// git will give an error.
  ///
  /// If [strict] is true, the exact [revision] must be tagged to return the
365
  /// version. If [strict] is not true, will look backwards in time starting at
366
  /// [revision] to find the most recent version tag.
367 368 369 370 371 372
  ///
  /// The version found as a git tag is added to the information given by
  /// `flutter --version --machine` with the `frameworkVersionFromGit` tag, and
  /// returned.
  Future<Map<String, String>> _getVersion() async {
    String gitVersion;
373 374
    if (strict) {
      try {
375
        gitVersion = await _runGit(<String>['describe', '--tags', '--exact-match', revision]);
376
      } on PreparePackageException catch (exception) {
377
        throw PreparePackageException(
378 379 380 381
          'Git error when checking for a version tag attached to revision $revision.\n'
          'Perhaps there is no tag at that revision?:\n'
          '$exception'
        );
382 383
      }
    } else {
384
      gitVersion = await _runGit(<String>['describe', '--tags', '--abbrev=0', revision]);
385
    }
386 387 388 389 390 391 392 393 394
    // Run flutter command twice, once to make sure the flutter command is built
    // and ready (and thus won't output any junk on stdout the second time), and
    // once to capture theJSON output. The second run should be fast.
    await _runFlutter(<String>['--version', '--machine']);
    final String versionJson = await _runFlutter(<String>['--version', '--machine']);
    final Map<String, String> versionMap = <String, String>{};
    final Map<String, dynamic> result = json.decode(versionJson) as Map<String, dynamic>;
    result.forEach((String key, dynamic value) => versionMap[key] = value.toString());
    versionMap[frameworkVersionTag] = gitVersion;
395
    versionMap[dartTargetArchTag] = await _dartArch;
396
    return versionMap;
397
  }
398 399 400

  /// Clone the Flutter repo and make sure that the git environment is sane
  /// for when the user will unpack it.
401
  Future<void> _checkoutFlutter() async {
402 403
    // We want the user to start out the in the specified branch instead of a
    // detached head. To do that, we need to make sure the branch points at the
404
    // desired revision.
405
    await _runGit(<String>['clone', '-b', branch.name, gobMirror], workingDirectory: tempDir);
406
    await _runGit(<String>['reset', '--hard', revision]);
407 408

    // Make the origin point to github instead of the chromium mirror.
409
    await _runGit(<String>['remote', 'set-url', 'origin', githubRepo]);
410 411 412

    // Minify `.git` footprint (saving about ~100 MB as of Oct 2022)
    await _runGit(<String>['gc', '--prune=now', '--aggressive']);
413 414 415
  }

  /// Retrieve the MinGit executable from storage and unpack it.
416
  Future<void> _installMinGitIfNeeded() async {
417
    if (!platform.isWindows) {
418 419
      return;
    }
420
    final Uint8List data = await httpReader(_minGitUri);
421
    final File gitFile = File(path.join(tempDir.absolute.path, 'mingit.zip'));
422 423
    await gitFile.writeAsBytes(data, flush: true);

424
    final Directory minGitPath = Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit'));
425
    await minGitPath.create(recursive: true);
426
    await _unzipArchive(gitFile, workingDirectory: minGitPath);
427 428 429
  }

  /// Prepare the archive repo so that it has all of the caches warmed up and
430
  /// is configured for the user to begin working.
431
  Future<void> _populateCaches() async {
432 433 434 435 436 437
    await _runFlutter(<String>['doctor']);
    await _runFlutter(<String>['update-packages']);
    await _runFlutter(<String>['precache']);
    await _runFlutter(<String>['ide-config']);

    // Create each of the templates, since they will call 'pub get' on
438 439
    // themselves when created, and this will warm the cache with their
    // dependencies too.
440
    for (final String template in <String>['app', 'package', 'plugin']) {
441
      final String createName = path.join(tempDir.path, 'create_$template');
442
      await _runFlutter(
443
        <String>['create', '--template=$template', createName],
444 445 446
        // Run it outside the cloned Flutter repo to not nest git repos, since
        // they'll be git repos themselves too.
        workingDirectory: tempDir,
447 448 449 450 451 452
      );
    }

    // Yes, we could just skip all .packages files when constructing
    // the archive, but some are checked in, and we don't want to skip
    // those.
453 454 455 456 457 458 459 460
    await _runGit(<String>[
      'clean',
      '-f',
      // Do not -X as it could lead to entire bin/cache getting cleaned
      '-x',
      '--',
      '**/.packages',
    ]);
461
    /// Remove package_config files and any contents in .dart_tool
462 463 464 465 466 467 468 469 470 471 472 473 474 475
    await _runGit(<String>[
      'clean',
      '-f',
      '-x',
      '--',
      '**/.dart_tool/',
    ]);

    // Ensure the above commands do not clean out the cache
    final Directory flutterCache = Directory(path.join(flutterRoot.absolute.path, 'bin', 'cache'));
    if (!flutterCache.existsSync()) {
      throw Exception('The flutter cache was not found at ${flutterCache.path}!');
    }

476 477 478 479 480 481
    /// Remove git subfolder from .pub-cache, this contains the flutter goldens
    /// and new flutter_gallery.
    final Directory gitCache = Directory(path.join(flutterRoot.absolute.path, '.pub-cache', 'git'));
    if (gitCache.existsSync()) {
      gitCache.deleteSync(recursive: true);
    }
482 483
  }

484
  /// Write the archive to the given output file.
485
  Future<void> _archiveFiles(File outputFile) async {
486
    if (outputFile.path.toLowerCase().endsWith('.zip')) {
487
      await _createZipArchive(outputFile, flutterRoot);
488
    } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
489
      await _createTarArchive(outputFile, flutterRoot);
490 491 492
    }
  }

493 494 495 496 497 498 499
  Future<String> _runDart(List<String> args, {Directory? workingDirectory}) {
    return _processRunner.runProcess(
      <String>[_dart, ...args],
      workingDirectory: workingDirectory ?? flutterRoot,
    );
  }

500
  Future<String> _runFlutter(List<String> args, {Directory? workingDirectory}) {
501 502
    return _processRunner.runProcess(
      <String>[_flutter, ...args],
503 504
      workingDirectory: workingDirectory ?? flutterRoot,
    );
505
  }
506

507
  Future<String> _runGit(List<String> args, {Directory? workingDirectory}) {
508 509 510 511
    return _processRunner.runProcess(
      <String>['git', ...args],
      workingDirectory: workingDirectory ?? flutterRoot,
    );
512 513
  }

514 515
  /// Unpacks the given zip file into the currentDirectory (if set), or the
  /// same directory as the archive.
516
  Future<String> _unzipArchive(File archive, {Directory? workingDirectory}) {
517
    workingDirectory ??= Directory(path.dirname(archive.absolute.path));
518 519 520 521 522 523 524 525 526 527 528 529 530
    List<String> commandLine;
    if (platform.isWindows) {
      commandLine = <String>[
        '7za',
        'x',
        archive.absolute.path,
      ];
    } else {
      commandLine = <String>[
        'unzip',
        archive.absolute.path,
      ];
    }
531
    return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
532 533
  }

534
  /// Create a zip archive from the directory source.
535
  Future<String> _createZipArchive(File output, Directory source) async {
536 537
    List<String> commandLine;
    if (platform.isWindows) {
538
      // Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib.
539
      await _processRunner.runProcess(
540 541 542
        <String>['attrib', '-h', '.git'],
        workingDirectory: Directory(source.absolute.path),
      );
543 544 545 546 547 548 549 550 551 552 553 554 555
      commandLine = <String>[
        '7za',
        'a',
        '-tzip',
        '-mx=9',
        output.absolute.path,
        path.basename(source.path),
      ];
    } else {
      commandLine = <String>[
        'zip',
        '-r',
        '-9',
556
        '--symlinks',
557 558 559 560
        output.absolute.path,
        path.basename(source.path),
      ];
    }
561
    return _processRunner.runProcess(
562
      commandLine,
563
      workingDirectory: Directory(path.dirname(source.absolute.path)),
564
    );
565 566
  }

567
  /// Create a tar archive from the directory source.
568 569 570 571 572 573 574
  Future<String> _createTarArchive(File output, Directory source) {
    return _processRunner.runProcess(<String>[
      'tar',
      'cJf',
      output.absolute.path,
      path.basename(source.absolute.path),
    ], workingDirectory: Directory(path.dirname(source.absolute.path)));
575
  }
576
}
577

578 579 580 581 582 583
class ArchivePublisher {
  ArchivePublisher(
    this.tempDir,
    this.revision,
    this.branch,
    this.version,
584 585
    this.outputFile,
    this.dryRun, {
586
    ProcessManager? processManager,
587 588
    bool subprocessOutput = true,
    this.platform = const LocalPlatform(),
589 590
  })  : assert(revision.length == 40),
        platformName = platform.operatingSystem.toLowerCase(),
591
        metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
592 593
        _processRunner = ProcessRunner(
          processManager: processManager,
594
          subprocessOutput: subprocessOutput,
595
        );
596 597 598 599 600 601

  final Platform platform;
  final String platformName;
  final String metadataGsPath;
  final Branch branch;
  final String revision;
602
  final Map<String, String> version;
603 604 605
  final Directory tempDir;
  final File outputFile;
  final ProcessRunner _processRunner;
606
  final bool dryRun;
607
  String get destinationArchivePath => '${branch.name}/$platformName/${path.basename(outputFile.path)}';
608
  static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json';
609

610 611 612 613 614 615 616 617 618 619 620 621
  Future<String> _getChecksum(File archiveFile) async {
    final DigestSink digestSink = DigestSink();
    final ByteConversionSink sink = sha256.startChunkedConversion(digestSink);

    final Stream<List<int>> stream = archiveFile.openRead();
    await stream.forEach((List<int> chunk) {
      sink.add(chunk);
    });
    sink.close();
    return digestSink.value.toString();
  }

622
  /// Publish the archive to Google Storage.
623 624 625 626
  ///
  /// This method will throw if the target archive already exists on cloud
  /// storage.
  Future<void> publishArchive([bool forceUpload = false]) async {
627 628 629 630 631 632
    final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
    if (!forceUpload) {
      if (await _cloudPathExists(destGsPath) && !dryRun) {
        throw PreparePackageException(
          'File $destGsPath already exists on cloud storage!',
        );
633 634
      }
    }
635 636 637 638 639 640
    await _cloudCopy(
      src: outputFile.absolute.path,
      dest: destGsPath,
    );
    assert(tempDir.existsSync());
    await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}');
641 642
  }

643 644
  Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
    jsonData['base_url'] = '$baseUrl$releaseFolder';
645 646 647
    if (!jsonData.containsKey('current_release')) {
      jsonData['current_release'] = <String, String>{};
    }
648
    (jsonData['current_release'] as Map<String, dynamic>)[branch.name] = revision;
649 650 651 652 653 654
    if (!jsonData.containsKey('releases')) {
      jsonData['releases'] = <Map<String, dynamic>>[];
    }

    final Map<String, dynamic> newEntry = <String, dynamic>{};
    newEntry['hash'] = revision;
655
    newEntry['channel'] = branch.name;
656 657
    newEntry['version'] = version[frameworkVersionTag];
    newEntry['dart_sdk_version'] = version[dartVersionTag];
658
    newEntry['dart_sdk_arch'] = version[dartTargetArchTag];
659
    newEntry['release_date'] = DateTime.now().toUtc().toIso8601String();
660
    newEntry['archive'] = destinationArchivePath;
661
    newEntry['sha256'] = await _getChecksum(outputFile);
662 663

    // Search for any entries with the same hash and channel and remove them.
664
    final List<dynamic> releases = jsonData['releases'] as List<dynamic>;
665
    jsonData['releases'] = <Map<String, dynamic>>[
666
      for (final Map<String, dynamic> entry in releases.cast<Map<String, dynamic>>())
667 668 669
        if (entry['hash'] != newEntry['hash'] ||
            entry['channel'] != newEntry['channel'] ||
            entry['dart_sdk_arch'] != newEntry['dart_sdk_arch'])
670 671 672
          entry,
      newEntry,
    ]..sort((Map<String, dynamic> a, Map<String, dynamic> b) {
673 674
      final DateTime aDate = DateTime.parse(a['release_date'] as String);
      final DateTime bDate = DateTime.parse(b['release_date'] as String);
675 676 677 678 679
      return bDate.compareTo(aDate);
    });
    return jsonData;
  }

680
  Future<void> _updateMetadata(String gsPath) async {
681 682 683 684
    // We can't just cat the metadata from the server with 'gsutil cat', because
    // Windows wants to echo the commands that execute in gsutil.bat to the
    // stdout when we do that. So, we copy the file locally and then read it
    // back in.
685
    final File metadataFile = File(
686 687
      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
    );
688
    await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
689
    Map<String, dynamic> jsonData = <String, dynamic>{};
690 691 692 693 694 695 696 697 698 699 700
    if (!dryRun) {
      final String currentMetadata = metadataFile.readAsStringSync();
      if (currentMetadata.isEmpty) {
        throw PreparePackageException('Empty metadata received from server');
      }
      try {
        jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
      } on FormatException catch (e) {
        throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
      }
    }
701 702 703 704 705 706 707
    // Run _addRelease, even on a dry run, so we can inspect the metadata on a
    // dry run. On a dry run, the only thing in the metadata file be the new
    // release.
    jsonData = await _addRelease(jsonData);

    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
    metadataFile.writeAsStringSync(encoder.convert(jsonData));
708 709 710 711 712 713 714 715
    await _cloudCopy(
      src: metadataFile.absolute.path,
      dest: gsPath,
      // This metadata file is used by the website, so we don't want a long
      // latency between publishing a release and it being available on the
      // site.
      cacheSeconds: shortCacheSeconds,
    );
716 717
  }

718 719
  Future<String> _runGsUtil(
    List<String> args, {
720
    Directory? workingDirectory,
721
    bool failOk = false,
722
  }) async {
723 724 725 726
    if (dryRun) {
      print('gsutil.py -- $args');
      return '';
    }
727
    if (platform.isWindows) {
728
      return _processRunner.runProcess(
729
        <String>['python3', path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'), '--', ...args],
730 731 732 733 734
        workingDirectory: workingDirectory,
        failOk: failOk,
      );
    }

735
    return _processRunner.runProcess(
736
      <String>['gsutil.py', '--', ...args],
737 738 739 740 741
      workingDirectory: workingDirectory,
      failOk: failOk,
    );
  }

742 743 744 745 746 747 748 749 750 751 752 753 754
  /// Determine if a file exists at a given [cloudPath].
  Future<bool> _cloudPathExists(String cloudPath) async {
    try {
      await _runGsUtil(
        <String>['stat', cloudPath],
      );
    } on PreparePackageException {
      // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist
      return false;
    }
    return true;
  }

755
  Future<String> _cloudCopy({
756 757 758
    required String src,
    required String dest,
    int? cacheSeconds,
759
  }) async {
760 761
    // We often don't have permission to overwrite, but
    // we have permission to remove, so that's what we do.
762
    await _runGsUtil(<String>['rm', dest], failOk: true);
763
    String? mimeType;
764 765 766 767 768 769 770 771 772
    if (dest.endsWith('.tar.xz')) {
      mimeType = 'application/x-gtar';
    }
    if (dest.endsWith('.zip')) {
      mimeType = 'application/zip';
    }
    if (dest.endsWith('.json')) {
      mimeType = 'application/json';
    }
773
    return _runGsUtil(<String>[
774 775 776
      // Use our preferred MIME type for the files we care about
      // and let gsutil figure it out for anything else.
      if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
777
      if (cacheSeconds != null) ...<String>['-h', 'Cache-Control:max-age=$cacheSeconds'],
778 779 780 781
      'cp',
      src,
      dest,
    ]);
782 783 784 785 786 787 788
  }
}

/// Prepares a flutter git repo to be packaged up for distribution.
/// It mainly serves to populate the .pub-cache with any appropriate Dart
/// packages, and the flutter cache in bin/cache with the appropriate
/// dependencies and snapshots.
789
///
Ian Hickson's avatar
Ian Hickson committed
790 791
/// Archives contain the executables and customizations for the platform that
/// they are created on.
Ian Hickson's avatar
Ian Hickson committed
792
Future<void> main(List<String> rawArguments) async {
793
  final ArgParser argParser = ArgParser();
794 795 796 797
  argParser.addOption(
    'temp_dir',
    help: 'A location where temporary files may be written. Defaults to a '
        'directory in the system temp folder. Will write a few GiB of data, '
798 799 800
        'so it should have sufficient free space. If a temp_dir is not '
        'specified, then the default temp_dir will be created, used, and '
        'removed automatically.',
801
  );
802 803 804
  argParser.addOption('revision',
      help: 'The Flutter git repo revision to build the '
          'archive with. Must be the full 40-character hash. Required.');
805
  argParser.addOption(
806
    'branch',
807
    allowed: Branch.values.map<String>((Branch branch) => branch.name),
808
    help: 'The Flutter branch to build the archive with. Required.',
809 810 811
  );
  argParser.addOption(
    'output',
812 813 814 815 816 817 818
    help: 'The path to the directory where the output archive should be '
        'written. If --output is not specified, the archive will be written to '
        "the current directory. If the output directory doesn't exist, it, and "
        'the path to it, will be created.',
  );
  argParser.addFlag(
    'publish',
819 820
    help: 'If set, will publish the archive to Google Cloud Storage upon '
        'successful creation of the archive. Will publish under this '
821
        'directory: $baseUrl$releaseFolder',
822
  );
823 824 825 826 827
  argParser.addFlag(
    'force',
    abbr: 'f',
    help: 'Overwrite a previously uploaded package.',
  );
828 829 830 831 832
  argParser.addFlag(
    'dry_run',
    negatable: false,
    help: 'Prints gsutil commands instead of executing them.',
  );
833 834 835 836
  argParser.addFlag(
    'help',
    negatable: false,
    help: 'Print help for this command.',
837
  );
838

Ian Hickson's avatar
Ian Hickson committed
839
  final ArgResults parsedArguments = argParser.parse(rawArguments);
840

841
  if (parsedArguments['help'] as bool) {
842 843 844 845
    print(argParser.usage);
    exit(0);
  }

846 847 848 849 850 851
  void errorExit(String message, {int exitCode = -1}) {
    stderr.write('Error: $message\n\n');
    stderr.write('${argParser.usage}\n');
    exit(exitCode);
  }

852
  if (!parsedArguments.wasParsed('revision')) {
853 854
    errorExit('Invalid argument: --revision must be specified.');
  }
855
  final String revision = parsedArguments['revision'] as String;
856 857 858 859
  if (revision.length != 40) {
    errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.');
  }

860
  if (!parsedArguments.wasParsed('branch')) {
861 862
    errorExit('Invalid argument: --branch must be specified.');
  }
863

864
  final String? tempDirArg = parsedArguments['temp_dir'] as String?;
865
  Directory tempDir;
866
  bool removeTempDir = false;
867
  if (tempDirArg == null || tempDirArg.isEmpty) {
868
    tempDir = Directory.systemTemp.createTempSync('flutter_package.');
869 870
    removeTempDir = true;
  } else {
871
    tempDir = Directory(tempDirArg);
872
    if (!tempDir.existsSync()) {
873
      errorExit("Temporary directory $tempDirArg doesn't exist.");
874 875 876
    }
  }

877
  Directory outputDir;
Ian Hickson's avatar
Ian Hickson committed
878
  if (parsedArguments['output'] == null) {
879
    outputDir = tempDir;
880
  } else {
881
    outputDir = Directory(parsedArguments['output'] as String);
882 883
    if (!outputDir.existsSync()) {
      outputDir.createSync(recursive: true);
884
    }
885 886
  }

887 888
  final bool publish = parsedArguments['publish'] as bool;
  final bool dryRun = parsedArguments['dry_run'] as bool;
889
  final Branch branch = Branch.values.byName(parsedArguments['branch'] as String);
890 891 892 893 894 895 896
  final ArchiveCreator creator = ArchiveCreator(
    tempDir,
    outputDir,
    revision,
    branch,
    strict: publish && !dryRun,
  );
897
  int exitCode = 0;
898
  late String message;
899
  try {
900
    final Map<String, String> version = await creator.initializeRepo();
901
    final File outputFile = await creator.createArchive();
902
    if (parsedArguments['publish'] as bool) {
903
      final ArchivePublisher publisher = ArchivePublisher(
904 905 906 907 908
        tempDir,
        revision,
        branch,
        version,
        outputFile,
909
        dryRun,
910
      );
911
      await publisher.publishArchive(parsedArguments['force'] as bool);
912
    }
913
  } on PreparePackageException catch (e) {
914
    exitCode = e.exitCode;
915 916
    message = e.message;
  } catch (e) {
917
    exitCode = -1;
918
    message = e.toString();
919 920
  } finally {
    if (removeTempDir) {
921
      tempDir.deleteSync(recursive: true);
922 923 924 925 926 927 928
    }
    if (exitCode != 0) {
      errorExit(message, exitCode: exitCode);
    }
    exit(0);
  }
}