prepare_package.dart 32.3 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:convert';
7
import 'dart:io' hide Platform;
8
import 'dart:typed_data';
9 10

import 'package:args/args.dart';
11 12
import 'package:crypto/crypto.dart';
import 'package:crypto/src/digest_sink.dart';
13
import 'package:http/http.dart' as http;
14
import 'package:path/path.dart' as path;
15
import 'package:platform/platform.dart' show LocalPlatform, Platform;
16
import 'package:process/process.dart';
17

18 19
const String gobMirror =
    'https://flutter.googlesource.com/mirrors/flutter';
20
const String githubRepo = 'https://github.com/flutter/flutter.git';
21
const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
22
    '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
23
const String releaseFolder = '/releases';
24 25 26
const String gsBase = 'gs://flutter_infra_release';
const String gsReleaseFolder = '$gsBase$releaseFolder';
const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
27
const int shortCacheSeconds = 60;
28 29
const String frameworkVersionTag = 'frameworkVersionFromGit';
const String dartVersionTag = 'dartSdkVersion';
30
const String dartTargetArchTag = 'dartTargetArch';
31

32
/// Exception class for when a process fails to run, so we can catch
33
/// it and provide something more readable than a stack trace.
34
class PreparePackageException implements Exception {
35
  PreparePackageException(this.message, [this.result]);
36

37
  final String message;
38
  final ProcessResult? result;
39
  int get exitCode => result?.exitCode ?? -1;
40 41

  @override
42 43 44 45 46
  String toString() {
    String output = runtimeType.toString();
    if (message != null) {
      output += ': $message';
    }
47
    final String stderr = result?.stderr as String? ?? '';
48 49 50 51 52
    if (stderr.isNotEmpty) {
      output += ':\n$stderr';
    }
    return output;
  }
53 54
}

55
enum Branch { dev, beta, stable }
56 57 58 59 60 61 62

String getBranchName(Branch branch) {
  switch (branch) {
    case Branch.beta:
      return 'beta';
    case Branch.dev:
      return 'dev';
63 64
    case Branch.stable:
      return 'stable';
65 66 67 68 69 70 71 72 73
  }
}

Branch fromBranchName(String name) {
  switch (name) {
    case 'beta':
      return Branch.beta;
    case 'dev':
      return Branch.dev;
74 75
    case 'stable':
      return Branch.stable;
76
    default:
77
      throw ArgumentError('Invalid branch name.');
78 79 80
  }
}

81 82 83 84 85
/// A helper class for classes that want to run a process, optionally have the
/// stderr and stdout reported as the process runs, and capture the stdout
/// properly without dropping any.
class ProcessRunner {
  ProcessRunner({
86
    ProcessManager? processManager,
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
    this.subprocessOutput = true,
    this.defaultWorkingDirectory,
    this.platform = const LocalPlatform(),
  }) : processManager = processManager ?? const LocalProcessManager() {
    environment = Map<String, String>.from(platform.environment);
  }

  /// The platform to use for a starting environment.
  final Platform platform;

  /// Set [subprocessOutput] to show output as processes run. Stdout from the
  /// process will be printed to stdout, and stderr printed to stderr.
  final bool subprocessOutput;

  /// Set the [processManager] in order to inject a test instance to perform
  /// testing.
  final ProcessManager processManager;

  /// Sets the default directory used when `workingDirectory` is not specified
  /// to [runProcess].
107
  final Directory? defaultWorkingDirectory;
108 109

  /// The environment to run processes with.
110
  late Map<String, String> environment;
111 112 113 114 115 116 117 118 119

  /// Run the command and arguments in `commandLine` as a sub-process from
  /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
  /// [Directory.current] if [defaultWorkingDirectory] is not set.
  ///
  /// Set `failOk` if [runProcess] should not throw an exception when the
  /// command completes with a non-zero exit code.
  Future<String> runProcess(
    List<String> commandLine, {
120
    Directory? workingDirectory,
121 122 123 124 125 126 127 128 129
    bool failOk = false,
  }) async {
    workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
    if (subprocessOutput) {
      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
    }
    final List<int> output = <int>[];
    final Completer<void> stdoutComplete = Completer<void>();
    final Completer<void> stderrComplete = Completer<void>();
130
    late Process process;
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
    Future<int> allComplete() async {
      await stderrComplete.future;
      await stdoutComplete.future;
      return process.exitCode;
    }

    try {
      process = await processManager.start(
        commandLine,
        workingDirectory: workingDirectory.absolute.path,
        environment: environment,
      );
      process.stdout.listen(
        (List<int> event) {
          output.addAll(event);
          if (subprocessOutput) {
            stdout.add(event);
          }
        },
        onDone: () async => stdoutComplete.complete(),
      );
      if (subprocessOutput) {
        process.stderr.listen(
          (List<int> event) {
            stderr.add(event);
          },
          onDone: () async => stderrComplete.complete(),
        );
      } else {
        stderrComplete.complete();
      }
    } on ProcessException catch (e) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
164
          'failed with:\n$e';
165 166 167
      throw PreparePackageException(message);
    } on ArgumentError catch (e) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
168
          'failed with:\n$e';
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
      throw PreparePackageException(message);
    }

    final int exitCode = await allComplete();
    if (exitCode != 0 && !failOk) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
      throw PreparePackageException(
        message,
        ProcessResult(0, exitCode, null, 'returned $exitCode'),
      );
    }
    return utf8.decoder.convert(output).trim();
  }
}

184
typedef HttpReader = Future<Uint8List> Function(Uri url, {Map<String, String> headers});
185

186 187
/// Creates a pre-populated Flutter archive from a git repo.
class ArchiveCreator {
188
  /// [tempDir] is the directory to use for creating the archive.  The script
189 190 191
  /// will place several GiB of data there, so it should have available space.
  ///
  /// The processManager argument is used to inject a mock of [ProcessManager] for
192
  /// testing purposes.
193 194 195
  ///
  /// If subprocessOutput is true, then output from processes invoked during
  /// archive creation is echoed to stderr and stdout.
196 197 198 199 200 201
  factory ArchiveCreator(
    Directory tempDir,
    Directory outputDir,
    String revision,
    Branch branch, {
    bool strict = true,
202
    ProcessManager? processManager,
203
    bool subprocessOutput = true,
204
    Platform platform = const LocalPlatform(),
205
    HttpReader? httpReader,
206 207 208 209 210 211 212 213 214 215
  }) {
    final Directory flutterRoot = Directory(path.join(tempDir.path, 'flutter'));
    final ProcessRunner processRunner = ProcessRunner(
      processManager: processManager,
      subprocessOutput: subprocessOutput,
      platform: platform,
    )..environment['PUB_CACHE'] = path.join(
      flutterRoot.absolute.path, '.pub-cache',
    );
    final String flutterExecutable = path.join(
216
      flutterRoot.absolute.path,
217
      'bin',
218
      'flutter',
219
    );
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
    final String dartExecutable = path.join(
      flutterRoot.absolute.path,
      'bin',
      'cache',
      'dart-sdk',
      'bin',
      'dart',
    );

    return ArchiveCreator._(
      tempDir: tempDir,
      platform: platform,
      flutterRoot: flutterRoot,
      outputDir: outputDir,
      revision: revision,
      branch: branch,
      strict: strict,
      processRunner: processRunner,
      httpReader: httpReader ?? http.readBytes,
      flutterExecutable: flutterExecutable,
      dartExecutable: dartExecutable,
    );
242 243
  }

244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
  ArchiveCreator._({
    required this.tempDir,
    required this.platform,
    required this.flutterRoot,
    required this.outputDir,
    required this.revision,
    required this.branch,
    required this.strict,
    required ProcessRunner processRunner,
    required this.httpReader,
    required String flutterExecutable,
    required String dartExecutable,
  }) :
    assert(revision.length == 40),
    _processRunner = processRunner,
    _flutter = flutterExecutable,
    _dart = dartExecutable;

262 263
  /// The platform to use for the environment and determining which
  /// platform we're running on.
264
  final Platform platform;
265 266

  /// The branch to build the archive for.  The branch must contain [revision].
267
  final Branch branch;
268 269 270 271 272

  /// The git revision hash to build the archive for. This revision has
  /// to be available in the [branch], although it doesn't have to be
  /// at HEAD, since we clone the branch and then reset to this revision
  /// to create the archive.
273
  final String revision;
274 275

  /// The flutter root directory in the [tempDir].
276
  final Directory flutterRoot;
277 278

  /// The temporary directory used to build the archive in.
279
  final Directory tempDir;
280 281

  /// The directory to write the output file to.
282
  final Directory outputDir;
283

284 285 286 287 288
  /// True if the creator should be strict about checking requirements or not.
  ///
  /// In strict mode, will insist that the [revision] be a tagged revision.
  final bool strict;

289
  final Uri _minGitUri = Uri.parse(mingitForWindowsUrl);
290 291
  final ProcessRunner _processRunner;

292 293 294 295 296
  /// Used to tell the [ArchiveCreator] which function to use for reading
  /// bytes from a URL. Used in tests to inject a fake reader. Defaults to
  /// [http.readBytes].
  final HttpReader httpReader;

297
  final Map<String, String> _version = <String, String>{};
298
  late String _flutter;
299 300 301 302 303 304 305
  late String _dart;

  late final Future<String> _dartArch = (() async {
    // Parse 'arch' out of a string like '... "os_arch"\n'.
    return (await _runDart(<String>['--version']))
        .trim().split(' ').last.replaceAll('"', '').split('_')[1];
  })();
306 307 308

  /// Get the name of the channel as a string.
  String get branchName => getBranchName(branch);
309 310 311

  /// Returns a default archive name when given a Git revision.
  /// Used when an output filename is not given.
312
  Future<String> get _archiveName async {
313
    final String os = platform.operatingSystem.toLowerCase();
314 315
    // Include the intended host archetecture in the file name for non-x64.
    final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_';
316 317 318 319 320 321 322 323
    // We don't use .tar.xz on Mac because although it can unpack them
    // on the command line (with tar), the "Archive Utility" that runs
    // when you double-click on them just does some crazy behavior (it
    // converts it to a compressed cpio archive, and when you double
    // click on that, it converts it back to .tar.xz, without ever
    // unpacking it!) So, we use .zip for Mac, and the files are about
    // 220MB larger than they need to be. :-(
    final String suffix = platform.isLinux ? 'tar.xz' : 'zip';
324
    return 'flutter_${os}_$arch${_version[frameworkVersionTag]}-$branchName.$suffix';
325 326 327 328
  }

  /// Checks out the flutter repo and prepares it for other operations.
  ///
329 330 331
  /// Returns the version for this release as obtained from the git tags, and
  /// the dart version as obtained from `flutter --version`.
  Future<Map<String, String>> initializeRepo() async {
332
    await _checkoutFlutter();
333 334 335
    if (_version.isEmpty) {
      _version.addAll(await _getVersion());
    }
336
    return _version;
337 338
  }

339
  /// Performs all of the steps needed to create an archive.
340
  Future<File> createArchive() async {
341
    assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive');
342 343 344 345
    final File outputFile = File(path.join(
      outputDir.absolute.path,
      await _archiveName,
    ));
346 347
    await _installMinGitIfNeeded();
    await _populateCaches();
348
    await _validate();
349 350
    await _archiveFiles(outputFile);
    return outputFile;
351 352
  }

353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
  /// Validates the integrity of the release package.
  ///
  /// Currently only checks that macOS binaries are codesigned. Will throw a
  /// [PreparePackageException] if the test failes.
  Future<void> _validate() async {
    // Only validate in strict mode, which means `--publish`
    if (!strict || !platform.isMacOS) {
      return;
    }
    // Validate that the dart binary is codesigned
    try {
      // TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701
      await _processRunner.runProcess(
        <String>[
          'codesign',
          '-vvvv',
          '--check-notarization',
370
          _dart,
371 372 373 374 375
        ],
        workingDirectory: flutterRoot,
      );
    } on PreparePackageException catch (e) {
      throw PreparePackageException(
376
        'The binary $_dart was not codesigned!\n${e.message}',
377 378 379 380
      );
    }
  }

381 382
  /// Returns the version map of this release, according the to tags in the
  /// repo and the output of `flutter --version --machine`.
383 384 385 386 387 388 389
  ///
  /// This looks for the tag attached to [revision] and, if it doesn't find one,
  /// git will give an error.
  ///
  /// If [strict] is true, the exact [revision] must be tagged to return the
  /// version.  If [strict] is not true, will look backwards in time starting at
  /// [revision] to find the most recent version tag.
390 391 392 393 394 395
  ///
  /// The version found as a git tag is added to the information given by
  /// `flutter --version --machine` with the `frameworkVersionFromGit` tag, and
  /// returned.
  Future<Map<String, String>> _getVersion() async {
    String gitVersion;
396 397
    if (strict) {
      try {
398
        gitVersion = await _runGit(<String>['describe', '--tags', '--exact-match', revision]);
399
      } on PreparePackageException catch (exception) {
400
        throw PreparePackageException(
401 402 403 404
          'Git error when checking for a version tag attached to revision $revision.\n'
          'Perhaps there is no tag at that revision?:\n'
          '$exception'
        );
405 406
      }
    } else {
407
      gitVersion = await _runGit(<String>['describe', '--tags', '--abbrev=0', revision]);
408
    }
409 410 411 412 413 414 415 416 417
    // Run flutter command twice, once to make sure the flutter command is built
    // and ready (and thus won't output any junk on stdout the second time), and
    // once to capture theJSON output. The second run should be fast.
    await _runFlutter(<String>['--version', '--machine']);
    final String versionJson = await _runFlutter(<String>['--version', '--machine']);
    final Map<String, String> versionMap = <String, String>{};
    final Map<String, dynamic> result = json.decode(versionJson) as Map<String, dynamic>;
    result.forEach((String key, dynamic value) => versionMap[key] = value.toString());
    versionMap[frameworkVersionTag] = gitVersion;
418
    versionMap[dartTargetArchTag] = await _dartArch;
419
    return versionMap;
420
  }
421 422 423

  /// Clone the Flutter repo and make sure that the git environment is sane
  /// for when the user will unpack it.
424
  Future<void> _checkoutFlutter() async {
425 426
    // We want the user to start out the in the specified branch instead of a
    // detached head. To do that, we need to make sure the branch points at the
427
    // desired revision.
428
    await _runGit(<String>['clone', '-b', branchName, gobMirror], workingDirectory: tempDir);
429
    await _runGit(<String>['reset', '--hard', revision]);
430 431

    // Make the origin point to github instead of the chromium mirror.
432
    await _runGit(<String>['remote', 'set-url', 'origin', githubRepo]);
433 434 435
  }

  /// Retrieve the MinGit executable from storage and unpack it.
436
  Future<void> _installMinGitIfNeeded() async {
437
    if (!platform.isWindows) {
438 439
      return;
    }
440
    final Uint8List data = await httpReader(_minGitUri);
441
    final File gitFile = File(path.join(tempDir.absolute.path, 'mingit.zip'));
442 443
    await gitFile.writeAsBytes(data, flush: true);

444
    final Directory minGitPath = Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit'));
445
    await minGitPath.create(recursive: true);
446
    await _unzipArchive(gitFile, workingDirectory: minGitPath);
447 448 449
  }

  /// Prepare the archive repo so that it has all of the caches warmed up and
450
  /// is configured for the user to begin working.
451
  Future<void> _populateCaches() async {
452 453 454 455 456 457
    await _runFlutter(<String>['doctor']);
    await _runFlutter(<String>['update-packages']);
    await _runFlutter(<String>['precache']);
    await _runFlutter(<String>['ide-config']);

    // Create each of the templates, since they will call 'pub get' on
458 459
    // themselves when created, and this will warm the cache with their
    // dependencies too.
460
    for (final String template in <String>['app', 'package', 'plugin']) {
461
      final String createName = path.join(tempDir.path, 'create_$template');
462
      await _runFlutter(
463
        <String>['create', '--template=$template', createName],
464 465 466
        // Run it outside the cloned Flutter repo to not nest git repos, since
        // they'll be git repos themselves too.
        workingDirectory: tempDir,
467 468 469 470 471 472
      );
    }

    // Yes, we could just skip all .packages files when constructing
    // the archive, but some are checked in, and we don't want to skip
    // those.
473 474 475 476 477 478 479 480
    await _runGit(<String>[
      'clean',
      '-f',
      // Do not -X as it could lead to entire bin/cache getting cleaned
      '-x',
      '--',
      '**/.packages',
    ]);
481
    /// Remove package_config files and any contents in .dart_tool
482 483 484 485 486 487 488 489 490 491 492 493 494 495
    await _runGit(<String>[
      'clean',
      '-f',
      '-x',
      '--',
      '**/.dart_tool/',
    ]);

    // Ensure the above commands do not clean out the cache
    final Directory flutterCache = Directory(path.join(flutterRoot.absolute.path, 'bin', 'cache'));
    if (!flutterCache.existsSync()) {
      throw Exception('The flutter cache was not found at ${flutterCache.path}!');
    }

496 497 498 499 500 501
    /// Remove git subfolder from .pub-cache, this contains the flutter goldens
    /// and new flutter_gallery.
    final Directory gitCache = Directory(path.join(flutterRoot.absolute.path, '.pub-cache', 'git'));
    if (gitCache.existsSync()) {
      gitCache.deleteSync(recursive: true);
    }
502 503
  }

504
  /// Write the archive to the given output file.
505
  Future<void> _archiveFiles(File outputFile) async {
506
    if (outputFile.path.toLowerCase().endsWith('.zip')) {
507
      await _createZipArchive(outputFile, flutterRoot);
508
    } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
509
      await _createTarArchive(outputFile, flutterRoot);
510 511 512
    }
  }

513 514 515 516 517 518 519
  Future<String> _runDart(List<String> args, {Directory? workingDirectory}) {
    return _processRunner.runProcess(
      <String>[_dart, ...args],
      workingDirectory: workingDirectory ?? flutterRoot,
    );
  }

520
  Future<String> _runFlutter(List<String> args, {Directory? workingDirectory}) {
521 522
    return _processRunner.runProcess(
      <String>[_flutter, ...args],
523 524
      workingDirectory: workingDirectory ?? flutterRoot,
    );
525
  }
526

527
  Future<String> _runGit(List<String> args, {Directory? workingDirectory}) {
528 529 530 531
    return _processRunner.runProcess(
      <String>['git', ...args],
      workingDirectory: workingDirectory ?? flutterRoot,
    );
532 533
  }

534 535
  /// Unpacks the given zip file into the currentDirectory (if set), or the
  /// same directory as the archive.
536
  Future<String> _unzipArchive(File archive, {Directory? workingDirectory}) {
537
    workingDirectory ??= Directory(path.dirname(archive.absolute.path));
538 539 540 541 542 543 544 545 546 547 548 549 550
    List<String> commandLine;
    if (platform.isWindows) {
      commandLine = <String>[
        '7za',
        'x',
        archive.absolute.path,
      ];
    } else {
      commandLine = <String>[
        'unzip',
        archive.absolute.path,
      ];
    }
551
    return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
552 553
  }

554
  /// Create a zip archive from the directory source.
555
  Future<String> _createZipArchive(File output, Directory source) async {
556 557
    List<String> commandLine;
    if (platform.isWindows) {
558
      // Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib.
559
      await _processRunner.runProcess(
560 561 562
        <String>['attrib', '-h', '.git'],
        workingDirectory: Directory(source.absolute.path),
      );
563 564 565 566 567 568 569 570 571 572 573 574 575
      commandLine = <String>[
        '7za',
        'a',
        '-tzip',
        '-mx=9',
        output.absolute.path,
        path.basename(source.path),
      ];
    } else {
      commandLine = <String>[
        'zip',
        '-r',
        '-9',
576
        '--symlinks',
577 578 579 580
        output.absolute.path,
        path.basename(source.path),
      ];
    }
581
    return _processRunner.runProcess(
582
      commandLine,
583
      workingDirectory: Directory(path.dirname(source.absolute.path)),
584
    );
585 586
  }

587
  /// Create a tar archive from the directory source.
588 589 590 591 592 593 594
  Future<String> _createTarArchive(File output, Directory source) {
    return _processRunner.runProcess(<String>[
      'tar',
      'cJf',
      output.absolute.path,
      path.basename(source.absolute.path),
    ], workingDirectory: Directory(path.dirname(source.absolute.path)));
595
  }
596
}
597

598 599 600 601 602 603
class ArchivePublisher {
  ArchivePublisher(
    this.tempDir,
    this.revision,
    this.branch,
    this.version,
604 605
    this.outputFile,
    this.dryRun, {
606
    ProcessManager? processManager,
607 608
    bool subprocessOutput = true,
    this.platform = const LocalPlatform(),
609 610
  })  : assert(revision.length == 40),
        platformName = platform.operatingSystem.toLowerCase(),
611
        metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
612 613
        _processRunner = ProcessRunner(
          processManager: processManager,
614
          subprocessOutput: subprocessOutput,
615
        );
616 617 618 619 620 621

  final Platform platform;
  final String platformName;
  final String metadataGsPath;
  final Branch branch;
  final String revision;
622
  final Map<String, String> version;
623 624 625
  final Directory tempDir;
  final File outputFile;
  final ProcessRunner _processRunner;
626
  final bool dryRun;
627
  String get branchName => getBranchName(branch);
628 629
  String get destinationArchivePath => '$branchName/$platformName/${path.basename(outputFile.path)}';
  static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json';
630

631 632 633 634 635 636 637 638 639 640 641 642
  Future<String> _getChecksum(File archiveFile) async {
    final DigestSink digestSink = DigestSink();
    final ByteConversionSink sink = sha256.startChunkedConversion(digestSink);

    final Stream<List<int>> stream = archiveFile.openRead();
    await stream.forEach((List<int> chunk) {
      sink.add(chunk);
    });
    sink.close();
    return digestSink.value.toString();
  }

643
  /// Publish the archive to Google Storage.
644 645 646 647
  ///
  /// This method will throw if the target archive already exists on cloud
  /// storage.
  Future<void> publishArchive([bool forceUpload = false]) async {
648 649 650 651 652 653
    final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
    if (!forceUpload) {
      if (await _cloudPathExists(destGsPath) && !dryRun) {
        throw PreparePackageException(
          'File $destGsPath already exists on cloud storage!',
        );
654 655
      }
    }
656 657 658 659 660 661
    await _cloudCopy(
      src: outputFile.absolute.path,
      dest: destGsPath,
    );
    assert(tempDir.existsSync());
    await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}');
662 663
  }

664 665
  Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
    jsonData['base_url'] = '$baseUrl$releaseFolder';
666 667 668
    if (!jsonData.containsKey('current_release')) {
      jsonData['current_release'] = <String, String>{};
    }
669
    (jsonData['current_release'] as Map<String, dynamic>)[branchName] = revision;
670 671 672 673 674 675 676
    if (!jsonData.containsKey('releases')) {
      jsonData['releases'] = <Map<String, dynamic>>[];
    }

    final Map<String, dynamic> newEntry = <String, dynamic>{};
    newEntry['hash'] = revision;
    newEntry['channel'] = branchName;
677 678
    newEntry['version'] = version[frameworkVersionTag];
    newEntry['dart_sdk_version'] = version[dartVersionTag];
679
    newEntry['dart_sdk_arch'] = version[dartTargetArchTag];
680
    newEntry['release_date'] = DateTime.now().toUtc().toIso8601String();
681
    newEntry['archive'] = destinationArchivePath;
682
    newEntry['sha256'] = await _getChecksum(outputFile);
683 684

    // Search for any entries with the same hash and channel and remove them.
685
    final List<dynamic> releases = jsonData['releases'] as List<dynamic>;
686
    jsonData['releases'] = <Map<String, dynamic>>[
687
      for (final Map<String, dynamic> entry in releases.cast<Map<String, dynamic>>())
688 689 690
        if (entry['hash'] != newEntry['hash'] ||
            entry['channel'] != newEntry['channel'] ||
            entry['dart_sdk_arch'] != newEntry['dart_sdk_arch'])
691 692 693
          entry,
      newEntry,
    ]..sort((Map<String, dynamic> a, Map<String, dynamic> b) {
694 695
      final DateTime aDate = DateTime.parse(a['release_date'] as String);
      final DateTime bDate = DateTime.parse(b['release_date'] as String);
696 697 698 699 700
      return bDate.compareTo(aDate);
    });
    return jsonData;
  }

701
  Future<void> _updateMetadata(String gsPath) async {
702 703 704 705
    // We can't just cat the metadata from the server with 'gsutil cat', because
    // Windows wants to echo the commands that execute in gsutil.bat to the
    // stdout when we do that. So, we copy the file locally and then read it
    // back in.
706
    final File metadataFile = File(
707 708
      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
    );
709
    await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
710
    Map<String, dynamic> jsonData = <String, dynamic>{};
711 712 713 714 715 716 717 718 719 720 721
    if (!dryRun) {
      final String currentMetadata = metadataFile.readAsStringSync();
      if (currentMetadata.isEmpty) {
        throw PreparePackageException('Empty metadata received from server');
      }
      try {
        jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
      } on FormatException catch (e) {
        throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
      }
    }
722 723 724 725 726 727 728
    // Run _addRelease, even on a dry run, so we can inspect the metadata on a
    // dry run. On a dry run, the only thing in the metadata file be the new
    // release.
    jsonData = await _addRelease(jsonData);

    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
    metadataFile.writeAsStringSync(encoder.convert(jsonData));
729 730 731 732 733 734 735 736
    await _cloudCopy(
      src: metadataFile.absolute.path,
      dest: gsPath,
      // This metadata file is used by the website, so we don't want a long
      // latency between publishing a release and it being available on the
      // site.
      cacheSeconds: shortCacheSeconds,
    );
737 738
  }

739 740
  Future<String> _runGsUtil(
    List<String> args, {
741
    Directory? workingDirectory,
742
    bool failOk = false,
743
  }) async {
744 745 746 747
    if (dryRun) {
      print('gsutil.py -- $args');
      return '';
    }
748
    if (platform.isWindows) {
749
      return _processRunner.runProcess(
750
        <String>['python3', path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'), '--', ...args],
751 752 753 754 755
        workingDirectory: workingDirectory,
        failOk: failOk,
      );
    }

756
    return _processRunner.runProcess(
757
      <String>['gsutil.py', '--', ...args],
758 759 760 761 762
      workingDirectory: workingDirectory,
      failOk: failOk,
    );
  }

763 764 765 766 767 768 769 770 771 772 773 774 775
  /// Determine if a file exists at a given [cloudPath].
  Future<bool> _cloudPathExists(String cloudPath) async {
    try {
      await _runGsUtil(
        <String>['stat', cloudPath],
      );
    } on PreparePackageException {
      // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist
      return false;
    }
    return true;
  }

776
  Future<String> _cloudCopy({
777 778 779
    required String src,
    required String dest,
    int? cacheSeconds,
780
  }) async {
781 782
    // We often don't have permission to overwrite, but
    // we have permission to remove, so that's what we do.
783
    await _runGsUtil(<String>['rm', dest], failOk: true);
784
    String? mimeType;
785 786 787 788 789 790 791 792 793
    if (dest.endsWith('.tar.xz')) {
      mimeType = 'application/x-gtar';
    }
    if (dest.endsWith('.zip')) {
      mimeType = 'application/zip';
    }
    if (dest.endsWith('.json')) {
      mimeType = 'application/json';
    }
794
    return _runGsUtil(<String>[
795 796 797
      // Use our preferred MIME type for the files we care about
      // and let gsutil figure it out for anything else.
      if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
798
      if (cacheSeconds != null) ...<String>['-h', 'Cache-Control:max-age=$cacheSeconds'],
799 800 801 802
      'cp',
      src,
      dest,
    ]);
803 804 805 806 807 808 809
  }
}

/// Prepares a flutter git repo to be packaged up for distribution.
/// It mainly serves to populate the .pub-cache with any appropriate Dart
/// packages, and the flutter cache in bin/cache with the appropriate
/// dependencies and snapshots.
810
///
Ian Hickson's avatar
Ian Hickson committed
811 812
/// Archives contain the executables and customizations for the platform that
/// they are created on.
Ian Hickson's avatar
Ian Hickson committed
813
Future<void> main(List<String> rawArguments) async {
814
  final ArgParser argParser = ArgParser();
815 816 817 818
  argParser.addOption(
    'temp_dir',
    help: 'A location where temporary files may be written. Defaults to a '
        'directory in the system temp folder. Will write a few GiB of data, '
819 820 821
        'so it should have sufficient free space. If a temp_dir is not '
        'specified, then the default temp_dir will be created, used, and '
        'removed automatically.',
822
  );
823 824 825
  argParser.addOption('revision',
      help: 'The Flutter git repo revision to build the '
          'archive with. Must be the full 40-character hash. Required.');
826
  argParser.addOption(
827
    'branch',
828
    allowed: Branch.values.map<String>((Branch branch) => getBranchName(branch)),
829
    help: 'The Flutter branch to build the archive with. Required.',
830 831 832
  );
  argParser.addOption(
    'output',
833 834 835 836 837 838 839
    help: 'The path to the directory where the output archive should be '
        'written. If --output is not specified, the archive will be written to '
        "the current directory. If the output directory doesn't exist, it, and "
        'the path to it, will be created.',
  );
  argParser.addFlag(
    'publish',
840 841
    help: 'If set, will publish the archive to Google Cloud Storage upon '
        'successful creation of the archive. Will publish under this '
842
        'directory: $baseUrl$releaseFolder',
843
  );
844 845 846 847 848
  argParser.addFlag(
    'force',
    abbr: 'f',
    help: 'Overwrite a previously uploaded package.',
  );
849 850 851 852 853
  argParser.addFlag(
    'dry_run',
    negatable: false,
    help: 'Prints gsutil commands instead of executing them.',
  );
854 855 856 857
  argParser.addFlag(
    'help',
    negatable: false,
    help: 'Print help for this command.',
858
  );
859

Ian Hickson's avatar
Ian Hickson committed
860
  final ArgResults parsedArguments = argParser.parse(rawArguments);
861

862
  if (parsedArguments['help'] as bool) {
863 864 865 866
    print(argParser.usage);
    exit(0);
  }

867 868 869 870 871 872
  void errorExit(String message, {int exitCode = -1}) {
    stderr.write('Error: $message\n\n');
    stderr.write('${argParser.usage}\n');
    exit(exitCode);
  }

873
  final String revision = parsedArguments['revision'] as String;
874
  if (!parsedArguments.wasParsed('revision')) {
875 876
    errorExit('Invalid argument: --revision must be specified.');
  }
877 878 879 880
  if (revision.length != 40) {
    errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.');
  }

881
  if (!parsedArguments.wasParsed('branch')) {
882 883
    errorExit('Invalid argument: --branch must be specified.');
  }
884

885
  final String tempDirArg = parsedArguments['temp_dir'] as String;
886
  Directory tempDir;
887
  bool removeTempDir = false;
888
  if (tempDirArg == null || tempDirArg.isEmpty) {
889
    tempDir = Directory.systemTemp.createTempSync('flutter_package.');
890 891
    removeTempDir = true;
  } else {
892
    tempDir = Directory(tempDirArg);
893
    if (!tempDir.existsSync()) {
894
      errorExit("Temporary directory $tempDirArg doesn't exist.");
895 896 897
    }
  }

898
  Directory outputDir;
Ian Hickson's avatar
Ian Hickson committed
899
  if (parsedArguments['output'] == null) {
900
    outputDir = tempDir;
901
  } else {
902
    outputDir = Directory(parsedArguments['output'] as String);
903 904
    if (!outputDir.existsSync()) {
      outputDir.createSync(recursive: true);
905
    }
906 907
  }

908 909
  final bool publish = parsedArguments['publish'] as bool;
  final bool dryRun = parsedArguments['dry_run'] as bool;
910
  final Branch branch = fromBranchName(parsedArguments['branch'] as String);
911 912 913 914 915 916 917
  final ArchiveCreator creator = ArchiveCreator(
    tempDir,
    outputDir,
    revision,
    branch,
    strict: publish && !dryRun,
  );
918
  int exitCode = 0;
919
  late String message;
920
  try {
921
    final Map<String, String> version = await creator.initializeRepo();
922
    final File outputFile = await creator.createArchive();
923
    if (parsedArguments['publish'] as bool) {
924
      final ArchivePublisher publisher = ArchivePublisher(
925 926 927 928 929
        tempDir,
        revision,
        branch,
        version,
        outputFile,
930
        dryRun,
931
      );
932
      await publisher.publishArchive(parsedArguments['force'] as bool);
933
    }
934
  } on PreparePackageException catch (e) {
935
    exitCode = e.exitCode;
936 937
    message = e.message;
  } catch (e) {
938
    exitCode = -1;
939
    message = e.toString();
940 941
  } finally {
    if (removeTempDir) {
942
      tempDir.deleteSync(recursive: true);
943 944 945 946 947 948 949
    }
    if (exitCode != 0) {
      errorExit(message, exitCode: exitCode);
    }
    exit(0);
  }
}