prepare_package.dart 29.3 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
import 'dart:async';
6
import 'dart:convert';
7
import 'dart:io' hide Platform;
8
import 'dart:typed_data';
9 10

import 'package:args/args.dart';
11 12
import 'package:crypto/crypto.dart';
import 'package:crypto/src/digest_sink.dart';
13
import 'package:http/http.dart' as http;
14
import 'package:path/path.dart' as path;
15
import 'package:platform/platform.dart' show Platform, LocalPlatform;
16
import 'package:process/process.dart';
17

18
const String chromiumRepo = 'https://chromium.googlesource.com/external/github.com/flutter/flutter';
19
const String githubRepo = 'https://github.com/flutter/flutter.git';
20
const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
21
    '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
22
const String releaseFolder = '/releases';
23 24 25
const String gsBase = 'gs://flutter_infra_release';
const String gsReleaseFolder = '$gsBase$releaseFolder';
const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
26
const int shortCacheSeconds = 60;
27

28
/// Exception class for when a process fails to run, so we can catch
29
/// it and provide something more readable than a stack trace.
30
class PreparePackageException implements Exception {
31
  PreparePackageException(this.message, [this.result]);
32

33
  final String message;
34
  final ProcessResult? result;
35
  int get exitCode => result?.exitCode ?? -1;
36 37

  @override
38 39 40 41 42
  String toString() {
    String output = runtimeType.toString();
    if (message != null) {
      output += ': $message';
    }
43
    final String stderr = result?.stderr as String? ?? '';
44 45 46 47 48
    if (stderr.isNotEmpty) {
      output += ':\n$stderr';
    }
    return output;
  }
49 50
}

51
enum Branch { dev, beta, stable }
52 53 54 55 56 57 58

String getBranchName(Branch branch) {
  switch (branch) {
    case Branch.beta:
      return 'beta';
    case Branch.dev:
      return 'dev';
59 60
    case Branch.stable:
      return 'stable';
61 62 63 64 65 66 67 68 69
  }
}

Branch fromBranchName(String name) {
  switch (name) {
    case 'beta':
      return Branch.beta;
    case 'dev':
      return Branch.dev;
70 71
    case 'stable':
      return Branch.stable;
72
    default:
73
      throw ArgumentError('Invalid branch name.');
74 75 76
  }
}

77 78 79 80 81
/// A helper class for classes that want to run a process, optionally have the
/// stderr and stdout reported as the process runs, and capture the stdout
/// properly without dropping any.
class ProcessRunner {
  ProcessRunner({
82
    ProcessManager? processManager,
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
    this.subprocessOutput = true,
    this.defaultWorkingDirectory,
    this.platform = const LocalPlatform(),
  }) : processManager = processManager ?? const LocalProcessManager() {
    environment = Map<String, String>.from(platform.environment);
  }

  /// The platform to use for a starting environment.
  final Platform platform;

  /// Set [subprocessOutput] to show output as processes run. Stdout from the
  /// process will be printed to stdout, and stderr printed to stderr.
  final bool subprocessOutput;

  /// Set the [processManager] in order to inject a test instance to perform
  /// testing.
  final ProcessManager processManager;

  /// Sets the default directory used when `workingDirectory` is not specified
  /// to [runProcess].
103
  final Directory? defaultWorkingDirectory;
104 105

  /// The environment to run processes with.
106
  late Map<String, String> environment;
107 108 109 110 111 112 113 114 115

  /// Run the command and arguments in `commandLine` as a sub-process from
  /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
  /// [Directory.current] if [defaultWorkingDirectory] is not set.
  ///
  /// Set `failOk` if [runProcess] should not throw an exception when the
  /// command completes with a non-zero exit code.
  Future<String> runProcess(
    List<String> commandLine, {
116
    Directory? workingDirectory,
117 118 119 120 121 122 123 124 125
    bool failOk = false,
  }) async {
    workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
    if (subprocessOutput) {
      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
    }
    final List<int> output = <int>[];
    final Completer<void> stdoutComplete = Completer<void>();
    final Completer<void> stderrComplete = Completer<void>();
126
    late Process process;
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
    Future<int> allComplete() async {
      await stderrComplete.future;
      await stdoutComplete.future;
      return process.exitCode;
    }

    try {
      process = await processManager.start(
        commandLine,
        workingDirectory: workingDirectory.absolute.path,
        environment: environment,
      );
      process.stdout.listen(
        (List<int> event) {
          output.addAll(event);
          if (subprocessOutput) {
            stdout.add(event);
          }
        },
        onDone: () async => stdoutComplete.complete(),
      );
      if (subprocessOutput) {
        process.stderr.listen(
          (List<int> event) {
            stderr.add(event);
          },
          onDone: () async => stderrComplete.complete(),
        );
      } else {
        stderrComplete.complete();
      }
    } on ProcessException catch (e) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
          'failed with:\n${e.toString()}';
      throw PreparePackageException(message);
    } on ArgumentError catch (e) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
          'failed with:\n${e.toString()}';
      throw PreparePackageException(message);
    }

    final int exitCode = await allComplete();
    if (exitCode != 0 && !failOk) {
      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
      throw PreparePackageException(
        message,
        ProcessResult(0, exitCode, null, 'returned $exitCode'),
      );
    }
    return utf8.decoder.convert(output).trim();
  }
}

180
typedef HttpReader = Future<Uint8List> Function(Uri url, {Map<String, String> headers});
181

182 183
/// Creates a pre-populated Flutter archive from a git repo.
class ArchiveCreator {
184
  /// [tempDir] is the directory to use for creating the archive.  The script
185 186 187
  /// will place several GiB of data there, so it should have available space.
  ///
  /// The processManager argument is used to inject a mock of [ProcessManager] for
188
  /// testing purposes.
189 190 191
  ///
  /// If subprocessOutput is true, then output from processes invoked during
  /// archive creation is echoed to stderr and stdout.
192 193 194 195 196
  ArchiveCreator(
    this.tempDir,
    this.outputDir,
    this.revision,
    this.branch, {
197
    this.strict = true,
198
    ProcessManager? processManager,
199 200
    bool subprocessOutput = true,
    this.platform = const LocalPlatform(),
201
    HttpReader? httpReader,
202 203 204 205
  })  : assert(revision.length == 40),
        flutterRoot = Directory(path.join(tempDir.path, 'flutter')),
        httpReader = httpReader ?? http.readBytes,
        _processRunner = ProcessRunner(
206 207 208
          processManager: processManager,
          subprocessOutput: subprocessOutput,
          platform: platform,
209
        ) {
210
    _flutter = path.join(
211
      flutterRoot.absolute.path,
212
      'bin',
213
      'flutter',
214
    );
215
    _processRunner.environment['PUB_CACHE'] = path.join(flutterRoot.absolute.path, '.pub-cache');
216 217
  }

218 219
  /// The platform to use for the environment and determining which
  /// platform we're running on.
220
  final Platform platform;
221 222

  /// The branch to build the archive for.  The branch must contain [revision].
223
  final Branch branch;
224 225 226 227 228

  /// The git revision hash to build the archive for. This revision has
  /// to be available in the [branch], although it doesn't have to be
  /// at HEAD, since we clone the branch and then reset to this revision
  /// to create the archive.
229
  final String revision;
230 231

  /// The flutter root directory in the [tempDir].
232
  final Directory flutterRoot;
233 234

  /// The temporary directory used to build the archive in.
235
  final Directory tempDir;
236 237

  /// The directory to write the output file to.
238
  final Directory outputDir;
239

240 241 242 243 244
  /// True if the creator should be strict about checking requirements or not.
  ///
  /// In strict mode, will insist that the [revision] be a tagged revision.
  final bool strict;

245
  final Uri _minGitUri = Uri.parse(mingitForWindowsUrl);
246 247
  final ProcessRunner _processRunner;

248 249 250 251 252
  /// Used to tell the [ArchiveCreator] which function to use for reading
  /// bytes from a URL. Used in tests to inject a fake reader. Defaults to
  /// [http.readBytes].
  final HttpReader httpReader;

253 254 255
  late File _outputFile;
  late String _version;
  late String _flutter;
256 257 258

  /// Get the name of the channel as a string.
  String get branchName => getBranchName(branch);
259 260 261

  /// Returns a default archive name when given a Git revision.
  /// Used when an output filename is not given.
262 263
  String get _archiveName {
    final String os = platform.operatingSystem.toLowerCase();
264 265 266 267 268 269 270 271
    // We don't use .tar.xz on Mac because although it can unpack them
    // on the command line (with tar), the "Archive Utility" that runs
    // when you double-click on them just does some crazy behavior (it
    // converts it to a compressed cpio archive, and when you double
    // click on that, it converts it back to .tar.xz, without ever
    // unpacking it!) So, we use .zip for Mac, and the files are about
    // 220MB larger than they need to be. :-(
    final String suffix = platform.isLinux ? 'tar.xz' : 'zip';
272 273 274 275 276 277 278 279 280 281
    return 'flutter_${os}_$_version-$branchName.$suffix';
  }

  /// Checks out the flutter repo and prepares it for other operations.
  ///
  /// Returns the version for this release, as obtained from the git tags.
  Future<String> initializeRepo() async {
    await _checkoutFlutter();
    _version = await _getVersion();
    return _version;
282 283
  }

284
  /// Performs all of the steps needed to create an archive.
285 286
  Future<File> createArchive() async {
    assert(_version != null, 'Must run initializeRepo before createArchive');
287
    _outputFile = File(path.join(outputDir.absolute.path, _archiveName));
288 289
    await _installMinGitIfNeeded();
    await _populateCaches();
290
    await _validate();
291 292 293 294
    await _archiveFiles(_outputFile);
    return _outputFile;
  }

295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
  /// Validates the integrity of the release package.
  ///
  /// Currently only checks that macOS binaries are codesigned. Will throw a
  /// [PreparePackageException] if the test failes.
  Future<void> _validate() async {
    // Only validate in strict mode, which means `--publish`
    if (!strict || !platform.isMacOS) {
      return;
    }
    // Validate that the dart binary is codesigned
    final String dartPath = path.join(
      flutterRoot.absolute.path,
      'bin',
      'cache',
      'dart-sdk',
      'bin',
      'dart',
    );
    try {
      // TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701
      await _processRunner.runProcess(
        <String>[
          'codesign',
          '-vvvv',
          '--check-notarization',
          dartPath,
        ],
        workingDirectory: flutterRoot,
      );
    } on PreparePackageException catch (e) {
      throw PreparePackageException(
        'The binary $dartPath was not codesigned!\n${e.message}',
      );
    }
  }

331 332 333 334 335 336 337 338 339
  /// Returns the version number of this release, according the to tags in the
  /// repo.
  ///
  /// This looks for the tag attached to [revision] and, if it doesn't find one,
  /// git will give an error.
  ///
  /// If [strict] is true, the exact [revision] must be tagged to return the
  /// version.  If [strict] is not true, will look backwards in time starting at
  /// [revision] to find the most recent version tag.
340
  Future<String> _getVersion() async {
341 342 343
    if (strict) {
      try {
        return _runGit(<String>['describe', '--tags', '--exact-match', revision]);
344
      } on PreparePackageException catch (exception) {
345
        throw PreparePackageException(
346 347 348 349
          'Git error when checking for a version tag attached to revision $revision.\n'
          'Perhaps there is no tag at that revision?:\n'
          '$exception'
        );
350 351 352 353
      }
    } else {
      return _runGit(<String>['describe', '--tags', '--abbrev=0', revision]);
    }
354
  }
355 356 357

  /// Clone the Flutter repo and make sure that the git environment is sane
  /// for when the user will unpack it.
358
  Future<void> _checkoutFlutter() async {
359 360
    // We want the user to start out the in the specified branch instead of a
    // detached head. To do that, we need to make sure the branch points at the
361
    // desired revision.
362
    await _runGit(<String>['clone', '-b', branchName, chromiumRepo], workingDirectory: tempDir);
363
    await _runGit(<String>['reset', '--hard', revision]);
364 365

    // Make the origin point to github instead of the chromium mirror.
366
    await _runGit(<String>['remote', 'set-url', 'origin', githubRepo]);
367 368 369
  }

  /// Retrieve the MinGit executable from storage and unpack it.
370
  Future<void> _installMinGitIfNeeded() async {
371
    if (!platform.isWindows) {
372 373
      return;
    }
374
    final Uint8List data = await httpReader(_minGitUri);
375
    final File gitFile = File(path.join(tempDir.absolute.path, 'mingit.zip'));
376 377
    await gitFile.writeAsBytes(data, flush: true);

378
    final Directory minGitPath = Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit'));
379
    await minGitPath.create(recursive: true);
380
    await _unzipArchive(gitFile, workingDirectory: minGitPath);
381 382 383
  }

  /// Prepare the archive repo so that it has all of the caches warmed up and
384
  /// is configured for the user to begin working.
385
  Future<void> _populateCaches() async {
386 387 388 389 390 391
    await _runFlutter(<String>['doctor']);
    await _runFlutter(<String>['update-packages']);
    await _runFlutter(<String>['precache']);
    await _runFlutter(<String>['ide-config']);

    // Create each of the templates, since they will call 'pub get' on
392 393
    // themselves when created, and this will warm the cache with their
    // dependencies too.
394
    for (final String template in <String>['app', 'package', 'plugin']) {
395
      final String createName = path.join(tempDir.path, 'create_$template');
396
      await _runFlutter(
397
        <String>['create', '--template=$template', createName],
398 399 400
        // Run it outside the cloned Flutter repo to not nest git repos, since
        // they'll be git repos themselves too.
        workingDirectory: tempDir,
401 402 403 404 405 406
      );
    }

    // Yes, we could just skip all .packages files when constructing
    // the archive, but some are checked in, and we don't want to skip
    // those.
407 408 409 410 411 412 413 414
    await _runGit(<String>[
      'clean',
      '-f',
      // Do not -X as it could lead to entire bin/cache getting cleaned
      '-x',
      '--',
      '**/.packages',
    ]);
415
    /// Remove package_config files and any contents in .dart_tool
416 417 418 419 420 421 422 423 424 425 426 427 428 429
    await _runGit(<String>[
      'clean',
      '-f',
      '-x',
      '--',
      '**/.dart_tool/',
    ]);

    // Ensure the above commands do not clean out the cache
    final Directory flutterCache = Directory(path.join(flutterRoot.absolute.path, 'bin', 'cache'));
    if (!flutterCache.existsSync()) {
      throw Exception('The flutter cache was not found at ${flutterCache.path}!');
    }

430 431 432 433 434 435
    /// Remove git subfolder from .pub-cache, this contains the flutter goldens
    /// and new flutter_gallery.
    final Directory gitCache = Directory(path.join(flutterRoot.absolute.path, '.pub-cache', 'git'));
    if (gitCache.existsSync()) {
      gitCache.deleteSync(recursive: true);
    }
436 437
  }

438
  /// Write the archive to the given output file.
439
  Future<void> _archiveFiles(File outputFile) async {
440
    if (outputFile.path.toLowerCase().endsWith('.zip')) {
441
      await _createZipArchive(outputFile, flutterRoot);
442
    } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
443
      await _createTarArchive(outputFile, flutterRoot);
444 445 446
    }
  }

447
  Future<String> _runFlutter(List<String> args, {Directory? workingDirectory}) {
448 449
    return _processRunner.runProcess(
      <String>[_flutter, ...args],
450 451
      workingDirectory: workingDirectory ?? flutterRoot,
    );
452
  }
453

454
  Future<String> _runGit(List<String> args, {Directory? workingDirectory}) {
455 456 457 458
    return _processRunner.runProcess(
      <String>['git', ...args],
      workingDirectory: workingDirectory ?? flutterRoot,
    );
459 460
  }

461 462
  /// Unpacks the given zip file into the currentDirectory (if set), or the
  /// same directory as the archive.
463
  Future<String> _unzipArchive(File archive, {Directory? workingDirectory}) {
464
    workingDirectory ??= Directory(path.dirname(archive.absolute.path));
465 466 467 468 469 470 471 472 473 474 475 476 477
    List<String> commandLine;
    if (platform.isWindows) {
      commandLine = <String>[
        '7za',
        'x',
        archive.absolute.path,
      ];
    } else {
      commandLine = <String>[
        'unzip',
        archive.absolute.path,
      ];
    }
478
    return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
479 480
  }

481
  /// Create a zip archive from the directory source.
482
  Future<String> _createZipArchive(File output, Directory source) async {
483 484
    List<String> commandLine;
    if (platform.isWindows) {
485
      // Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib.
486
      await _processRunner.runProcess(
487 488 489
        <String>['attrib', '-h', '.git'],
        workingDirectory: Directory(source.absolute.path),
      );
490 491 492 493 494 495 496 497 498 499 500 501 502
      commandLine = <String>[
        '7za',
        'a',
        '-tzip',
        '-mx=9',
        output.absolute.path,
        path.basename(source.path),
      ];
    } else {
      commandLine = <String>[
        'zip',
        '-r',
        '-9',
503
        '--symlinks',
504 505 506 507
        output.absolute.path,
        path.basename(source.path),
      ];
    }
508
    return _processRunner.runProcess(
509
      commandLine,
510
      workingDirectory: Directory(path.dirname(source.absolute.path)),
511
    );
512 513
  }

514
  /// Create a tar archive from the directory source.
515 516 517 518 519 520 521
  Future<String> _createTarArchive(File output, Directory source) {
    return _processRunner.runProcess(<String>[
      'tar',
      'cJf',
      output.absolute.path,
      path.basename(source.absolute.path),
    ], workingDirectory: Directory(path.dirname(source.absolute.path)));
522
  }
523
}
524

525 526 527 528 529 530
class ArchivePublisher {
  ArchivePublisher(
    this.tempDir,
    this.revision,
    this.branch,
    this.version,
531 532
    this.outputFile,
    this.dryRun, {
533
    ProcessManager? processManager,
534 535
    bool subprocessOutput = true,
    this.platform = const LocalPlatform(),
536 537
  })  : assert(revision.length == 40),
        platformName = platform.operatingSystem.toLowerCase(),
538
        metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
539 540
        _processRunner = ProcessRunner(
          processManager: processManager,
541
          subprocessOutput: subprocessOutput,
542
        );
543 544 545 546 547 548 549 550 551 552

  final Platform platform;
  final String platformName;
  final String metadataGsPath;
  final Branch branch;
  final String revision;
  final String version;
  final Directory tempDir;
  final File outputFile;
  final ProcessRunner _processRunner;
553
  final bool dryRun;
554
  String get branchName => getBranchName(branch);
555 556
  String get destinationArchivePath => '$branchName/$platformName/${path.basename(outputFile.path)}';
  static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json';
557

558 559 560 561 562 563 564 565 566 567 568 569
  Future<String> _getChecksum(File archiveFile) async {
    final DigestSink digestSink = DigestSink();
    final ByteConversionSink sink = sha256.startChunkedConversion(digestSink);

    final Stream<List<int>> stream = archiveFile.openRead();
    await stream.forEach((List<int> chunk) {
      sink.add(chunk);
    });
    sink.close();
    return digestSink.value.toString();
  }

570
  /// Publish the archive to Google Storage.
571 572 573 574
  ///
  /// This method will throw if the target archive already exists on cloud
  /// storage.
  Future<void> publishArchive([bool forceUpload = false]) async {
575 576 577 578 579 580
    final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
    if (!forceUpload) {
      if (await _cloudPathExists(destGsPath) && !dryRun) {
        throw PreparePackageException(
          'File $destGsPath already exists on cloud storage!',
        );
581 582
      }
    }
583 584 585 586 587 588
    await _cloudCopy(
      src: outputFile.absolute.path,
      dest: destGsPath,
    );
    assert(tempDir.existsSync());
    await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}');
589 590
  }

591 592
  Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
    jsonData['base_url'] = '$baseUrl$releaseFolder';
593 594 595
    if (!jsonData.containsKey('current_release')) {
      jsonData['current_release'] = <String, String>{};
    }
596
    (jsonData['current_release'] as Map<String, dynamic>)[branchName] = revision;
597 598 599 600 601 602 603 604
    if (!jsonData.containsKey('releases')) {
      jsonData['releases'] = <Map<String, dynamic>>[];
    }

    final Map<String, dynamic> newEntry = <String, dynamic>{};
    newEntry['hash'] = revision;
    newEntry['channel'] = branchName;
    newEntry['version'] = version;
605
    newEntry['release_date'] = DateTime.now().toUtc().toIso8601String();
606
    newEntry['archive'] = destinationArchivePath;
607
    newEntry['sha256'] = await _getChecksum(outputFile);
608 609

    // Search for any entries with the same hash and channel and remove them.
610
    final List<dynamic> releases = jsonData['releases'] as List<dynamic>;
611
    jsonData['releases'] = <Map<String, dynamic>>[
612
      for (final Map<String, dynamic> entry in releases.cast<Map<String, dynamic>>())
613 614 615 616
        if (entry['hash'] != newEntry['hash'] || entry['channel'] != newEntry['channel'])
          entry,
      newEntry,
    ]..sort((Map<String, dynamic> a, Map<String, dynamic> b) {
617 618
      final DateTime aDate = DateTime.parse(a['release_date'] as String);
      final DateTime bDate = DateTime.parse(b['release_date'] as String);
619 620 621 622 623
      return bDate.compareTo(aDate);
    });
    return jsonData;
  }

624
  Future<void> _updateMetadata(String gsPath) async {
625 626 627 628
    // We can't just cat the metadata from the server with 'gsutil cat', because
    // Windows wants to echo the commands that execute in gsutil.bat to the
    // stdout when we do that. So, we copy the file locally and then read it
    // back in.
629
    final File metadataFile = File(
630 631
      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
    );
632 633 634 635 636 637
    await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
    if (!dryRun) {
      final String currentMetadata = metadataFile.readAsStringSync();
      if (currentMetadata.isEmpty) {
        throw PreparePackageException('Empty metadata received from server');
      }
638

639 640 641 642 643 644
      Map<String, dynamic> jsonData;
      try {
        jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
      } on FormatException catch (e) {
        throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
      }
645

646
      jsonData = await _addRelease(jsonData);
647

648 649 650
      const JsonEncoder encoder = JsonEncoder.withIndent('  ');
      metadataFile.writeAsStringSync(encoder.convert(jsonData));
    }
651 652 653 654 655 656 657 658
    await _cloudCopy(
      src: metadataFile.absolute.path,
      dest: gsPath,
      // This metadata file is used by the website, so we don't want a long
      // latency between publishing a release and it being available on the
      // site.
      cacheSeconds: shortCacheSeconds,
    );
659 660
  }

661 662
  Future<String> _runGsUtil(
    List<String> args, {
663
    Directory? workingDirectory,
664
    bool failOk = false,
665
  }) async {
666 667 668 669
    if (dryRun) {
      print('gsutil.py -- $args');
      return '';
    }
670
    if (platform.isWindows) {
671
      return _processRunner.runProcess(
672
        <String>['python3', path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'), '--', ...args],
673 674 675 676 677
        workingDirectory: workingDirectory,
        failOk: failOk,
      );
    }

678
    return _processRunner.runProcess(
679
      <String>['gsutil.py', '--', ...args],
680 681 682 683 684
      workingDirectory: workingDirectory,
      failOk: failOk,
    );
  }

685 686 687 688 689 690 691 692 693 694 695 696 697 698
  /// Determine if a file exists at a given [cloudPath].
  Future<bool> _cloudPathExists(String cloudPath) async {
    try {
      await _runGsUtil(
        <String>['stat', cloudPath],
        failOk: false,
      );
    } on PreparePackageException {
      // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist
      return false;
    }
    return true;
  }

699
  Future<String> _cloudCopy({
700 701 702
    required String src,
    required String dest,
    int? cacheSeconds,
703
  }) async {
704 705
    // We often don't have permission to overwrite, but
    // we have permission to remove, so that's what we do.
706
    await _runGsUtil(<String>['rm', dest], failOk: true);
707
    String? mimeType;
708 709 710 711 712 713 714 715 716
    if (dest.endsWith('.tar.xz')) {
      mimeType = 'application/x-gtar';
    }
    if (dest.endsWith('.zip')) {
      mimeType = 'application/zip';
    }
    if (dest.endsWith('.json')) {
      mimeType = 'application/json';
    }
717
    return _runGsUtil(<String>[
718 719 720
      // Use our preferred MIME type for the files we care about
      // and let gsutil figure it out for anything else.
      if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
721
      if (cacheSeconds != null) ...<String>['-h', 'Cache-Control:max-age=$cacheSeconds'],
722 723 724 725
      'cp',
      src,
      dest,
    ]);
726 727 728 729 730 731 732
  }
}

/// Prepares a flutter git repo to be packaged up for distribution.
/// It mainly serves to populate the .pub-cache with any appropriate Dart
/// packages, and the flutter cache in bin/cache with the appropriate
/// dependencies and snapshots.
733
///
Ian Hickson's avatar
Ian Hickson committed
734 735
/// Archives contain the executables and customizations for the platform that
/// they are created on.
Ian Hickson's avatar
Ian Hickson committed
736
Future<void> main(List<String> rawArguments) async {
737
  final ArgParser argParser = ArgParser();
738 739 740 741 742
  argParser.addOption(
    'temp_dir',
    defaultsTo: null,
    help: 'A location where temporary files may be written. Defaults to a '
        'directory in the system temp folder. Will write a few GiB of data, '
743 744 745
        'so it should have sufficient free space. If a temp_dir is not '
        'specified, then the default temp_dir will be created, used, and '
        'removed automatically.',
746
  );
747 748 749 750
  argParser.addOption('revision',
      defaultsTo: null,
      help: 'The Flutter git repo revision to build the '
          'archive with. Must be the full 40-character hash. Required.');
751
  argParser.addOption(
752 753
    'branch',
    defaultsTo: null,
754
    allowed: Branch.values.map<String>((Branch branch) => getBranchName(branch)),
755
    help: 'The Flutter branch to build the archive with. Required.',
756 757 758 759
  );
  argParser.addOption(
    'output',
    defaultsTo: null,
760 761 762 763 764 765 766 767
    help: 'The path to the directory where the output archive should be '
        'written. If --output is not specified, the archive will be written to '
        "the current directory. If the output directory doesn't exist, it, and "
        'the path to it, will be created.',
  );
  argParser.addFlag(
    'publish',
    defaultsTo: false,
768 769
    help: 'If set, will publish the archive to Google Cloud Storage upon '
        'successful creation of the archive. Will publish under this '
770
        'directory: $baseUrl$releaseFolder',
771
  );
772 773 774 775 776 777
  argParser.addFlag(
    'force',
    abbr: 'f',
    defaultsTo: false,
    help: 'Overwrite a previously uploaded package.',
  );
778 779 780 781 782 783
  argParser.addFlag(
    'dry_run',
    defaultsTo: false,
    negatable: false,
    help: 'Prints gsutil commands instead of executing them.',
  );
784 785 786 787 788
  argParser.addFlag(
    'help',
    defaultsTo: false,
    negatable: false,
    help: 'Print help for this command.',
789
  );
790

Ian Hickson's avatar
Ian Hickson committed
791
  final ArgResults parsedArguments = argParser.parse(rawArguments);
792

793
  if (parsedArguments['help'] as bool) {
794 795 796 797
    print(argParser.usage);
    exit(0);
  }

798 799 800 801 802 803
  void errorExit(String message, {int exitCode = -1}) {
    stderr.write('Error: $message\n\n');
    stderr.write('${argParser.usage}\n');
    exit(exitCode);
  }

804
  final String revision = parsedArguments['revision'] as String;
805
  if (!parsedArguments.wasParsed('revision')) {
806 807
    errorExit('Invalid argument: --revision must be specified.');
  }
808 809 810 811
  if (revision.length != 40) {
    errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.');
  }

812
  if (!parsedArguments.wasParsed('branch')) {
813 814
    errorExit('Invalid argument: --branch must be specified.');
  }
815

816
  final String tempDirArg = parsedArguments['temp_dir'] as String;
817
  Directory tempDir;
818
  bool removeTempDir = false;
819
  if (tempDirArg == null || tempDirArg.isEmpty) {
820
    tempDir = Directory.systemTemp.createTempSync('flutter_package.');
821 822
    removeTempDir = true;
  } else {
823
    tempDir = Directory(tempDirArg);
824
    if (!tempDir.existsSync()) {
825
      errorExit("Temporary directory $tempDirArg doesn't exist.");
826 827 828
    }
  }

829
  Directory outputDir;
Ian Hickson's avatar
Ian Hickson committed
830
  if (parsedArguments['output'] == null) {
831
    outputDir = tempDir;
832
  } else {
833
    outputDir = Directory(parsedArguments['output'] as String);
834 835
    if (!outputDir.existsSync()) {
      outputDir.createSync(recursive: true);
836
    }
837 838
  }

839
  final Branch branch = fromBranchName(parsedArguments['branch'] as String);
840
  final ArchiveCreator creator = ArchiveCreator(tempDir, outputDir, revision, branch, strict: parsedArguments['publish'] as bool);
841
  int exitCode = 0;
842
  late String message;
843
  try {
844 845
    final String version = await creator.initializeRepo();
    final File outputFile = await creator.createArchive();
846
    if (parsedArguments['publish'] as bool) {
847
      final ArchivePublisher publisher = ArchivePublisher(
848 849 850 851 852
        tempDir,
        revision,
        branch,
        version,
        outputFile,
853
        parsedArguments['dry_run'] as bool,
854
      );
855
      await publisher.publishArchive(parsedArguments['force'] as bool);
856
    }
857
  } on PreparePackageException catch (e) {
858
    exitCode = e.exitCode;
859 860
    message = e.message;
  } catch (e) {
861
    exitCode = -1;
862
    message = e.toString();
863 864
  } finally {
    if (removeTempDir) {
865
      tempDir.deleteSync(recursive: true);
866 867 868 869 870 871 872
    }
    if (exitCode != 0) {
      errorExit(message, exitCode: exitCode);
    }
    exit(0);
  }
}