Create packages only for release builds, and publish when created. (#14476)

This changes the publishing of archives so that it happens on the chrome_infra bots when they build a packaged branch instead of as part of the dev_roll process.

It uses the tagged version in the branch, and leaves the git repo that it clones checked out on the branch and hash used to build the package.

It updates metadata located at gs://flutter_infra/releases/releases_.json (where is one of macos, linux, or windows) once published, since it would be complex to do the proper locking to keep them all in one shared .json file safely.

A separate [change to the chrome_infra bots](https://chromium-review.googlesource.com/c/chromium/tools/build/+/902823) was made to instruct them to build packaged for the dev, beta, and release branches (but not master anymore).

dev/bots/prepare_package.dart

@@ -4,13 +4,14 @@
 
 import 'dart:async';
 import 'dart:convert';
-import 'dart:io';
+import 'dart:io' hide Platform;
 import 'dart:typed_data';
 
 import 'package:args/args.dart';
 import 'package:http/http.dart' as http;
 import 'package:path/path.dart' as path;
 import 'package:process/process.dart';
+import 'package:platform/platform.dart' show Platform, LocalPlatform;
 
 const String CHROMIUM_REPO =
     'https://chromium.googlesource.com/external/github.com/flutter/flutter';
@@ -18,21 +19,157 @@ const String GITHUB_REPO = 'https://github.com/flutter/flutter.git';
 const String MINGIT_FOR_WINDOWS_URL = 'https://storage.googleapis.com/flutter_infra/mingit/'
     '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
 
-/// Error class for when a process fails to run, so we can catch
+/// Exception class for when a process fails to run, so we can catch
 /// it and provide something more readable than a stack trace.
-class ProcessFailedException extends Error {
-  ProcessFailedException([this.message, this.exitCode]);
+class ProcessRunnerException implements Exception {
+  ProcessRunnerException(this.message, [this.result]);
 
-  String message = '';
-  int exitCode = 0;
+  final String message;
+  final ProcessResult result;
+  int get exitCode => result.exitCode ?? -1;
 
   @override
-  String toString() => message;
+  String toString() {
+    String output = runtimeType.toString();
+    if (message != null) {
+      output += ': $message';
+    }
+    final String stderr = result?.stderr ?? '';
+    if (stderr.isNotEmpty) {
+      output += ':\n${result.stderr}';
+    }
+    return output;
+  }
+}
+
+enum Branch { dev, beta, release }
+
+String getBranchName(Branch branch) {
+  switch (branch) {
+    case Branch.beta:
+      return 'beta';
+    case Branch.dev:
+      return 'dev';
+    case Branch.release:
+      return 'release';
+  }
+  return null;
+}
+
+Branch fromBranchName(String name) {
+  switch (name) {
+    case 'beta':
+      return Branch.beta;
+    case 'dev':
+      return Branch.dev;
+    case 'release':
+      return Branch.release;
+    default:
+      throw new ArgumentError('Invalid branch name.');
+  }
+}
+
+/// A helper class for classes that want to run a process, optionally have the
+/// stderr and stdout reported as the process runs, and capture the stdout
+/// properly without dropping any.
+class ProcessRunner {
+  ProcessRunner({
+    this.processManager: const LocalProcessManager(),
+    this.subprocessOutput: true,
+    this.defaultWorkingDirectory,
+    this.platform: const LocalPlatform(),
+  }) {
+    environment = new Map<String, String>.from(platform.environment);
+  }
+
+  /// The platform to use for a starting environment.
+  final Platform platform;
+
+  /// Set [subprocessOutput] to show output as processes run. Stdout from the
+  /// process will be printed to stdout, and stderr printed to stderr.
+  final bool subprocessOutput;
+
+  /// Set the [processManager] in order to inject a test instance to perform
+  /// testing.
+  final ProcessManager processManager;
+
+  /// Sets the default directory used when `workingDirectory` is not specified
+  /// to [runProcess].
+  final Directory defaultWorkingDirectory;
+
+  /// The environment to run processes with.
+  Map<String, String> environment;
+
+  /// Run the command and arguments in `commandLine` as a sub-process from
+  /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
+  /// [Directory.current] if [defaultWorkingDirectory] is not set.
+  ///
+  /// Set `failOk` if [runProcess] should not throw an exception when the
+  /// command completes with a a non-zero exit code.
+  Future<String> runProcess(
+    List<String> commandLine, {
+    Directory workingDirectory,
+    bool failOk: false,
+  }) async {
+    workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
+    if (subprocessOutput) {
+      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
+    }
+    final List<int> output = <int>[];
+    final Completer<Null> stdoutComplete = new Completer<Null>();
+    final Completer<Null> stderrComplete = new Completer<Null>();
+    Process process;
+    Future<int> allComplete() async {
+      await stderrComplete.future;
+      await stdoutComplete.future;
+      return process.exitCode;
+    }
+
+    try {
+      process = await processManager.start(
+        commandLine,
+        workingDirectory: workingDirectory.absolute.path,
+        environment: environment,
+      );
+      process.stdout.listen(
+        (List<int> event) {
+          output.addAll(event);
+          if (subprocessOutput) {
+            stdout.add(event);
+          }
+        },
+        onDone: () async => stdoutComplete.complete(),
+      );
+      if (subprocessOutput) {
+        process.stderr.listen(
+          (List<int> event) {
+            stderr.add(event);
+          },
+          onDone: () async => stderrComplete.complete(),
+        );
+      } else {
+        stderrComplete.complete();
+      }
+    } on ProcessException catch (e) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
+          'failed with:\n${e.toString()}';
+      throw new ProcessRunnerException(message);
+    }
+
+    final int exitCode = await allComplete();
+    if (exitCode != 0 && !failOk) {
+      final String message =
+          'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
+      throw new ProcessRunnerException(
+          message, new ProcessResult(0, exitCode, null, 'returned $exitCode'));
+    }
+    return UTF8.decoder.convert(output).trim();
+  }
 }
 
 /// Creates a pre-populated Flutter archive from a git repo.
 class ArchiveCreator {
-  /// [_tempDir] is the directory to use for creating the archive. The script
+  /// [tempDir] is the directory to use for creating the archive.  The script
   /// will place several GiB of data there, so it should have available space.
   ///
   /// The processManager argument is used to inject a mock of [ProcessManager] for
@@ -40,52 +177,85 @@ class ArchiveCreator {
   ///
   /// If subprocessOutput is true, then output from processes invoked during
   /// archive creation is echoed to stderr and stdout.
-  ArchiveCreator(this._tempDir, {ProcessManager processManager, bool subprocessOutput: true})
-      : _flutterRoot = new Directory(path.join(_tempDir.path, 'flutter')),
-        _processManager = processManager ?? const LocalProcessManager(),
-        _subprocessOutput = subprocessOutput {
+  ArchiveCreator(
+    this.tempDir,
+    this.outputDir,
+    this.revision,
+    this.branch, {
+    ProcessManager processManager,
+    bool subprocessOutput: true,
+    this.platform: const LocalPlatform(),
+  }) : assert(revision.length == 40),
+       flutterRoot = new Directory(path.join(tempDir.path, 'flutter')),
+       _processRunner = new ProcessRunner(
+         processManager: processManager,
+         subprocessOutput: subprocessOutput,
+         platform: platform,
+       ) {
     _flutter = path.join(
-      _flutterRoot.absolute.path,
+      flutterRoot.absolute.path,
       'bin',
       'flutter',
     );
-    _environment = new Map<String, String>.from(Platform.environment);
-    _environment['PUB_CACHE'] = path.join(_flutterRoot.absolute.path, '.pub-cache');
+    _processRunner.environment['PUB_CACHE'] = path.join(flutterRoot.absolute.path, '.pub-cache');
   }
 
-  final Directory _flutterRoot;
-  final Directory _tempDir;
-  final bool _subprocessOutput;
-  final ProcessManager _processManager;
-  String _flutter;
+  final Platform platform;
+  final Branch branch;
+  final String revision;
+  final Directory flutterRoot;
+  final Directory tempDir;
+  final Directory outputDir;
   final Uri _minGitUri = Uri.parse(MINGIT_FOR_WINDOWS_URL);
-  Map<String, String> _environment;
+  final ProcessRunner _processRunner;
+
+  File _outputFile;
+  String _version;
+  String _flutter;
+
+  /// Get the name of the channel as a string.
+  String get branchName => getBranchName(branch);
 
   /// Returns a default archive name when given a Git revision.
   /// Used when an output filename is not given.
-  static String defaultArchiveName(String revision) {
-    final String os = Platform.operatingSystem.toLowerCase();
-    final String id = revision.length > 10 ? revision.substring(0, 10) : revision;
-    final String suffix = Platform.isWindows ? 'zip' : 'tar.xz';
-    return 'flutter_${os}_$id.$suffix';
+  String get _archiveName {
+    final String os = platform.operatingSystem.toLowerCase();
+    final String suffix = platform.isWindows ? 'zip' : 'tar.xz';
+    return 'flutter_${os}_$_version-$branchName.$suffix';
+  }
+
+  /// Checks out the flutter repo and prepares it for other operations.
+  ///
+  /// Returns the version for this release, as obtained from the git tags.
+  Future<String> initializeRepo() async {
+    await _checkoutFlutter();
+    _version = await _getVersion();
+    return _version;
   }
 
   /// Performs all of the steps needed to create an archive.
-  Future<File> createArchive(String revision, File outputFile) async {
-    await _checkoutFlutter(revision);
+  Future<File> createArchive() async {
+    assert(_version != null, 'Must run initializeRepo before createArchive');
+    _outputFile = new File(path.join(outputDir.absolute.path, _archiveName));
     await _installMinGitIfNeeded();
     await _populateCaches();
-    await _archiveFiles(outputFile);
-    return outputFile;
+    await _archiveFiles(_outputFile);
+    return _outputFile;
+  }
+
+  /// Returns the version number of this release, according the to tags in
+  /// the repo.
+  Future<String> _getVersion() async {
+    return _runGit(<String>['describe', '--tags', '--abbrev=0']);
   }
 
   /// Clone the Flutter repo and make sure that the git environment is sane
   /// for when the user will unpack it.
-  Future<Null> _checkoutFlutter(String revision) async {
-    // We want the user to start out the in the 'master' branch instead of a
-    // detached head. To do that, we need to make sure master points at the
+  Future<Null> _checkoutFlutter() async {
+    // We want the user to start out the in the specified branch instead of a
+    // detached head. To do that, we need to make sure the branch points at the
     // desired revision.
-    await _runGit(<String>['clone', '-b', 'master', CHROMIUM_REPO], workingDirectory: _tempDir);
+    await _runGit(<String>['clone', '-b', branchName, CHROMIUM_REPO], workingDirectory: tempDir);
     await _runGit(<String>['reset', '--hard', revision]);
 
     // Make the origin point to github instead of the chromium mirror.
@@ -95,16 +265,17 @@ class ArchiveCreator {
 
   /// Retrieve the MinGit executable from storage and unpack it.
   Future<Null> _installMinGitIfNeeded() async {
-    if (!Platform.isWindows) {
+    if (!platform.isWindows) {
       return;
     }
     final Uint8List data = await http.readBytes(_minGitUri);
-    final File gitFile = new File(path.join(_tempDir.path, 'mingit.zip'));
+    final File gitFile = new File(path.join(tempDir.absolute.path, 'mingit.zip'));
     await gitFile.writeAsBytes(data, flush: true);
 
-    final Directory minGitPath = new Directory(path.join(_flutterRoot.path, 'bin', 'mingit'));
+    final Directory minGitPath =
+        new Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit'));
     await minGitPath.create(recursive: true);
-    await _unzipArchive(gitFile, currentDirectory: minGitPath);
+    await _unzipArchive(gitFile, workingDirectory: minGitPath);
   }
 
   /// Prepare the archive repo so that it has all of the caches warmed up and
@@ -118,8 +289,11 @@ class ArchiveCreator {
     // Create each of the templates, since they will call 'pub get' on
     // themselves when created, and this will warm the cache with their
     // dependencies too.
-    for (String template in <String>['app', 'package', 'plugin']) {
-      final String createName = path.join(_tempDir.path, 'create_$template');
+    // TODO(gspencer): 'package' is broken on dev branch right now!
+    // Add it back in once the following is fixed:
+    // https://github.com/flutter/flutter/issues/14448
+    for (String template in <String>['app', 'plugin']) {
+      final String createName = path.join(tempDir.path, 'create_$template');
       await _runFlutter(
         <String>['create', '--template=$template', createName],
       );
@@ -134,111 +308,154 @@ class ArchiveCreator {
   /// Write the archive to the given output file.
   Future<Null> _archiveFiles(File outputFile) async {
     if (outputFile.path.toLowerCase().endsWith('.zip')) {
-      await _createZipArchive(outputFile, _flutterRoot);
+      await _createZipArchive(outputFile, flutterRoot);
     } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
-      await _createTarArchive(outputFile, _flutterRoot);
+      await _createTarArchive(outputFile, flutterRoot);
     }
   }
 
-  Future<String> _runFlutter(List<String> args) => _runProcess(<String>[_flutter]..addAll(args));
+  Future<String> _runFlutter(List<String> args, {Directory workingDirectory}) {
+    return _processRunner.runProcess(<String>[_flutter]..addAll(args),
+        workingDirectory: workingDirectory ?? flutterRoot);
+  }
 
   Future<String> _runGit(List<String> args, {Directory workingDirectory}) {
-    return _runProcess(<String>['git']..addAll(args), workingDirectory: workingDirectory);
+    return _processRunner.runProcess(<String>['git']..addAll(args),
+        workingDirectory: workingDirectory ?? flutterRoot);
   }
 
   /// Unpacks the given zip file into the currentDirectory (if set), or the
   /// same directory as the archive.
   ///
   /// May only be run on Windows (since 7Zip is not available on other platforms).
-  Future<String> _unzipArchive(File archive, {Directory currentDirectory}) {
-    assert(Platform.isWindows); // 7Zip is only available on Windows.
-    currentDirectory ??= new Directory(path.dirname(archive.absolute.path));
+  Future<String> _unzipArchive(File archive, {Directory workingDirectory}) {
+    assert(platform.isWindows); // 7Zip is only available on Windows.
+    workingDirectory ??= new Directory(path.dirname(archive.absolute.path));
     final List<String> commandLine = <String>['7za', 'x', archive.absolute.path];
-    return _runProcess(commandLine, workingDirectory: currentDirectory);
+    return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
   }
 
   /// Create a zip archive from the directory source.
   ///
   /// May only be run on Windows (since 7Zip is not available on other platforms).
   Future<String> _createZipArchive(File output, Directory source) {
-    assert(Platform.isWindows); // 7Zip is only available on Windows.
+    assert(platform.isWindows); // 7Zip is only available on Windows.
     final List<String> commandLine = <String>[
       '7za',
       'a',
       '-tzip',
       '-mx=9',
       output.absolute.path,
-      path.basename(source.absolute.path),
+      path.basename(source.path),
     ];
-    return _runProcess(commandLine,
+    return _processRunner.runProcess(commandLine,
         workingDirectory: new Directory(path.dirname(source.absolute.path)));
   }
 
   /// Create a tar archive from the directory source.
   Future<String> _createTarArchive(File output, Directory source) {
-    return _runProcess(<String>[
+    return _processRunner.runProcess(<String>[
       'tar',
       'cJf',
       output.absolute.path,
       path.basename(source.absolute.path),
     ], workingDirectory: new Directory(path.dirname(source.absolute.path)));
   }
+}
 
-  /// Run the command and arguments in commandLine as a sub-process from
-  /// workingDirectory if set, or the current directory if not.
-  Future<String> _runProcess(List<String> commandLine, {Directory workingDirectory}) async {
-    workingDirectory ??= _flutterRoot;
-    if (_subprocessOutput) {
-      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
-    }
-    final List<int> output = <int>[];
-    final Completer<Null> stdoutComplete = new Completer<Null>();
-    final Completer<Null> stderrComplete = new Completer<Null>();
-    Process process;
-    Future<int> allComplete() async {
-      await stderrComplete.future;
-      await stdoutComplete.future;
-      return process.exitCode;
-    }
+class ArchivePublisher {
+  ArchivePublisher(
+    this.tempDir,
+    this.revision,
+    this.branch,
+    this.version,
+    this.outputFile, {
+    ProcessManager processManager,
+    bool subprocessOutput: true,
+    this.platform: const LocalPlatform(),
+  }) : assert(revision.length == 40),
+       platformName = platform.operatingSystem.toLowerCase(),
+       metadataGsPath = '$gsReleaseFolder/releases_${platform.operatingSystem.toLowerCase()}.json',
+       _processRunner = new ProcessRunner(
+         processManager: processManager,
+         subprocessOutput: subprocessOutput,
+       );
 
+  static String gsBase = 'gs://flutter_infra';
+  static String releaseFolder = '/releases';
+  static String gsReleaseFolder = '$gsBase$releaseFolder';
+  static String baseUrl = 'https://storage.googleapis.com/flutter_infra';
+
+  final Platform platform;
+  final String platformName;
+  final String metadataGsPath;
+  final Branch branch;
+  final String revision;
+  final String version;
+  final Directory tempDir;
+  final File outputFile;
+  final ProcessRunner _processRunner;
+  String get branchName => getBranchName(branch);
+  String get destinationArchivePath =>
+      '$branchName/$platformName/${path.basename(outputFile.path)}';
+
+  /// Publish the archive to Google Storage.
+  Future<Null> publishArchive() async {
+    final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
+    await _cloudCopy(outputFile.absolute.path, destGsPath);
+    assert(tempDir.existsSync());
+    return _updateMetadata();
+  }
+
+  Future<Null> _updateMetadata() async {
+    final String currentMetadata = await _runGsUtil(<String>['cat', metadataGsPath]);
+    if (currentMetadata.isEmpty) {
+      throw new ProcessRunnerException('Empty metadata received from server');
+    }
+
+    Map<String, dynamic> jsonData;
     try {
-      process = await _processManager.start(
-        commandLine,
-        workingDirectory: workingDirectory.absolute.path,
-        environment: _environment,
-      );
-      process.stdout.listen(
-        (List<int> event) {
-          output.addAll(event);
-          if (_subprocessOutput) {
-            stdout.add(event);
+      jsonData = json.decode(currentMetadata);
+    } on FormatException catch (e) {
+      throw new ProcessRunnerException('Unable to parse JSON metadata received from cloud: $e');
     }
-        },
-        onDone: () async => stdoutComplete.complete(),
-      );
-      if (_subprocessOutput) {
-        process.stderr.listen(
-          (List<int> event) {
-            stderr.add(event);
-          },
-          onDone: () async => stderrComplete.complete(),
-        );
-      } else {
-        stderrComplete.complete();
+
+    // Update the metadata file with the data for this package.
+    jsonData['base_url'] = '$baseUrl$releaseFolder';
+    if (!jsonData.containsKey('current_release')) {
+      jsonData['current_release'] = <String, String>{};
     }
-    } on ProcessException catch (e) {
-      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
-          'failed with:\n${e.toString()}';
-      throw new ProcessFailedException(message, -1);
+    jsonData['current_release'][branchName] = revision;
+    if (!jsonData.containsKey('releases')) {
+      jsonData['releases'] = <String, dynamic>{};
     }
+    if (!jsonData['releases'].containsKey(revision)) {
+      jsonData['releases'][revision] = <String, Map<String, String>>{};
+    }
+    final Map<String, String> metadata = <String, String>{};
+    metadata['${platformName}_archive'] = destinationArchivePath;
+    metadata['release_date'] = new DateTime.now().toUtc().toIso8601String();
+    metadata['version'] = version;
+    jsonData['releases'][revision][branchName] = metadata;
 
-    final int exitCode = await allComplete();
-    if (exitCode != 0) {
-      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
-          'failed with $exitCode.';
-      throw new ProcessFailedException(message, exitCode);
+    final File tempFile = new File(path.join(tempDir.absolute.path, 'releases_$platformName.json'));
+    final JsonEncoder encoder = const JsonEncoder.withIndent('  ');
+    tempFile.writeAsStringSync(encoder.convert(jsonData));
+    await _cloudCopy(tempFile.absolute.path, metadataGsPath);
   }
-    return UTF8.decoder.convert(output).trim();
+
+  Future<String> _runGsUtil(List<String> args,
+      {Directory workingDirectory, bool failOk: false}) async {
+    return _processRunner.runProcess(
+      <String>['gsutil']..addAll(args),
+      workingDirectory: workingDirectory,
+      failOk: failOk,
+    );
+  }
+
+  Future<String> _cloudCopy(String src, String dest) async {
+    await _runGsUtil(<String>['rm', dest], failOk: true);
+    return _runGsUtil(<String>['cp', src, dest]);
   }
 }
 
@@ -256,23 +473,37 @@ Future<Null> main(List<String> argList) async {
     defaultsTo: null,
     help: 'A location where temporary files may be written. Defaults to a '
         'directory in the system temp folder. Will write a few GiB of data, '
-        'so it should have sufficient free space.',
+        'so it should have sufficient free space. If a temp_dir is not '
+        'specified, then the default temp_dir will be created, used, and '
+        'removed automatically.',
   );
+  argParser.addOption('revision',
+      defaultsTo: null,
+      help: 'The Flutter git repo revision to build the '
+          'archive with. Must be the full 40-character hash. Required.');
   argParser.addOption(
-    'revision',
-    defaultsTo: 'master',
-    help: 'The Flutter revision to build the archive with. Defaults to the '
-        "master branch's HEAD revision.",
+    'branch',
+    defaultsTo: null,
+    allowed: Branch.values.map((Branch branch) => getBranchName(branch)),
+    help: 'The Flutter branch to build the archive with. Required.',
   );
   argParser.addOption(
     'output',
     defaultsTo: null,
-    help: 'The path to the file where the output archive should be '
-        'written. The output file must end in ".tar.xz" on Linux and Mac, '
-        'and ".zip" on Windows. If --output is not specified, the archive will '
-        "be written to the current directory. If the output directory doesn't "
-        'exist, it, and the path to it, will be created.',
+    help: 'The path to the directory where the output archive should be '
+        'written. If --output is not specified, the archive will be written to '
+        "the current directory. If the output directory doesn't exist, it, and "
+        'the path to it, will be created.',
+  );
+  argParser.addFlag(
+    'publish',
+    defaultsTo: false,
+    help: 'The path to the directory where the output archive should be '
+        'written. If --output is not specified, the archive will be written to '
+        "the current directory. If the output directory doesn't exist, it, and "
+        'the path to it, will be created.',
   );
+
   final ArgResults args = argParser.parse(argList);
 
   void errorExit(String message, {int exitCode = -1}) {
@@ -281,9 +512,17 @@ Future<Null> main(List<String> argList) async {
     exit(exitCode);
   }
 
-  if (args['revision'].isEmpty) {
+  final String revision = args['revision'];
+  if (revision.isEmpty) {
     errorExit('Invalid argument: --revision must be specified.');
   }
+  if (revision.length != 40) {
+    errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.');
+  }
+
+  if (args['branch'].isEmpty) {
+    errorExit('Invalid argument: --branch must be specified.');
+  }
 
   Directory tempDir;
   bool removeTempDir = false;
@@ -297,34 +536,34 @@ Future<Null> main(List<String> argList) async {
     }
   }
 
-  final String output = (args['output'] == null || args['output'].isEmpty)
-      ? path.join(path.current, ArchiveCreator.defaultArchiveName(args['revision']))
-      : args['output'];
-
-  /// Sanity check the output filename.
-  final String outputFilename = path.basename(output);
-  if (Platform.isWindows) {
-    if (!outputFilename.endsWith('.zip')) {
-      errorExit('The argument to --output must end in .zip on Windows.');
-    }
+  Directory outputDir;
+  if (args['output'] == null) {
+    outputDir = tempDir;
   } else {
-    if (!outputFilename.endsWith('.tar.xz')) {
-      errorExit('The argument to --output must end in .tar.xz on Linux and Mac.');
+    outputDir = new Directory(args['output']);
+    if (!outputDir.existsSync()) {
+      outputDir.createSync(recursive: true);
     }
   }
 
-  final Directory outputDirectory = new Directory(path.dirname(output));
-  if (!outputDirectory.existsSync()) {
-    outputDirectory.createSync(recursive: true);
-  }
-  final File outputFile = new File(path.join(outputDirectory.absolute.path, outputFilename));
-
-  final ArchiveCreator preparer = new ArchiveCreator(tempDir);
+  final Branch branch = fromBranchName(args['branch']);
+  final ArchiveCreator creator = new ArchiveCreator(tempDir, outputDir, revision, branch);
   int exitCode = 0;
   String message;
   try {
-    await preparer.createArchive(args['revision'], outputFile);
-  } on ProcessFailedException catch (e) {
+    final String version = await creator.initializeRepo();
+    final File outputFile = await creator.createArchive();
+    if (args['publish']) {
+      final ArchivePublisher publisher = new ArchivePublisher(
+        tempDir,
+        revision,
+        branch,
+        version,
+        outputFile,
+      );
+      await publisher.publishArchive();
+    }
+  } on ProcessRunnerException catch (e) {
     exitCode = e.exitCode;
     message = e.message;
   } finally {

dev/tools/test/fake_process_manager.dart → dev/bots/test/fake_process_manager.dart

@@ -29,111 +29,88 @@ class FakeProcessManager extends Mock implements ProcessManager {
   /// The list of results that will be sent back, organized by the command line
   /// that will produce them. Each command line has a list of returned stdout
   /// output that will be returned on each successive call.
-  Map<String, List<ProcessResult>> fakeResults = <String, List<ProcessResult>>{};
+  Map<String, List<ProcessResult>> _fakeResults = <String, List<ProcessResult>>{};
+  Map<String, List<ProcessResult>> get fakeResults => _fakeResults;
+  set fakeResults(Map<String, List<ProcessResult>> value) {
+    _fakeResults = <String, List<ProcessResult>>{};
+    for (String key in value.keys) {
+      _fakeResults[key] = <ProcessResult>[]
+        ..addAll(value[key] ?? <ProcessResult>[new ProcessResult(0, 0, '', '')]);
+    }
+  }
 
   /// The list of invocations that occurred, in the order they occurred.
   List<Invocation> invocations = <Invocation>[];
 
-  /// Verify that the given command lines were called, in the given order.
+  /// Verify that the given command lines were called, in the given order, and that the
+  /// parameters were in the same order.
   void verifyCalls(List<String> calls) {
     int index = 0;
-    expect(invocations.length, equals(calls.length));
     for (String call in calls) {
       expect(call.split(' '), orderedEquals(invocations[index].positionalArguments[0]));
       index++;
     }
+    expect(invocations.length, equals(calls.length));
   }
 
-  /// Sets the list of results that will be returned from each successive call.
-  void setResults(Map<String, List<String>> results) {
-    final Map<String, List<ProcessResult>> resultCodeUnits = <String, List<ProcessResult>>{};
-    for (String key in results.keys) {
-      resultCodeUnits[key] =
-          results[key].map((String result) => new ProcessResult(0, 0, result, '')).toList();
-    }
-    fakeResults = resultCodeUnits;
-  }
-
-  ProcessResult _popResult(String key) {
+  ProcessResult _popResult(List<String> command) {
+    final String key = command.join(' ');
     expect(fakeResults, isNotEmpty);
     expect(fakeResults, contains(key));
     expect(fakeResults[key], isNotEmpty);
     return fakeResults[key].removeAt(0);
   }
 
-  FakeProcess _popProcess(String key) =>
-      new FakeProcess(_popResult(key), stdinResults: stdinResults);
+  FakeProcess _popProcess(List<String> command) =>
+      new FakeProcess(_popResult(command), stdinResults: stdinResults);
 
   Future<Process> _nextProcess(Invocation invocation) async {
     invocations.add(invocation);
-    return new Future<Process>.value(_popProcess(invocation.positionalArguments[0].join(' ')));
+    return new Future<Process>.value(_popProcess(invocation.positionalArguments[0]));
   }
 
   ProcessResult _nextResultSync(Invocation invocation) {
     invocations.add(invocation);
-    return _popResult(invocation.positionalArguments[0].join(' '));
+    return _popResult(invocation.positionalArguments[0]);
   }
 
   Future<ProcessResult> _nextResult(Invocation invocation) async {
     invocations.add(invocation);
-    return new Future<ProcessResult>.value(_popResult(invocation.positionalArguments[0].join(' ')));
+    return new Future<ProcessResult>.value(_popResult(invocation.positionalArguments[0]));
   }
 
   void _setupMock() {
     // Note that not all possible types of invocations are covered here, just the ones
     // expected to be called.
     // TODO(gspencer): make this more general so that any call will be captured.
-    when(
-      start(
+    when(start(
       typed(captureAny),
       environment: typed(captureAny, named: 'environment'),
       workingDirectory: typed(captureAny, named: 'workingDirectory'),
-      ),
-    ).thenAnswer(_nextProcess);
+    )).thenAnswer(_nextProcess);
 
-    when(
-      start(
-        typed(captureAny),
-      ),
-    ).thenAnswer(_nextProcess);
+    when(start(typed(captureAny))).thenAnswer(_nextProcess);
 
-    when(
-      run(
+    when(run(
       typed(captureAny),
       environment: typed(captureAny, named: 'environment'),
       workingDirectory: typed(captureAny, named: 'workingDirectory'),
-      ),
-    ).thenAnswer(_nextResult);
+    )).thenAnswer(_nextResult);
 
-    when(
-      run(
-        typed(captureAny),
-      ),
-    ).thenAnswer(_nextResult);
+    when(run(typed(captureAny))).thenAnswer(_nextResult);
 
-    when(
-      runSync(
+    when(runSync(
       typed(captureAny),
       environment: typed(captureAny, named: 'environment'),
-        workingDirectory: typed(captureAny, named: 'workingDirectory'),
-      ),
-    ).thenAnswer(_nextResultSync);
+      workingDirectory: typed(captureAny, named: 'workingDirectory')
+    )).thenAnswer(_nextResultSync);
 
-    when(
-      runSync(
-        typed(captureAny),
-      ),
-    ).thenAnswer(_nextResultSync);
+    when(runSync(typed(captureAny))).thenAnswer(_nextResultSync);
 
     when(killPid(typed(captureAny), typed(captureAny))).thenReturn(true);
 
-    when(
-      canRun(captureAny,
-          workingDirectory: typed(
-            captureAny,
-            named: 'workingDirectory',
-          )),
-    ).thenReturn(true);
+    when(canRun(captureAny, workingDirectory: typed(captureAny, named: 'workingDirectory')))
+        .thenReturn(true);
   }
 }
 
@@ -190,9 +167,11 @@ class StringStreamConsumer implements StreamConsumer<List<int>> {
   Future<dynamic> addStream(Stream<List<int>> value) {
     streams.add(value);
     completers.add(new Completer<dynamic>());
-    subscriptions.add(value.listen((List<int> data) {
+    subscriptions.add(
+      value.listen((List<int> data) {
         sendString(utf8.decode(data));
-    }));
+      }),
+    );
     subscriptions.last.onDone(() => completers.last.complete(null));
     return new Future<dynamic>.value(null);
   }

dev/tools/test/fake_process_manager_test.dart → dev/bots/test/fake_process_manager_test.dart

@@ -25,11 +25,15 @@ void main() {
     tearDown(() async {});
 
     test('start works', () async {
-      final Map<String, List<String>> calls = <String, List<String>>{
-        'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
-        'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
+      final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+        'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output1', '')
+        ],
+        'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output2', '')
+        ],
       };
-      processManager.setResults(calls);
+      processManager.fakeResults = calls;
       for (String key in calls.keys) {
         final Process process = await processManager.start(key.split(' '));
         String output = '';
@@ -37,56 +41,68 @@ void main() {
           output += utf8.decode(item);
         });
         await process.exitCode;
-        expect(output, equals(calls[key][0]));
+        expect(output, equals(calls[key][0].stdout));
       }
-      processManager.verifyCalls(calls.keys);
+      processManager.verifyCalls(calls.keys.toList());
     });
 
     test('run works', () async {
-      final Map<String, List<String>> calls = <String, List<String>>{
-        'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
-        'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
+      final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+        'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output1', '')
+        ],
+        'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output2', '')
+        ],
       };
-      processManager.setResults(calls);
+      processManager.fakeResults = calls;
       for (String key in calls.keys) {
         final ProcessResult result = await processManager.run(key.split(' '));
-        expect(result.stdout, equals(calls[key][0]));
+        expect(result.stdout, equals(calls[key][0].stdout));
       }
-      processManager.verifyCalls(calls.keys);
+      processManager.verifyCalls(calls.keys.toList());
     });
 
     test('runSync works', () async {
-      final Map<String, List<String>> calls = <String, List<String>>{
-        'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
-        'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
+      final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+        'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output1', '')
+        ],
+        'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output2', '')
+        ],
       };
-      processManager.setResults(calls);
+      processManager.fakeResults = calls;
       for (String key in calls.keys) {
         final ProcessResult result = processManager.runSync(key.split(' '));
-        expect(result.stdout, equals(calls[key][0]));
+        expect(result.stdout, equals(calls[key][0].stdout));
       }
-      processManager.verifyCalls(calls.keys);
+      processManager.verifyCalls(calls.keys.toList());
     });
 
     test('captures stdin', () async {
-      final Map<String, List<String>> calls = <String, List<String>>{
-        'gsutil acl get gs://flutter_infra/releases/releases.json': <String>['output1'],
-        'gsutil cat gs://flutter_infra/releases/releases.json': <String>['test'],
+      final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+        'gsutil acl get gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output1', '')
+        ],
+        'gsutil cat gs://flutter_infra/releases/releases.json': <ProcessResult>[
+          new ProcessResult(0, 0, 'output2', '')
+        ],
       };
-      processManager.setResults(calls);
+      processManager.fakeResults = calls;
       for (String key in calls.keys) {
         final Process process = await processManager.start(key.split(' '));
         String output = '';
         process.stdout.listen((List<int> item) {
           output += utf8.decode(item);
         });
-        final String testInput = '${calls[key][0]} input';
+        final String testInput = '${calls[key][0].stdout} input';
         process.stdin.add(testInput.codeUnits);
         await process.exitCode;
-        expect(output, equals(calls[key][0]));
+        expect(output, equals(calls[key][0].stdout));
         expect(stdinCaptured.last, equals(testInput));
       }
-      processManager.verifyCalls(calls.keys);
+      processManager.verifyCalls(calls.keys.toList());
     });
   });
 }

dev/bots/test/prepare_package_test.dart

@@ -2,77 +2,94 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:async';
-import 'dart:io';
+import 'dart:convert';
+import 'dart:io' hide Platform;
 
 import 'package:mockito/mockito.dart';
 import 'package:test/test.dart';
 import 'package:path/path.dart' as path;
-import 'package:process/process.dart';
+import 'package:platform/platform.dart' show FakePlatform;
 
 import '../prepare_package.dart';
+import 'fake_process_manager.dart';
 
 void main() {
-  group('ArchiveCreator', () {
-    ArchiveCreator preparer;
+  final String testRef = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef';
+  for (String platformName in <String>['macos', 'linux', 'windows']) {
+    final FakePlatform platform = new FakePlatform(
+      operatingSystem: platformName,
+      environment: <String, String>{},
+    );
+    group('ArchiveCreator for $platformName', () {
+      ArchiveCreator creator;
       Directory tmpDir;
       Directory flutterDir;
-    File outputFile;
-    MockProcessManager processManager;
-    List<MockProcess> results = <MockProcess>[];
+      FakeProcessManager processManager;
       final List<List<String>> args = <List<String>>[];
       final List<Map<Symbol, dynamic>> namedArgs = <Map<Symbol, dynamic>>[];
-    String flutterExe;
-
-    void _verifyCommand(List<dynamic> args, String expected) {
-      final List<String> expectedList = expected.split(' ');
-      expect(args[0], orderedEquals(expectedList));
-    }
-
-    Future<Process> _nextResult(Invocation invocation) async {
-      args.add(invocation.positionalArguments);
-      namedArgs.add(invocation.namedArguments);
-      final Process result = results.isEmpty ? new MockProcess('', '', 0) : results.removeAt(0);
-      return new Future<Process>.value(result);
-    }
-
-    void _answerWithResults() {
-      when(
-        processManager.start(
-          typed(captureAny),
-          environment: typed(captureAny, named: 'environment'),
-          workingDirectory: typed(captureAny, named: 'workingDirectory'),
-        ),
-      ).thenAnswer(_nextResult);
-    }
+      String flutter;
 
       setUp(() async {
-      processManager = new MockProcessManager();
+        processManager = new FakeProcessManager();
         args.clear();
         namedArgs.clear();
         tmpDir = await Directory.systemTemp.createTemp('flutter_');
-      outputFile =
-          new File(path.join(tmpDir.absolute.path, ArchiveCreator.defaultArchiveName('master')));
         flutterDir = new Directory(path.join(tmpDir.path, 'flutter'));
         flutterDir.createSync(recursive: true);
-      flutterExe =
-          path.join(flutterDir.path, 'bin', 'flutter');
+        creator = new ArchiveCreator(
+          tmpDir,
+          tmpDir,
+          testRef,
+          Branch.dev,
+          processManager: processManager,
+          subprocessOutput: false,
+          platform: platform,
+        );
+        flutter = path.join(creator.flutterRoot.absolute.path, 'bin', 'flutter');
       });
 
       tearDown(() async {
-      // On Windows, the directory is locked and not able to be deleted, because it is a
-      // temporary directory. So we just leave some (very small, because we're not actually
-      // building archives here) trash around to be deleted at the next reboot.
-      if (!Platform.isWindows) {
+        // On Windows, the directory is locked and not able to be deleted yet. So
+        // we just leave some (very small, because we're not actually building
+        // archives here) trash around to be deleted at the next reboot.
+        if (!platform.isWindows) {
           await tmpDir.delete(recursive: true);
         }
       });
 
       test('sets PUB_CACHE properly', () async {
-      preparer =
-          new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false);
-      _answerWithResults();
-      await preparer.createArchive('master', outputFile);
+        final String createBase = path.join(tmpDir.absolute.path, 'create_');
+        final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+          'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter':
+              null,
+          'git reset --hard $testRef': null,
+          'git remote remove origin': null,
+          'git remote add origin https://github.com/flutter/flutter.git': null,
+          'git describe --tags --abbrev=0': <ProcessResult>[new ProcessResult(0, 0, 'v1.2.3', '')],
+        };
+        if (platform.isWindows) {
+          calls['7za x ${path.join(tmpDir.path, 'mingit.zip')}'] = null;
+        }
+        calls.addAll(<String, List<ProcessResult>>{
+          '$flutter doctor': null,
+          '$flutter update-packages': null,
+          '$flutter precache': null,
+          '$flutter ide-config': null,
+          '$flutter create --template=app ${createBase}app': null,
+          '$flutter create --template=package ${createBase}package': null,
+          '$flutter create --template=plugin ${createBase}plugin': null,
+          'git clean -f -X **/.packages': null,
+        });
+        final String archiveName = path.join(tmpDir.absolute.path,
+            'flutter_${platformName}_v1.2.3-dev${platform.isWindows ? '.zip' : '.tar.xz'}');
+        if (platform.isWindows) {
+          calls['7za a -tzip -mx=9 $archiveName flutter'] = null;
+        } else {
+          calls['tar cJf $archiveName flutter'] = null;
+        }
+        processManager.fakeResults = calls;
+        await creator.initializeRepo();
+        await creator.createArchive();
         expect(
           verify(processManager.start(
             captureAny,
@@ -84,72 +101,148 @@ void main() {
       });
 
       test('calls the right commands for archive output', () async {
-      preparer =
-          new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false);
-      _answerWithResults();
-      await preparer.createArchive('master', outputFile);
-      final List<String> commands = <String>[
-        'git clone -b master https://chromium.googlesource.com/external/github.com/flutter/flutter',
-        'git reset --hard master',
-        'git remote remove origin',
-        'git remote add origin https://github.com/flutter/flutter.git',
-      ];
-      if (Platform.isWindows) {
-        commands.add('7za x ${path.join(tmpDir.path, 'mingit.zip')}');
+        final String createBase = path.join(tmpDir.absolute.path, 'create_');
+        final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+          'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter':
+              null,
+          'git reset --hard $testRef': null,
+          'git remote remove origin': null,
+          'git remote add origin https://github.com/flutter/flutter.git': null,
+          'git describe --tags --abbrev=0': <ProcessResult>[new ProcessResult(0, 0, 'v1.2.3', '')],
+        };
+        if (platform.isWindows) {
+          calls['7za x ${path.join(tmpDir.path, 'mingit.zip')}'] = null;
         }
-      commands.addAll(<String>[
-        '$flutterExe doctor',
-        '$flutterExe update-packages',
-        '$flutterExe precache',
-        '$flutterExe ide-config',
-        '$flutterExe create --template=app ${path.join(tmpDir.path, 'create_app')}',
-        '$flutterExe create --template=package ${path.join(tmpDir.path, 'create_package')}',
-        '$flutterExe create --template=plugin ${path.join(tmpDir.path, 'create_plugin')}',
-        'git clean -f -X **/.packages',
-      ]);
-      if (Platform.isWindows) {
-        commands.add('7za a -tzip -mx=9 ${outputFile.absolute.path} flutter');
+        calls.addAll(<String, List<ProcessResult>>{
+          '$flutter doctor': null,
+          '$flutter update-packages': null,
+          '$flutter precache': null,
+          '$flutter ide-config': null,
+          '$flutter create --template=app ${createBase}app': null,
+          // TODO(gspencer): Re-enable this when package works again:
+          // https://github.com/flutter/flutter/issues/14448
+          // '$flutter create --template=package ${createBase}package': null,
+          '$flutter create --template=plugin ${createBase}plugin': null,
+          'git clean -f -X **/.packages': null,
+        });
+        final String archiveName = path.join(tmpDir.absolute.path,
+            'flutter_${platformName}_v1.2.3-dev${platform.isWindows ? '.zip' : '.tar.xz'}');
+        if (platform.isWindows) {
+          calls['7za a -tzip -mx=9 $archiveName flutter'] = null;
         } else {
-        commands.add('tar cJf ${outputFile.absolute.path} flutter');
-      }
-      int step = 0;
-      for (String command in commands) {
-        _verifyCommand(args[step++], command);
+          calls['tar cJf $archiveName flutter'] = null;
         }
+        processManager.fakeResults = calls;
+        creator = new ArchiveCreator(
+          tmpDir,
+          tmpDir,
+          testRef,
+          Branch.dev,
+          processManager: processManager,
+          subprocessOutput: false,
+          platform: platform,
+        );
+        await creator.initializeRepo();
+        await creator.createArchive();
+        processManager.verifyCalls(calls.keys.toList());
       });
 
       test('throws when a command errors out', () async {
-      preparer =
-          new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false);
-
-      results = <MockProcess>[
-        new MockProcess('', '', 0),
-        new MockProcess('', "Don't panic.\n", -1)
-      ];
-      _answerWithResults();
-      expect(expectAsync2<Null, String, File>(preparer.createArchive)('master', new File('foo')),
-          throwsA(const isInstanceOf<ProcessFailedException>()));
+        final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+          'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter':
+              <ProcessResult>[new ProcessResult(0, 0, 'output1', '')],
+          'git reset --hard $testRef': <ProcessResult>[new ProcessResult(0, -1, 'output2', '')],
+        };
+        processManager.fakeResults = calls;
+        expect(expectAsync0(creator.initializeRepo),
+            throwsA(const isInstanceOf<ProcessRunnerException>()));
       });
     });
-}
-
-class MockProcessManager extends Mock implements ProcessManager {}
 
-class MockProcess extends Mock implements Process {
-  MockProcess(this._stdout, [this._stderr, this._exitCode]);
+    group('ArchivePublisher for $platformName', () {
+      FakeProcessManager processManager;
+      Directory tempDir;
 
-  String _stdout;
-  String _stderr;
-  int _exitCode;
-
-  @override
-  Stream<List<int>> get stdout =>
-      new Stream<List<int>>.fromIterable(<List<int>>[_stdout.codeUnits]);
+      setUp(() async {
+        processManager = new FakeProcessManager();
+        tempDir = await Directory.systemTemp.createTemp('flutter_');
+        tempDir.createSync();
+      });
 
-  @override
-  Stream<List<int>> get stderr =>
-      new Stream<List<int>>.fromIterable(<List<int>>[_stderr.codeUnits]);
+      tearDown(() async {
+        // On Windows, the directory is locked and not able to be deleted yet. So
+        // we just leave some (very small, because we're not actually building
+        // archives here) trash around to be deleted at the next reboot.
+        if (!platform.isWindows) {
+          await tempDir.delete(recursive: true);
+        }
+      });
 
-  @override
-  Future<int> get exitCode => new Future<int>.value(_exitCode);
+      test('calls the right processes', () async {
+        final String releasesName = 'releases_$platformName.json';
+        final String archivePath = path.join(tempDir.absolute.path, 'output_archive');
+        final String gsArchivePath = 'gs://flutter_infra/releases/dev/$platformName/output_archive';
+        final String jsonPath = path.join(tempDir.absolute.path, releasesName);
+        final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
+        final String releasesJson = '''{
+    "base_url": "https://storage.googleapis.com/flutter_infra/releases",
+    "current_release": {
+        "beta": "6da8ec6bd0c4801b80d666869e4069698561c043",
+        "dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba"
+    },
+    "releases": {
+        "6da8ec6bd0c4801b80d666869e4069698561c043": {
+            "${platformName}_archive": "dev/linux/flutter_${platformName}_0.21.0-beta.tar.xz",
+            "release_date": "2017-12-19T10:30:00,847287019-08:00",
+            "version": "0.21.0-beta"
+        },
+        "f88c60b38c3a5ef92115d24e3da4175b4890daba": {
+            "${platformName}_archive": "dev/linux/flutter_${platformName}_0.22.0-dev.tar.xz",
+            "release_date": "2018-01-19T13:30:09,728487019-08:00",
+            "version": "0.22.0-dev"
+        }
+    }
+}
+''';
+        final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
+          'gsutil rm $gsArchivePath': null,
+          'gsutil cp $archivePath $gsArchivePath': null,
+          'gsutil cat $gsJsonPath': <ProcessResult>[new ProcessResult(0, 0, releasesJson, '')],
+          'gsutil rm $gsJsonPath': null,
+          'gsutil cp $jsonPath $gsJsonPath': null,
+        };
+        processManager.fakeResults = calls;
+        final File outputFile = new File(path.join(tempDir.absolute.path, 'output_archive'));
+        assert(tempDir.existsSync());
+        final ArchivePublisher publisher = new ArchivePublisher(
+          tempDir,
+          testRef,
+          Branch.dev,
+          '1.2.3',
+          outputFile,
+          processManager: processManager,
+          subprocessOutput: false,
+          platform: platform,
+        );
+        assert(tempDir.existsSync());
+        await publisher.publishArchive();
+        processManager.verifyCalls(calls.keys.toList());
+        final File releaseFile = new File(jsonPath);
+        expect(releaseFile.existsSync(), isTrue);
+        final String contents = releaseFile.readAsStringSync();
+        // Make sure new data is added.
+        expect(contents, contains('"dev": "$testRef"'));
+        expect(contents, contains('"$testRef": {'));
+        expect(contents, contains('"${platformName}_archive": "dev/$platformName/output_archive"'));
+        // Make sure existing entries are preserved.
+        expect(contents, contains('"6da8ec6bd0c4801b80d666869e4069698561c043": {'));
+        expect(contents, contains('"f88c60b38c3a5ef92115d24e3da4175b4890daba": {'));
+        expect(contents, contains('"beta": "6da8ec6bd0c4801b80d666869e4069698561c043"'));
+        // Make sure it's valid JSON, and in the right format.
+        final Map<String, dynamic> jsonData = json.decode(contents);
+        final JsonEncoder encoder = const JsonEncoder.withIndent('  ');
+        expect(contents, equals(encoder.convert(jsonData)));
+      });
+    });
+  }
 }

dev/tools/lib/archive_publisher.dart

-// Copyright 2018 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:path/path.dart' as path;
-import 'package:process/process.dart';
-
-class ArchivePublisherException implements Exception {
-  ArchivePublisherException(this.message, [this.result]);
-
-  final String message;
-  final ProcessResult result;
-
-  @override
-  String toString() {
-    String output = 'ArchivePublisherException';
-    if (message != null) {
-      output += ': $message';
-    }
-    final String stderr = result?.stderr ?? '';
-    if (stderr.isNotEmpty) {
-      output += ':\n${result.stderr}';
-    }
-    return output;
-  }
-}
-
-enum Channel { dev, beta }
-
-/// Publishes the archive created for a particular version and git hash to
-/// the releases directory on cloud storage, and updates the metadata for
-/// releases.
-///
-/// See https://github.com/flutter/flutter/wiki/Release-process for more
-/// information on the release process.
-class ArchivePublisher {
-  ArchivePublisher(
-    this.revision,
-    this.version,
-    this.channel, {
-    this.processManager = const LocalProcessManager(),
-    this.tempDir,
-  }) : assert(revision.length == 40, 'Git hash must be 40 characters long (i.e. the entire hash).');
-
-  /// A git hash describing the revision to publish. It should be the complete
-  /// hash, not just a prefix.
-  final String revision;
-
-  /// A version number for the release (e.g. "1.2.3").
-  final String version;
-
-  /// The channel to publish to.
-  // TODO(gspencer): support Channel.beta: it is currently unimplemented.
-  final Channel channel;
-
-  /// Get the name of the channel as a string.
-  String get channelName {
-    switch (channel) {
-      case Channel.beta:
-        return 'beta';
-      case Channel.dev:
-      default:
-        return 'dev';
-    }
-  }
-
-  /// The process manager to use for invoking commands. Typically only
-  /// used for testing purposes.
-  final ProcessManager processManager;
-
-  /// The temporary directory used for this publisher. If not set, one will
-  /// be created, used, and then removed automatically. If set, it will not be
-  /// deleted when done: that is left to the caller. Typically used by tests.
-  Directory tempDir;
-
-  static String gsBase = 'gs://flutter_infra';
-  static String releaseFolder = '/releases';
-  static String baseUrl = 'https://storage.googleapis.com/flutter_infra';
-  static String archivePrefix = 'flutter_';
-  static String releaseNotesPrefix = 'release_notes_';
-
-  final String metadataGsPath = '$gsBase$releaseFolder/releases.json';
-
-  /// Publishes the archive for the given constructor parameters.
-  bool publishArchive() {
-    assert(channel == Channel.dev, 'Channel must be dev (beta not yet supported)');
-    final List<String> platforms = <String>['linux', 'mac', 'win'];
-    final Map<String, String> metadata = <String, String>{};
-    for (String platform in platforms) {
-      final String src = _builtArchivePath(platform);
-      final String dest = _destinationArchivePath(platform);
-      final String srcGsPath = '$gsBase$src';
-      final String destGsPath = '$gsBase$releaseFolder$dest';
-      _cloudCopy(srcGsPath, destGsPath);
-      metadata['${platform}_archive'] = '$channelName/$platform$dest';
-    }
-    metadata['release_date'] = new DateTime.now().toUtc().toIso8601String();
-    metadata['version'] = version;
-    _updateMetadata(metadata);
-    return true;
-  }
-
-  /// Checks to make sure the user has access to the Google Storage bucket
-  /// required to publish. Will throw an [ArchivePublisherException] if not.
-  void checkForGSUtilAccess() {
-    // Fetching ACLs requires FULL_CONTROL access.
-    final ProcessResult result = _runGsUtil(<String>['acl', 'get', metadataGsPath]);
-    if (result.exitCode != 0) {
-      throw new ArchivePublisherException(
-        'GSUtil cannot get ACLs for metadata file $metadataGsPath',
-        result,
-      );
-    }
-  }
-
-  void _updateMetadata(Map<String, String> metadata) {
-    final ProcessResult result = _runGsUtil(<String>['cat', metadataGsPath]);
-    if (result.exitCode != 0) {
-      throw new ArchivePublisherException(
-          'Unable to get existing metadata at $metadataGsPath', result);
-    }
-    final String currentMetadata = result.stdout;
-    if (currentMetadata.isEmpty) {
-      throw new ArchivePublisherException('Empty metadata received from server', result);
-    }
-    Map<String, dynamic> jsonData;
-    try {
-      jsonData = json.decode(currentMetadata);
-    } on FormatException catch (e) {
-      throw new ArchivePublisherException('Unable to parse JSON metadata received from cloud: $e');
-    }
-    jsonData['current_$channelName'] = revision;
-    if (!jsonData.containsKey('releases')) {
-      jsonData['releases'] = <String, dynamic>{};
-    }
-    if (jsonData['releases'].containsKey(revision)) {
-      throw new ArchivePublisherException(
-          'Revision $revision already exists in metadata! Aborting.');
-    }
-    jsonData['releases'][revision] = metadata;
-    final Directory localTempDir = tempDir ?? Directory.systemTemp.createTempSync('flutter_');
-    final File tempFile = new File(path.join(localTempDir.absolute.path, 'releases.json'));
-    const JsonEncoder encoder = const JsonEncoder.withIndent('  ');
-    tempFile.writeAsStringSync(encoder.convert(jsonData));
-    _cloudCopy(tempFile.absolute.path, metadataGsPath);
-    if (tempDir == null) {
-      localTempDir.delete(recursive: true);
-    }
-  }
-
-  String _getArchiveSuffix(String platform) {
-    switch (platform) {
-      case 'linux':
-      case 'mac':
-        return '.tar.xz';
-      case 'win':
-        return '.zip';
-      default:
-        assert(false, 'platform $platform not recognized.');
-        return null;
-    }
-  }
-
-  String _builtArchivePath(String platform) {
-    final String shortRevision = revision.substring(0, revision.length > 10 ? 10 : revision.length);
-    final String archivePathBase = '/flutter/$revision/$archivePrefix';
-    final String suffix = _getArchiveSuffix(platform);
-    return '$archivePathBase${platform}_$shortRevision$suffix';
-  }
-
-  String _destinationArchivePath(String platform) {
-    final String archivePathBase = '/$channelName/$platform/$archivePrefix';
-    final String suffix = _getArchiveSuffix(platform);
-    return '$archivePathBase${platform}_$version-$channelName$suffix';
-  }
-
-  ProcessResult _runGsUtil(List<String> args) {
-    return processManager.runSync(<String>['gsutil']..addAll(args));
-  }
-
-  void _cloudCopy(String src, String dest) {
-    final ProcessResult result = _runGsUtil(<String>['cp', src, dest]);
-    if (result.exitCode != 0) {
-      throw new ArchivePublisherException('GSUtil copy command failed: ${result.stderr}', result);
-    }
-  }
-}

dev/tools/lib/roll_dev.dart

@@ -10,7 +10,6 @@
 import 'dart:io';
 
 import 'package:args/args.dart';
-import 'archive_publisher.dart';
 
 const String kIncrement = 'increment';
 const String kX = 'x';
@@ -114,19 +113,6 @@ void main(List<String> args) {
 
   final String hash = getGitOutput('rev-parse HEAD', 'Get git hash for $commit');
 
-  final ArchivePublisher publisher = new ArchivePublisher(hash, version, Channel.dev);
-
-  // Check for access early so that we don't try to publish things if the
-  // user doesn't have access to the metadata file.
-  try {
-    publisher.checkForGSUtilAccess();
-  } on ArchivePublisherException {
-    print('You do not appear to have the credentials required to update the archive links.');
-    print('Make sure you have "gsutil" installed, then run "gsutil config".');
-    print('Talk to @gspencergoog for details on which project to use.');
-    exit(1);
-  }
-
   runGit('tag v$version', 'tag the commit with the version label');
 
   // PROMPT
@@ -140,17 +126,6 @@ void main(List<String> args) {
     exit(0);
   }
 
-  // Publish the archive before pushing the tag so that if something fails in
-  // the publish step, we can clean up.
-  try {
-    publisher.publishArchive();
-  } on ArchivePublisherException catch (e) {
-    print('Archive publishing failed.\n$e');
-    runGit('tag -d v$version', 'remove the tag that was not published');
-    print('The dev roll has been aborted.');
-    exit(1);
-  }
-
   runGit('push upstream v$version', 'publish the version');
   runGit('push upstream HEAD:dev', 'land the new version on the "dev" branch');
   print('Flutter version $version has been rolled to the "dev" channel!');

dev/tools/test/archive_publisher_test.dart

-// Copyright 2018 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'package:test/test.dart';
-
-import 'package:path/path.dart' as path;
-
-import '../lib/archive_publisher.dart';
-import 'fake_process_manager.dart';
-
-void main() {
-  group('ArchivePublisher', () {
-    final List<String> emptyStdout = <String>[''];
-    FakeProcessManager processManager;
-    Directory tempDir;
-
-    setUp(() async {
-      processManager = new FakeProcessManager();
-      tempDir = await Directory.systemTemp.createTemp('flutter_');
-    });
-
-    tearDown(() async {
-      // On Windows, the directory is locked and not able to be deleted, because it is a
-      // temporary directory. So we just leave some (very small, because we're not actually
-      // building archives here) trash around to be deleted at the next reboot.
-      if (!Platform.isWindows) {
-        await tempDir.delete(recursive: true);
-      }
-    });
-
-    test('calls the right processes', () {
-      final Map<String, List<String>> calls = <String, List<String>>{
-        'gsutil acl get gs://flutter_infra/releases/releases.json': emptyStdout,
-        'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_linux_deadbeef.tar.xz '
-            'gs://flutter_infra/releases/dev/linux/flutter_linux_1.2.3-dev.tar.xz': emptyStdout,
-        'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_mac_deadbeef.tar.xz '
-            'gs://flutter_infra/releases/dev/mac/flutter_mac_1.2.3-dev.tar.xz': emptyStdout,
-        'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_win_deadbeef.zip '
-            'gs://flutter_infra/releases/dev/win/flutter_win_1.2.3-dev.zip': emptyStdout,
-        'gsutil cat gs://flutter_infra/releases/releases.json': <String>[
-          '''{
-    "base_url": "https://storage.googleapis.com/flutter_infra/releases",
-    "current_beta": "6da8ec6bd0c4801b80d666869e4069698561c043",
-    "current_dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba",
-    "releases": {
-        "6da8ec6bd0c4801b80d666869e4069698561c043": {
-            "linux_archive": "beta/linux/flutter_linux_0.21.0-beta.tar.xz",
-            "mac_archive": "beta/mac/flutter_mac_0.21.0-beta.tar.xz",
-            "windows_archive": "beta/win/flutter_win_0.21.0-beta.tar.xz",
-            "release_date": "2017-12-19T10:30:00,847287019-08:00",
-            "release_notes": "beta/release_notes_0.21.0-beta.html",
-            "version": "0.21.0-beta"
-        },
-        "f88c60b38c3a5ef92115d24e3da4175b4890daba": {
-            "linux_archive": "dev/linux/flutter_linux_0.22.0-dev.tar.xz",
-            "mac_archive": "dev/mac/flutter_mac_0.22.0-dev.tar.xz",
-            "windows_archive": "dev/win/flutter_win_0.22.0-dev.tar.xz",
-            "release_date": "2018-01-19T13:30:09,728487019-08:00",
-            "release_notes": "dev/release_notes_0.22.0-dev.html",
-            "version": "0.22.0-dev"
-        }
-    }
-}
-'''],
-        'gsutil cp ${tempDir.path}/releases.json gs://flutter_infra/releases/releases.json':
-            emptyStdout,
-      };
-      processManager.setResults(calls);
-      new ArchivePublisher('deadbeef', '1.2.3', Channel.dev,
-          processManager: processManager, tempDir: tempDir)
-        ..publishArchive();
-      processManager.verifyCalls(calls.keys);
-      final File outputFile = new File(path.join(tempDir.path, 'releases.json'));
-      expect(outputFile.existsSync(), isTrue);
-      final String contents = outputFile.readAsStringSync();
-      expect(contents, contains('"current_dev": "deadbeef"'));
-      expect(contents, contains('"deadbeef": {'));
-    });
-  });
-}