Add unpublish_package script. (#33267)

This adds a maintenance script that will allow us to "unpublish" a release: basically remove it from the cloud storage so that we're no longer serving it from the website.

Obviously, gsutil access to the cloud storage server is required for this to function.

dev/bots/README.md

@@ -190,3 +190,38 @@ not a direct result of changes made in your PR or that breaking this
 test is absolutely necessary, escalate this issue by [submitting an
 issue](https://github.com/material-components/material-components-flutter-codelabs/issues/new?title=%5BURGENT%5D%20Flutter%20Framework%20breaking%20PR)
 to the MDC-Flutter Team.
+
+## Unpublishing published archives
+
+Flutter downloadable archives are built for each release by our continuous
+integration systems using the [`prepare_package.dart`](prepare_package.dart)
+script, but if something goes very wrong, and a release is published that wasn't
+intended to be published, the [`unpublish_package.dart`](unpublish_package.dart)
+script may be used to remove the package or packages from the channels in which
+they were published.
+
+For example To remove a published package corresponding to the git hash
+`d444a455de87a2e40b7f576dc12ffd9ab82fd491`, first do a dry run of the script to
+see what it will do:
+
+```
+$ dart ./unpublish_pacakge.dart --temp_dir=/tmp/foo --revision d444a455de87a2e40b7f576dc12ffd9ab82fd491
+```
+
+And once you've verified the output of the dry run to be sure it is what you
+want to do, run:
+
+```
+$ dart ./unpublish_pacakge.dart --confirm --temp_dir=/tmp/foo --revision d444a455de87a2e40b7f576dc12ffd9ab82fd491
+```
+
+and it will actually perform the actions. You will of course need to have access
+to the cloud storage server and have gsutil installed in order to perform this
+operation. Only runs on Linux or macOS systems.
+
+See `dart ./unpublish_pacakge.dart --help` for more details.
+
+Once the package is unpublished, it will not be available from the website for
+download, and will not be rebuilt (even though there is a tagged revision in the
+repo still) unless someone forces the packaging build to run again at that
+revision to rebuild the package.

dev/bots/unpublish_package.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+/// This script removes published archives from the cloud storage and the
+/// corresponding JSON metadata file that the website uses to determine what
+/// releases are available.
+///
+/// If asked to remove a release that is currently the release on that channel,
+/// it will replace that release with the next most recent release on that
+/// channel.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io' hide Platform;
+import 'dart:typed_data';
+
+import 'package:args/args.dart';
+import 'package:path/path.dart' as path;
+import 'package:platform/platform.dart' show Platform, LocalPlatform;
+import 'package:process/process.dart';
+
+const String gsBase = 'gs://flutter_infra';
+const String releaseFolder = '/releases';
+const String gsReleaseFolder = '$gsBase$releaseFolder';
+const String baseUrl = 'https://storage.googleapis.com/flutter_infra';
+
+/// Exception class for when a process fails to run, so we can catch
+/// it and provide something more readable than a stack trace.
+class UnpublishException implements Exception {
+  UnpublishException(this.message, [this.result]);
+
+  final String message;
+  final ProcessResult result;
+  int get exitCode => result?.exitCode ?? -1;
+
+  @override
+  String toString() {
+    String output = runtimeType.toString();
+    if (message != null) {
+      output += ': $message';
+    }
+    final String stderr = result?.stderr ?? '';
+    if (stderr.isNotEmpty) {
+      output += ':\n$stderr';
+    }
+    return output;
+  }
+}
+
+enum Channel { dev, beta, stable }
+
+String getChannelName(Channel channel) {
+  switch (channel) {
+    case Channel.beta:
+      return 'beta';
+    case Channel.dev:
+      return 'dev';
+    case Channel.stable:
+      return 'stable';
+  }
+  return null;
+}
+
+Channel fromChannelName(String name) {
+  switch (name) {
+    case 'beta':
+      return Channel.beta;
+    case 'dev':
+      return Channel.dev;
+    case 'stable':
+      return Channel.stable;
+    default:
+      throw ArgumentError('Invalid channel name.');
+  }
+}
+
+enum PublishedPlatform { linux, macos, windows }
+
+String getPublishedPlatform(PublishedPlatform platform) {
+  switch (platform) {
+    case PublishedPlatform.linux:
+      return 'linux';
+    case PublishedPlatform.macos:
+      return 'macos';
+    case PublishedPlatform.windows:
+      return 'windows';
+  }
+  return null;
+}
+
+PublishedPlatform fromPublishedPlatform(String name) {
+  switch (name) {
+    case 'linux':
+      return PublishedPlatform.linux;
+    case 'macos':
+      return PublishedPlatform.macos;
+    case 'windows':
+      return PublishedPlatform.windows;
+    default:
+      throw ArgumentError('Invalid published platform name.');
+  }
+}
+
+/// A helper class for classes that want to run a process, optionally have the
+/// stderr and stdout reported as the process runs, and capture the stdout
+/// properly without dropping any.
+class ProcessRunner {
+  /// Creates a [ProcessRunner].
+  ///
+  /// The [processManager], [subprocessOutput], and [platform] arguments must
+  /// not be null.
+  ProcessRunner({
+    this.processManager = const LocalProcessManager(),
+    this.subprocessOutput = true,
+    this.defaultWorkingDirectory,
+    this.platform = const LocalPlatform(),
+  }) : assert(subprocessOutput != null),
+       assert(processManager != null),
+       assert(platform != null) {
+    environment = Map<String, String>.from(platform.environment);
+  }
+
+  /// The platform to use for a starting environment.
+  final Platform platform;
+
+  /// Set [subprocessOutput] to show output as processes run. Stdout from the
+  /// process will be printed to stdout, and stderr printed to stderr.
+  final bool subprocessOutput;
+
+  /// Set the [processManager] in order to inject a test instance to perform
+  /// testing.
+  final ProcessManager processManager;
+
+  /// Sets the default directory used when `workingDirectory` is not specified
+  /// to [runProcess].
+  final Directory defaultWorkingDirectory;
+
+  /// The environment to run processes with.
+  Map<String, String> environment;
+
+  /// Run the command and arguments in `commandLine` as a sub-process from
+  /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
+  /// [Directory.current] if [defaultWorkingDirectory] is not set.
+  ///
+  /// Set `failOk` if [runProcess] should not throw an exception when the
+  /// command completes with a a non-zero exit code.
+  Future<String> runProcess(
+    List<String> commandLine, {
+    Directory workingDirectory,
+    bool failOk = false,
+  }) async {
+    workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
+    if (subprocessOutput) {
+      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
+    }
+    final List<int> output = <int>[];
+    final Completer<void> stdoutComplete = Completer<void>();
+    final Completer<void> stderrComplete = Completer<void>();
+    Process process;
+    Future<int> allComplete() async {
+      await stderrComplete.future;
+      await stdoutComplete.future;
+      return process.exitCode;
+    }
+
+    try {
+      process = await processManager.start(
+        commandLine,
+        workingDirectory: workingDirectory.absolute.path,
+        environment: environment,
+      );
+      process.stdout.listen(
+        (List<int> event) {
+          output.addAll(event);
+          if (subprocessOutput) {
+            stdout.add(event);
+          }
+        },
+        onDone: () async => stdoutComplete.complete(),
+      );
+      if (subprocessOutput) {
+        process.stderr.listen(
+          (List<int> event) {
+            stderr.add(event);
+          },
+          onDone: () async => stderrComplete.complete(),
+        );
+      } else {
+        stderrComplete.complete();
+      }
+    } on ProcessException catch (e) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
+          'failed with:\n${e.toString()}';
+      throw UnpublishException(message);
+    } on ArgumentError catch (e) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
+          'failed with:\n${e.toString()}';
+      throw UnpublishException(message);
+    }
+
+    final int exitCode = await allComplete();
+    if (exitCode != 0 && !failOk) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
+      throw UnpublishException(
+        message,
+        ProcessResult(0, exitCode, null, 'returned $exitCode'),
+      );
+    }
+    return utf8.decoder.convert(output).trim();
+  }
+}
+
+typedef HttpReader = Future<Uint8List> Function(Uri url, {Map<String, String> headers});
+
+class ArchiveUnpublisher {
+  ArchiveUnpublisher(
+    this.tempDir,
+    this.revisionsBeingRemoved,
+    this.channels,
+    this.platform, {
+    this.confirmed = false,
+    ProcessManager processManager,
+    bool subprocessOutput = true,
+  })  : assert(revisionsBeingRemoved.length == 40),
+        metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
+        _processRunner = ProcessRunner(
+          processManager: processManager,
+          subprocessOutput: subprocessOutput,
+        );
+
+  final PublishedPlatform platform;
+  final String metadataGsPath;
+  final Set<Channel> channels;
+  final Set<String> revisionsBeingRemoved;
+  final bool confirmed;
+  final Directory tempDir;
+  final ProcessRunner _processRunner;
+  static String getMetadataFilename(PublishedPlatform platform) => 'releases_${getPublishedPlatform(platform)}.json';
+
+  /// Remove the archive from Google Storage.
+  Future<void> unpublishArchive() async {
+    final Map<String, dynamic> jsonData = await _loadMetadata();
+    final List<Map<String, String>> releases = jsonData['releases'].map<Map<String, String>>((dynamic entry) {
+      final Map<String, dynamic> mapEntry = entry;
+      return mapEntry.cast<String, String>();
+    }).toList();
+    final Map<Channel, Map<String, String>> paths = await _getArchivePaths(releases);
+    releases.removeWhere((Map<String, String> value) => revisionsBeingRemoved.contains(value['hash']) && channels.contains(fromChannelName(value['channel'])));
+    releases.sort((Map<String, String> a, Map<String, String> b) {
+      final DateTime aDate = DateTime.parse(a['release_date']);
+      final DateTime bDate = DateTime.parse(b['release_date']);
+      return bDate.compareTo(aDate);
+    });
+    jsonData['releases'] = releases;
+    for (Channel channel in channels) {
+      if (!revisionsBeingRemoved.contains(jsonData['current_release'][getChannelName(channel)])) {
+        // Don't replace the current release if it's not one of the revisions we're removing.
+        continue;
+      }
+      final Map<String, String> replacementRelease = releases.firstWhere((Map<String, String> value) => value['channel'] == getChannelName(channel));
+      if (replacementRelease == null) {
+        throw UnpublishException('Unable to find previous release for channel ${getChannelName(channel)}.');
+      }
+      jsonData['current_release'][getChannelName(channel)] = replacementRelease['hash'];
+      print(
+        '${confirmed ? 'Reverting' : 'Would revert'} current ${getChannelName(channel)} '
+        '${getPublishedPlatform(platform)} release to ${replacementRelease['hash']} (version ${replacementRelease['version']}).'
+      );
+    }
+    await _cloudRemoveArchive(paths);
+    await _updateMetadata(jsonData);
+  }
+
+  Future<Map<Channel, Map<String, String>>> _getArchivePaths(List<Map<String, String>> releases) async {
+    final Set<String> hashes = <String>{};
+    final Map<Channel, Map<String, String>> paths = <Channel, Map<String, String>>{};
+    for (Map<String, String> revision in releases) {
+      final String hash = revision['hash'];
+      final Channel channel = fromChannelName(revision['channel']);
+      hashes.add(hash);
+      if (revisionsBeingRemoved.contains(hash) && channels.contains(channel)) {
+        paths[channel] ??= <String, String>{};
+        paths[channel][hash] = revision['archive'];
+      }
+    }
+    final Set<String> missingRevisions = revisionsBeingRemoved.difference(hashes.intersection(revisionsBeingRemoved));
+    if (missingRevisions.isNotEmpty) {
+      final bool plural = missingRevisions.length > 1;
+      throw UnpublishException('Revision${plural ? 's' : ''} $missingRevisions ${plural ? 'are' : 'is'} not present in the server metadata.');
+    }
+    return paths;
+  }
+
+  Future<Map<String, dynamic>> _loadMetadata() async {
+    final File metadataFile = File(
+      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
+    );
+    // Always run this, even in dry runs.
+    await _runGsUtil(<String>['cp', metadataGsPath, metadataFile.absolute.path], confirm: true);
+    final String currentMetadata = metadataFile.readAsStringSync();
+    if (currentMetadata.isEmpty) {
+      throw UnpublishException('Empty metadata received from server');
+    }
+
+    Map<String, dynamic> jsonData;
+    try {
+      jsonData = json.decode(currentMetadata);
+    } on FormatException catch (e) {
+      throw UnpublishException('Unable to parse JSON metadata received from cloud: $e');
+    }
+
+    return jsonData;
+  }
+
+  Future<void> _updateMetadata(Map<String, dynamic> jsonData) async {
+    // We can't just cat the metadata from the server with 'gsutil cat', because
+    // Windows wants to echo the commands that execute in gsutil.bat to the
+    // stdout when we do that. So, we copy the file locally and then read it
+    // back in.
+    final File metadataFile = File(
+      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
+    );
+    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
+    metadataFile.writeAsStringSync(encoder.convert(jsonData));
+    print('${confirmed ? 'Overwriting' : 'Would overwrite'} $metadataGsPath with contents of ${metadataFile.absolute.path}');
+    await _cloudReplaceDest(metadataFile.absolute.path, metadataGsPath);
+  }
+
+  Future<String> _runGsUtil(
+    List<String> args, {
+    Directory workingDirectory,
+    bool failOk = false,
+    bool confirm = false,
+  }) async {
+    final List<String> command = <String>['gsutil', '--']..addAll(args);
+    if (confirm) {
+      return _processRunner.runProcess(
+        command,
+        workingDirectory: workingDirectory,
+        failOk: failOk,
+      );
+    } else {
+      print('Would run: ${command.join(' ')}');
+      return '';
+    }
+  }
+
+  Future<void> _cloudRemoveArchive(Map<Channel, Map<String, String>> paths) async {
+    final List<String> files = <String>[];
+    print('${confirmed ? 'Removing' : 'Would remove'} the following release archives:');
+    for (Channel channel in paths.keys) {
+      final Map<String, String> hashes = paths[channel];
+      for (String hash in hashes.keys) {
+        final String file = '$gsReleaseFolder/${hashes[hash]}';
+        files.add(file);
+        print('  $file');
+      }
+    }
+    await _runGsUtil(<String>['rm']..addAll(files), failOk: true, confirm: confirmed);
+  }
+
+  Future<String> _cloudReplaceDest(String src, String dest) async {
+    assert(dest.startsWith('gs:'), '_cloudReplaceDest must have a destination in cloud storage.');
+    assert(!src.startsWith('gs:'), '_cloudReplaceDest must have a local source file.');
+    // We often don't have permission to overwrite, but
+    // we have permission to remove, so that's what we do first.
+    await _runGsUtil(<String>['rm', dest], failOk: true, confirm: confirmed);
+    String mimeType;
+    if (dest.endsWith('.tar.xz')) {
+      mimeType = 'application/x-gtar';
+    }
+    if (dest.endsWith('.zip')) {
+      mimeType = 'application/zip';
+    }
+    if (dest.endsWith('.json')) {
+      mimeType = 'application/json';
+    }
+    final List<String> args = <String>[];
+    // Use our preferred MIME type for the files we care about
+    // and let gsutil figure it out for anything else.
+    if (mimeType != null) {
+      args.addAll(<String>['-h', 'Content-Type:$mimeType']);
+    }
+    args.addAll(<String>['cp', src, dest]);
+    return await _runGsUtil(args, confirm: confirmed);
+  }
+}
+
+void _printBanner(String message) {
+  final String banner = '*** $message ***';
+  print('\n');
+  print('*' * banner.length);
+  print('$banner');
+  print('*' * banner.length);
+  print('\n');
+}
+
+/// Prepares a flutter git repo to be removed from the published cloud storage.
+Future<void> main(List<String> rawArguments) async {
+  final List<String> allowedChannelValues = Channel.values.map<String>((Channel channel) => getChannelName(channel)).toList();
+  final List<String> allowedPlatformNames = PublishedPlatform.values.map<String>((PublishedPlatform platform) => getPublishedPlatform(platform)).toList();
+  final ArgParser argParser = ArgParser();
+  argParser.addOption(
+    'temp_dir',
+    defaultsTo: null,
+    help: 'A location where temporary files may be written. Defaults to a '
+        'directory in the system temp folder. If a temp_dir is not '
+        'specified, then by default a generated temporary directory will be '
+        'created, used, and removed automatically when the script exits.',
+  );
+  argParser.addMultiOption('revision',
+      help: 'The Flutter git repo revisions to remove from the published site. '
+          'Must be full 40-character hashes. More than one may be specified, '
+          'either by giving the option more than once, or by giving a comma '
+          'separated list. Required.');
+  argParser.addMultiOption(
+    'channel',
+    allowed: allowedChannelValues,
+    help: 'The Flutter channels to remove the archives corresponding to the '
+        'revisions given with --revision. More than one may be specified, '
+        'either by giving the option more than once, or by giving a '
+        'comma separated list. If not specified, then the archives from all '
+        'channels that a revision appears in will be removed.',
+  );
+  argParser.addMultiOption(
+    'platform',
+    allowed: allowedPlatformNames,
+    help: 'The Flutter platforms to remove the archive from. May specify more '
+        'than one, either by giving the option more than once, or by giving a '
+        'comma separated list. If not specified, then the archives from all '
+        'platforms that a revision appears in will be removed.',
+  );
+  argParser.addFlag(
+    'confirm',
+    defaultsTo: false,
+    help: 'If set, will actually remove the archive from Google Cloud Storage '
+        'upon successful execution of this script. Published archives will be '
+        'removed from this directory: $baseUrl$releaseFolder.  This option '
+        'must be set to perform any action on the server, otherwise only a dry '
+        'run is performed.',
+  );
+  argParser.addFlag(
+    'help',
+    defaultsTo: false,
+    negatable: false,
+    help: 'Print help for this command.',
+  );
+
+  final ArgResults parsedArguments = argParser.parse(rawArguments);
+
+  if (parsedArguments['help']) {
+    print(argParser.usage);
+    exit(0);
+  }
+
+  void errorExit(String message, {int exitCode = -1}) {
+    stderr.write('Error: $message\n\n');
+    stderr.write('${argParser.usage}\n');
+    exit(exitCode);
+  }
+
+  final List<String> revisions = parsedArguments['revision'];
+  if (revisions.isEmpty) {
+    errorExit('Invalid argument: at least one --revision must be specified.');
+  }
+  for (String revision in revisions) {
+    if (revision.length != 40) {
+      errorExit('Invalid argument: --revision "$revision" must be the entire hash, not just a prefix.');
+    }
+    if (revision.contains(RegExp(r'[^a-fA-F0-9]'))) {
+      errorExit('Invalid argument: --revision "$revision" contains non-hex characters.');
+    }
+  }
+
+  Directory tempDir;
+  bool removeTempDir = false;
+  if (parsedArguments['temp_dir'] == null || parsedArguments['temp_dir'].isEmpty) {
+    tempDir = Directory.systemTemp.createTempSync('flutter_package.');
+    removeTempDir = true;
+  } else {
+    tempDir = Directory(parsedArguments['temp_dir']);
+    if (!tempDir.existsSync()) {
+      errorExit("Temporary directory ${parsedArguments['temp_dir']} doesn't exist.");
+    }
+  }
+
+  if (!parsedArguments['confirm']) {
+    _printBanner('This will be just a dry run.  To actually perform the changes below, re-run with --confirm argument.');
+  }
+
+  final List<String> channelOptions = parsedArguments['channel'].isNotEmpty ? parsedArguments['channel'] : allowedChannelValues;
+  final Set<Channel> channels = channelOptions.map<Channel>((String value) => fromChannelName(value)).toSet();
+  final List<String> platformOptions = parsedArguments['platform'].isNotEmpty ? parsedArguments['platform'] : allowedPlatformNames;
+  final List<PublishedPlatform> platforms = platformOptions.map<PublishedPlatform>((String value) => fromPublishedPlatform(value)).toList();
+  int exitCode = 0;
+  String message;
+  String stack;
+  try {
+    for (PublishedPlatform platform in platforms) {
+      final ArchiveUnpublisher publisher = ArchiveUnpublisher(
+        tempDir,
+        revisions.toSet(),
+        channels,
+        platform,
+        confirmed: parsedArguments['confirm'],
+      );
+      await publisher.unpublishArchive();
+    }
+  } on UnpublishException catch (e, s) {
+    exitCode = e.exitCode;
+    message = e.message;
+    stack = s.toString();
+  } catch (e, s) {
+    exitCode = -1;
+    message = e.toString();
+    stack = s.toString();
+  } finally {
+    if (removeTempDir) {
+      tempDir.deleteSync(recursive: true);
+    }
+    if (exitCode != 0) {
+      errorExit('$message\n$stack', exitCode: exitCode);
+    }
+    if (!parsedArguments['confirm']) {
+      _printBanner('This was just a dry run.  To actually perform the above changes, re-run with --confirm argument.');
+    }
+    exit(0);
+  }
+}