Refactor prepare_package.dart (#139277)

I plan to extend the prepare_package.dart script to upload the flutter preview device ([design doc](https://docs.google.com/document/d/1AzI-_Uk2v1LA2kKKFJ7gVD4xcakXJ6yVZiS5Ek6RHtg/edit#heading=h.byp03plw7mg9)).

However, given that that script is one large >1k line file, I decided to organize it into smaller libraries in this PR. There should be no behavioral change in this PR, this is a cleanup only. I made the following changes:

1. Created a //dev/bots/prepare_package/ directory to contain helper libraries
2. Moved everything but the `main()` function in //dev/bots/prepare_package.dart into one of 4 helper libraries under the new directory from step 1:
  a. archive_creator.dart which contains the code that creates archive directory locally on disk
  b. archive_publisher.dart which contains the code that uploads the archive to cloud storage
  c. common.dart for shared constants and definitions
  d. process_runner.dart for an abstraction over running sub-processes
3. Changed all definitions to `File` and `Directory` from `dart:io` to use the testable versions from `package:file`. This allowed me to use the `MemoryFileSystem` in the unit tests, rather than creating real temp file system directories.

dev/bots/prepare_package/archive_publisher.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:crypto/crypto.dart';
+import 'package:crypto/src/digest_sink.dart';
+import 'package:file/file.dart';
+import 'package:path/path.dart' as path;
+import 'package:platform/platform.dart' show LocalPlatform, Platform;
+import 'package:process/process.dart';
+
+import 'common.dart';
+import 'process_runner.dart';
+
+class ArchivePublisher {
+  ArchivePublisher(
+    this.tempDir,
+    this.revision,
+    this.branch,
+    this.version,
+    this.outputFile,
+    this.dryRun, {
+    ProcessManager? processManager,
+    bool subprocessOutput = true,
+    required this.fs,
+    this.platform = const LocalPlatform(),
+  })  : assert(revision.length == 40),
+        platformName = platform.operatingSystem.toLowerCase(),
+        metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
+        _processRunner = ProcessRunner(
+          processManager: processManager,
+          subprocessOutput: subprocessOutput,
+        );
+
+  final Platform platform;
+  final FileSystem fs;
+  final String platformName;
+  final String metadataGsPath;
+  final Branch branch;
+  final String revision;
+  final Map<String, String> version;
+  final Directory tempDir;
+  final File outputFile;
+  final ProcessRunner _processRunner;
+  final bool dryRun;
+  String get destinationArchivePath => '${branch.name}/$platformName/${path.basename(outputFile.path)}';
+  static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json';
+
+  Future<String> _getChecksum(File archiveFile) async {
+    final DigestSink digestSink = DigestSink();
+    final ByteConversionSink sink = sha256.startChunkedConversion(digestSink);
+
+    final Stream<List<int>> stream = archiveFile.openRead();
+    await stream.forEach((List<int> chunk) {
+      sink.add(chunk);
+    });
+    sink.close();
+    return digestSink.value.toString();
+  }
+
+  /// Publish the archive to Google Storage.
+  ///
+  /// This method will throw if the target archive already exists on cloud
+  /// storage.
+  Future<void> publishArchive([bool forceUpload = false]) async {
+    final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
+    if (!forceUpload) {
+      if (await _cloudPathExists(destGsPath) && !dryRun) {
+        throw PreparePackageException(
+          'File $destGsPath already exists on cloud storage!',
+        );
+      }
+    }
+    await _cloudCopy(
+      src: outputFile.absolute.path,
+      dest: destGsPath,
+    );
+    assert(tempDir.existsSync());
+    final String gcsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}';
+    await _publishMetadata(gcsPath);
+  }
+
+  /// Downloads and updates the metadata file without publishing it.
+  Future<void> generateLocalMetadata() async {
+    await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}');
+  }
+
+  Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
+    jsonData['base_url'] = '$baseUrl$releaseFolder';
+    if (!jsonData.containsKey('current_release')) {
+      jsonData['current_release'] = <String, String>{};
+    }
+    (jsonData['current_release'] as Map<String, dynamic>)[branch.name] = revision;
+    if (!jsonData.containsKey('releases')) {
+      jsonData['releases'] = <Map<String, dynamic>>[];
+    }
+
+    final Map<String, dynamic> newEntry = <String, dynamic>{};
+    newEntry['hash'] = revision;
+    newEntry['channel'] = branch.name;
+    newEntry['version'] = version[frameworkVersionTag];
+    newEntry['dart_sdk_version'] = version[dartVersionTag];
+    newEntry['dart_sdk_arch'] = version[dartTargetArchTag];
+    newEntry['release_date'] = DateTime.now().toUtc().toIso8601String();
+    newEntry['archive'] = destinationArchivePath;
+    newEntry['sha256'] = await _getChecksum(outputFile);
+
+    // Search for any entries with the same hash and channel and remove them.
+    final List<dynamic> releases = jsonData['releases'] as List<dynamic>;
+    jsonData['releases'] = <Map<String, dynamic>>[
+      for (final Map<String, dynamic> entry in releases.cast<Map<String, dynamic>>())
+        if (entry['hash'] != newEntry['hash'] ||
+            entry['channel'] != newEntry['channel'] ||
+            entry['dart_sdk_arch'] != newEntry['dart_sdk_arch'])
+          entry,
+      newEntry,
+    ]..sort((Map<String, dynamic> a, Map<String, dynamic> b) {
+      final DateTime aDate = DateTime.parse(a['release_date'] as String);
+      final DateTime bDate = DateTime.parse(b['release_date'] as String);
+      return bDate.compareTo(aDate);
+    });
+    return jsonData;
+  }
+
+  Future<void> _updateMetadata(String gsPath) async {
+    // We can't just cat the metadata from the server with 'gsutil cat', because
+    // Windows wants to echo the commands that execute in gsutil.bat to the
+    // stdout when we do that. So, we copy the file locally and then read it
+    // back in.
+    final File metadataFile = fs.file(
+      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
+    );
+    await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
+    Map<String, dynamic> jsonData = <String, dynamic>{};
+    if (!dryRun) {
+      final String currentMetadata = metadataFile.readAsStringSync();
+      if (currentMetadata.isEmpty) {
+        throw PreparePackageException('Empty metadata received from server');
+      }
+      try {
+        jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
+      } on FormatException catch (e) {
+        throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
+      }
+    }
+    // Run _addRelease, even on a dry run, so we can inspect the metadata on a
+    // dry run. On a dry run, the only thing in the metadata file be the new
+    // release.
+    jsonData = await _addRelease(jsonData);
+
+    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
+    metadataFile.writeAsStringSync(encoder.convert(jsonData));
+  }
+
+  /// Publishes the metadata file to GCS.
+  Future<void> _publishMetadata(String gsPath) async {
+    final File metadataFile = fs.file(
+      path.join(tempDir.absolute.path, getMetadataFilename(platform)),
+    );
+    await _cloudCopy(
+      src: metadataFile.absolute.path,
+      dest: gsPath,
+      // This metadata file is used by the website, so we don't want a long
+      // latency between publishing a release and it being available on the
+      // site.
+      cacheSeconds: shortCacheSeconds,
+    );
+  }
+
+  Future<String> _runGsUtil(
+    List<String> args, {
+    Directory? workingDirectory,
+    bool failOk = false,
+  }) async {
+    if (dryRun) {
+      print('gsutil.py -- $args');
+      return '';
+    }
+    return _processRunner.runProcess(
+      <String>['python3', path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'), '--', ...args],
+      workingDirectory: workingDirectory,
+      failOk: failOk,
+    );
+  }
+
+  /// Determine if a file exists at a given [cloudPath].
+  Future<bool> _cloudPathExists(String cloudPath) async {
+    try {
+      await _runGsUtil(
+        <String>['stat', cloudPath],
+      );
+    } on PreparePackageException {
+      // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist
+      return false;
+    }
+    return true;
+  }
+
+  Future<String> _cloudCopy({
+    required String src,
+    required String dest,
+    int? cacheSeconds,
+  }) async {
+    // We often don't have permission to overwrite, but
+    // we have permission to remove, so that's what we do.
+    await _runGsUtil(<String>['rm', dest], failOk: true);
+    String? mimeType;
+    if (dest.endsWith('.tar.xz')) {
+      mimeType = 'application/x-gtar';
+    }
+    if (dest.endsWith('.zip')) {
+      mimeType = 'application/zip';
+    }
+    if (dest.endsWith('.json')) {
+      mimeType = 'application/json';
+    }
+    return _runGsUtil(<String>[
+      // Use our preferred MIME type for the files we care about
+      // and let gsutil figure it out for anything else.
+      if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
+      if (cacheSeconds != null) ...<String>['-h', 'Cache-Control:max-age=$cacheSeconds'],
+      'cp',
+      src,
+      dest,
+    ]);
+  }
+}

dev/bots/prepare_package/common.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' hide Platform;
+
+const String gobMirror = 'https://flutter.googlesource.com/mirrors/flutter';
+const String githubRepo = 'https://github.com/flutter/flutter.git';
+const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
+    '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
+const String releaseFolder = '/releases';
+const String gsBase = 'gs://flutter_infra_release';
+const String gsReleaseFolder = '$gsBase$releaseFolder';
+const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
+const int shortCacheSeconds = 60;
+const String frameworkVersionTag = 'frameworkVersionFromGit';
+const String dartVersionTag = 'dartSdkVersion';
+const String dartTargetArchTag = 'dartTargetArch';
+
+enum Branch {
+  beta,
+  stable,
+  master,
+  main;
+}
+
+/// Exception class for when a process fails to run, so we can catch
+/// it and provide something more readable than a stack trace.
+class PreparePackageException implements Exception {
+  PreparePackageException(this.message, [this.result]);
+
+  final String message;
+  final ProcessResult? result;
+  int get exitCode => result?.exitCode ?? -1;
+
+  @override
+  String toString() {
+    String output = runtimeType.toString();
+    output += ': $message';
+    final String stderr = result?.stderr as String? ?? '';
+    if (stderr.isNotEmpty) {
+      output += ':\n$stderr';
+    }
+    return output;
+  }
+}

dev/bots/prepare_package/process_runner.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io' hide Platform;
+
+import 'package:platform/platform.dart' show LocalPlatform, Platform;
+import 'package:process/process.dart';
+
+import 'common.dart';
+
+/// A helper class for classes that want to run a process.
+///
+/// The stderr and stdout can optionally be reported as the process runs, and
+/// capture the stdout properly without dropping any.
+class ProcessRunner {
+  ProcessRunner({
+    ProcessManager? processManager,
+    this.subprocessOutput = true,
+    this.defaultWorkingDirectory,
+    this.platform = const LocalPlatform(),
+  }) : processManager = processManager ?? const LocalProcessManager() {
+    environment = Map<String, String>.from(platform.environment);
+  }
+
+  /// The platform to use for a starting environment.
+  final Platform platform;
+
+  /// Set [subprocessOutput] to show output as processes run. Stdout from the
+  /// process will be printed to stdout, and stderr printed to stderr.
+  final bool subprocessOutput;
+
+  /// Set the [processManager] in order to inject a test instance to perform
+  /// testing.
+  final ProcessManager processManager;
+
+  /// Sets the default directory used when `workingDirectory` is not specified
+  /// to [runProcess].
+  final Directory? defaultWorkingDirectory;
+
+  /// The environment to run processes with.
+  late Map<String, String> environment;
+
+  /// Run the command and arguments in `commandLine` as a sub-process from
+  /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
+  /// [Directory.current] if [defaultWorkingDirectory] is not set.
+  ///
+  /// Set `failOk` if [runProcess] should not throw an exception when the
+  /// command completes with a non-zero exit code.
+  Future<String> runProcess(
+    List<String> commandLine, {
+    Directory? workingDirectory,
+    bool failOk = false,
+  }) async {
+    workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
+    if (subprocessOutput) {
+      stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
+    }
+    final List<int> output = <int>[];
+    final Completer<void> stdoutComplete = Completer<void>();
+    final Completer<void> stderrComplete = Completer<void>();
+    late Process process;
+    Future<int> allComplete() async {
+      await stderrComplete.future;
+      await stdoutComplete.future;
+      return process.exitCode;
+    }
+
+    try {
+      process = await processManager.start(
+        commandLine,
+        workingDirectory: workingDirectory.absolute.path,
+        environment: environment,
+      );
+      process.stdout.listen(
+        (List<int> event) {
+          output.addAll(event);
+          if (subprocessOutput) {
+            stdout.add(event);
+          }
+        },
+        onDone: () async => stdoutComplete.complete(),
+      );
+      if (subprocessOutput) {
+        process.stderr.listen(
+          (List<int> event) {
+            stderr.add(event);
+          },
+          onDone: () async => stderrComplete.complete(),
+        );
+      } else {
+        stderrComplete.complete();
+      }
+    } on ProcessException catch (e) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
+          'failed with:\n$e';
+      throw PreparePackageException(message);
+    } on ArgumentError catch (e) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
+          'failed with:\n$e';
+      throw PreparePackageException(message);
+    }
+
+    final int exitCode = await allComplete();
+    if (exitCode != 0 && !failOk) {
+      final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
+      throw PreparePackageException(
+        message,
+        ProcessResult(0, exitCode, null, 'returned $exitCode'),
+      );
+    }
+    return utf8.decoder.convert(output).trim();
+  }
+}