Update prepare_package to upload artifacts to old and new buckets. (#75389)

* Update prepare_package to upload to old and new infra buckets.

Bug: https://github.com/flutter/flutter/issues/75363

* Fix failing tests.

* Add duplicated code to a loop.

dev/bots/prepare_package.dart

@@ -17,12 +17,15 @@ import 'package:process/process.dart';
 
 const String chromiumRepo = 'https://chromium.googlesource.com/external/github.com/flutter/flutter';
 const String githubRepo = 'https://github.com/flutter/flutter.git';
-const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra/mingit/'
+const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
     '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
-const String gsBase = 'gs://flutter_infra';
+const String oldGsBase = 'gs://flutter_infra';
 const String releaseFolder = '/releases';
-const String gsReleaseFolder = '$gsBase$releaseFolder';
-const String baseUrl = 'https://storage.googleapis.com/flutter_infra';
+const String oldGsReleaseFolder = '$oldGsBase$releaseFolder';
+const String oldBaseUrl = 'https://storage.googleapis.com/flutter_infra';
+const String newGsBase = 'gs://flutter_infra_release';
+const String newGsReleaseFolder = '$newGsBase$releaseFolder';
+const String newBaseUrl = 'https://storage.googleapis.com/flutter_infra_release';
 
 /// Exception class for when a process fails to run, so we can catch
 /// it and provide something more readable than a stack trace.
@@ -470,13 +473,14 @@ class ArchivePublisher {
     this.revision,
     this.branch,
     this.version,
-    this.outputFile, {
+    this.outputFile,
+    this.dryRun, {
     ProcessManager processManager,
     bool subprocessOutput = true,
     this.platform = const LocalPlatform(),
   })  : assert(revision.length == 40),
         platformName = platform.operatingSystem.toLowerCase(),
-        metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
+        metadataGsPath = '$newGsReleaseFolder/${getMetadataFilename(platform)}',
         _processRunner = ProcessRunner(
           processManager: processManager,
           subprocessOutput: subprocessOutput,
@@ -491,6 +495,7 @@ class ArchivePublisher {
   final Directory tempDir;
   final File outputFile;
   final ProcessRunner _processRunner;
+  final bool dryRun;
   String get branchName => getBranchName(branch);
   String get destinationArchivePath => '$branchName/$platformName/${path.basename(outputFile.path)}';
   static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json';
@@ -512,21 +517,24 @@ class ArchivePublisher {
   /// This method will throw if the target archive already exists on cloud
   /// storage.
   Future<void> publishArchive([bool forceUpload = false]) async {
-    final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
-    if (!forceUpload) {
-      if (await _cloudPathExists(destGsPath)) {
-        throw PreparePackageException(
-          'File $destGsPath already exists on cloud storage!',
-        );
+    for (final String releaseFolder in <String>[oldGsReleaseFolder, newGsReleaseFolder]) {
+      final String destGsPath = '$releaseFolder/$destinationArchivePath';
+      if (!forceUpload) {
+        if (await _cloudPathExists(destGsPath) && !dryRun) {
+          throw PreparePackageException(
+            'File $destGsPath already exists on cloud storage!',
+          );
+        }
       }
+      await _cloudCopy(outputFile.absolute.path, destGsPath);
+      assert(tempDir.existsSync());
+      await _updateMetadata('$releaseFolder/${getMetadataFilename(platform)}', newBucket: false);
     }
-    await _cloudCopy(outputFile.absolute.path, destGsPath);
-    assert(tempDir.existsSync());
-    await _updateMetadata();
   }
 
-  Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
-    jsonData['base_url'] = '$baseUrl$releaseFolder';
+  Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData, {bool newBucket=true}) async {
+    final String tmpBaseUrl = newBucket ? newBaseUrl : oldBaseUrl;
+    jsonData['base_url'] = '$tmpBaseUrl$releaseFolder';
     if (!jsonData.containsKey('current_release')) {
       jsonData['current_release'] = <String, String>{};
     }
@@ -558,7 +566,7 @@ class ArchivePublisher {
     return jsonData;
   }
 
-  Future<void> _updateMetadata() async {
+  Future<void> _updateMetadata(String gsPath, {bool newBucket=true}) async {
     // We can't just cat the metadata from the server with 'gsutil cat', because
     // Windows wants to echo the commands that execute in gsutil.bat to the
     // stdout when we do that. So, we copy the file locally and then read it
@@ -566,24 +574,26 @@ class ArchivePublisher {
     final File metadataFile = File(
       path.join(tempDir.absolute.path, getMetadataFilename(platform)),
     );
-    await _runGsUtil(<String>['cp', metadataGsPath, metadataFile.absolute.path]);
-    final String currentMetadata = metadataFile.readAsStringSync();
-    if (currentMetadata.isEmpty) {
-      throw PreparePackageException('Empty metadata received from server');
-    }
+    await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
+    if (!dryRun) {
+      final String currentMetadata = metadataFile.readAsStringSync();
+      if (currentMetadata.isEmpty) {
+        throw PreparePackageException('Empty metadata received from server');
+      }
 
-    Map<String, dynamic> jsonData;
-    try {
-      jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
-    } on FormatException catch (e) {
-      throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
-    }
+      Map<String, dynamic> jsonData;
+      try {
+        jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
+      } on FormatException catch (e) {
+        throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
+      }
 
-    jsonData = await _addRelease(jsonData);
+      jsonData = await _addRelease(jsonData, newBucket: newBucket);
 
-    const JsonEncoder encoder = JsonEncoder.withIndent('  ');
-    metadataFile.writeAsStringSync(encoder.convert(jsonData));
-    await _cloudCopy(metadataFile.absolute.path, metadataGsPath);
+      const JsonEncoder encoder = JsonEncoder.withIndent('  ');
+      metadataFile.writeAsStringSync(encoder.convert(jsonData));
+    }
+    await _cloudCopy(metadataFile.absolute.path, gsPath);
   }
 
   Future<String> _runGsUtil(
@@ -591,6 +601,10 @@ class ArchivePublisher {
     Directory workingDirectory,
     bool failOk = false,
   }) async {
+    if (dryRun) {
+      print('gsutil.py -- $args');
+      return '';
+    }
     if (platform.isWindows) {
       return _processRunner.runProcess(
         <String>['python', path.join(platform.environment['DEPOT_TOOLS'], 'gsutil.py'), '--', ...args],
@@ -686,7 +700,7 @@ Future<void> main(List<String> rawArguments) async {
     defaultsTo: false,
     help: 'If set, will publish the archive to Google Cloud Storage upon '
         'successful creation of the archive. Will publish under this '
-        'directory: $baseUrl$releaseFolder',
+        'directory: $newBaseUrl$releaseFolder',
   );
   argParser.addFlag(
     'force',
@@ -694,6 +708,12 @@ Future<void> main(List<String> rawArguments) async {
     defaultsTo: false,
     help: 'Overwrite a previously uploaded package.',
   );
+  argParser.addFlag(
+    'dry_run',
+    defaultsTo: false,
+    negatable: false,
+    help: 'Prints gsutil commands instead of executing them.',
+  );
   argParser.addFlag(
     'help',
     defaultsTo: false,
@@ -763,6 +783,7 @@ Future<void> main(List<String> rawArguments) async {
         branch,
         version,
         outputFile,
+	parsedArguments['dry_run'] as bool,
       );
       await publisher.publishArchive(parsedArguments['force'] as bool);
     }

dev/bots/test/prepare_package_test.dart

@@ -241,6 +241,7 @@ void main() {
       final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
       final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip';
       final String gsArchivePath = 'gs://flutter_infra/releases/stable/$platformName/$archiveName';
+      final String newGsArchivePath = 'gs://flutter_infra_release/releases/stable/$platformName/$archiveName';
 
       setUp(() async {
         processManager = FakeProcessManager();
@@ -255,6 +256,7 @@ void main() {
         final String archivePath = path.join(tempDir.absolute.path, archiveName);
         final String jsonPath = path.join(tempDir.absolute.path, releasesName);
         final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
+        final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
         final String releasesJson = '''
 {
   "base_url": "https://storage.googleapis.com/flutter_infra/releases",
@@ -300,9 +302,16 @@ void main() {
           '$gsutilCall -- cp $gsJsonPath $jsonPath': null,
           '$gsutilCall -- rm $gsJsonPath': null,
           '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null,
+          '$gsutilCall -- stat $newGsArchivePath': <ProcessResult>[ProcessResult(0, 1, '', '')],
+          '$gsutilCall -- rm $newGsArchivePath': null,
+          '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
+          '$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
+          '$gsutilCall -- rm $newGsJsonPath': null,
+          '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $newGsJsonPath': null,
         };
         processManager.fakeResults = calls;
         final File outputFile = File(path.join(tempDir.absolute.path, archiveName));
+        outputFile.createSync();
         assert(tempDir.existsSync());
         final ArchivePublisher publisher = ArchivePublisher(
           tempDir,
@@ -310,6 +319,7 @@ void main() {
           Branch.stable,
           'v1.2.3',
           outputFile,
+          false,
           processManager: processManager,
           subprocessOutput: false,
           platform: platform,
@@ -354,6 +364,7 @@ void main() {
           Branch.stable,
           'v1.2.3',
           outputFile,
+          false,
           processManager: processManager,
           subprocessOutput: false,
           platform: platform,
@@ -376,6 +387,7 @@ void main() {
           Branch.stable,
           'v1.2.3',
           outputFile,
+          false,
           processManager: processManager,
           subprocessOutput: false,
           platform: platform,
@@ -383,6 +395,7 @@ void main() {
         final String archivePath = path.join(tempDir.absolute.path, archiveName);
         final String jsonPath = path.join(tempDir.absolute.path, releasesName);
         final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
+        final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
         final String releasesJson = '''
 {
   "base_url": "https://storage.googleapis.com/flutter_infra/releases",
@@ -426,6 +439,11 @@ void main() {
           '$gsutilCall -- cp $gsJsonPath $jsonPath': null,
           '$gsutilCall -- rm $gsJsonPath': null,
           '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null,
+          '$gsutilCall -- rm $newGsArchivePath': null,
+          '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
+          '$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
+          '$gsutilCall -- rm $newGsJsonPath': null,
+          '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $newGsJsonPath': null,
         };
         processManager.fakeResults = calls;
         assert(tempDir.existsSync());