Merge pull request #494 from jason-simmons/apk_download_all_artifacts

Download build artifacts from a zip file

packages/flutter/pubspec.yaml

@@ -7,8 +7,8 @@ dependencies:
   collection: '>=1.1.3 <2.0.0'
   intl: '>=0.12.4+2 <0.13.0'
   material_design_icons: '>=0.0.3 <0.1.0'
-  sky_engine: 0.0.57
-  sky_services: 0.0.57
+  sky_engine: 0.0.58
+  sky_services: 0.0.58
   vector_math: '>=1.4.3 <2.0.0'
 
   cassowary:

packages/flutter_tools/lib/src/artifacts.dart

@@ -5,6 +5,7 @@
 import 'dart:async';
 import 'dart:io';
 
+import 'package:archive/archive.dart';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart' as path;
 
@@ -14,9 +15,6 @@ import 'process.dart';
 
 final Logger _logging = new Logger('flutter_tools.artifacts');
 
-const String _kShellCategory = 'shell';
-const String _kViewerCategory = 'viewer';
-
 String _getNameForHostPlatform(HostPlatform platform) {
   switch (platform) {
     case HostPlatform.linux:
@@ -43,14 +41,12 @@ String _getNameForTargetPlatform(TargetPlatform platform) {
 
 // Keep in sync with https://github.com/flutter/engine/blob/master/sky/tools/release_engine.py
 // and https://github.com/flutter/buildbot/blob/master/travis/build.sh
-String _getCloudStorageBaseUrl({String category, String platform, String revision}) {
-  if (platform == 'darwin-x64') {
+String _getCloudStorageBaseUrl({String platform, String revision}) {
   // In the fullness of time, we'll have a consistent URL pattern for all of
   // our artifacts, but, for the time being, Mac OS X artifacts are stored in a
   // different cloud storage bucket.
-    return 'https://storage.googleapis.com/mojo_infra/flutter/$platform/$revision/';
-  }
-  return 'https://storage.googleapis.com/mojo/sky/$category/$platform/$revision/';
+  String bucket = (platform == 'darwin-x64') ? "mojo_infra" : "mojo";
+  return 'https://storage.googleapis.com/$bucket/flutter/$revision/$platform/';
 }
 
 enum ArtifactType {
@@ -63,7 +59,6 @@ class Artifact {
   const Artifact._({
     this.name,
     this.fileName,
-    this.category,
     this.type,
     this.hostPlatform,
     this.targetPlatform
@@ -71,7 +66,6 @@ class Artifact {
 
   final String name;
   final String fileName;
-  final String category; // TODO(abarth): Remove categories.
   final ArtifactType type;
   final HostPlatform hostPlatform;
   final TargetPlatform targetPlatform;
@@ -87,7 +81,6 @@ class Artifact {
 
   String getUrl(String revision) {
     return _getCloudStorageBaseUrl(
-      category: category,
       platform: platform,
       revision: revision
     ) + fileName;
@@ -102,35 +95,30 @@ class ArtifactStore {
     const Artifact._(
       name: 'Sky Shell',
       fileName: 'SkyShell.apk',
-      category: _kShellCategory,
       type: ArtifactType.shell,
       targetPlatform: TargetPlatform.android
     ),
     const Artifact._(
       name: 'Sky Snapshot',
       fileName: 'sky_snapshot',
-      category: _kShellCategory,
       type: ArtifactType.snapshot,
       hostPlatform: HostPlatform.linux
     ),
     const Artifact._(
       name: 'Sky Snapshot',
       fileName: 'sky_snapshot',
-      category: _kShellCategory,
       type: ArtifactType.snapshot,
       hostPlatform: HostPlatform.mac
     ),
     const Artifact._(
       name: 'Flutter for Mojo',
       fileName: 'flutter.mojo',
-      category: _kShellCategory,
       type: ArtifactType.mojo,
       targetPlatform: TargetPlatform.android
     ),
     const Artifact._(
       name: 'Flutter for Mojo',
       fileName: 'flutter.mojo',
-      category: _kShellCategory,
       type: ArtifactType.mojo,
       targetPlatform: TargetPlatform.linux
     ),
@@ -192,28 +180,64 @@ class ArtifactStore {
     return _engineRevision;
   }
 
-  static String getCloudStorageBaseUrl(String category, String platform) {
+  static String getCloudStorageBaseUrl(String platform) {
     return _getCloudStorageBaseUrl(
-      category: category,
       platform: platform,
       revision: engineRevision
     );
   }
 
-  static Future _downloadFile(String url, File file) async {
-    _logging.info('Downloading $url to ${file.path}.');
+  /// Download the artifacts.zip archive for the given platform from GCS
+  /// and extract it to the local cache.
+  static Future _doDownloadArtifactsFromZip(String platform) async {
+    String url = getCloudStorageBaseUrl(platform) + 'artifacts.zip';
+    _logging.info('Downloading $url.');
+
     HttpClient httpClient = new HttpClient();
     HttpClientRequest request = await httpClient.getUrl(Uri.parse(url));
     HttpClientResponse response = await request.close();
     _logging.fine('Received response statusCode=${response.statusCode}');
     if (response.statusCode != 200)
       throw new Exception(response.reasonPhrase);
-    IOSink sink = file.openWrite();
-    await sink.addStream(response);
+
+    BytesBuilder responseBody = new BytesBuilder(copy: false);
+    await for (List<int> chunk in response) {
+      responseBody.add(chunk);
+    }
+
+    Archive archive = new ZipDecoder().decodeBytes(responseBody.takeBytes());
+    Directory cacheDir = _getCacheDirForPlatform(platform);
+    for (ArchiveFile archiveFile in archive) {
+      File cacheFile = new File(path.join(cacheDir.path, archiveFile.name));
+      IOSink sink = cacheFile.openWrite();
+      sink.add(archiveFile.content);
       await sink.close();
-    _logging.fine('Wrote file');
     }
 
+    for (Artifact artifact in knownArtifacts) {
+      if (artifact.platform == platform && artifact.executable) {
+        ProcessResult result = osUtils.makeExecutable(
+            new File(path.join(cacheDir.path, artifact.fileName)));
+        if (result.exitCode != 0)
+          throw new Exception(result.stderr);
+      }
+    }
+  }
+
+  /// A wrapper ensuring that a platform's ZIP is not downloaded multiple times
+  /// concurrently.
+  static Future _downloadArtifactsFromZip(String platform) {
+    if (_pendingZipDownloads.containsKey(platform)) {
+      return _pendingZipDownloads[platform];
+    }
+    print('Downloading $platform artifacts from the cloud, one moment please...');
+    Future future = _doDownloadArtifactsFromZip(platform);
+    _pendingZipDownloads[platform] = future;
+    return future.then((_) => _pendingZipDownloads.remove(platform));
+  }
+
+  static final Map<String, Future> _pendingZipDownloads = new Map<String, Future>();
+
   static Directory _getBaseCacheDir() {
     if (flutterRoot == null) {
       _logging.severe('FLUTTER_ROOT not specified. Cannot find artifact cache.');
@@ -225,27 +249,25 @@ class ArtifactStore {
     return cacheDir;
   }
 
-  static Directory _getCacheDirForArtifact(Artifact artifact) {
+  static Directory _getCacheDirForPlatform(String platform) {
     Directory baseDir = _getBaseCacheDir();
     // TODO(jamesr): Add support for more configurations.
     String config = 'Release';
     Directory artifactSpecificDir = new Directory(path.join(
-        baseDir.path, 'sky_engine', engineRevision, config, artifact.platform));
+        baseDir.path, 'sky_engine', engineRevision, config, platform));
     if (!artifactSpecificDir.existsSync())
       artifactSpecificDir.createSync(recursive: true);
     return artifactSpecificDir;
   }
 
   static Future<String> getPath(Artifact artifact) async {
-    Directory cacheDir = _getCacheDirForArtifact(artifact);
+    Directory cacheDir = _getCacheDirForPlatform(artifact.platform);
     File cachedFile = new File(path.join(cacheDir.path, artifact.fileName));
     if (!cachedFile.existsSync()) {
-      print('Downloading ${artifact.name} from the cloud, one moment please...');
-      await _downloadFile(artifact.getUrl(engineRevision), cachedFile);
-      if (artifact.executable) {
-        ProcessResult result = osUtils.makeExecutable(cachedFile);
-        if (result.exitCode != 0)
-          throw new Exception(result.stderr);
+      await _downloadArtifactsFromZip(artifact.platform);
+      if (!cachedFile.existsSync()) {
+        _logging.severe('File not found in the platform artifacts: ${cachedFile.path}');
+        throw new ProcessExit(2);
       }
     }
     return cachedFile.path;

packages/flutter_tools/lib/src/commands/run_mojo.dart

@@ -89,7 +89,7 @@ class RunMojoCommand extends FlutterCommand {
 
     if (argResults['android']) {
       args.add('--android');
-      final String cloudStorageBaseUrl = ArtifactStore.getCloudStorageBaseUrl('shell', 'android-arm');
+      final String cloudStorageBaseUrl = ArtifactStore.getCloudStorageBaseUrl('android-arm');
       final String appPath = _makePathAbsolute(bundlePath);
       final String appName = path.basename(appPath);
       final String appDir = path.dirname(appPath);

travis/download_tester.py

@@ -7,6 +7,7 @@ import argparse
 import os
 import subprocess
 import sys
+import zipfile
 
 def download(base_url, out_dir, name):
     url = '%s/%s' % (base_url, name)
@@ -28,10 +29,12 @@ def main():
     with open(args.revision_file, 'r') as f:
         revision = f.read()
 
-    base_url = 'https://storage.googleapis.com/mojo/sky/shell/linux-x64/%s' % revision
-    download(base_url, out_dir, 'sky_shell')
-    download(base_url, out_dir, 'icudtl.dat')
-    download(base_url, out_dir, 'sky_snapshot')
+    base_url = 'https://storage.googleapis.com/mojo/flutter/%s/linux-x64' % revision
+    download(base_url, out_dir, 'artifacts.zip')
+
+    artifacts_zip = zipfile.ZipFile(os.path.join(out_dir, 'artifacts.zip'))
+    artifacts_zip.extractall(out_dir)
+    artifacts_zip.close()
 
     subprocess.call([ 'chmod', 'a+x', os.path.join(out_dir, 'sky_shell' )])
     subprocess.call([ 'chmod', 'a+x', os.path.join(out_dir, 'sky_snapshot' )])