Download sky_engine and sky_services from cloud storage

Downloading these packages from cloud storage simplifies our deployment
story because we can upload to cloud storage automatically from the
buildbot.

This patch also switches the responsibility for downloading the engine
artifacts to update_engine.sh. Centralizing this responsibility ensures
that the packages and the binaries are always in sync.

bin/cache/.gitignore

 *.snapshot
 *.stamp
-artifacts
-dart-sdk
+artifacts/
+dart-sdk/
+pkg/

bin/cache/engine.version

+6e736e7a76600640c5bd854c2fc9ed0f664ee26e

bin/cache/update_dart_sdk.sh

@@ -5,6 +5,8 @@
 
 set -e
 
+FLUTTER_ROOT=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
+
 DART_SDK_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk"
 DART_SDK_STAMP_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk.stamp"
 DART_SDK_VERSION=`cat "$FLUTTER_ROOT/bin/cache/dart-sdk.version"`

bin/cache/update_engine.sh

+#!/bin/bash
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+FLUTTER_ROOT=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
+
+ENGINE_STAMP_PATH="$FLUTTER_ROOT/bin/cache/engine.stamp"
+ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/cache/engine.version"`
+
+if [ ! -f "$ENGINE_STAMP_PATH" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP_PATH"` ]; then
+  echo "Downloading Flutter engine $ENGINE_VERSION..."
+
+  BASE_URL="https://storage.googleapis.com/flutter_infra/flutter/$ENGINE_VERSION"
+  PKG_PATH="$FLUTTER_ROOT/bin/cache/pkg"
+  mkdir -p -- "$PKG_PATH"
+
+  # sky_engine Package
+
+  ENGINE_PKG_URL="$BASE_URL/sky_engine.zip"
+  ENGINE_PKG_ZIP="$FLUTTER_ROOT/bin/cache/sky_engine.zip"
+
+  curl --progress-bar -continue-at=- --location --output "$ENGINE_PKG_ZIP" "$ENGINE_PKG_URL"
+  rm -rf -- "$PKG_PATH/sky_engine"
+  unzip -o -q "$ENGINE_PKG_ZIP" -d "$PKG_PATH"
+  rm -f -- "$ENGINE_PKG_ZIP"
+
+  # sky_services Package
+
+  SERVICES_PKG_URL="$BASE_URL/sky_services.zip"
+  SERVICES_PKG_ZIP="$FLUTTER_ROOT/bin/cache/sky_services.zip"
+
+  curl --progress-bar -continue-at=- --location --output "$SERVICES_PKG_ZIP" "$SERVICES_PKG_URL"
+  rm -rf -- "$PKG_PATH/sky_services"
+  unzip -o -q "$SERVICES_PKG_ZIP" -d "$PKG_PATH"
+  rm -f -- "$SERVICES_PKG_ZIP"
+
+  # Binary artifacts
+
+  ENGINE_ARTIFACT_PATH="$FLUTTER_ROOT/bin/cache/artifacts/engine"
+  rm -rf -- "$ENGINE_ARTIFACT_PATH"
+
+  download_artifacts() {
+    PLATFORM="$1"
+    PLATFORM_PATH="$ENGINE_ARTIFACT_PATH/$PLATFORM"
+
+    mkdir -p -- "$PLATFORM_PATH"
+
+    ARTIFACTS_URL="$BASE_URL/$PLATFORM/artifacts.zip"
+    ARTIFACTS_ZIP="$PLATFORM_PATH/artifacts.zip"
+
+    curl --progress-bar -continue-at=- --location --output "$ARTIFACTS_ZIP" "$ARTIFACTS_URL"
+    unzip -o -q "$ARTIFACTS_ZIP" -d "$PLATFORM_PATH"
+    rm -f -- "$ARTIFACTS_ZIP"
+  }
+
+  download_artifacts android-arm
+
+  case "$(uname -s)" in
+    Darwin)
+      download_artifacts darwin-x64
+      chmod a+x "$ENGINE_ARTIFACT_PATH/darwin-x64/sky_snapshot"
+      ;;
+    Linux)
+      download_artifacts linux-x64
+      chmod a+x "$ENGINE_ARTIFACT_PATH/linux-x64/sky_shell"
+      chmod a+x "$ENGINE_ARTIFACT_PATH/linux-x64/sky_snapshot"
+      ;;
+  esac
+
+  echo "$ENGINE_VERSION" > "$ENGINE_STAMP_PATH"
+fi

bin/flutter

@@ -17,12 +17,11 @@ DART="$DART_SDK_PATH/bin/dart"
 REVISION=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)`
 if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -f "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$REVISION" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then
   "$FLUTTER_ROOT/bin/cache/update_dart_sdk.sh"
+  "$FLUTTER_ROOT/bin/cache/update_engine.sh"
 
   echo Building flutter tool...
   FLUTTER_DIR="$FLUTTER_ROOT/packages/flutter"
-
   (cd "$FLUTTER_TOOLS_DIR"; "../../bin/cache/dart-sdk/bin/pub" get > /dev/null)
-  (cd "$FLUTTER_DIR"; "../../bin/cache/dart-sdk/bin/pub" get > /dev/null)  # Allows us to check if sky_engine's REVISION is correct
   "$DART" --snapshot="$SNAPSHOT_PATH" --package-root="$FLUTTER_TOOLS_DIR/packages" "$SCRIPT_PATH"
   echo $REVISION > "$STAMP_PATH"
 fi

packages/flutter/pubspec.yaml

@@ -8,8 +8,6 @@ dependencies:
   collection: '>=1.1.3 <2.0.0'
   intl: '>=0.12.4+2 <0.13.0'
   material_design_icons: '>=0.0.3 <0.1.0'
-  sky_engine: 0.0.99
-  sky_services: 0.0.99
   vector_math: '>=1.4.5 <2.0.0'
   quiver: '>=0.21.4 <0.22.0'
 
@@ -18,6 +16,11 @@ dependencies:
   # on test.
   test: 0.12.6+1
 
+  sky_engine:
+    path: ../../bin/cache/pkg/sky_engine
+  sky_services:
+    path: ../../bin/cache/pkg/sky_services
+
   cassowary:
     path: ../cassowary
   newton:

packages/flutter_tools/lib/executable.dart

@@ -14,7 +14,6 @@ import 'src/base/process.dart';
 import 'src/commands/analyze.dart';
 import 'src/commands/apk.dart';
 import 'src/commands/build.dart';
-import 'src/commands/cache.dart';
 import 'src/commands/create.dart';
 import 'src/commands/daemon.dart';
 import 'src/commands/devices.dart';
@@ -47,7 +46,6 @@ Future main(List<String> args) async {
     ..addCommand(new AnalyzeCommand())
     ..addCommand(new ApkCommand())
     ..addCommand(new BuildCommand())
-    ..addCommand(new CacheCommand())
     ..addCommand(new CreateCommand())
     ..addCommand(new DaemonCommand(hideCommand: !verboseHelp))
     ..addCommand(new DevicesCommand())

packages/flutter_tools/lib/src/artifacts.dart

@@ -8,7 +8,6 @@ import 'dart:io';
 import 'package:archive/archive.dart';
 import 'package:path/path.dart' as path;
 
-import 'base/os.dart';
 import 'base/process.dart';
 import 'build_configuration.dart';
 import 'globals.dart';
@@ -37,12 +36,6 @@ String _getNameForTargetPlatform(TargetPlatform platform) {
   }
 }
 
-// Keep in sync with https://github.com/flutter/engine/blob/master/sky/tools/release_engine.py
-// and https://github.com/flutter/buildbot/blob/master/travis/build.sh
-String _getCloudStorageBaseUrl({String platform, String revision}) {
-  return 'https://storage.googleapis.com/mojo_infra/flutter/$revision/$platform/';
-}
-
 enum ArtifactType {
   snapshot,
   shell,
@@ -76,12 +69,6 @@ class Artifact {
     assert(false);
     return null;
   }
-
-  // Whether the artifact needs to be marked as executable on disk.
-  bool get executable {
-    return type == ArtifactType.snapshot ||
-      (type == ArtifactType.shell && targetPlatform == TargetPlatform.linux);
-  }
 }
 
 class ArtifactStore {
@@ -192,45 +179,39 @@ class ArtifactStore {
     }
   }
 
-  static void validateSkyEnginePackage() {
-    if (engineRevision == null) {
-      printError("Cannot locate the sky_engine package; did you include 'flutter' in your pubspec.yaml file?");
-      throw new ProcessExit(2);
-    }
-    if (engineRevision != expectedEngineRevision) {
-      printError("Error: incompatible sky_engine package; please run 'pub get' to get the correct one.\n");
-      throw new ProcessExit(2);
-    }
-  }
-
   static String _engineRevision;
 
   static String get engineRevision {
     if (_engineRevision == null) {
-      File revisionFile = new File(path.join(packageRoot, 'sky_engine', 'REVISION'));
+      File revisionFile = new File(path.join(flutterRoot, 'bin', 'cache', 'engine.version'));
       if (revisionFile.existsSync())
-        _engineRevision = revisionFile.readAsStringSync();
+        _engineRevision = revisionFile.readAsStringSync().trim();
     }
     return _engineRevision;
   }
 
-  static String _expectedEngineRevision;
-
-  static String get expectedEngineRevision {
-    if (_expectedEngineRevision == null) {
-      // TODO(jackson): Parse the .packages file and use the path from there instead
-      File revisionFile = new File(path.join(flutterRoot, 'packages', 'flutter', 'packages', 'sky_engine', 'REVISION'));
-      if (revisionFile.existsSync())
-        _expectedEngineRevision = revisionFile.readAsStringSync();
+  static Directory getBaseCacheDir() {
+    if (flutterRoot == null) {
+      printError('FLUTTER_ROOT not specified. Cannot find artifact cache.');
+      throw new ProcessExit(2);
+    }
+    Directory cacheDir = new Directory(path.join(flutterRoot, 'bin', 'cache', 'artifacts'));
+    if (!cacheDir.existsSync()) {
+      printError('${cacheDir.path} does not exist. Cannot find artifact cache.');
+      throw new ProcessExit(2);
     }
-    return _expectedEngineRevision;
+    return cacheDir;
   }
 
-  static String getCloudStorageBaseUrl(String platform) {
-    return _getCloudStorageBaseUrl(
-      platform: platform,
-      revision: engineRevision
-    );
+  static Future<String> getPath(Artifact artifact) async {
+    File cachedFile = new File(path.join(
+        getBaseCacheDir().path, 'engine', artifact.platform, artifact.fileName
+    ));
+    if (!cachedFile.existsSync()) {
+      printError('File not found in the platform artifacts: ${cachedFile.path}');
+      throw new ProcessExit(2);
+    }
+    return cachedFile.path;
   }
 
   /// Download a file from the given URL and return the bytes.
@@ -245,9 +226,8 @@ class ArtifactStore {
       throw new Exception(response.reasonPhrase);
 
     BytesBuilder responseBody = new BytesBuilder(copy: false);
-    await for (List<int> chunk in response) {
+    await for (List<int> chunk in response)
       responseBody.add(chunk);
-    }
 
     return responseBody.takeBytes();
   }
@@ -275,73 +255,6 @@ class ArtifactStore {
     }
   }
 
-  /// Download the artifacts.zip archive for the given platform from GCS
-  /// and extract it to the local cache.
-  static Future _doDownloadArtifactsFromZip(String platform) async {
-    String url = getCloudStorageBaseUrl(platform) + 'artifacts.zip';
-    Directory cacheDir = _getCacheDirForPlatform(platform);
-    await _downloadFileToCache(Uri.parse(url), cacheDir, true);
-
-    for (Artifact artifact in knownArtifacts) {
-      if (artifact.platform == platform && artifact.executable) {
-        ProcessResult result = os.makeExecutable(
-            new File(path.join(cacheDir.path, artifact.fileName)));
-        if (result.exitCode != 0)
-          throw new Exception(result.stderr);
-      }
-    }
-  }
-
-  /// A wrapper ensuring that a platform's ZIP is not downloaded multiple times
-  /// concurrently.
-  static Future _downloadArtifactsFromZip(String platform) {
-    if (_pendingZipDownloads.containsKey(platform)) {
-      return _pendingZipDownloads[platform];
-    }
-    printStatus('Downloading $platform artifacts from the cloud, one moment please...');
-    Future future = _doDownloadArtifactsFromZip(platform);
-    _pendingZipDownloads[platform] = future;
-    return future.then((_) => _pendingZipDownloads.remove(platform));
-  }
-
-  static final Map<String, Future> _pendingZipDownloads = new Map<String, Future>();
-
-  static Directory getBaseCacheDir() {
-    if (flutterRoot == null) {
-      printError('FLUTTER_ROOT not specified. Cannot find artifact cache.');
-      throw new ProcessExit(2);
-    }
-    Directory cacheDir = new Directory(path.join(flutterRoot, 'bin', 'cache', 'artifacts'));
-    if (!cacheDir.existsSync())
-      cacheDir.createSync(recursive: true);
-    return cacheDir;
-  }
-
-  static Directory _getCacheDirForPlatform(String platform) {
-    validateSkyEnginePackage();
-    Directory baseDir = getBaseCacheDir();
-    // TODO(jamesr): Add support for more configurations.
-    String config = 'Release';
-    Directory artifactSpecificDir = new Directory(path.join(
-        baseDir.path, 'sky_engine', engineRevision, config, platform));
-    if (!artifactSpecificDir.existsSync())
-      artifactSpecificDir.createSync(recursive: true);
-    return artifactSpecificDir;
-  }
-
-  static Future<String> getPath(Artifact artifact) async {
-    Directory cacheDir = _getCacheDirForPlatform(artifact.platform);
-    File cachedFile = new File(path.join(cacheDir.path, artifact.fileName));
-    if (!cachedFile.existsSync()) {
-      await _downloadArtifactsFromZip(artifact.platform);
-      if (!cachedFile.existsSync()) {
-        printError('File not found in the platform artifacts: ${cachedFile.path}');
-        throw new ProcessExit(2);
-      }
-    }
-    return cachedFile.path;
-  }
-
   static Future<String> getThirdPartyFile(String urlStr, String cacheSubdir, bool unzip) async {
     Uri url = Uri.parse(urlStr);
     Directory baseDir = getBaseCacheDir();
@@ -359,14 +272,4 @@ class ArtifactStore {
     }
     return cachedFile.path;
   }
-
-  static void clear() {
-    Directory cacheDir = getBaseCacheDir();
-    printTrace('Clearing cache directory ${cacheDir.path}');
-    cacheDir.deleteSync(recursive: true);
-  }
-
-  static Future populate() {
-    return Future.wait(knownArtifacts.map((artifact) => getPath(artifact)));
-  }
 }

packages/flutter_tools/lib/src/commands/cache.dart

-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:async';
-
-import 'package:args/command_runner.dart';
-
-import '../artifacts.dart';
-import '../globals.dart';
-
-class CacheCommand extends Command {
-  final String name = 'cache';
-  final String description = 'Manages Flutter\'s cache of binary artifacts.';
-
-  CacheCommand() {
-    addSubcommand(new _ClearCommand());
-    addSubcommand(new _PopulateCommand());
-  }
-}
-
-class _ClearCommand extends Command {
-  final String name = 'clear';
-  final String description = 'Clears all artifacts from the cache.';
-
-  @override
-  Future<int> run() async {
-    await ArtifactStore.clear();
-    printStatus('Cleared cache directory ${ArtifactStore.getBaseCacheDir().path}.');
-    return 0;
-  }
-}
-
-class _PopulateCommand extends Command {
-  final String name = 'populate';
-  final String description = 'Populates the cache with all known artifacts.';
-
-  @override
-  Future<int> run() async {
-    await ArtifactStore.populate();
-    return 0;
-  }
-}

packages/flutter_tools/lib/src/runner/flutter_command.dart

@@ -8,7 +8,6 @@ import 'dart:io';
 import 'package:args/command_runner.dart';
 
 import '../application_package.dart';
-import '../artifacts.dart';
 import '../build_configuration.dart';
 import '../device.dart';
 import '../globals.dart';
@@ -59,7 +58,6 @@ abstract class FlutterCommand extends Command {
         'Do not run this command from the root of your git clone of Flutter.');
       return false;
     }
-    ArtifactStore.validateSkyEnginePackage();
     return true;
   };
 

travis/setup.sh

@@ -7,8 +7,9 @@ fi
 
 set -x
 
-dart dev/update_packages.dart
-(cd packages/flutter; ../../bin/flutter cache populate)
+# Download dependencies flutter
+./bin/flutter --version
+./bin/cache/dart-sdk/bin/dart ./dev/update_packages.dart
 
 if [ $TRAVIS_PULL_REQUEST = "false" ]; then
   export CLOUDSDK_CORE_DISABLE_PROMPTS=1