Merge pull request #2059 from abarth/cloud_package

Download sky_engine and sky_services from cloud storage

bin/cache/.gitignore

 *.snapshot
 *.stamp
-artifacts
-dart-sdk
+artifacts/
+dart-sdk/
+pkg/

bin/cache/engine.version

+6e736e7a76600640c5bd854c2fc9ed0f664ee26e

bin/cache/update_dart_sdk.sh

@@ -5,6 +5,8 @@
 
 set -e
 
+FLUTTER_ROOT=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
+
 DART_SDK_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk"
 DART_SDK_STAMP_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk.stamp"
 DART_SDK_VERSION=`cat "$FLUTTER_ROOT/bin/cache/dart-sdk.version"`

bin/cache/update_engine.sh

+#!/bin/bash
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+FLUTTER_ROOT=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
+
+ENGINE_STAMP_PATH="$FLUTTER_ROOT/bin/cache/engine.stamp"
+ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/cache/engine.version"`
+
+if [ ! -f "$ENGINE_STAMP_PATH" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP_PATH"` ]; then
+  echo "Downloading Flutter engine $ENGINE_VERSION..."
+
+  BASE_URL="https://storage.googleapis.com/flutter_infra/flutter/$ENGINE_VERSION"
+  PKG_PATH="$FLUTTER_ROOT/bin/cache/pkg"
+  mkdir -p -- "$PKG_PATH"
+
+  # sky_engine Package
+
+  ENGINE_PKG_URL="$BASE_URL/sky_engine.zip"
+  ENGINE_PKG_ZIP="$FLUTTER_ROOT/bin/cache/sky_engine.zip"
+
+  curl --progress-bar -continue-at=- --location --output "$ENGINE_PKG_ZIP" "$ENGINE_PKG_URL"
+  rm -rf -- "$PKG_PATH/sky_engine"
+  unzip -o -q "$ENGINE_PKG_ZIP" -d "$PKG_PATH"
+  rm -f -- "$ENGINE_PKG_ZIP"
+
+  # sky_services Package
+
+  SERVICES_PKG_URL="$BASE_URL/sky_services.zip"
+  SERVICES_PKG_ZIP="$FLUTTER_ROOT/bin/cache/sky_services.zip"
+
+  curl --progress-bar -continue-at=- --location --output "$SERVICES_PKG_ZIP" "$SERVICES_PKG_URL"
+  rm -rf -- "$PKG_PATH/sky_services"
+  unzip -o -q "$SERVICES_PKG_ZIP" -d "$PKG_PATH"
+  rm -f -- "$SERVICES_PKG_ZIP"
+
+  # Binary artifacts
+
+  ENGINE_ARTIFACT_PATH="$FLUTTER_ROOT/bin/cache/artifacts/engine"
+  rm -rf -- "$ENGINE_ARTIFACT_PATH"
+
+  download_artifacts() {
+    PLATFORM="$1"
+    PLATFORM_PATH="$ENGINE_ARTIFACT_PATH/$PLATFORM"
+
+    mkdir -p -- "$PLATFORM_PATH"
+
+    ARTIFACTS_URL="$BASE_URL/$PLATFORM/artifacts.zip"
+    ARTIFACTS_ZIP="$PLATFORM_PATH/artifacts.zip"
+
+    curl --progress-bar -continue-at=- --location --output "$ARTIFACTS_ZIP" "$ARTIFACTS_URL"
+    unzip -o -q "$ARTIFACTS_ZIP" -d "$PLATFORM_PATH"
+    rm -f -- "$ARTIFACTS_ZIP"
+  }
+
+  download_artifacts android-arm
+
+  case "$(uname -s)" in
+    Darwin)
+      download_artifacts darwin-x64
+      chmod a+x "$ENGINE_ARTIFACT_PATH/darwin-x64/sky_snapshot"
+      ;;
+    Linux)
+      download_artifacts linux-x64
+      chmod a+x "$ENGINE_ARTIFACT_PATH/linux-x64/sky_shell"
+      chmod a+x "$ENGINE_ARTIFACT_PATH/linux-x64/sky_snapshot"
+      ;;
+  esac
+
+  echo "$ENGINE_VERSION" > "$ENGINE_STAMP_PATH"
+fi

bin/flutter

@@ -17,12 +17,11 @@ DART="$DART_SDK_PATH/bin/dart"
 REVISION=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)`
 if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -f "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$REVISION" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then
   "$FLUTTER_ROOT/bin/cache/update_dart_sdk.sh"
+  "$FLUTTER_ROOT/bin/cache/update_engine.sh"
 
   echo Building flutter tool...
   FLUTTER_DIR="$FLUTTER_ROOT/packages/flutter"
-
   (cd "$FLUTTER_TOOLS_DIR"; "../../bin/cache/dart-sdk/bin/pub" get > /dev/null)
-  (cd "$FLUTTER_DIR"; "../../bin/cache/dart-sdk/bin/pub" get > /dev/null)  # Allows us to check if sky_engine's REVISION is correct
   "$DART" --snapshot="$SNAPSHOT_PATH" --package-root="$FLUTTER_TOOLS_DIR/packages" "$SCRIPT_PATH"
   echo $REVISION > "$STAMP_PATH"
 fi

packages/flutter/pubspec.yaml

@@ -8,8 +8,6 @@ dependencies:
   collection: '>=1.1.3 <2.0.0'
   intl: '>=0.12.4+2 <0.13.0'
   material_design_icons: '>=0.0.3 <0.1.0'
-  sky_engine: 0.0.99
-  sky_services: 0.0.99
   vector_math: '>=1.4.5 <2.0.0'
   quiver: '>=0.21.4 <0.22.0'
 
@@ -18,6 +16,11 @@ dependencies:
   # on test.
   test: 0.12.6+1
 
+  sky_engine:
+    path: ../../bin/cache/pkg/sky_engine
+  sky_services:
+    path: ../../bin/cache/pkg/sky_services
+
   cassowary:
     path: ../cassowary
   newton:

packages/flutter_tools/lib/executable.dart

@@ -14,7 +14,6 @@ import 'src/base/process.dart';
 import 'src/commands/analyze.dart';
 import 'src/commands/apk.dart';
 import 'src/commands/build.dart';
-import 'src/commands/cache.dart';
 import 'src/commands/create.dart';
 import 'src/commands/daemon.dart';
 import 'src/commands/devices.dart';
@@ -47,7 +46,6 @@ Future main(List<String> args) async {
     ..addCommand(new AnalyzeCommand())
     ..addCommand(new ApkCommand())
     ..addCommand(new BuildCommand())
-    ..addCommand(new CacheCommand())
     ..addCommand(new CreateCommand())
     ..addCommand(new DaemonCommand(hideCommand: !verboseHelp))
     ..addCommand(new DevicesCommand())

packages/flutter_tools/lib/src/artifacts.dart

@@ -8,7 +8,6 @@ import 'dart:io';
 import 'package:archive/archive.dart';
 import 'package:path/path.dart' as path;
 
-import 'base/os.dart';
 import 'base/process.dart';
 import 'build_configuration.dart';
 import 'globals.dart';
@@ -37,12 +36,6 @@ String _getNameForTargetPlatform(TargetPlatform platform) {
   }
 }
 
-// Keep in sync with https://github.com/flutter/engine/blob/master/sky/tools/release_engine.py
-// and https://github.com/flutter/buildbot/blob/master/travis/build.sh
-String _getCloudStorageBaseUrl({String platform, String revision}) {
-  return 'https://storage.googleapis.com/mojo_infra/flutter/$revision/$platform/';
-}
-
 enum ArtifactType {
   snapshot,
   shell,
@@ -76,12 +69,6 @@ class Artifact {
     assert(false);
     return null;
   }
-
-  // Whether the artifact needs to be marked as executable on disk.
-  bool get executable {
-    return type == ArtifactType.snapshot ||
-      (type == ArtifactType.shell && targetPlatform == TargetPlatform.linux);
-  }
 }
 
 class ArtifactStore {
@@ -192,45 +179,39 @@ class ArtifactStore {
     }
   }
 
-  static void validateSkyEnginePackage() {
-    if (engineRevision == null) {
-      printError("Cannot locate the sky_engine package; did you include 'flutter' in your pubspec.yaml file?");
-      throw new ProcessExit(2);
-    }
-    if (engineRevision != expectedEngineRevision) {
-      printError("Error: incompatible sky_engine package; please run 'pub get' to get the correct one.\n");
-      throw new ProcessExit(2);
-    }
-  }
-
   static String _engineRevision;
 
   static String get engineRevision {
     if (_engineRevision == null) {
-      File revisionFile = new File(path.join(packageRoot, 'sky_engine', 'REVISION'));
+      File revisionFile = new File(path.join(flutterRoot, 'bin', 'cache', 'engine.version'));
       if (revisionFile.existsSync())
-        _engineRevision = revisionFile.readAsStringSync();
+        _engineRevision = revisionFile.readAsStringSync().trim();
     }
     return _engineRevision;
   }
 
-  static String _expectedEngineRevision;
-
-  static String get expectedEngineRevision {
-    if (_expectedEngineRevision == null) {
-      // TODO(jackson): Parse the .packages file and use the path from there instead
-      File revisionFile = new File(path.join(flutterRoot, 'packages', 'flutter', 'packages', 'sky_engine', 'REVISION'));
-      if (revisionFile.existsSync())
-        _expectedEngineRevision = revisionFile.readAsStringSync();
+  static Directory getBaseCacheDir() {
+    if (flutterRoot == null) {
+      printError('FLUTTER_ROOT not specified. Cannot find artifact cache.');
+      throw new ProcessExit(2);
+    }
+    Directory cacheDir = new Directory(path.join(flutterRoot, 'bin', 'cache', 'artifacts'));
+    if (!cacheDir.existsSync()) {
+      printError('${cacheDir.path} does not exist. Cannot find artifact cache.');
+      throw new ProcessExit(2);
     }
-    return _expectedEngineRevision;
+    return cacheDir;
   }
 
-  static String getCloudStorageBaseUrl(String platform) {
-    return _getCloudStorageBaseUrl(
-      platform: platform,
-      revision: engineRevision
-    );
+  static Future<String> getPath(Artifact artifact) async {
+    File cachedFile = new File(path.join(
+        getBaseCacheDir().path, 'engine', artifact.platform, artifact.fileName
+    ));
+    if (!cachedFile.existsSync()) {
+      printError('File not found in the platform artifacts: ${cachedFile.path}');
+      throw new ProcessExit(2);
+    }
+    return cachedFile.path;
   }
 
   /// Download a file from the given URL and return the bytes.
@@ -245,9 +226,8 @@ class ArtifactStore {
       throw new Exception(response.reasonPhrase);
 
     BytesBuilder responseBody = new BytesBuilder(copy: false);
-    await for (List<int> chunk in response) {
+    await for (List<int> chunk in response)
       responseBody.add(chunk);
-    }
 
     return responseBody.takeBytes();
   }
@@ -275,73 +255,6 @@ class ArtifactStore {
     }
   }
 
-  /// Download the artifacts.zip archive for the given platform from GCS
-  /// and extract it to the local cache.
-  static Future _doDownloadArtifactsFromZip(String platform) async {
-    String url = getCloudStorageBaseUrl(platform) + 'artifacts.zip';
-    Directory cacheDir = _getCacheDirForPlatform(platform);
-    await _downloadFileToCache(Uri.parse(url), cacheDir, true);
-
-    for (Artifact artifact in knownArtifacts) {
-      if (artifact.platform == platform && artifact.executable) {
-        ProcessResult result = os.makeExecutable(
-            new File(path.join(cacheDir.path, artifact.fileName)));
-        if (result.exitCode != 0)
-          throw new Exception(result.stderr);
-      }
-    }
-  }
-
-  /// A wrapper ensuring that a platform's ZIP is not downloaded multiple times
-  /// concurrently.
-  static Future _downloadArtifactsFromZip(String platform) {
-    if (_pendingZipDownloads.containsKey(platform)) {
-      return _pendingZipDownloads[platform];
-    }
-    printStatus('Downloading $platform artifacts from the cloud, one moment please...');
-    Future future = _doDownloadArtifactsFromZip(platform);
-    _pendingZipDownloads[platform] = future;
-    return future.then((_) => _pendingZipDownloads.remove(platform));
-  }
-
-  static final Map<String, Future> _pendingZipDownloads = new Map<String, Future>();
-
-  static Directory getBaseCacheDir() {
-    if (flutterRoot == null) {
-      printError('FLUTTER_ROOT not specified. Cannot find artifact cache.');
-      throw new ProcessExit(2);
-    }
-    Directory cacheDir = new Directory(path.join(flutterRoot, 'bin', 'cache', 'artifacts'));
-    if (!cacheDir.existsSync())
-      cacheDir.createSync(recursive: true);
-    return cacheDir;
-  }
-
-  static Directory _getCacheDirForPlatform(String platform) {
-    validateSkyEnginePackage();
-    Directory baseDir = getBaseCacheDir();
-    // TODO(jamesr): Add support for more configurations.
-    String config = 'Release';
-    Directory artifactSpecificDir = new Directory(path.join(
-        baseDir.path, 'sky_engine', engineRevision, config, platform));
-    if (!artifactSpecificDir.existsSync())
-      artifactSpecificDir.createSync(recursive: true);
-    return artifactSpecificDir;
-  }
-
-  static Future<String> getPath(Artifact artifact) async {
-    Directory cacheDir = _getCacheDirForPlatform(artifact.platform);
-    File cachedFile = new File(path.join(cacheDir.path, artifact.fileName));
-    if (!cachedFile.existsSync()) {
-      await _downloadArtifactsFromZip(artifact.platform);
-      if (!cachedFile.existsSync()) {
-        printError('File not found in the platform artifacts: ${cachedFile.path}');
-        throw new ProcessExit(2);
-      }
-    }
-    return cachedFile.path;
-  }
-
   static Future<String> getThirdPartyFile(String urlStr, String cacheSubdir, bool unzip) async {
     Uri url = Uri.parse(urlStr);
     Directory baseDir = getBaseCacheDir();
@@ -359,14 +272,4 @@ class ArtifactStore {
     }
     return cachedFile.path;
   }
-
-  static void clear() {
-    Directory cacheDir = getBaseCacheDir();
-    printTrace('Clearing cache directory ${cacheDir.path}');
-    cacheDir.deleteSync(recursive: true);
-  }
-
-  static Future populate() {
-    return Future.wait(knownArtifacts.map((artifact) => getPath(artifact)));
-  }
 }

packages/flutter_tools/lib/src/commands/cache.dart

-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:async';
-
-import 'package:args/command_runner.dart';
-
-import '../artifacts.dart';
-import '../globals.dart';
-
-class CacheCommand extends Command {
-  final String name = 'cache';
-  final String description = 'Manages Flutter\'s cache of binary artifacts.';
-
-  CacheCommand() {
-    addSubcommand(new _ClearCommand());
-    addSubcommand(new _PopulateCommand());
-  }
-}
-
-class _ClearCommand extends Command {
-  final String name = 'clear';
-  final String description = 'Clears all artifacts from the cache.';
-
-  @override
-  Future<int> run() async {
-    await ArtifactStore.clear();
-    printStatus('Cleared cache directory ${ArtifactStore.getBaseCacheDir().path}.');
-    return 0;
-  }
-}
-
-class _PopulateCommand extends Command {
-  final String name = 'populate';
-  final String description = 'Populates the cache with all known artifacts.';
-
-  @override
-  Future<int> run() async {
-    await ArtifactStore.populate();
-    return 0;
-  }
-}

packages/flutter_tools/lib/src/runner/flutter_command.dart

@@ -8,7 +8,6 @@ import 'dart:io';
 import 'package:args/command_runner.dart';
 
 import '../application_package.dart';
-import '../artifacts.dart';
 import '../build_configuration.dart';
 import '../device.dart';
 import '../globals.dart';
@@ -59,7 +58,6 @@ abstract class FlutterCommand extends Command {
         'Do not run this command from the root of your git clone of Flutter.');
       return false;
     }
-    ArtifactStore.validateSkyEnginePackage();
     return true;
   };
 

travis/setup.sh

@@ -7,8 +7,9 @@ fi
 
 set -x
 
-dart dev/update_packages.dart
-(cd packages/flutter; ../../bin/flutter cache populate)
+# Download dependencies flutter
+./bin/flutter --version
+./bin/cache/dart-sdk/bin/dart ./dev/update_packages.dart
 
 if [ $TRAVIS_PULL_REQUEST = "false" ]; then
   export CLOUDSDK_CORE_DISABLE_PROMPTS=1