move the engine download code to dart (#3180)

* move the engine download code to dart

* fix populating the bin/cache/pkg dir

bin/cache/update_engine.sh

-#!/bin/bash
-# Copyright 2016 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-set -e
-
-FLUTTER_ROOT=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
-
-ENGINE_STAMP_PATH="$FLUTTER_ROOT/bin/cache/engine.stamp"
-ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/cache/engine.version"`
-
-if [ ! -f "$ENGINE_STAMP_PATH" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP_PATH"` ]; then
-
-  BASE_URL="https://storage.googleapis.com/flutter_infra/flutter/$ENGINE_VERSION"
-  PKG_PATH="$FLUTTER_ROOT/bin/cache/pkg"
-  mkdir -p -- "$PKG_PATH"
-
-  # sky_engine Package
-
-  echo "Downloading Flutter engine $ENGINE_VERSION..."
-  ENGINE_PKG_URL="$BASE_URL/sky_engine.zip"
-  ENGINE_PKG_ZIP="$FLUTTER_ROOT/bin/cache/sky_engine.zip"
-  curl --progress-bar -continue-at=- --location --output "$ENGINE_PKG_ZIP" "$ENGINE_PKG_URL"
-  rm -rf -- "$PKG_PATH/sky_engine"
-  unzip -o -q "$ENGINE_PKG_ZIP" -d "$PKG_PATH"
-  rm -f -- "$ENGINE_PKG_ZIP"
-
-  # sky_services Package
-
-  echo "  And corresponding services package..."
-  SERVICES_PKG_URL="$BASE_URL/sky_services.zip"
-  SERVICES_PKG_ZIP="$FLUTTER_ROOT/bin/cache/sky_services.zip"
-  curl --progress-bar -continue-at=- --location --output "$SERVICES_PKG_ZIP" "$SERVICES_PKG_URL"
-  rm -rf -- "$PKG_PATH/sky_services"
-  unzip -o -q "$SERVICES_PKG_ZIP" -d "$PKG_PATH"
-  rm -f -- "$SERVICES_PKG_ZIP"
-
-  # Binary artifacts
-
-  ENGINE_ARTIFACT_PATH="$FLUTTER_ROOT/bin/cache/artifacts/engine"
-  rm -rf -- "$ENGINE_ARTIFACT_PATH"
-
-  download_artifacts() {
-    PLATFORM="$1"
-
-    PLATFORM_PATH="$ENGINE_ARTIFACT_PATH/$PLATFORM"
-    mkdir -p -- "$PLATFORM_PATH"
-
-    echo "  And corresponding toolchain for $PLATFORM..."
-    ARTIFACTS_URL="$BASE_URL/$PLATFORM/artifacts.zip"
-    ARTIFACTS_ZIP="$PLATFORM_PATH/artifacts.zip"
-    curl --progress-bar -continue-at=- --location --output "$ARTIFACTS_ZIP" "$ARTIFACTS_URL"
-    unzip -o -q "$ARTIFACTS_ZIP" -d "$PLATFORM_PATH"
-    rm -f -- "$ARTIFACTS_ZIP"
-  }
-
-  download_artifacts android-arm
-
-  case "$(uname -s)" in
-    Darwin)
-      download_artifacts darwin-x64
-      chmod a+x "$ENGINE_ARTIFACT_PATH/darwin-x64/sky_snapshot"
-      download_artifacts ios
-      ;;
-    Linux)
-      download_artifacts linux-x64
-      chmod a+x "$ENGINE_ARTIFACT_PATH/linux-x64/sky_shell"
-      chmod a+x "$ENGINE_ARTIFACT_PATH/linux-x64/sky_snapshot"
-      ;;
-  esac
-
-  echo "$ENGINE_VERSION" > "$ENGINE_STAMP_PATH"
-fi

bin/flutter

@@ -29,7 +29,6 @@ DART="$DART_SDK_PATH/bin/dart"
 REVISION=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)`
 if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -f "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$REVISION" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then
   "$FLUTTER_ROOT/bin/cache/update_dart_sdk.sh"
-  "$FLUTTER_ROOT/bin/cache/update_engine.sh"
 
   echo Building flutter tool...
   FLUTTER_DIR="$FLUTTER_ROOT/packages/flutter"

packages/flutter_tools/lib/src/base/os.dart

@@ -40,7 +40,7 @@ class _PosixUtils extends OperatingSystemUtils {
 
   @override
   ProcessResult makeExecutable(File file) {
-    return Process.runSync('chmod', ['u+x', file.path]);
+    return Process.runSync('chmod', ['a+x', file.path]);
   }
 
   /// Return the path (with symlinks resolved) to the given executable, or `null`

packages/flutter_tools/lib/src/cache.dart

@@ -10,25 +10,27 @@ import 'package:path/path.dart' as path;
 import 'artifacts.dart';
 import 'base/context.dart';
 import 'base/logger.dart';
+import 'base/os.dart';
 import 'base/process.dart';
 import 'globals.dart';
 
-// update_material_fonts
-
 class Cache {
   static Cache get instance => context[Cache] ?? (context[Cache] = new Cache());
 
   /// Return the top-level directory in the cache; this is `bin/cache`.
   Directory getRoot() => new Directory(path.join(ArtifactStore.flutterRoot, 'bin', 'cache'));
 
-  /// Return the top-level mutable directory in the cache; this is `bin/cache/artifacts`.
-  Directory getCacheArtifacts() {
-    Directory artifacts = new Directory(path.join(getRoot().path, 'artifacts'));
-    if (!artifacts.existsSync())
-      artifacts.createSync();
-    return artifacts;
+  /// Return a directory in the cache dir. For `pkg`, this will return `bin/cache/pkg`.
+  Directory getCacheDir(String name) {
+    Directory dir = new Directory(path.join(getRoot().path, name));
+    if (!dir.existsSync())
+      dir.createSync();
+    return dir;
   }
 
+  /// Return the top-level mutable directory in the cache; this is `bin/cache/artifacts`.
+  Directory getCacheArtifacts() => getCacheDir('artifacts');
+
   /// Get a named directory from with the cache's artifact directory; for example,
   /// `material_fonts` would return `bin/cache/artifacts/material_fonts`.
   Directory getArtifactDirectory(String name) {
@@ -50,6 +52,16 @@ class Cache {
     stampFile.writeAsStringSync(version);
   }
 
+  Future<Null> updateAll() async {
+    MaterialFonts materialFonts = new MaterialFonts(cache);
+    if (!materialFonts.isUpToDate())
+      await materialFonts.download();
+
+    FlutterEngine engine = new FlutterEngine(cache);
+    if (!engine.isUpToDate())
+      await engine.download();
+  }
+
   /// Download a file from the given URL and return the bytes.
   static Future<List<int>> _downloadFile(Uri url) async {
     printTrace('Downloading $url.');
@@ -120,3 +132,93 @@ class MaterialFonts {
     });
   }
 }
+
+// TODO(devoncarew): Move the services download to here.
+// gs://flutter_infra/flutter/ed3014b3d337d025393bd894ffa2897e05d43e91/firebase/
+// gs://flutter_infra/flutter/ed3014b3d337d025393bd894ffa2897e05d43e91/gcm/
+
+class FlutterEngine {
+  FlutterEngine(this.cache);
+
+  static const String kName = 'engine';
+
+  final Cache cache;
+
+  List<String> _getEngineDirs() {
+    List<String> dirs = <String>['android-arm', 'android-x64'];
+
+    if (Platform.isMacOS)
+      dirs.addAll(<String>['ios', 'darwin-x64']);
+    else if (Platform.isLinux)
+      dirs.add('linux-x64');
+
+    return dirs;
+  }
+
+  List<String> _getPackageDirs() => <String>['sky_engine', 'sky_services'];
+
+  bool isUpToDate() {
+    Directory pkgDir = cache.getCacheDir('pkg');
+    for (String pkgName in _getPackageDirs()) {
+      Directory dir = new Directory(path.join(pkgDir.path, pkgName));
+      if (!dir.existsSync())
+        return false;
+    }
+
+    Directory engineDir = cache.getArtifactDirectory(kName);
+    for (String dirName in _getEngineDirs()) {
+      Directory dir = new Directory(path.join(engineDir.path, dirName));
+      if (!dir.existsSync())
+        return false;
+    }
+
+    return cache.getVersionFor(kName) == cache.getStampFor(kName);
+  }
+
+  Future<Null> download() async {
+    String engineVersion = cache.getVersionFor(kName);
+    String url = 'https://storage.googleapis.com/flutter_infra/flutter/$engineVersion/';
+
+    bool allDirty = engineVersion != cache.getStampFor(kName);
+
+    Directory pkgDir = cache.getCacheDir('pkg');
+    for (String pkgName in _getPackageDirs()) {
+      Directory dir = new Directory(path.join(pkgDir.path, pkgName));
+      if (!dir.existsSync() || allDirty) {
+        await _downloadItem('Downloading engine package $pkgName...',
+          url + pkgName + '.zip', pkgDir);
+      }
+    }
+
+    Directory engineDir = cache.getArtifactDirectory(kName);
+    for (String dirName in _getEngineDirs()) {
+      Directory dir = new Directory(path.join(engineDir.path, dirName));
+      if (!dir.existsSync() || allDirty) {
+        await _downloadItem('Downloading engine artifacts $dirName...',
+          url + dirName + '/artifacts.zip', dir);
+        _makeFilesExecutable(dir);
+      }
+    }
+
+    cache.setStampFor(kName, cache.getVersionFor(kName));
+  }
+
+  void _makeFilesExecutable(Directory dir) {
+    for (FileSystemEntity entity in dir.listSync()) {
+      if (entity is File) {
+        String name = path.basename(entity.path);
+        if (name == 'sky_snapshot' || name == 'sky_shell')
+          os.makeExecutable(entity);
+      }
+    }
+  }
+
+  Future<Null> _downloadItem(String message, String url, Directory dest) {
+    Status status = logger.startProgress(message);
+    return Cache._downloadFileToCache(Uri.parse(url), dest, true).then((_) {
+      status.stop(showElapsedTime: true);
+    }).whenComplete(() {
+      status.cancel();
+    });
+  }
+}

packages/flutter_tools/lib/src/commands/upgrade.dart

@@ -54,6 +54,9 @@ class UpgradeCommand extends FlutterCommand {
     if (code != 0)
       return code;
 
+    // Check for and download any engine updates.
+    await cache.updateAll();
+
     if (FileSystemEntity.isFileSync('pubspec.yaml')) {
       printStatus('');
       code = await pubGet(upgrade: true, checkLastModified: false);

packages/flutter_tools/lib/src/runner/flutter_command.dart

@@ -9,7 +9,6 @@ import 'package:args/command_runner.dart';
 
 import '../application_package.dart';
 import '../build_configuration.dart';
-import '../cache.dart';
 import '../dart/pub.dart';
 import '../device.dart';
 import '../flx.dart' as flx;
@@ -128,9 +127,7 @@ abstract class FlutterCommand extends Command {
     }
 
     // Populate the cache.
-    MaterialFonts materialFonts = new MaterialFonts(cache);
-    if (!materialFonts.isUpToDate())
-      await materialFonts.download();
+    await cache.updateAll();
 
     _setupToolchain();
     _setupApplicationPackages();