Use .pub-cache from Flutter root, if it exists. (#13358)

The purpose of this PR is to make it so that when the user runs 'flutter', if they have a .pub-cache directory in their flutter root, we use that instead of the default location for the pub cache. Otherwise, it should act as before.

The eventual goal is to support a pre-populated flutter .zip/.tar.gz file that has everything the developer needs in one bundle. In order for that to actually work, we need to have the pub cache be self-contained, and not in the user's home dir.

Another advantage of this is that if you have multiple flutter repos that you're switching between, then the versions in the pub cache will remain static when you switch between them.

This is an attempt to re-land: #13248. Includes a fix for the test that makes it work on bots in the presence of PUB_CACHE being set, and no other changes.

.gitignore

@@ -32,6 +32,7 @@
 **/doc/api/
 .flutter-plugins
 .packages
+.pub-cache/
 .pub/
 build/
 flutter_*.png

bin/flutter

@@ -16,7 +16,7 @@ set -e
 
 function follow_links() {
   cd -P "${1%/*}"
-  file="$PWD/${1##*/}"
+  local file="$PWD/${1##*/}"
   while [ -h "$file" ]; do
     # On Mac OS, readlink -f doesn't work.
     cd -P "${file%/*}"
@@ -33,6 +33,39 @@ function path_uri() {
   echo "$1" | sed -E -e "s,^/+,/,"
 }
 
+function upgrade_flutter () {
+  if hash flock 2>/dev/null; then
+    flock 3 # ensures that we don't simultaneously update Dart in multiple parallel instances
+    # some platforms (e.g. Mac) don't have flock or any reliable alternative
+  fi
+
+  local revision=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)`
+  if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -s "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$revision" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then
+    mkdir -p "$FLUTTER_ROOT/bin/cache"
+    touch "$FLUTTER_ROOT/bin/cache/.dartignore"
+    "$FLUTTER_ROOT/bin/internal/update_dart_sdk.sh"
+
+    echo Building flutter tool...
+    if [ "$TRAVIS" == "true" ] || [ "$BOT" == "true" ] || [ "$CONTINUOUS_INTEGRATION" == "true" ] || [ "$CHROME_HEADLESS" == "1" ] || [ "$APPVEYOR" == "true" ] || [ "$CI" == "true" ]; then
+      PUB_ENVIRONMENT="$PUB_ENVIRONMENT:flutter_bot"
+    fi
+    export PUB_ENVIRONMENT="$PUB_ENVIRONMENT:flutter_install"
+
+    if [ -d "$FLUTTER_ROOT/.pub-cache" ]; then
+      export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_ROOT/.pub-cache"}"
+    fi
+
+    while : ; do
+      cd "$FLUTTER_TOOLS_DIR"
+      "$PUB" upgrade --verbosity=error --no-packages-dir && break
+      echo Error: Unable to 'pub upgrade' flutter tool. Retrying in five seconds...
+      sleep 5
+    done
+    "$DART" --snapshot="$SNAPSHOT_PATH" --packages="$FLUTTER_TOOLS_DIR/.packages" "$SCRIPT_PATH"
+    echo "$revision" > "$STAMP_PATH"
+  fi
+}
+
 PROG_NAME="$(path_uri "$(follow_links "$BASH_SOURCE")")"
 BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
 export FLUTTER_ROOT="$(cd "${BIN_DIR}/.." ; pwd -P)"
@@ -70,33 +103,7 @@ FLUTTER_TOOL_ARGS="--assert-initializer $FLUTTER_TOOL_ARGS"
 # FLUTTER_TOOL_ARGS="--checked $FLUTTER_TOOL_ARGS"
 # FLUTTER_TOOL_ARGS="$FLUTTER_TOOL_ARGS --observe=65432"
 
-(
-  if hash flock 2>/dev/null; then
-    flock 3 # ensures that we don't simultaneously update Dart in multiple parallel instances
-    # some platforms (e.g. Mac) don't have flock or any reliable alternative
-  fi
-  REVISION=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)`
-  if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -s "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$REVISION" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then
-    mkdir -p "$FLUTTER_ROOT/bin/cache"
-    touch "$FLUTTER_ROOT/bin/cache/.dartignore"
-    "$FLUTTER_ROOT/bin/internal/update_dart_sdk.sh"
-
-    echo Building flutter tool...
-    LOCAL_PUB_ENV="$PUB_ENVIRONMENT"
-    if [ "$TRAVIS" == "true" ] || [ "$BOT" == "true" ] || [ "$CONTINUOUS_INTEGRATION" == "true" ] || [ "$CHROME_HEADLESS" == "1" ] || [ "$APPVEYOR" == "true" ] || [ "$CI" == "true" ]; then
-      LOCAL_PUB_ENV="$LOCAL_PUB_ENV:flutter_bot"
-    fi
-    LOCAL_PUB_ENV="$LOCAL_PUB_ENV:flutter_install"
-    while : ; do
-      cd "$FLUTTER_TOOLS_DIR"
-      PUB_ENVIRONMENT="$LOCAL_PUB_ENV" "$PUB" upgrade --verbosity=error --no-packages-dir && break
-      echo Error: Unable to 'pub upgrade' flutter tool. Retrying in five seconds...
-      sleep 5
-    done
-    "$DART" --snapshot="$SNAPSHOT_PATH" --packages="$FLUTTER_TOOLS_DIR/.packages" "$SCRIPT_PATH"
-    echo $REVISION > "$STAMP_PATH"
-  fi
-) 3< "$PROG_NAME"
+(upgrade_flutter) 3< "$PROG_NAME"
 
 set +e
 "$DART" $FLUTTER_TOOL_ARGS "$SNAPSHOT_PATH" "$@"

bin/flutter.bat

@@ -24,6 +24,7 @@ SET script_path=%flutter_tools_dir%\bin\flutter_tools.dart
 SET dart_sdk_path=%cache_dir%\dart-sdk
 SET dart_stamp_path=%cache_dir%\dart-sdk.stamp
 SET dart_version_path=%FLUTTER_ROOT%\bin\internal\dart-sdk.version
+SET pub_cache_path=%FLUTTER_ROOT%\.pub-cache
 
 SET dart=%dart_sdk_path%\bin\dart.exe
 SET pub=%dart_sdk_path%\bin\pub.bat
@@ -106,6 +107,9 @@ GOTO :after_subroutine
         SET PUB_ENVIRONMENT=%PUB_ENVIRONMENT%:flutter_bot
       :not_on_bot
       SET PUB_ENVIRONMENT=%PUB_ENVIRONMENT%:flutter_install
+      IF "%PUB_CACHE%" == "" (
+       IF EXIST "%pub_cache_path%" SET PUB_CACHE=%pub_cache_path%
+      )
       :retry_pub_upgrade
       CALL "%pub%" upgrade --verbosity=error --no-packages-dir
       IF "%ERRORLEVEL%" NEQ "0" (

dev/bots/docs.sh

@@ -3,20 +3,29 @@ set -e
 
 # If you want to run this script locally, make sure you run it from
 # the root of the flutter repository.
+export FLUTTER_ROOT="$PWD"
 
 # This is called from travis_upload.sh on Travis.
 
 # Make sure dart is installed
 bin/flutter --version
 
+# If the pub cache directory exists in the root, then use that.
+FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache"
+if [ -d "$FLUTTER_PUB_CACHE" ]; then
+  # This has to be exported, because pub interprets setting it
+  # to the empty string in the same way as setting it to ".".
+  export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
+fi
+
 # Install dartdoc.
 bin/cache/dart-sdk/bin/pub global activate dartdoc 0.14.1
 
 # This script generates a unified doc set, and creates
 # a custom index.html, placing everything into dev/docs/doc.
 (cd dev/tools; ../../bin/cache/dart-sdk/bin/pub get)
-FLUTTER_ROOT=$PWD bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart
-FLUTTER_ROOT=$PWD bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart
+bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart
+bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart
 
 # Ensure google webmaster tools can verify our site.
 cp dev/docs/google2ed1af765c529f57.html dev/docs/doc

dev/bots/test.dart

@@ -14,6 +14,7 @@ final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(P
 final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
 final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
 final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
+final String pubCache = path.join(flutterRoot, '.pub-cache');
 final String flutterTestArgs = Platform.environment['FLUTTER_TEST_ARGS'];
 final bool hasColor = stdout.supportsAnsiEscapes;
 
@@ -204,8 +205,12 @@ Future<Null> _pubRunTest(
   final List<String> args = <String>['run', 'test', '-j1', '-rexpanded'];
   if (testPath != null)
     args.add(testPath);
+  final Map<String, String> pubEnvironment = <String, String>{'DART_VM_OPTIONS': '--assert-initializer'};
+  if (new Directory(pubCache).existsSync()) {
+    pubEnvironment['PUB_CACHE'] = pubCache;
+  }
   return _runCommand(pub, args, workingDirectory: workingDirectory,
-      environment: <String, String>{'DART_VM_OPTIONS': '--assert-initializer'});
+      environment: pubEnvironment);
 }
 
 class EvalResult {

dev/tools/dartdoc.dart

@@ -65,14 +65,25 @@ dependencies:
   }
   new File('dev/docs/lib/temp_doc.dart').writeAsStringSync(contents.toString());
 
+  final String flutterRoot = Directory.current.path;
+  final Map<String, String> pubEnvironment = <String, String>{
+    'FLUTTER_ROOT': flutterRoot,
+  };
+
+  // If there's a .pub-cache dir in the flutter root, use that.
+  final String pubCachePath = '$flutterRoot/.pub-cache';
+  if (new Directory(pubCachePath).existsSync()) {
+    pubEnvironment['PUB_CACHE'] = pubCachePath;
+  }
+
+  final String pubExecutable = '$flutterRoot/bin/cache/dart-sdk/bin/pub';
+
   // Run pub.
   Process process = await Process.start(
-    '../../bin/cache/dart-sdk/bin/pub',
+    pubExecutable,
     <String>['get'],
     workingDirectory: 'dev/docs',
-    environment: <String, String>{
-      'FLUTTER_ROOT': Directory.current.path,
-    },
+    environment: pubEnvironment,
   );
   printStream(process.stdout, prefix: 'pub:stdout: ');
   printStream(process.stderr, prefix: 'pub:stderr: ');
@@ -84,9 +95,10 @@ dependencies:
 
   // Verify which version of dartdoc we're using.
   final ProcessResult result = Process.runSync(
-    '../../bin/cache/dart-sdk/bin/pub',
+    pubExecutable,
     <String>['global', 'run', 'dartdoc', '--version'],
     workingDirectory: 'dev/docs',
+    environment: pubEnvironment,
   );
   print('\n${result.stdout}');
 
@@ -113,9 +125,10 @@ dependencies:
   }
 
   process = await Process.start(
-    '../../bin/cache/dart-sdk/bin/pub',
+    pubExecutable,
     args,
     workingDirectory: 'dev/docs',
+    environment: pubEnvironment,
   );
   printStream(process.stdout, prefix: 'dartdoc:stdout: ',
     filter: kVerbose ? const <Pattern>[] : <Pattern>[

packages/flutter_tools/lib/src/dart/pub.dart

@@ -148,16 +148,26 @@ List<String> _pubCommand(List<String> arguments) {
 ///
 /// [context] provides extra information to package server requests to
 /// understand usage. It must match the regular expression `[a-z][a-z_]*[a-z]`.
-Map<String, String> _createPubEnvironment(String context) => <String, String>{
-  'FLUTTER_ROOT': Cache.flutterRoot,
-  _pubEnvironmentKey: _getPubEnvironmentValue(context),
-};
+Map<String, String> _createPubEnvironment(String context) {
+  final Map<String, String> environment = <String, String>{
+    'FLUTTER_ROOT': Cache.flutterRoot,
+    _pubEnvironmentKey: _getPubEnvironmentValue(context),
+  };
+  final String pubCache = _getRootPubCacheIfAvailable();
+  if (pubCache != null) {
+    environment[_pubCacheEnvironmentKey] = pubCache;
+  }
+  return environment;
+}
 
 final RegExp _analyzerWarning = new RegExp(r'^! \w+ [^ ]+ from path \.\./\.\./bin/cache/dart-sdk/lib/\w+$');
 
 /// The console environment key used by the pub tool.
 const String _pubEnvironmentKey = 'PUB_ENVIRONMENT';
 
+/// The console environment key used by the pub tool to find the cache directory.
+const String _pubCacheEnvironmentKey = 'PUB_CACHE';
+
 final RegExp _validContext = new RegExp('[a-z][a-z_]*[a-z]');
 
 /// Returns the environment value that should be used when running pub.
@@ -189,6 +199,21 @@ String _getPubEnvironmentValue(String pubContext) {
   return values.join(':');
 }
 
+String _getRootPubCacheIfAvailable() {
+  if (platform.environment.containsKey(_pubCacheEnvironmentKey)) {
+    return platform.environment[_pubCacheEnvironmentKey];
+  }
+
+  final String cachePath = fs.path.join(Cache.flutterRoot, '.pub-cache');
+  if (fs.directory(cachePath).existsSync()) {
+    printTrace('Using $cachePath for the pub cache.');
+    return cachePath;
+  }
+
+  // Use pub's default location by returning null.
+  return null;
+}
+
 String _filterOverrideWarnings(String message) {
   // This function filters out these three messages:
   //   Warning: You are using these overridden dependencies:

packages/flutter_tools/test/dart/pub_get_test.dart

@@ -8,7 +8,9 @@ import 'package:file/file.dart';
 import 'package:file/memory.dart';
 import 'package:flutter_tools/src/base/context.dart';
 import 'package:flutter_tools/src/base/io.dart';
+import 'package:flutter_tools/src/base/platform.dart';
 import 'package:flutter_tools/src/dart/pub.dart';
+
 import 'package:mockito/mockito.dart';
 import 'package:process/process.dart';
 import 'package:quiver/testing/async.dart';
@@ -26,8 +28,8 @@ void main() {
       expect(processMock.lastPubEnvironmment, isNull);
       pubGet(context: 'flutter_tests', checkLastModified: false).then((Null value) {
         error = 'test completed unexpectedly';
-      }, onError: (dynamic error) {
-        error = 'test failed unexpectedly';
+      }, onError: (dynamic thrownError) {
+        error = 'test failed unexpectedly: $thrownError';
       });
       expect(testLogger.statusText, '');
       time.elapse(const Duration(milliseconds: 500));
@@ -36,6 +38,7 @@ void main() {
         'pub get failed (69) -- attempting retry 1 in 1 second...\n'
       );
       expect(processMock.lastPubEnvironmment, contains('flutter_cli:ctx_flutter_tests'));
+      expect(processMock.lastPubCache, isNull);
       time.elapse(const Duration(milliseconds: 500));
       expect(testLogger.statusText,
         'Running "flutter packages get" in /...\n'
@@ -79,6 +82,61 @@ void main() {
   }, overrides: <Type, Generator>{
     ProcessManager: () => new MockProcessManager(69),
     FileSystem: () => new MockFileSystem(),
+    Platform: () => new FakePlatform(
+      environment: <String, String>{},
+    ),
+  });
+
+  testUsingContext('pub cache in root is used', () async {
+    String error;
+
+    final MockProcessManager processMock = context.getVariable(ProcessManager);
+
+    new FakeAsync().run((FakeAsync time) {
+      MockDirectory.findCache = true;
+      expect(processMock.lastPubEnvironmment, isNull);
+      expect(processMock.lastPubCache, isNull);
+      pubGet(context: 'flutter_tests', checkLastModified: false).then((Null value) {
+        error = 'test completed unexpectedly';
+      }, onError: (dynamic thrownError) {
+        error = 'test failed unexpectedly: $thrownError';
+      });
+      time.elapse(const Duration(milliseconds: 500));
+      expect(processMock.lastPubCache, endsWith('flutter/.pub-cache'));
+      expect(error, isNull);
+    });
+  }, overrides: <Type, Generator>{
+    ProcessManager: () => new MockProcessManager(69),
+    FileSystem: () => new MockFileSystem(),
+    Platform: () => new FakePlatform(
+      environment: <String, String>{},
+    ),
+  });
+
+  testUsingContext('pub cache in environment is used', () async {
+    String error;
+
+    final MockProcessManager processMock = context.getVariable(ProcessManager);
+
+    new FakeAsync().run((FakeAsync time) {
+      MockDirectory.findCache = false;
+      expect(processMock.lastPubEnvironmment, isNull);
+      expect(processMock.lastPubCache, isNull);
+      pubGet(context: 'flutter_tests', checkLastModified: false).then((Null value) {
+        error = 'test completed unexpectedly';
+      }, onError: (dynamic thrownError) {
+        error = 'test failed unexpectedly: $thrownError';
+      });
+      time.elapse(const Duration(milliseconds: 500));
+      expect(processMock.lastPubCache, equals('path/to/pub-cache'));
+      expect(error, isNull);
+    });
+  }, overrides: <Type, Generator>{
+    ProcessManager: () => new MockProcessManager(69),
+    FileSystem: () => new MockFileSystem(),
+    Platform: () => new FakePlatform(
+      environment: <String, String>{'PUB_CACHE': 'path/to/pub-cache'},
+    ),
   });
 }
 
@@ -90,6 +148,7 @@ class MockProcessManager implements ProcessManager {
   final int fakeExitCode;
 
   String lastPubEnvironmment;
+  String lastPubCache;
 
   @override
   Future<Process> start(
@@ -101,6 +160,7 @@ class MockProcessManager implements ProcessManager {
     ProcessStartMode mode: ProcessStartMode.NORMAL,
   }) {
     lastPubEnvironmment = environment['PUB_ENVIRONMENT'];
+    lastPubCache = environment['PUB_CACHE'];
     return new Future<Process>.value(new MockProcess(fakeExitCode));
   }
 
@@ -159,6 +219,11 @@ class MockFileSystem extends MemoryFileSystem {
   File file(dynamic path) {
     return new MockFile();
   }
+
+  @override
+  Directory directory(dynamic path) {
+    return new MockDirectory(path);
+  }
 }
 
 class MockFile implements File {
@@ -177,4 +242,19 @@ class MockFile implements File {
   dynamic noSuchMethod(Invocation invocation) => null;
 }
 
+class MockDirectory implements Directory {
+  static bool findCache = false;
+
+  MockDirectory(this.path);
+
+  @override
+  final String path;
+
+  @override
+  bool existsSync() => findCache && path.endsWith('.pub-cache');
+
+  @override
+  dynamic noSuchMethod(Invocation invocation) => null;
+}
+
 class MockRandomAccessFile extends Mock implements RandomAccessFile {}