Hot reload: Only sync Dart sources necessary for startup on first launch (#5333)

- [x] Update engine to bring in new snapshotter.
- [x] Use the new snapshotter to quickly determine the minimal set of files necessary to run.
- [x] On first DevFS sync, only sync files necessary to run the application.
- [x] Fix a DevFS unit test failure.
- [x] Include DevFS tests in all.dart.

bin/cache/engine.version

-9f6294c4f389633b9bfb9bea324337ab5ac49558
+607d379c23379ab29948d116ed0e431ef6b7d86a

packages/flutter_tools/lib/src/devfs.dart

@@ -286,13 +286,16 @@ class DevFS {
 
   Future<dynamic> update({ DevFSProgressReporter progressReporter,
                            AssetBundle bundle,
-                           bool bundleDirty: false }) async {
+                           bool bundleDirty: false,
+                           Set<String> fileFilter}) async {
     _reset();
     printTrace('DevFS: Starting sync from $rootDirectory');
     Status status;
     status = logger.startProgress('Scanning project files...');
     Directory directory = rootDirectory;
-    await _scanDirectory(directory, recursive: true);
+    await _scanDirectory(directory,
+                         recursive: true,
+                         fileFilter: fileFilter);
     status.stop(showElapsedTime: true);
 
     status = logger.startProgress('Scanning package files...');
@@ -310,7 +313,8 @@ class DevFS {
         bool packageExists =
             await _scanDirectory(directory,
                                  directoryName: 'packages/$packageName',
-                                 recursive: true);
+                                 recursive: true,
+                                 fileFilter: fileFilter);
         if (packageExists) {
           sb ??= new StringBuffer();
           sb.writeln('$packageName:packages/$packageName');
@@ -440,7 +444,7 @@ class DevFS {
     List<String> ignoredPrefixes = <String>['android/',
                                             'build/',
                                             'ios/',
-                                            'packages/analyzer'];
+                                            '.pub/'];
     for (String ignoredPrefix in ignoredPrefixes) {
       if (devicePath.startsWith(ignoredPrefix))
         return true;
@@ -451,7 +455,8 @@ class DevFS {
   Future<bool> _scanDirectory(Directory directory,
                               {String directoryName,
                                bool recursive: false,
-                               bool ignoreDotFiles: true}) async {
+                               bool ignoreDotFiles: true,
+                               Set<String> fileFilter}) async {
     String prefix = directoryName;
     if (prefix == null) {
       prefix = path.relative(directory.path, from: rootDirectory.path);
@@ -472,6 +477,15 @@ class DevFS {
         }
         final String devicePath =
             path.join(prefix, path.relative(file.path, from: directory.path));
+        if ((fileFilter != null) &&
+            !fileFilter.contains(devicePath)) {
+          // Skip files that are not included in the filter.
+          continue;
+        }
+        if (ignoreDotFiles && devicePath.startsWith('.')) {
+          // Skip directories that start with a dot.
+          continue;
+        }
         if (!_shouldIgnore(devicePath))
           _scanFile(devicePath, file);
       }

packages/flutter_tools/lib/src/hot.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:path/path.dart' as path;
@@ -10,15 +11,18 @@ import 'package:path/path.dart' as path;
 import 'application_package.dart';
 import 'asset.dart';
 import 'base/logger.dart';
+import 'base/process.dart';
 import 'base/utils.dart';
 import 'cache.dart';
 import 'commands/build_apk.dart';
 import 'commands/install.dart';
+import 'dart/package_map.dart';
 import 'device.dart';
 import 'globals.dart';
 import 'devfs.dart';
 import 'observatory.dart';
 import 'resident_runner.dart';
+import 'toolchain.dart';
 
 String getDevFSLoaderScript() {
   return path.absolute(path.join(Cache.flutterRoot,
@@ -29,6 +33,51 @@ String getDevFSLoaderScript() {
                                  'loader_app.dart'));
 }
 
+class StartupDependencySetBuilder {
+  StartupDependencySetBuilder(this.mainScriptPath,
+                              this.projectRootPath);
+
+  final String mainScriptPath;
+  final String projectRootPath;
+
+  Set<String> build() {
+    final String skySnapshotPath =
+        ToolConfiguration.instance.getHostToolPath(HostTool.SkySnapshot);
+
+    final List<String> args = <String>[
+      skySnapshotPath,
+      '--packages=${path.absolute(PackageMap.globalPackagesPath)}',
+      '--print-deps',
+      mainScriptPath
+    ];
+
+    String output;
+    try {
+      output = runCheckedSync(args);
+    } catch (e) {
+      return null;
+    }
+
+    final List<String> lines = LineSplitter.split(output).toList();
+    final Set<String> minimalDependencies = new Set<String>();
+    for (String line in lines) {
+      // We need to convert the uris so that they are relative to the project
+      // root and tweak package: uris so that they reflect their devFS location.
+      if (line.startsWith('package:')) {
+        // Swap out package: for packages/ because we place all package sources
+        // under packages/.
+        line = line.replaceFirst('package:', 'packages/');
+      } else {
+        // Ensure paths are relative to the project root.
+        line = path.relative(line, from: projectRootPath);
+      }
+      minimalDependencies.add(line);
+    }
+    return minimalDependencies;
+  }
+}
+
+
 class FirstFrameTimer {
   FirstFrameTimer(this.serviceProtocol);
 
@@ -81,6 +130,7 @@ class HotRunner extends ResidentRunner {
   ApplicationPackage _package;
   String _mainPath;
   String _projectRootPath;
+  Set<String> _startupDependencies;
   final AssetBundle bundle = new AssetBundle();
   final File pipe;
 
@@ -194,7 +244,8 @@ class HotRunner extends ResidentRunner {
       printStatus('Launching ${getDisplayPath(_mainPath)} on ${device.name}...');
     }
 
-    LaunchResult result = await device.startApp(
+    // Start the loader.
+    Future<LaunchResult> futureResult = device.startApp(
       _package,
       debuggingOptions.buildMode,
       mainPath: device.needsDevFS ? getDevFSLoaderScript() : _mainPath,
@@ -203,6 +254,17 @@ class HotRunner extends ResidentRunner {
       route: route
     );
 
+    // In parallel, compute the minimal dependency set.
+    StartupDependencySetBuilder startupDependencySetBuilder =
+        new StartupDependencySetBuilder(_mainPath, _projectRootPath);
+    _startupDependencies = startupDependencySetBuilder.build();
+    if (_startupDependencies == null) {
+      printError('Error determining the set of Dart sources necessary to start '
+                 'the application. Initial file upload may take a long time.');
+    }
+
+    LaunchResult result = await futureResult;
+
     if (!result.started) {
       if (device.needsDevFS) {
         printError('Error launching DevFS loader on ${device.name}.');
@@ -243,6 +305,10 @@ class HotRunner extends ResidentRunner {
 
     registerSignalHandlers();
 
+    printStatus('Finishing file synchronization...');
+    // Finish the file sync now.
+    await _updateDevFS();
+
     return await waitForAppToFinish();
   }
 
@@ -298,8 +364,11 @@ class HotRunner extends ResidentRunner {
     Status devFSStatus = logger.startProgress('Syncing files to device...');
     await _devFS.update(progressReporter: progressReporter,
                         bundle: bundle,
-                        bundleDirty: rebuildBundle);
+                        bundleDirty: rebuildBundle,
+                        fileFilter: _startupDependencies);
     devFSStatus.stop(showElapsedTime: true);
+    // Clear the minimal set after the first sync.
+    _startupDependencies = null;
     if (progressReporter != null)
       printStatus('Synced ${getSizeAsMB(_devFS.bytes)}.');
     else

packages/flutter_tools/test/all.dart

@@ -21,6 +21,7 @@ import 'config_test.dart' as config_test;
 import 'context_test.dart' as context_test;
 import 'create_test.dart' as create_test;
 import 'daemon_test.dart' as daemon_test;
+import 'devfs_test.dart' as devfs_test;
 import 'device_test.dart' as device_test;
 // import 'devices_test.dart' as devices_test;
 import 'drive_test.dart' as drive_test;
@@ -51,6 +52,7 @@ void main() {
   context_test.main();
   create_test.main();
   daemon_test.main();
+  devfs_test.main();
   device_test.main();
   // devices_test.main(); // https://github.com/flutter/flutter/issues/4480
   drive_test.main();

packages/flutter_tools/test/devfs_test.dart

@@ -58,17 +58,17 @@ void main() {
       expect(devFSOperations.contains('deleteFile test bar/foo.txt'), isTrue);
     });
     testUsingContext('add file in an asset bundle', () async {
-      await devFS.update(bundle: assetBundle);
+      await devFS.update(bundle: assetBundle, bundleDirty: true);
       expect(devFSOperations.contains('writeFile test build/flx/a.txt'), isTrue);
     });
     testUsingContext('add a file to the asset bundle', () async {
       assetBundle.entries.add(new AssetBundleEntry.fromString('b.txt', ''));
-      await devFS.update(bundle: assetBundle);
+      await devFS.update(bundle: assetBundle, bundleDirty: true);
       expect(devFSOperations.contains('writeFile test build/flx/b.txt'), isTrue);
     });
     testUsingContext('delete a file from the asset bundle', () async {
       assetBundle.entries.clear();
-      await devFS.update(bundle: assetBundle);
+      await devFS.update(bundle: assetBundle, bundleDirty: true);
       expect(devFSOperations.contains('deleteFile test build/flx/b.txt'), isTrue);
     });
     testUsingContext('delete dev file system', () async {