Add flutter tool support for creating app-specific VM snapshots. (#18417)

Add flutter tool support for creating app-specific VM snapshots.

packages/flutter_tools/bin/xcode_backend.sh

@@ -166,8 +166,10 @@ BuildApp() {
   RunCommand "${FLUTTER_ROOT}/bin/flutter" --suppress-analytics             \
     ${verbose_flag}                                                         \
     build bundle                                                            \
+    --target-platform=ios                                                   \
     --target="${target_path}"                                               \
     --snapshot="${build_dir}/snapshot_blob.bin"                             \
+    --${build_mode}                                                         \
     --depfile="${build_dir}/snapshot_blob.bin.d"                            \
     --asset-dir="${derived_dir}/flutter_assets"                             \
     ${precompilation_flag}                                                  \

packages/flutter_tools/gradle/flutter.gradle

@@ -276,7 +276,10 @@ class FlutterPlugin implements Plugin<Project> {
         if (project.hasProperty('track-widget-creation')) {
             trackWidgetCreationValue = project.property('track-widget-creation').toBoolean()
         }
-
+        Boolean buildSnapshotValue = false
+        if (project.hasProperty('build-snapshot')) {
+            buildSnapshotValue = project.property('build-snapshot').toBoolean()
+        }
         String extraFrontEndOptionsValue = null
         if (project.hasProperty('extra-front-end-options')) {
             extraFrontEndOptionsValue = project.property('extra-front-end-options')
@@ -319,6 +322,7 @@ class FlutterPlugin implements Plugin<Project> {
                 fileSystemRoots fileSystemRootsValue
                 fileSystemScheme fileSystemSchemeValue
                 trackWidgetCreation trackWidgetCreationValue
+                buildSnapshot buildSnapshotValue
                 buildSharedLibrary buildSharedLibraryValue
                 targetPlatform targetPlatformValue
                 sourceDir project.file(project.flutter.source)
@@ -368,6 +372,8 @@ abstract class BaseFlutterTask extends DefaultTask {
     @Optional @Input
     Boolean trackWidgetCreation
     @Optional @Input
+    Boolean buildSnapshot
+    @Optional @Input
     Boolean buildSharedLibrary
     @Optional @Input
     String targetPlatform
@@ -380,8 +386,7 @@ abstract class BaseFlutterTask extends DefaultTask {
 
     @OutputFiles
     FileCollection getDependenciesFiles() {
-        if (buildMode != 'debug') {
-            // For AOT builds, include the gen_snapshot depfile.
+        // For AOT and Core JIT builds, include the gen_snapshot depfile.
         FileCollection depfiles = project.files("${intermediateDir}/snapshot.d")
         if (previewDart2) {
             // For Dart 2, also include the kernel compiler depfile, since
@@ -389,10 +394,13 @@ abstract class BaseFlutterTask extends DefaultTask {
             // and it includes all the Dart sources.
             depfiles += project.files("${intermediateDir}/kernel_compile.d")
         }
+
+        // Include Core JIT kernel compiler depfile, since kernel compile is
+        // the first stage of JIT builds in this mode, and it includes all the
+        // Dart sources.
+        depfiles += project.files("${intermediateDir}/snapshot_blob.bin.d")
         return depfiles
     }
-        return project.files("${intermediateDir}/snapshot_blob.bin.d")
-    }
 
     void buildBundle() {
         if (!sourceDir.isDirectory()) {
@@ -468,9 +476,18 @@ abstract class BaseFlutterTask extends DefaultTask {
             if (trackWidgetCreation) {
                 args "--track-widget-creation"
             }
+            if (buildSnapshot) {
+                args "--build-snapshot"
+            }
             if (extraFrontEndOptions != null) {
                 args "--extra-front-end-options", "${extraFrontEndOptions}"
             }
+            if (extraGenSnapshotOptions != null) {
+                args "--extra-gen-snapshot-options", "${extraGenSnapshotOptions}"
+            }
+            if (targetPlatform != null) {
+                args "--target-platform", "${targetPlatform}"
+            }
             if (buildMode != "debug") {
                 args "--precompiled"
             } else {
@@ -480,6 +497,7 @@ abstract class BaseFlutterTask extends DefaultTask {
                 }
             }
             args "--asset-dir", "${intermediateDir}/flutter_assets"
+            args "--${buildMode}"
         }
     }
 }
@@ -496,7 +514,7 @@ class FlutterTask extends BaseFlutterTask {
 
             include "flutter_assets/**" // the working dir and its files
 
-            if (buildMode != 'debug') {
+            if (buildMode != 'debug' || buildSnapshot) {
                 if (buildSharedLibrary) {
                     include "app.so"
                 } else {

packages/flutter_tools/lib/src/android/gradle.dart

@@ -331,6 +331,8 @@ Future<Null> _buildGradleProjectV2(String gradle, BuildInfo buildInfo, String ta
     command.add('-Ppreview-dart-2=true');
     if (buildInfo.trackWidgetCreation)
       command.add('-Ptrack-widget-creation=true');
+    if (buildInfo.buildSnapshot)
+      command.add('-Pbuild-snapshot=true');
     if (buildInfo.extraFrontEndOptions != null)
       command.add('-Pextra-front-end-options=${buildInfo.extraFrontEndOptions}');
     if (buildInfo.extraGenSnapshotOptions != null)
@@ -345,8 +347,8 @@ Future<Null> _buildGradleProjectV2(String gradle, BuildInfo buildInfo, String ta
   if (buildInfo.buildSharedLibrary && androidSdk.ndk != null) {
     command.add('-Pbuild-shared-library=true');
   }
-  if (buildInfo.targetPlatform == TargetPlatform.android_arm64)
-    command.add('-Ptarget-platform=android-arm64');
+  if (buildInfo.targetPlatform != null)
+    command.add('-Ptarget-platform=${getNameForTargetPlatform(buildInfo.targetPlatform)}');
 
   command.add(assembleTask);
   final int exitCode = await runCommandAndStreamOutput(

packages/flutter_tools/lib/src/base/build.dart

@@ -416,3 +416,119 @@ class AOTSnapshotter {
     return fs.path.dirname(fs.path.fromUri(packageMap.map[package]));
   }
 }
+
+class CoreJITSnapshotter {
+  /// Builds a "Core JIT" VM snapshot of the specified kernel. This snapshot
+  /// includes data as well as either machine code or DBC, depending on build
+  /// configuration.
+  Future<int> build({
+    @required TargetPlatform platform,
+    @required BuildMode buildMode,
+    @required String mainPath,
+    @required String packagesPath,
+    @required String outputPath,
+    List<String> extraGenSnapshotOptions = const <String>[],
+  }) async {
+    if (!_isValidCoreJitPlatform(platform)) {
+      printError('${getNameForTargetPlatform(platform)} does not support Core JIT compilation.');
+      return 1;
+    }
+
+    final Directory outputDir = fs.directory(outputPath);
+    outputDir.createSync(recursive: true);
+
+    final List<String> inputPaths = <String>[mainPath];
+    final Set<String> outputPaths = new Set<String>();
+
+    final String depfilePath = fs.path.join(outputDir.path, 'snapshot.d');
+    final List<String> genSnapshotArgs = <String>[
+      '--reify-generic-functions',
+      '--strong',
+    ];
+    if (buildMode == BuildMode.debug) {
+      genSnapshotArgs.add('--enable_asserts');
+    }
+    if (extraGenSnapshotOptions != null && extraGenSnapshotOptions.isNotEmpty) {
+      printTrace('Extra gen_snapshot options: $extraGenSnapshotOptions');
+      genSnapshotArgs.addAll(extraGenSnapshotOptions);
+    }
+
+    // Blob Core JIT snapshot.
+    final String vmSnapshotData = fs.path.join(outputDir.path, 'vm_snapshot_data');
+    final String isolateSnapshotData = fs.path.join(outputDir.path, 'isolate_snapshot_data');
+    final String vmSnapshotInstructions = fs.path.join(outputDir.path, 'vm_snapshot_instr');
+    final String isolateSnapshotInstructions = fs.path.join(outputDir.path, 'isolate_snapshot_instr');
+    outputPaths.addAll(<String>[vmSnapshotData, isolateSnapshotData, vmSnapshotInstructions, isolateSnapshotInstructions]);
+    genSnapshotArgs.addAll(<String>[
+      '--snapshot_kind=core-jit',
+      '--vm_snapshot_data=$vmSnapshotData',
+      '--isolate_snapshot_data=$isolateSnapshotData',
+      '--vm_snapshot_instructions=$vmSnapshotInstructions',
+      '--isolate_snapshot_instructions=$isolateSnapshotInstructions',
+      '--load_compilation_trace=trace.txt',
+    ]);
+
+    if (platform == TargetPlatform.android_arm) {
+      // Use softfp for Android armv7 devices.
+      // TODO(cbracken) eliminate this when we fix https://github.com/flutter/flutter/issues/17489
+      genSnapshotArgs.add('--no-sim-use-hardfp');
+
+      // Not supported by the Pixel in 32-bit mode.
+      genSnapshotArgs.add('--no-use-integer-division');
+    }
+
+    genSnapshotArgs.add(mainPath);
+
+    // Verify that all required inputs exist.
+    final Iterable<String> missingInputs = inputPaths.where((String p) => !fs.isFileSync(p));
+    if (missingInputs.isNotEmpty) {
+      printError('Missing input files: $missingInputs from $inputPaths');
+      return 1;
+    }
+
+    // If inputs and outputs have not changed since last run, skip the build.
+    final Fingerprinter fingerprinter = new Fingerprinter(
+      fingerprintPath: '$depfilePath.fingerprint',
+      paths: <String>[mainPath]..addAll(inputPaths)..addAll(outputPaths),
+      properties: <String, String>{
+        'buildMode': buildMode.toString(),
+        'targetPlatform': platform.toString(),
+        'entryPoint': mainPath,
+        'extraGenSnapshotOptions': extraGenSnapshotOptions.join(' '),
+      },
+      depfilePaths: <String>[depfilePath],
+    );
+    if (await fingerprinter.doesFingerprintMatch()) {
+      printTrace('Skipping Core JIT snapshot build. Fingerprint match.');
+      return 0;
+    }
+
+    final SnapshotType snapshotType = new SnapshotType(platform, buildMode);
+    final int genSnapshotExitCode = await genSnapshot.run(
+      snapshotType: snapshotType,
+      packagesPath: packagesPath,
+      depfilePath: depfilePath,
+      additionalArgs: genSnapshotArgs,
+    );
+    if (genSnapshotExitCode != 0) {
+      printError('Dart snapshot generator failed with exit code $genSnapshotExitCode');
+      return genSnapshotExitCode;
+    }
+
+    // Write path to gen_snapshot, since snapshots have to be re-generated when we roll
+    // the Dart SDK.
+    final String genSnapshotPath = GenSnapshot.getSnapshotterPath(snapshotType);
+    await outputDir.childFile('gen_snapshot.d').writeAsString('snapshot.d: $genSnapshotPath\n');
+
+    // Compute and record build fingerprint.
+    await fingerprinter.writeFingerprint();
+    return 0;
+  }
+
+  bool _isValidCoreJitPlatform(TargetPlatform platform) {
+    return const <TargetPlatform>[
+      TargetPlatform.android_arm,
+      TargetPlatform.android_arm64,
+    ].contains(platform);
+  }
+}

packages/flutter_tools/lib/src/build_info.dart

@@ -13,6 +13,7 @@ class BuildInfo {
   const BuildInfo(this.mode, this.flavor, {
     this.previewDart2 = false,
     this.trackWidgetCreation = false,
+    this.buildSnapshot = false,
     this.extraFrontEndOptions,
     this.extraGenSnapshotOptions,
     this.buildSharedLibrary,
@@ -42,6 +43,9 @@ class BuildInfo {
   /// Whether the build should track widget creation locations.
   final bool trackWidgetCreation;
 
+  /// Whether the build should create VM snapshot instead of using prebuilt one from engine.
+  final bool buildSnapshot;
+
   /// Extra command-line options for front-end.
   final String extraFrontEndOptions;
 
@@ -95,6 +99,7 @@ class BuildInfo {
       new BuildInfo(mode, flavor,
           previewDart2: previewDart2,
           trackWidgetCreation: trackWidgetCreation,
+          buildSnapshot: buildSnapshot,
           extraFrontEndOptions: extraFrontEndOptions,
           extraGenSnapshotOptions: extraGenSnapshotOptions,
           buildSharedLibrary: buildSharedLibrary,

packages/flutter_tools/lib/src/bundle.dart

@@ -26,11 +26,15 @@ const String defaultPrivateKeyPath = 'privatekey.der';
 const String _kKernelKey = 'kernel_blob.bin';
 const String _kSnapshotKey = 'snapshot_blob.bin';
 const String _kVMSnapshotData = 'vm_snapshot_data';
+const String _kVMSnapshotInstr = 'vm_snapshot_instr';
 const String _kIsolateSnapshotData = 'isolate_snapshot_data';
+const String _kIsolateSnapshotInstr = 'isolate_snapshot_instr';
 const String _kDylibKey = 'libapp.so';
 const String _kPlatformKernelKey = 'platform.dill';
 
 Future<void> build({
+  TargetPlatform platform,
+  BuildMode buildMode,
   String mainPath = defaultMainPath,
   String manifestPath = defaultManifestPath,
   String snapshotPath,
@@ -43,7 +47,9 @@ Future<void> build({
   bool precompiledSnapshot = false,
   bool reportLicensedPackages = false,
   bool trackWidgetCreation = false,
+  bool buildSnapshot = false,
   List<String> extraFrontEndOptions = const <String>[],
+  List<String> extraGenSnapshotOptions = const <String>[],
   List<String> fileSystemRoots,
   String fileSystemScheme,
 }) async {
@@ -78,7 +84,8 @@ Future<void> build({
     ensureDirectoryExists(applicationKernelFilePath);
     final CompilerOutput compilerOutput = await kernelCompiler.compile(
       sdkRoot: artifacts.getArtifactPath(Artifact.flutterPatchedSdkPath),
-      incrementalCompilerByteStorePath: fs.path.absolute(getIncrementalCompilerByteStoreDirectory()),
+      incrementalCompilerByteStorePath: buildSnapshot ? null :
+          fs.path.absolute(getIncrementalCompilerByteStoreDirectory()),
       mainPath: fs.file(mainPath).absolute.path,
       outputFilePath: applicationKernelFilePath,
       depFilePath: depfilePath,
@@ -87,6 +94,7 @@ Future<void> build({
       fileSystemRoots: fileSystemRoots,
       fileSystemScheme: fileSystemScheme,
       packagesPath: packagesPath,
+      linkPlatformKernelIn: buildSnapshot,
     );
     if (compilerOutput?.outputFilename == null) {
       throwToolExit('Compiler failed on $mainPath');
@@ -95,6 +103,22 @@ Future<void> build({
 
     await fs.directory(getBuildDirectory()).childFile('frontend_server.d')
         .writeAsString('frontend_server.d: ${artifacts.getArtifactPath(Artifact.frontendServerSnapshotForEngineDartSdk)}\n');
+
+    if (buildSnapshot) {
+      final CoreJITSnapshotter snapshotter = new CoreJITSnapshotter();
+      final int snapshotExitCode = await snapshotter.build(
+        platform: platform,
+        buildMode: buildMode,
+        mainPath: applicationKernelFilePath,
+        outputPath: getBuildDirectory(),
+        packagesPath: packagesPath,
+        extraGenSnapshotOptions: extraGenSnapshotOptions,
+      );
+      if (snapshotExitCode != 0) {
+        printError('Snapshotting exited with non-zero exit code: $snapshotExitCode');
+        return;
+      }
+    }
   }
 
   final AssetBundle assets = await buildAssets(
@@ -112,6 +136,7 @@ Future<void> build({
     snapshotFile: snapshotFile,
     privateKeyPath: privateKeyPath,
     assetDirPath: assetDirPath,
+    buildSnapshot: buildSnapshot,
   );
 }
 
@@ -147,22 +172,35 @@ Future<void> assemble({
   File dylibFile,
   String privateKeyPath = defaultPrivateKeyPath,
   String assetDirPath,
+  bool buildSnapshot,
 }) async {
   assetDirPath ??= getAssetBuildDirectory();
   printTrace('Building bundle');
 
   final Map<String, DevFSContent> assetEntries = new Map<String, DevFSContent>.from(assetBundle.entries);
-  final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
-  final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
-
   if (kernelContent != null) {
+    if (buildSnapshot) {
+      final String vmSnapshotData = fs.path.join(getBuildDirectory(), _kVMSnapshotData);
+      final String vmSnapshotInstr = fs.path.join(getBuildDirectory(), _kVMSnapshotInstr);
+      final String isolateSnapshotData = fs.path.join(getBuildDirectory(), _kIsolateSnapshotData);
+      final String isolateSnapshotInstr = fs.path.join(getBuildDirectory(), _kIsolateSnapshotInstr);
+      assetEntries[_kVMSnapshotData] = new DevFSFileContent(fs.file(vmSnapshotData));
+      assetEntries[_kVMSnapshotInstr] = new DevFSFileContent(fs.file(vmSnapshotInstr));
+      assetEntries[_kIsolateSnapshotData] = new DevFSFileContent(fs.file(isolateSnapshotData));
+      assetEntries[_kIsolateSnapshotInstr] = new DevFSFileContent(fs.file(isolateSnapshotInstr));
+    } else {
       final String platformKernelDill = artifacts.getArtifactPath(Artifact.platformKernelDill);
+      final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
+      final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
       assetEntries[_kKernelKey] = kernelContent;
       assetEntries[_kPlatformKernelKey] = new DevFSFileContent(fs.file(platformKernelDill));
       assetEntries[_kVMSnapshotData] = new DevFSFileContent(fs.file(vmSnapshotData));
       assetEntries[_kIsolateSnapshotData] = new DevFSFileContent(fs.file(isolateSnapshotData));
     }
+  }
   if (snapshotFile != null) {
+    final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
+    final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
     assetEntries[_kSnapshotKey] = new DevFSFileContent(snapshotFile);
     assetEntries[_kVMSnapshotData] = new DevFSFileContent(fs.file(vmSnapshotData));
     assetEntries[_kIsolateSnapshotData] = new DevFSFileContent(fs.file(isolateSnapshotData));
@@ -190,5 +228,3 @@ Future<void> writeBundle(
     await file.writeAsBytes(await entry.value.contentsAsBytes());
   }));
 }
-
-

packages/flutter_tools/lib/src/commands/build_bundle.dart

@@ -4,6 +4,7 @@
 
 import 'dart:async';
 
+import '../base/common.dart';
 import '../build_info.dart';
 import '../bundle.dart';
 import '../runner/flutter_command.dart' show FlutterOptions;
@@ -12,6 +13,7 @@ import 'build.dart';
 class BuildBundleCommand extends BuildSubCommand {
   BuildBundleCommand({bool verboseHelp = false}) {
     usesTargetOption();
+    addBuildModeFlags();
     argParser
       ..addFlag('precompiled', negatable: false)
       // This option is still referenced by the iOS build scripts. We should
@@ -27,14 +29,28 @@ class BuildBundleCommand extends BuildSubCommand {
         hide: !verboseHelp,
         help: 'Preview Dart 2.0 functionality.',
       )
+      ..addOption('target-platform',
+        defaultsTo: 'android-arm',
+        allowed: <String>['android-arm', 'android-arm64', 'ios']
+      )
       ..addFlag('track-widget-creation',
         hide: !verboseHelp,
         help: 'Track widget creation locations. Requires Dart 2.0 functionality.',
       )
+      ..addFlag('build-snapshot',
+        hide: !verboseHelp,
+        defaultsTo: false,
+        help: 'Build and use application-specific VM snapshot instead of\n'
+            'prebuilt one provided by the engine.',
+      )
       ..addMultiOption(FlutterOptions.kExtraFrontEndOptions,
         splitCommas: true,
         hide: true,
       )
+      ..addMultiOption(FlutterOptions.kExtraGenSnapshotOptions,
+        splitCommas: true,
+        hide: true,
+      )
       ..addOption('asset-dir', defaultsTo: getAssetBuildDirectory())
       ..addFlag('report-licensed-packages',
         help: 'Whether to report the names of all the packages that are included '
@@ -69,7 +85,16 @@ class BuildBundleCommand extends BuildSubCommand {
   Future<Null> runCommand() async {
     await super.runCommand();
 
+    final String targetPlatform = argResults['target-platform'];
+    final TargetPlatform platform = getTargetPlatformForName(targetPlatform);
+    if (platform == null)
+      throwToolExit('Unknown platform: $targetPlatform');
+
+    final BuildMode buildMode = getBuildMode();
+
     await build(
+      platform: platform,
+      buildMode: buildMode,
       mainPath: targetFile,
       manifestPath: argResults['manifest'],
       snapshotPath: argResults['snapshot'],
@@ -81,7 +106,9 @@ class BuildBundleCommand extends BuildSubCommand {
       precompiledSnapshot: argResults['precompiled'],
       reportLicensedPackages: argResults['report-licensed-packages'],
       trackWidgetCreation: argResults['track-widget-creation'],
+      buildSnapshot: argResults['build-snapshot'],
       extraFrontEndOptions: argResults[FlutterOptions.kExtraFrontEndOptions],
+      extraGenSnapshotOptions: argResults[FlutterOptions.kExtraGenSnapshotOptions],
       fileSystemScheme: argResults['filesystem-scheme'],
       fileSystemRoots: argResults['filesystem-root'],
     );

packages/flutter_tools/lib/src/commands/run.dart

@@ -125,6 +125,12 @@ class RunCommand extends RunCommandBase {
         hide: !verboseHelp,
         help: 'Preview Dart 2.0 functionality.',
       )
+      ..addFlag('build-snapshot',
+        hide: !verboseHelp,
+        defaultsTo: false,
+        help: 'Build and use application-specific VM snapshot instead of\n'
+              'prebuilt one provided by the engine.',
+      )
       ..addFlag('track-widget-creation',
         hide: !verboseHelp,
         help: 'Track widget creation locations. Requires Dart 2.0 functionality.',

packages/flutter_tools/lib/src/runner/flutter_command.dart

@@ -216,6 +216,9 @@ abstract class FlutterCommand extends Command<Null> {
         : null,
       previewDart2: previewDart2,
       trackWidgetCreation: trackWidgetCreation,
+      buildSnapshot: argParser.options.containsKey('build-snapshot')
+          ? argResults['build-snapshot']
+          : false,
       extraFrontEndOptions: argParser.options.containsKey(FlutterOptions.kExtraFrontEndOptions)
           ? argResults[FlutterOptions.kExtraFrontEndOptions]
           : null,