Extract snapshotting logic to Snapshotter class (#11820)

Extract a Snapshotter class that can be shared between FLX snapshotting,
AOT snapshotting, and assembly AOT snapshotting. Allows for better
testability of snapshotting logic.

* Extracts script snapshotting used in FLX build.
* Adds tests for snapshot checksumming, build invalidation/skipping.

Remaining work: disentangle + extract AOT snapshotting and Assembly AOT
snapshotting logic from build_aot.dart.

packages/flutter_tools/lib/src/base/build.dart

@@ -6,11 +6,54 @@ import 'dart:async';
 import 'dart:convert' show JSON;
 
 import 'package:crypto/crypto.dart' show md5;
+import 'package:meta/meta.dart';
 import 'package:quiver/core.dart' show hash3;
 
+import '../artifacts.dart';
 import '../build_info.dart';
+import '../globals.dart';
 import '../version.dart';
+import 'context.dart';
 import 'file_system.dart';
+import 'process.dart';
+
+GenSnapshot get genSnapshot => context.putIfAbsent(GenSnapshot, () => const GenSnapshot());
+
+/// A snapshot build configuration.
+class SnapshotType {
+  const SnapshotType(this.platform, this.mode);
+
+  final TargetPlatform platform;
+  final BuildMode mode;
+}
+
+/// Interface to the gen_snapshot command-line tool.
+class GenSnapshot {
+  const GenSnapshot();
+
+  Future<int> run({
+    @required SnapshotType snapshotType,
+    @required String packagesPath,
+    @required String depfilePath,
+    Iterable<String> additionalArgs: const <String>[],
+  }) {
+    final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
+    final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
+    final List<String> args = <String>[
+      '--assert_initializer',
+      '--await_is_keyword',
+      '--causal_async_stacks',
+      '--vm_snapshot_data=$vmSnapshotData',
+      '--isolate_snapshot_data=$isolateSnapshotData',
+      '--packages=$packagesPath',
+      '--dependencies=$depfilePath',
+      '--print_snapshot_sizes',
+    ]..addAll(additionalArgs);
+
+    final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, snapshotType.platform, snapshotType.mode);
+    return runCommandAndStreamOutput(<String>[snapshotterPath]..addAll(args));
+  }
+}
 
 /// A collection of checksums for a set of input files.
 ///
@@ -19,14 +62,14 @@ import 'file_system.dart';
 /// build step. This assumes that build outputs are strictly a product of the
 /// input files.
 class Checksum {
-  Checksum.fromFiles(BuildMode buildMode, TargetPlatform targetPlatform, Set<String> inputPaths) {
+  Checksum.fromFiles(SnapshotType type, Set<String> inputPaths) {
     final Iterable<File> files = inputPaths.map(fs.file);
     final Iterable<File> missingInputs = files.where((File file) => !file.existsSync());
     if (missingInputs.isNotEmpty)
       throw new ArgumentError('Missing input files:\n' + missingInputs.join('\n'));
 
-    _buildMode = buildMode.toString();
-    _targetPlatform = targetPlatform?.toString() ?? '';
+    _buildMode = type.mode.toString();
+    _targetPlatform = type.platform?.toString() ?? '';
     _checksums = <String, String>{};
     for (File file in files) {
       final List<int> bytes = file.readAsBytesSync();
@@ -109,3 +152,111 @@ Future<Set<String>> readDepfile(String depfilePath) async {
       .where((String path) => path.isNotEmpty)
       .toSet();
 }
+
+/// Dart snapshot builder.
+///
+/// Builds Dart snapshots in one of three modes:
+///   * Script snapshot: architecture-independent snapshot of a Dart script
+///     and core libraries.
+///   * AOT snapshot: architecture-specific ahead-of-time compiled snapshot
+///     suitable for loading with `mmap`.
+///   * Assembly AOT snapshot: architecture-specific ahead-of-time compile to
+///     assembly suitable for compilation as a static or dynamic library.
+class Snapshotter {
+  /// Builds an architecture-independent snapshot of the specified script.
+  Future<int> buildScriptSnapshot({
+    @required String mainPath,
+    @required String snapshotPath,
+    @required String depfilePath,
+    @required String packagesPath
+  }) async {
+    const SnapshotType type = const SnapshotType(null, BuildMode.debug);
+    final List<String> args = <String>[
+      '--snapshot_kind=script',
+      '--script_snapshot=$snapshotPath',
+      mainPath,
+    ];
+
+    final String checksumsPath = '$depfilePath.checksums';
+    final int exitCode = await _build(
+      snapshotType: type,
+      outputSnapshotPath: snapshotPath,
+      packagesPath: packagesPath,
+      snapshotArgs: args,
+      depfilePath: depfilePath,
+      mainPath: mainPath,
+      checksumsPath: checksumsPath,
+    );
+    if (exitCode != 0)
+      return exitCode;
+    await _writeChecksum(type, snapshotPath, depfilePath, mainPath, checksumsPath);
+    return exitCode;
+  }
+
+  /// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script.
+  Future<Null> buildAotSnapshot() async {
+    throw new UnimplementedError('AOT snapshotting not yet implemented');
+  }
+
+  Future<int> _build({
+    @required SnapshotType snapshotType,
+    @required List<String> snapshotArgs,
+    @required String outputSnapshotPath,
+    @required String packagesPath,
+    @required String depfilePath,
+    @required String mainPath,
+    @required String checksumsPath,
+  }) async {
+    if (!await _isBuildRequired(snapshotType, outputSnapshotPath, depfilePath, mainPath, checksumsPath)) {
+      printTrace('Skipping snapshot build. Checksums match.');
+      return 0;
+    }
+
+    // Build the snapshot.
+    final int exitCode = await genSnapshot.run(
+        snapshotType: snapshotType,
+        packagesPath: packagesPath,
+        depfilePath: depfilePath,
+        additionalArgs: snapshotArgs,
+    );
+    if (exitCode != 0)
+      return exitCode;
+
+    _writeChecksum(snapshotType, outputSnapshotPath, depfilePath, mainPath, checksumsPath);
+    return 0;
+  }
+
+  Future<bool> _isBuildRequired(SnapshotType type, String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
+    final File checksumFile = fs.file(checksumsPath);
+    final File outputSnapshotFile = fs.file(outputSnapshotPath);
+    final File depfile = fs.file(depfilePath);
+    if (!outputSnapshotFile.existsSync() || !depfile.existsSync() || !checksumFile.existsSync())
+      return true;
+
+    try {
+      if (checksumFile.existsSync()) {
+        final Checksum oldChecksum = new Checksum.fromJson(await checksumFile.readAsString());
+        final Set<String> checksumPaths = await readDepfile(depfilePath)
+          ..addAll(<String>[outputSnapshotPath, mainPath]);
+        final Checksum newChecksum = new Checksum.fromFiles(type, checksumPaths);
+        return oldChecksum != newChecksum;
+      }
+    } catch (e, s) {
+      // Log exception and continue, this step is a performance improvement only.
+      printTrace('Error during snapshot checksum output: $e\n$s');
+    }
+    return true;
+  }
+
+  Future<Null> _writeChecksum(SnapshotType type, String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
+    try {
+      final Set<String> checksumPaths = await readDepfile(depfilePath)
+        ..addAll(<String>[outputSnapshotPath, mainPath]);
+      final Checksum checksum = new Checksum.fromFiles(type, checksumPaths);
+      await fs.file(checksumsPath).writeAsString(checksum.toJson());
+    } catch (e, s) {
+      // Log exception and continue, this step is a performance improvement only.
+      print('Error during snapshot checksum output: $e\n$s');
+    }
+  }
+}

packages/flutter_tools/lib/src/commands/build_aot.dart

@@ -108,6 +108,7 @@ Future<String> buildAotSnapshot(
   }
 }
 
+// TODO(cbracken): split AOT and Assembly AOT snapshotting logic and migrate to Snapshotter class.
 Future<String> _buildAotSnapshot(
   String mainPath,
   TargetPlatform platform,
@@ -267,6 +268,7 @@ Future<String> _buildAotSnapshot(
 
   genSnapshotCmd.add(mainPath);
 
+  final SnapshotType snapshotType = new SnapshotType(platform, buildMode);
   final File checksumFile = fs.file('$dependencies.checksum');
   final List<File> checksumFiles = <File>[checksumFile, fs.file(dependencies)]
       ..addAll(inputPaths.map(fs.file))
@@ -278,7 +280,7 @@ Future<String> _buildAotSnapshot(
       final Set<String> snapshotInputPaths = await readDepfile(dependencies)
         ..add(mainPath)
         ..addAll(outputPaths);
-      final Checksum newChecksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
+      final Checksum newChecksum = new Checksum.fromFiles(snapshotType, snapshotInputPaths);
       if (oldChecksum == newChecksum) {
         printStatus('Skipping AOT snapshot build. Checksums match.');
         return outputPath;
@@ -356,7 +358,7 @@ Future<String> _buildAotSnapshot(
     final Set<String> snapshotInputPaths = await readDepfile(dependencies)
       ..add(mainPath)
       ..addAll(outputPaths);
-    final Checksum checksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
+    final Checksum checksum = new Checksum.fromFiles(snapshotType, snapshotInputPaths);
     await checksumFile.writeAsString(checksum.toJson());
   } catch (e, s) {
     // Log exception and continue, this step is a performance improvement only.

packages/flutter_tools/lib/src/flx.dart

@@ -4,14 +4,10 @@
 
 import 'dart:async';
 
-import 'package:meta/meta.dart' show required;
-
-import 'artifacts.dart';
 import 'asset.dart';
 import 'base/build.dart';
 import 'base/common.dart';
 import 'base/file_system.dart';
-import 'base/process.dart';
 import 'build_info.dart';
 import 'dart/package_map.dart';
 import 'devfs.dart';
@@ -30,75 +26,6 @@ const String _kKernelKey = 'kernel_blob.bin';
 const String _kSnapshotKey = 'snapshot_blob.bin';
 const String _kDylibKey = 'libapp.so';
 
-Future<int> _createSnapshot({
-  @required String mainPath,
-  @required String snapshotPath,
-  @required String depfilePath,
-  @required String packages
-}) async {
-  assert(mainPath != null);
-  assert(snapshotPath != null);
-  assert(depfilePath != null);
-  assert(packages != null);
-  final BuildMode buildMode = BuildMode.debug;
-  final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, null, buildMode);
-  final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
-  final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
-
-  final List<String> args = <String>[
-    snapshotterPath,
-    '--snapshot_kind=script',
-    '--vm_snapshot_data=$vmSnapshotData',
-    '--isolate_snapshot_data=$isolateSnapshotData',
-    '--packages=$packages',
-    '--script_snapshot=$snapshotPath',
-    '--dependencies=$depfilePath',
-    mainPath,
-  ];
-
-  // Write the depfile path to disk.
-  fs.file(depfilePath).parent.childFile('gen_snapshot.d').writeAsString('$depfilePath: $snapshotterPath\n');
-
-  final File checksumFile = fs.file('$depfilePath.checksums');
-  final File snapshotFile = fs.file(snapshotPath);
-  final File depfile = fs.file(depfilePath);
-  if (snapshotFile.existsSync() && depfile.existsSync() && checksumFile.existsSync()) {
-    try {
-        final String json = await checksumFile.readAsString();
-        final Checksum oldChecksum = new Checksum.fromJson(json);
-        final Set<String> inputPaths = await readDepfile(depfilePath);
-        inputPaths.add(snapshotPath);
-        inputPaths.add(mainPath);
-        final Checksum newChecksum = new Checksum.fromFiles(buildMode, null, inputPaths);
-        if (oldChecksum == newChecksum) {
-          printTrace('Skipping snapshot build. Checksums match.');
-          return 0;
-        }
-    } catch (e, s) {
-      // Log exception and continue, this step is a performance improvement only.
-      printTrace('Error during snapshot checksum check: $e\n$s');
-    }
-  }
-
-  // Build the snapshot.
-  final int exitCode = await runCommandAndStreamOutput(args);
-  if (exitCode != 0)
-    return exitCode;
-
-  // Compute and record input file checksums.
-  try {
-    final Set<String> inputPaths = await readDepfile(depfilePath);
-    inputPaths.add(snapshotPath);
-    inputPaths.add(mainPath);
-    final Checksum checksum = new Checksum.fromFiles(buildMode, null, inputPaths);
-    await checksumFile.writeAsString(checksum.toJson());
-  } catch (e, s) {
-    // Log exception and continue, this step is a performance improvement only.
-    printTrace('Error during snapshot checksum output: $e\n$s');
-  }
-  return 0;
-}
-
 Future<Null> build({
   String mainPath: defaultMainPath,
   String manifestPath: defaultManifestPath,
@@ -124,11 +51,12 @@ Future<Null> build({
 
     // In a precompiled snapshot, the instruction buffer contains script
     // content equivalents
-    final int result = await _createSnapshot(
+    final Snapshotter snapshotter = new Snapshotter();
+    final int result = await snapshotter.buildScriptSnapshot(
       mainPath: mainPath,
       snapshotPath: snapshotPath,
       depfilePath: depfilePath,
-      packages: packagesPath
+      packagesPath: packagesPath,
     );
     if (result != 0)
       throwToolExit('Failed to run the Flutter compiler. Exit code: $result', exitCode: result);