Extract snapshotting logic to Snapshotter class (#11591)

First step in eliminating code duplication between script snapshotting
(in FLX build) and AOT, assembly AOT snapshotting.

packages/flutter_tools/lib/src/base/build.dart

@@ -6,8 +6,44 @@ import 'dart:async';
 import 'dart:convert' show JSON;
 
 import 'package:crypto/crypto.dart' show md5;
+import 'package:meta/meta.dart';
 
+import '../artifacts.dart';
+import '../build_info.dart';
+import '../globals.dart';
+import 'context.dart';
 import 'file_system.dart';
+import 'process.dart';
+
+GenSnapshot get genSnapshot => context.putIfAbsent(GenSnapshot, () => const GenSnapshot());
+
+class GenSnapshot {
+  const GenSnapshot();
+
+  Future<int> run({
+    @required TargetPlatform targetPlatform,
+    @required BuildMode buildMode,
+    @required String packagesPath,
+    @required String depfilePath,
+    Iterable<String> additionalArgs: const <String>[],
+  }) {
+    final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
+    final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
+    final List<String> args = <String>[
+      '--assert_initializer',
+      '--await_is_keyword',
+      '--causal_async_stacks',
+      '--vm_snapshot_data=$vmSnapshotData',
+      '--isolate_snapshot_data=$isolateSnapshotData',
+      '--packages=$packagesPath',
+      '--dependencies=$depfilePath',
+      '--print_snapshot_sizes',
+    ]..addAll(additionalArgs);
+
+    final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, targetPlatform, buildMode);
+    return runCommandAndStreamOutput(<String>[snapshotterPath]..addAll(args));
+  }
+}
 
 /// A collection of checksums for a set of input files.
 ///
@@ -68,3 +104,109 @@ Future<Set<String>> readDepfile(String depfilePath) async {
       .where((String path) => path.isNotEmpty)
       .toSet();
 }
+
+/// Dart snapshot builder.
+///
+/// Builds Dart snapshots in one of three modes:
+///   * Script snapshot: architecture-independent snapshot of a Dart script core
+///     libraries.
+///   * AOT snapshot: architecture-specific ahead-of-time compiled snapshot
+///     suitable for loading with `mmap`.
+///   * Assembly AOT snapshot: architecture-specific ahead-of-time compile to
+///     assembly suitable for compilation as a static or dynamic library.
+class Snapshotter {
+  /// Builds an architecture-independent snapshot of the specified script.
+  Future<int> buildScriptSnapshot({
+    @required String mainPath,
+    @required String snapshotPath,
+    @required String depfilePath,
+    @required String packagesPath
+  }) async {
+    final List<String> args = <String>[
+      '--snapshot_kind=script',
+      '--script_snapshot=$snapshotPath',
+      mainPath,
+    ];
+
+    final String checksumsPath = '$depfilePath.checksums';
+    final int exitCode = await _build(
+      outputSnapshotPath: snapshotPath,
+      packagesPath: packagesPath,
+      snapshotArgs: args,
+      depfilePath: depfilePath,
+      mainPath: mainPath,
+      checksumsPath: checksumsPath,
+    );
+    if (exitCode != 0)
+      return exitCode;
+    await _writeChecksum(snapshotPath, depfilePath, mainPath, checksumsPath);
+    return exitCode;
+  }
+
+  /// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script.
+  Future<Null> buildAotSnapshot() async {
+    throw new UnimplementedError('AOT snapshotting not yet implemented');
+  }
+
+  Future<int> _build({
+    @required List<String> snapshotArgs,
+    @required String outputSnapshotPath,
+    @required String packagesPath,
+    @required String depfilePath,
+    @required String mainPath,
+    @required String checksumsPath,
+  }) async {
+    if (!await _isBuildRequired(outputSnapshotPath, depfilePath, mainPath, checksumsPath)) {
+      printTrace('Skipping snapshot build. Checksums match.');
+      return 0;
+    }
+
+    // Build the snapshot.
+    final int exitCode = await genSnapshot.run(
+        targetPlatform: null,
+        buildMode: BuildMode.debug,
+        packagesPath: packagesPath,
+        depfilePath: depfilePath,
+        additionalArgs: snapshotArgs,
+    );
+    if (exitCode != 0)
+      return exitCode;
+
+    _writeChecksum(outputSnapshotPath, depfilePath, mainPath, checksumsPath);
+    return 0;
+  }
+
+  Future<bool> _isBuildRequired(String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
+    final File checksumFile = fs.file(checksumsPath);
+    final File outputSnapshotFile = fs.file(outputSnapshotPath);
+    final File depfile = fs.file(depfilePath);
+    if (!outputSnapshotFile.existsSync() || !depfile.existsSync() || !checksumFile.existsSync())
+      return true;
+
+    try {
+      if (checksumFile.existsSync()) {
+        final Checksum oldChecksum = new Checksum.fromJson(await checksumFile.readAsString());
+        final Set<String> checksumPaths = await readDepfile(depfilePath)
+          ..addAll(<String>[outputSnapshotPath, mainPath]);
+        final Checksum newChecksum = new Checksum.fromFiles(checksumPaths);
+        return oldChecksum != newChecksum;
+      }
+    } catch (e, s) {
+      // Log exception and continue, this step is a performance improvement only.
+      printTrace('Error during snapshot checksum output: $e\n$s');
+    }
+    return true;
+  }
+
+  Future<Null> _writeChecksum(String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
+    try {
+      final Set<String> checksumPaths = await readDepfile(depfilePath)
+        ..addAll(<String>[outputSnapshotPath, mainPath]);
+      final Checksum checksum = new Checksum.fromFiles(checksumPaths);
+      await fs.file(checksumsPath).writeAsString(checksum.toJson());
+    } catch (e, s) {
+      // Log exception and continue, this step is a performance improvement only.
+      print('Error during snapshot checksum output: $e\n$s');
+    }
+  }
+}

packages/flutter_tools/lib/src/flx.dart

@@ -4,14 +4,10 @@
 
 import 'dart:async';
 
-import 'package:meta/meta.dart' show required;
-
-import 'artifacts.dart';
 import 'asset.dart';
 import 'base/build.dart';
 import 'base/common.dart';
 import 'base/file_system.dart';
-import 'base/process.dart';
 import 'build_info.dart';
 import 'dart/package_map.dart';
 import 'devfs.dart';
@@ -30,74 +26,6 @@ const String _kKernelKey = 'kernel_blob.bin';
 const String _kSnapshotKey = 'snapshot_blob.bin';
 const String _kDylibKey = 'libapp.so';
 
-Future<int> _createSnapshot({
-  @required String mainPath,
-  @required String snapshotPath,
-  @required String depfilePath,
-  @required String packages
-}) async {
-  assert(mainPath != null);
-  assert(snapshotPath != null);
-  assert(depfilePath != null);
-  assert(packages != null);
-  final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, null, BuildMode.debug);
-  final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
-  final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
-
-  final List<String> args = <String>[
-    snapshotterPath,
-    '--snapshot_kind=script',
-    '--vm_snapshot_data=$vmSnapshotData',
-    '--isolate_snapshot_data=$isolateSnapshotData',
-    '--packages=$packages',
-    '--script_snapshot=$snapshotPath',
-    '--dependencies=$depfilePath',
-    mainPath,
-  ];
-
-  // Write the depfile path to disk.
-  fs.file(depfilePath).parent.childFile('gen_snapshot.d').writeAsString('$depfilePath: $snapshotterPath\n');
-
-  final File checksumFile = fs.file('$depfilePath.checksums');
-  final File snapshotFile = fs.file(snapshotPath);
-  final File depfile = fs.file(depfilePath);
-  if (snapshotFile.existsSync() && depfile.existsSync() && checksumFile.existsSync()) {
-    try {
-        final String json = await checksumFile.readAsString();
-        final Checksum oldChecksum = new Checksum.fromJson(json);
-        final Set<String> inputPaths = await readDepfile(depfilePath);
-        inputPaths.add(snapshotPath);
-        inputPaths.add(mainPath);
-        final Checksum newChecksum = new Checksum.fromFiles(inputPaths);
-        if (oldChecksum == newChecksum) {
-          printTrace('Skipping snapshot build. Checksums match.');
-          return 0;
-        }
-    } catch (e, s) {
-      // Log exception and continue, this step is a performance improvement only.
-      printTrace('Error during snapshot checksum check: $e\n$s');
-    }
-  }
-
-  // Build the snapshot.
-  final int exitCode = await runCommandAndStreamOutput(args);
-  if (exitCode != 0)
-    return exitCode;
-
-  // Compute and record input file checksums.
-  try {
-    final Set<String> inputPaths = await readDepfile(depfilePath);
-    inputPaths.add(snapshotPath);
-    inputPaths.add(mainPath);
-    final Checksum checksum = new Checksum.fromFiles(inputPaths);
-    await checksumFile.writeAsString(checksum.toJson());
-  } catch (e, s) {
-    // Log exception and continue, this step is a performance improvement only.
-    printTrace('Error during snapshot checksum output: $e\n$s');
-  }
-  return 0;
-}
-
 Future<Null> build({
   String mainPath: defaultMainPath,
   String manifestPath: defaultManifestPath,
@@ -123,11 +51,12 @@ Future<Null> build({
 
     // In a precompiled snapshot, the instruction buffer contains script
     // content equivalents
-    final int result = await _createSnapshot(
+    final Snapshotter snapshotter = new Snapshotter();
+    final int result = await snapshotter.buildScriptSnapshot(
       mainPath: mainPath,
       snapshotPath: snapshotPath,
       depfilePath: depfilePath,
-      packages: packagesPath
+      packagesPath: packagesPath,
     );
     if (result != 0)
       throwToolExit('Failed to run the Flutter compiler. Exit code: $result', exitCode: result);

packages/flutter_tools/test/base/build_test.dart

@@ -2,13 +2,54 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:convert' show JSON;
+
 import 'package:file/memory.dart';
 import 'package:flutter_tools/src/base/build.dart';
+import 'package:flutter_tools/src/base/context.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/build_info.dart';
 import 'package:test/test.dart';
 
 import '../src/context.dart';
 
+class _FakeGenSnapshot implements GenSnapshot {
+  _FakeGenSnapshot({
+    this.succeed: true,
+    this.snapshotPath: 'output.snapshot',
+    this.snapshotContent: '',
+    this.depfilePath: 'output.snapshot.d',
+    this.depfileContent: 'output.snapshot.d : main.dart',
+  });
+
+  final bool succeed;
+  final String snapshotPath;
+  final String snapshotContent;
+  final String depfilePath;
+  final String depfileContent;
+  int _callCount = 0;
+
+  int get callCount => _callCount;
+
+  @override
+  Future<int> run({
+    TargetPlatform targetPlatform,
+    BuildMode buildMode,
+    String packagesPath,
+    String depfilePath,
+    Iterable<String> additionalArgs,
+  }) async {
+    _callCount += 1;
+
+    if (!succeed)
+      return 1;
+    await fs.file(snapshotPath).writeAsString(snapshotContent);
+    await fs.file(depfilePath).writeAsString(depfileContent);
+    return 0;
+  }
+}
+
 void main() {
   group('Checksum', () {
     group('fromFiles', () {
@@ -103,4 +144,146 @@ void main() {
       ]));
     }, overrides: <Type, Generator>{ FileSystem: () => fs });
   });
+
+  group('Snapshotter', () {
+    _FakeGenSnapshot genSnapshot;
+    MemoryFileSystem fs;
+    Snapshotter snapshotter;
+
+    setUp(() {
+      fs = new MemoryFileSystem();
+      genSnapshot = new _FakeGenSnapshot();
+      snapshotter = new Snapshotter();
+    });
+
+    testUsingContext('builds snapshot and checksums when no checksums are present', () async {
+      await fs.file('main.dart').writeAsString('void main() {}');
+      await fs.file('output.snapshot').create();
+      await fs.file('output.snapshot.d').writeAsString('snapshot : main.dart');
+      await snapshotter.buildScriptSnapshot(
+        mainPath: 'main.dart',
+        snapshotPath: 'output.snapshot',
+        depfilePath: 'output.snapshot.d',
+        packagesPath: '.packages',
+      );
+
+      expect(genSnapshot.callCount, 1);
+
+      final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
+      expect(json, hasLength(2));
+      expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
+      expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      GenSnapshot: () => genSnapshot,
+    });
+
+    testUsingContext('builds snapshot and checksums when checksums differ', () async {
+      await fs.file('main.dart').writeAsString('void main() {}');
+      await fs.file('output.snapshot').create();
+      await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
+      await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
+        'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
+        'output.snapshot': 'deadbeef01234567890abcdef0123456',
+      }));
+      await snapshotter.buildScriptSnapshot(
+        mainPath: 'main.dart',
+        snapshotPath: 'output.snapshot',
+        depfilePath: 'output.snapshot.d',
+        packagesPath: '.packages',
+      );
+
+      expect(genSnapshot.callCount, 1);
+
+      final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
+      expect(json, hasLength(2));
+      expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
+      expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      GenSnapshot: () => genSnapshot,
+    });
+
+    testUsingContext('builds snapshot and checksums when checksums match but previous snapshot not present', () async {
+      await fs.file('main.dart').writeAsString('void main() {}');
+      await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
+      await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
+        'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
+        'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
+      }));
+      await snapshotter.buildScriptSnapshot(
+        mainPath: 'main.dart',
+        snapshotPath: 'output.snapshot',
+        depfilePath: 'output.snapshot.d',
+        packagesPath: '.packages',
+      );
+
+      expect(genSnapshot.callCount, 1);
+
+      final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
+      expect(json, hasLength(2));
+      expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
+      expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      GenSnapshot: () => genSnapshot,
+    });
+
+    testUsingContext('builds snapshot and checksums when main entry point changes', () async {
+      final _FakeGenSnapshot genSnapshot = new _FakeGenSnapshot(
+        snapshotPath: 'output.snapshot',
+        depfilePath: 'output.snapshot.d',
+        depfileContent: 'output.snapshot : other.dart',
+      );
+      context.setVariable(GenSnapshot, genSnapshot);
+
+      await fs.file('main.dart').writeAsString('void main() {}');
+      await fs.file('other.dart').writeAsString('void main() { print("Kanpai ima kimi wa jinsei no ookina ookina butai ni tachi"); }');
+      await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
+      await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
+        'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
+        'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
+      }));
+      await snapshotter.buildScriptSnapshot(
+        mainPath: 'other.dart',
+        snapshotPath: 'output.snapshot',
+        depfilePath: 'output.snapshot.d',
+        packagesPath: '.packages',
+      );
+
+      expect(genSnapshot.callCount, 1);
+      final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
+      expect(json, hasLength(2));
+      expect(json['other.dart'], '3238d0ae341339b1731d3c2e195ad177');
+      expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+    });
+
+    testUsingContext('skips snapshot when checksums match and previous snapshot is present', () async {
+      await fs.file('main.dart').writeAsString('void main() {}');
+      await fs.file('output.snapshot').create();
+      await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
+      await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
+        'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
+        'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
+      }));
+      await snapshotter.buildScriptSnapshot(
+        mainPath: 'main.dart',
+        snapshotPath: 'output.snapshot',
+        depfilePath: 'output.snapshot.d',
+        packagesPath: '.packages',
+      );
+
+      expect(genSnapshot.callCount, 0);
+
+      final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
+      expect(json, hasLength(2));
+      expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
+      expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
+    }, overrides: <Type, Generator>{
+      FileSystem: () => fs,
+      GenSnapshot: () => genSnapshot,
+    });
+  });
 }