Avoid rebuilding snapshots if no change to source (#11551)

This change re-introduces skipping snapshot builds if input sources (and
outputs) have not changed since the last snapshot build, with a bugfix
to include the entry-point source in the checksum used to check whether
rebuild can be skipped. This ensures that the following sequence
invalidates the cached build, resulting in two snapshot builds:

  flutter build ios lib/foo.dart
  flutter build ios lib/bar.dart

This reverts commit 3d5afb5a.

packages/flutter_tools/lib/src/base/build.dart

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:convert' show JSON;
 
 import 'package:crypto/crypto.dart' show md5;
@@ -42,3 +43,28 @@ class Checksum {
   @override
   int get hashCode => _checksums.hashCode;
 }
+
+final RegExp _separatorExpr = new RegExp(r'([^\\]) ');
+final RegExp _escapeExpr = new RegExp(r'\\(.)');
+
+/// Parses a VM snapshot dependency file.
+///
+/// Snapshot dependency files are a single line mapping the output snapshot to a
+/// space-separated list of input files used to generate that output. Spaces and
+/// backslashes are escaped with a backslash. e.g,
+///
+/// outfile : file1.dart fil\\e2.dart fil\ e3.dart
+///
+/// will return a set containing: 'file1.dart', 'fil\e2.dart', 'fil e3.dart'.
+Future<Set<String>> readDepfile(String depfilePath) async {
+  // Depfile format:
+  // outfile1 outfile2 : file1.dart file2.dart file3.dart
+  final String contents = await fs.file(depfilePath).readAsString();
+  final String dependencies = contents.split(': ')[1];
+  return dependencies
+      .replaceAllMapped(_separatorExpr, (Match match) => '${match.group(1)}\n')
+      .split('\n')
+      .map((String path) => path.replaceAllMapped(_escapeExpr, (Match match) => match.group(1)).trim())
+      .where((String path) => path.isNotEmpty)
+      .toSet();
+}

packages/flutter_tools/lib/src/commands/build_aot.dart

@@ -5,6 +5,7 @@
 import 'dart:async';
 
 import '../artifacts.dart';
+import '../base/build.dart';
 import '../base/common.dart';
 import '../base/file_system.dart';
 import '../base/logger.dart';
@@ -149,13 +150,16 @@ Future<String> _buildAotSnapshot(
   final String uiPath = fs.path.join(skyEnginePkg, 'lib', 'ui', 'ui.dart');
   final String vmServicePath = fs.path.join(skyEnginePkg, 'sdk_ext', 'vmservice_io.dart');
 
-  final List<String> filePaths = <String>[
+  final List<String> inputPaths = <String>[
     vmEntryPoints,
     ioEntryPoints,
     uiPath,
     vmServicePath,
+    mainPath,
   ];
 
+  final Set<String> outputPaths = new Set<String>();
+
   // These paths are used only on iOS.
   String snapshotDartIOS;
   String assembly;
@@ -164,13 +168,15 @@ Future<String> _buildAotSnapshot(
     case TargetPlatform.android_arm:
     case TargetPlatform.android_x64:
     case TargetPlatform.android_x86:
+      outputPaths.addAll(<String>[
+        vmSnapshotData,
+        isolateSnapshotData,
+      ]);
       break;
     case TargetPlatform.ios:
       snapshotDartIOS = artifacts.getArtifactPath(Artifact.snapshotDart, platform, buildMode);
       assembly = fs.path.join(outputDir.path, 'snapshot_assembly.S');
-      filePaths.addAll(<String>[
-        snapshotDartIOS,
-      ]);
+      inputPaths.add(snapshotDartIOS);
       break;
     case TargetPlatform.darwin_x64:
     case TargetPlatform.linux_x64:
@@ -179,9 +185,9 @@ Future<String> _buildAotSnapshot(
       assert(false);
   }
 
-  final List<String> missingFiles = filePaths.where((String p) => !fs.isFileSync(p)).toList();
-  if (missingFiles.isNotEmpty) {
-    printError('Missing files: $missingFiles');
+  final Iterable<String> missingInputs = inputPaths.where((String p) => !fs.isFileSync(p));
+  if (missingInputs.isNotEmpty) {
+    printError('Missing input files: $missingInputs');
     return null;
   }
   if (!processManager.canRun(genSnapshot)) {
@@ -208,6 +214,17 @@ Future<String> _buildAotSnapshot(
     genSnapshotCmd.add('--embedder_entry_points_manifest=$ioEntryPoints');
   }
 
+  // iOS symbols used to load snapshot data in the engine.
+  const String kVmSnapshotData = 'kDartVmSnapshotData';
+  const String kIsolateSnapshotData = 'kDartIsolateSnapshotData';
+
+  // iOS snapshot generated files, compiled object files.
+  final String kVmSnapshotDataC = fs.path.join(outputDir.path, '$kVmSnapshotData.c');
+  final String kIsolateSnapshotDataC = fs.path.join(outputDir.path, '$kIsolateSnapshotData.c');
+  final String kVmSnapshotDataO = fs.path.join(outputDir.path, '$kVmSnapshotData.o');
+  final String kIsolateSnapshotDataO = fs.path.join(outputDir.path, '$kIsolateSnapshotData.o');
+  final String assemblyO = fs.path.join(outputDir.path, 'snapshot_assembly.o');
+
   switch (platform) {
     case TargetPlatform.android_arm:
     case TargetPlatform.android_x64:
@@ -224,9 +241,14 @@ Future<String> _buildAotSnapshot(
       if (interpreter) {
         genSnapshotCmd.add('--snapshot_kind=core');
         genSnapshotCmd.add(snapshotDartIOS);
+        outputPaths.addAll(<String>[
+          kVmSnapshotDataO,
+          kIsolateSnapshotDataO,
+        ]);
       } else {
         genSnapshotCmd.add('--snapshot_kind=app-aot-assembly');
         genSnapshotCmd.add('--assembly=$assembly');
+        outputPaths.add(assemblyO);
       }
       break;
     case TargetPlatform.darwin_x64:
@@ -245,6 +267,27 @@ Future<String> _buildAotSnapshot(
 
   genSnapshotCmd.add(mainPath);
 
+  final File checksumFile = fs.file('$dependencies.checksum');
+  final List<File> checksumFiles = <File>[checksumFile, fs.file(dependencies)]
+      ..addAll(inputPaths.map(fs.file))
+      ..addAll(outputPaths.map(fs.file));
+  if (checksumFiles.every((File file) => file.existsSync())) {
+    try {
+      final String json = await checksumFile.readAsString();
+      final Checksum oldChecksum = new Checksum.fromJson(json);
+      final Set<String> snapshotInputPaths = await readDepfile(dependencies);
+      snapshotInputPaths.addAll(outputPaths);
+      final Checksum newChecksum = new Checksum.fromFiles(snapshotInputPaths);
+      if (oldChecksum == newChecksum) {
+        printTrace('Skipping AOT snapshot build. Checksums match.');
+        return outputPath;
+      }
+    } catch (e, s) {
+      // Log exception and continue, this step is a performance improvement only.
+      printTrace('Error during AOT snapshot checksum check: $e\n$s');
+    }
+  }
+
   final RunResult results = await runAsync(genSnapshotCmd);
   if (results.exitCode != 0) {
     printError('Dart snapshot generator failed with exit code ${results.exitCode}');
@@ -261,16 +304,6 @@ Future<String> _buildAotSnapshot(
   if (platform == TargetPlatform.ios) {
     printStatus('Building App.framework...');
 
-    // These names are known to from the engine.
-    const String kVmSnapshotData = 'kDartVmSnapshotData';
-    const String kIsolateSnapshotData = 'kDartIsolateSnapshotData';
-
-    final String kVmSnapshotDataC = fs.path.join(outputDir.path, '$kVmSnapshotData.c');
-    final String kIsolateSnapshotDataC = fs.path.join(outputDir.path, '$kIsolateSnapshotData.c');
-    final String kVmSnapshotDataO = fs.path.join(outputDir.path, '$kVmSnapshotData.o');
-    final String kIsolateSnapshotDataO = fs.path.join(outputDir.path, '$kIsolateSnapshotData.o');
-    final String assemblyO = fs.path.join(outputDir.path, 'snapshot_assembly.o');
-
     final List<String> commonBuildOptions = <String>['-arch', 'arm64', '-miphoneos-version-min=8.0'];
 
     if (interpreter) {
@@ -317,5 +350,16 @@ Future<String> _buildAotSnapshot(
     await runCheckedAsync(linkCommand);
   }
 
+  // Compute and record checksums.
+  try {
+    final Set<String> snapshotInputPaths = await readDepfile(dependencies);
+    snapshotInputPaths..addAll(outputPaths);
+    final Checksum checksum = new Checksum.fromFiles(snapshotInputPaths);
+    await checksumFile.writeAsString(checksum.toJson());
+  } catch (e, s) {
+    // Log exception and continue, this step is a performance improvement only.
+    printTrace('Error during AOT snapshot checksum output: $e\n$s');
+  }
+
   return outputPath;
 }

packages/flutter_tools/lib/src/flx.dart

@@ -65,8 +65,9 @@ Future<int> _createSnapshot({
     try {
         final String json = await checksumFile.readAsString();
         final Checksum oldChecksum = new Checksum.fromJson(json);
-        final Set<String> inputPaths = await _readDepfile(depfilePath);
+        final Set<String> inputPaths = await readDepfile(depfilePath);
         inputPaths.add(snapshotPath);
+        inputPaths.add(mainPath);
         final Checksum newChecksum = new Checksum.fromFiles(inputPaths);
         if (oldChecksum == newChecksum) {
           printTrace('Skipping snapshot build. Checksums match.');
@@ -85,8 +86,9 @@ Future<int> _createSnapshot({
 
   // Compute and record input file checksums.
   try {
-    final Set<String> inputPaths = await _readDepfile(depfilePath);
+    final Set<String> inputPaths = await readDepfile(depfilePath);
     inputPaths.add(snapshotPath);
+    inputPaths.add(mainPath);
     final Checksum checksum = new Checksum.fromFiles(inputPaths);
     await checksumFile.writeAsString(checksum.toJson());
   } catch (e, s) {
@@ -96,24 +98,6 @@ Future<int> _createSnapshot({
   return 0;
 }
 
-/// Parses a VM snapshot dependency file.
-///
-/// Snapshot dependency files are a single line mapping the output snapshot to a
-/// space-separated list of input files used to generate that output. e.g,
-///
-/// outfile : file1.dart file2.dart file3.dart
-Future<Set<String>> _readDepfile(String depfilePath) async {
-  // Depfile format:
-  // outfile : file1.dart file2.dart file3.dart
-  final String contents = await fs.file(depfilePath).readAsString();
-  final String dependencies = contents.split(': ')[1];
-  return dependencies
-      .split(' ')
-      .map((String path) => path.trim())
-      .where((String path) => path.isNotEmpty)
-      .toSet();
-}
-
 Future<Null> build({
   String mainPath: defaultMainPath,
   String manifestPath: defaultManifestPath,

packages/flutter_tools/test/base/build_test.dart

@@ -64,4 +64,43 @@ void main() {
       });
     });
   });
+
+  group('readDepfile', () {
+    MemoryFileSystem fs;
+
+    setUp(() {
+      fs = new MemoryFileSystem();
+    });
+
+    testUsingContext('returns one file if only one is listed', () async {
+      await fs.file('a.d').writeAsString('snapshot.d: /foo/a.dart');
+      expect(await readDepfile('a.d'), unorderedEquals(<String>['/foo/a.dart']));
+    }, overrides: <Type, Generator>{ FileSystem: () => fs});
+
+    testUsingContext('returns multiple files', () async {
+      await fs.file('a.d').writeAsString('snapshot.d: /foo/a.dart /foo/b.dart');
+      expect(await readDepfile('a.d'), unorderedEquals(<String>[
+        '/foo/a.dart',
+        '/foo/b.dart',
+      ]));
+    }, overrides: <Type, Generator>{ FileSystem: () => fs});
+
+    testUsingContext('trims extra spaces between files', () async {
+      await fs.file('a.d').writeAsString('snapshot.d: /foo/a.dart    /foo/b.dart  /foo/c.dart');
+      expect(await readDepfile('a.d'), unorderedEquals(<String>[
+        '/foo/a.dart',
+        '/foo/b.dart',
+        '/foo/c.dart',
+      ]));
+    }, overrides: <Type, Generator>{ FileSystem: () => fs});
+
+    testUsingContext('returns files with spaces and backslashes', () async {
+      await fs.file('a.d').writeAsString(r'snapshot.d: /foo/a\ a.dart /foo/b\\b.dart /foo/c\\ c.dart');
+      expect(await readDepfile('a.d'), unorderedEquals(<String>[
+        r'/foo/a a.dart',
+        r'/foo/b\b.dart',
+        r'/foo/c\ c.dart',
+      ]));
+    }, overrides: <Type, Generator>{ FileSystem: () => fs});
+  });
 }