Skip to content

F
Front-End
Commit 1ee94001 authored by Chris Bracken's avatar Chris Bracken Committed by GitHub
Browse files
Options

Avoid rebuilding snapshots if no change to source (#11551) (#11793) 

This change re-introduces skipping AOT snapshot builds if input sources
and outputs have not changed since the last snapshot build, assuming a
build for the same platform in the same build mode.

This reverts commit 3d5afb5a.
It includes the following changes relative to the original:
  1. Include the entrypoint source in the checksums
  2. include the build mode in the checksums
  3. include the target platform in the checksums
parent f2653015
Hide whitespace changes
Inline Side-by-side
Showing with 194 additions and 55 deletions
packages/flutter_tools/lib/src/base/build.dart
View file @ 1ee94001
......@@ -2,10 +2,11 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show JSON;
import 'package:crypto/crypto.dart' show md5;
import 'package:quiver/core.dart' show hash2;
import 'package:quiver/core.dart' show hash3;
import '../build_info.dart';
import '../version.dart';
......@@ -18,13 +19,14 @@ import 'file_system.dart';
/// build step. This assumes that build outputs are strictly a product of the
/// input files.
class Checksum {
Checksum.fromFiles(BuildMode buildMode, Set<String> inputPaths) {
Checksum.fromFiles(BuildMode buildMode, TargetPlatform targetPlatform, Set<String> inputPaths) {
final Iterable<File> files = inputPaths.map(fs.file);
final Iterable<File> missingInputs = files.where((File file) => !file.existsSync());
if (missingInputs.isNotEmpty)
throw new ArgumentError('Missing input files:\n' + missingInputs.join('\n'));
_buildMode = buildMode.toString();
_targetPlatform = targetPlatform?.toString() ?? '';
_checksums = <String, String>{};
for (File file in files) {
final List<int> bytes = file.readAsBytesSync();
......@@ -36,7 +38,8 @@ class Checksum {
///
/// Throws [ArgumentError] in the following cases:
/// * Version mismatch between the serializing framework and this framework.
/// * BuildMode is unspecified.
/// * buildMode is unspecified.
/// * targetPlatform is unspecified.
/// * File checksum map is unspecified.
Checksum.fromJson(String json) {
final Map<String, dynamic> content = JSON.decode(json);
......@@ -47,7 +50,11 @@ class Checksum {
_buildMode = content['buildMode'];
if (_buildMode == null || _buildMode.isEmpty)
throw new ArgumentError('BuildMode unspecified in checksum JSON');
throw new ArgumentError('Build mode unspecified in checksum JSON');
_targetPlatform = content['targetPlatform'];
if (_targetPlatform == null)
throw new ArgumentError('Target platform unspecified in checksum JSON');
_checksums = content['files'];
if (_checksums == null)
......@@ -55,11 +62,13 @@ class Checksum {
}
String _buildMode;
String _targetPlatform;
Map<String, String> _checksums;
String toJson() => JSON.encode(<String, dynamic>{
'version': FlutterVersion.instance.frameworkRevision,
'buildMode': _buildMode,
'targetPlatform': _targetPlatform,
'files': _checksums,
});
......@@ -67,10 +76,36 @@ class Checksum {
bool operator==(dynamic other) {
return other is Checksum &&
_buildMode == other._buildMode &&
_targetPlatform == other._targetPlatform &&
_checksums.length == other._checksums.length &&
_checksums.keys.every((String key) => _checksums[key] == other._checksums[key]);
}
@override
int get hashCode => hash2(_buildMode, _checksums);
int get hashCode => hash3(_buildMode, _targetPlatform, _checksums);
}
final RegExp _separatorExpr = new RegExp(r'([^\\]) ');
final RegExp _escapeExpr = new RegExp(r'\\(.)');
/// Parses a VM snapshot dependency file.
///
/// Snapshot dependency files are a single line mapping the output snapshot to a
/// space-separated list of input files used to generate that output. Spaces and
/// backslashes are escaped with a backslash. e.g,
///
/// outfile : file1.dart fil\\e2.dart fil\ e3.dart
///
/// will return a set containing: 'file1.dart', 'fil\e2.dart', 'fil e3.dart'.
Future<Set<String>> readDepfile(String depfilePath) async {
// Depfile format:
// outfile1 outfile2 : file1.dart file2.dart file3.dart
final String contents = await fs.file(depfilePath).readAsString();
final String dependencies = contents.split(': ')[1];
return dependencies
.replaceAllMapped(_separatorExpr, (Match match) => '${match.group(1)}\n')
.split('\n')
.map((String path) => path.replaceAllMapped(_escapeExpr, (Match match) => match.group(1)).trim())
.where((String path) => path.isNotEmpty)
.toSet();
}
packages/flutter_tools/lib/src/commands/build_aot.dart
View file @ 1ee94001
......@@ -5,6 +5,7 @@
import 'dart:async';
import '../artifacts.dart';
import '../base/build.dart';
import '../base/common.dart';
import '../base/file_system.dart';
import '../base/logger.dart';
......@@ -149,13 +150,16 @@ Future<String> _buildAotSnapshot(
final String uiPath = fs.path.join(skyEnginePkg, 'lib', 'ui', 'ui.dart');
final String vmServicePath = fs.path.join(skyEnginePkg, 'sdk_ext', 'vmservice_io.dart');
final List<String> filePaths = <String>[
final List<String> inputPaths = <String>[
vmEntryPoints,
ioEntryPoints,
uiPath,
vmServicePath,
mainPath,
];
final Set<String> outputPaths = new Set<String>();
// These paths are used only on iOS.
String snapshotDartIOS;
String assembly;
......@@ -164,13 +168,15 @@ Future<String> _buildAotSnapshot(
case TargetPlatform.android_arm:
case TargetPlatform.android_x64:
case TargetPlatform.android_x86:
outputPaths.addAll(<String>[
vmSnapshotData,
isolateSnapshotData,
]);
break;
case TargetPlatform.ios:
snapshotDartIOS = artifacts.getArtifactPath(Artifact.snapshotDart, platform, buildMode);
assembly = fs.path.join(outputDir.path, 'snapshot_assembly.S');
filePaths.addAll(<String>[
snapshotDartIOS,
]);
inputPaths.add(snapshotDartIOS);
break;
case TargetPlatform.darwin_x64:
case TargetPlatform.linux_x64:
......@@ -179,9 +185,9 @@ Future<String> _buildAotSnapshot(
assert(false);
}
final List<String> missingFiles = filePaths.where((String p) => !fs.isFileSync(p)).toList();
if (missingFiles.isNotEmpty) {
printError('Missing files: $missingFiles');
final Iterable<String> missingInputs = inputPaths.where((String p) => !fs.isFileSync(p));
if (missingInputs.isNotEmpty) {
printError('Missing input files: $missingInputs');
return null;
}
if (!processManager.canRun(genSnapshot)) {
......@@ -208,6 +214,17 @@ Future<String> _buildAotSnapshot(
genSnapshotCmd.add('--embedder_entry_points_manifest=$ioEntryPoints');
}
// iOS symbols used to load snapshot data in the engine.
const String kVmSnapshotData = 'kDartVmSnapshotData';
const String kIsolateSnapshotData = 'kDartIsolateSnapshotData';
// iOS snapshot generated files, compiled object files.
final String kVmSnapshotDataC = fs.path.join(outputDir.path, '$kVmSnapshotData.c');
final String kIsolateSnapshotDataC = fs.path.join(outputDir.path, '$kIsolateSnapshotData.c');
final String kVmSnapshotDataO = fs.path.join(outputDir.path, '$kVmSnapshotData.o');
final String kIsolateSnapshotDataO = fs.path.join(outputDir.path, '$kIsolateSnapshotData.o');
final String assemblyO = fs.path.join(outputDir.path, 'snapshot_assembly.o');
switch (platform) {
case TargetPlatform.android_arm:
case TargetPlatform.android_x64:
......@@ -224,9 +241,14 @@ Future<String> _buildAotSnapshot(
if (interpreter) {
genSnapshotCmd.add('--snapshot_kind=core');
genSnapshotCmd.add(snapshotDartIOS);
outputPaths.addAll(<String>[
kVmSnapshotDataO,
kIsolateSnapshotDataO,
]);
} else {
genSnapshotCmd.add('--snapshot_kind=app-aot-assembly');
genSnapshotCmd.add('--assembly=$assembly');
outputPaths.add(assemblyO);
}
break;
case TargetPlatform.darwin_x64:
......@@ -245,6 +267,28 @@ Future<String> _buildAotSnapshot(
genSnapshotCmd.add(mainPath);
final File checksumFile = fs.file('$dependencies.checksum');
final List<File> checksumFiles = <File>[checksumFile, fs.file(dependencies)]
..addAll(inputPaths.map(fs.file))
..addAll(outputPaths.map(fs.file));
if (checksumFiles.every((File file) => file.existsSync())) {
try {
final String json = await checksumFile.readAsString();
final Checksum oldChecksum = new Checksum.fromJson(json);
final Set<String> snapshotInputPaths = await readDepfile(dependencies)
..add(mainPath)
..addAll(outputPaths);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
if (oldChecksum == newChecksum) {
printStatus('Skipping AOT snapshot build. Checksums match.');
return outputPath;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printStatus('Error during AOT snapshot checksum check: $e\n$s');
}
}
final RunResult results = await runAsync(genSnapshotCmd);
if (results.exitCode != 0) {
printError('Dart snapshot generator failed with exit code ${results.exitCode}');
......@@ -261,16 +305,6 @@ Future<String> _buildAotSnapshot(
if (platform == TargetPlatform.ios) {
printStatus('Building App.framework...');
// These names are known to from the engine.
const String kVmSnapshotData = 'kDartVmSnapshotData';
const String kIsolateSnapshotData = 'kDartIsolateSnapshotData';
final String kVmSnapshotDataC = fs.path.join(outputDir.path, '$kVmSnapshotData.c');
final String kIsolateSnapshotDataC = fs.path.join(outputDir.path, '$kIsolateSnapshotData.c');
final String kVmSnapshotDataO = fs.path.join(outputDir.path, '$kVmSnapshotData.o');
final String kIsolateSnapshotDataO = fs.path.join(outputDir.path, '$kIsolateSnapshotData.o');
final String assemblyO = fs.path.join(outputDir.path, 'snapshot_assembly.o');
final List<String> commonBuildOptions = <String>['-arch', 'arm64', '-miphoneos-version-min=8.0'];
if (interpreter) {
......@@ -317,5 +351,17 @@ Future<String> _buildAotSnapshot(
await runCheckedAsync(linkCommand);
}
// Compute and record checksums.
try {
final Set<String> snapshotInputPaths = await readDepfile(dependencies)
..add(mainPath)
..addAll(outputPaths);
final Checksum checksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printStatus('Error during AOT snapshot checksum output: $e\n$s');
}
return outputPath;
}
packages/flutter_tools/lib/src/flx.dart
View file @ 1ee94001
......@@ -66,10 +66,10 @@ Future<int> _createSnapshot({
try {
final String json = await checksumFile.readAsString();
final Checksum oldChecksum = new Checksum.fromJson(json);
final Set<String> inputPaths = await _readDepfile(depfilePath);
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, inputPaths);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, null, inputPaths);
if (oldChecksum == newChecksum) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
......@@ -87,10 +87,10 @@ Future<int> _createSnapshot({
// Compute and record input file checksums.
try {
final Set<String> inputPaths = await _readDepfile(depfilePath);
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum checksum = new Checksum.fromFiles(buildMode, inputPaths);
final Checksum checksum = new Checksum.fromFiles(buildMode, null, inputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
......@@ -99,24 +99,6 @@ Future<int> _createSnapshot({
return 0;
}
/// Parses a VM snapshot dependency file.
///
/// Snapshot dependency files are a single line mapping the output snapshot to a
/// space-separated list of input files used to generate that output. e.g,
///
/// outfile : file1.dart file2.dart file3.dart
Future<Set<String>> _readDepfile(String depfilePath) async {
// Depfile format:
// outfile : file1.dart file2.dart file3.dart
final String contents = await fs.file(depfilePath).readAsString();
final String dependencies = contents.split(': ')[1];
return dependencies
.split(' ')
.map((String path) => path.trim())
.where((String path) => path.isNotEmpty)
.toSet();
}
Future<Null> build({
String mainPath: defaultMainPath,
String manifestPath: defaultManifestPath,
......
packages/flutter_tools/test/base/build_test.dart
View file @ 1ee94001
......@@ -35,18 +35,38 @@ void main() {
testUsingContext('throws if any input file does not exist', () async {
await fs.file('a.dart').create();
expect(() => new Checksum.fromFiles(BuildMode.debug, <String>['a.dart', 'b.dart'].toSet()), throwsA(anything));
expect(
() => new Checksum.fromFiles(BuildMode.debug, TargetPlatform.ios, <String>['a.dart', 'b.dart'].toSet()),
throwsA(anything),
);
}, overrides: <Type, Generator>{ FileSystem: () => fs});
testUsingContext('throws if any build mode is null', () async {
await fs.file('a.dart').create();
expect(
() => new Checksum.fromFiles(null, TargetPlatform.ios, <String>['a.dart', 'b.dart'].toSet()),
throwsA(anything),
);
}, overrides: <Type, Generator>{ FileSystem: () => fs});
testUsingContext('does not throw if any target platform is null', () async {
await fs.file('a.dart').create();
expect(
new Checksum.fromFiles(BuildMode.debug, null, <String>['a.dart'].toSet()),
isNotNull,
);
}, overrides: <Type, Generator>{ FileSystem: () => fs});
testUsingContext('populates checksums for valid files', () async {
await fs.file('a.dart').writeAsString('This is a');
await fs.file('b.dart').writeAsString('This is b');
final Checksum checksum = new Checksum.fromFiles(BuildMode.debug, <String>['a.dart', 'b.dart'].toSet());
final Checksum checksum = new Checksum.fromFiles(BuildMode.debug, TargetPlatform.ios, <String>['a.dart', 'b.dart'].toSet());
final Map<String, dynamic> json = JSON.decode(checksum.toJson());
expect(json, hasLength(3));
expect(json, hasLength(4));
expect(json['version'], mockVersion.frameworkRevision);
expect(json['buildMode'], BuildMode.debug.toString());
expect(json['targetPlatform'], TargetPlatform.ios.toString());
expect(json['files'], hasLength(2));
expect(json['files']['a.dart'], '8a21a15fad560b799f6731d436c1b698');
expect(json['files']['b.dart'], '6f144e08b58cd0925328610fad7ac07c');
......@@ -64,13 +84,14 @@ void main() {
});
testUsingContext('populates checksums for valid JSON', () async {
final String json = '{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}';
final String json = '{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}';
final Checksum checksum = new Checksum.fromJson(json);
final Map<String, dynamic> content = JSON.decode(checksum.toJson());
expect(content, hasLength(3));
expect(content, hasLength(4));
expect(content['version'], mockVersion.frameworkRevision);
expect(content['buildMode'], BuildMode.release.toString());
expect(content['targetPlatform'], TargetPlatform.ios.toString());
expect(content['files']['a.dart'], '8a21a15fad560b799f6731d436c1b698');
expect(content['files']['b.dart'], '6f144e08b58cd0925328610fad7ac07c');
}, overrides: <Type, Generator>{
......@@ -86,29 +107,84 @@ void main() {
});
group('operator ==', () {
testUsingContext('reports not equal if build modes do not match', () async {
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.debug","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
expect(a == b, isFalse);
}, overrides: <Type, Generator>{
FlutterVersion: () => mockVersion,
});
testUsingContext('reports not equal if target platforms do not match', () async {
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.fuchsia","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
expect(a == b, isFalse);
}, overrides: <Type, Generator>{
FlutterVersion: () => mockVersion,
});
testUsingContext('reports not equal if checksums do not match', () async {
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07d"}}');
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07d"}}');
expect(a == b, isFalse);
}, overrides: <Type, Generator>{
FlutterVersion: () => mockVersion,
});
testUsingContext('reports not equal if keys do not match', () async {
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","c.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","c.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
expect(a == b, isFalse);
}, overrides: <Type, Generator>{
FlutterVersion: () => mockVersion,
});
testUsingContext('reports equal if all checksums match', () async {
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum a = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
final Checksum b = new Checksum.fromJson('{"version":"$kVersion","buildMode":"BuildMode.release","targetPlatform":"TargetPlatform.ios","files":{"a.dart":"8a21a15fad560b799f6731d436c1b698","b.dart":"6f144e08b58cd0925328610fad7ac07c"}}');
expect(a == b, isTrue);
}, overrides: <Type, Generator>{
FlutterVersion: () => mockVersion,
});
});
});
group('readDepfile', () {
MemoryFileSystem fs;
setUp(() {
fs = new MemoryFileSystem();
});
testUsingContext('returns one file if only one is listed', () async {
await fs.file('a.d').writeAsString('snapshot.d: /foo/a.dart');
expect(await readDepfile('a.d'), unorderedEquals(<String>['/foo/a.dart']));
}, overrides: <Type, Generator>{ FileSystem: () => fs});
testUsingContext('returns multiple files', () async {
await fs.file('a.d').writeAsString('snapshot.d: /foo/a.dart /foo/b.dart');
expect(await readDepfile('a.d'), unorderedEquals(<String>[
'/foo/a.dart',
'/foo/b.dart',
]));
}, overrides: <Type, Generator>{ FileSystem: () => fs});
testUsingContext('trims extra spaces between files', () async {
await fs.file('a.d').writeAsString('snapshot.d: /foo/a.dart /foo/b.dart /foo/c.dart');
expect(await readDepfile('a.d'), unorderedEquals(<String>[
'/foo/a.dart',
'/foo/b.dart',
'/foo/c.dart',
]));
}, overrides: <Type, Generator>{ FileSystem: () => fs});
testUsingContext('returns files with spaces and backslashes', () async {
await fs.file('a.d').writeAsString(r'snapshot.d: /foo/a\ a.dart /foo/b\\b.dart /foo/c\\ c.dart');
expect(await readDepfile('a.d'), unorderedEquals(<String>[
r'/foo/a a.dart',
r'/foo/b\b.dart',
r'/foo/c\ c.dart',
]));
}, overrides: <Type, Generator>{ FileSystem: () => fs});
});
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab