Skip to content

F
Front-End
Unverified Commit 66c7b6a9 authored by Chris Bracken's avatar Chris Bracken Committed by GitHub
Browse files
Options

Add Fingerprinter class (#17255) 

Adds a Fingerprinter utility class that can be used to compute unique
fingerprints for a set of input paths and build options, compare to the
output of a previous run, and skip the build action if no inputs or
options have changed. The existing Fingerprint class still does all the
heavy lifting. Fingerprinter adds common operations such as
reading/writing/comparing fingerprints and parsing depfiles.

This migrates existing uses of Fingerprint over to Fingerprinter.

This also adds better fingerprinting to AOT snapshotting, which
previously failed to include several options in its fingerprint
(--preview-dart-2, --prefer-shared-library).
parent 322eb81a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 601 additions and 486 deletions
packages/flutter_tools/lib/src/base/build.dart
View file @ 66c7b6a9
......@@ -3,11 +3,8 @@
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show json;
import 'package:crypto/crypto.dart' show md5;
import 'package:meta/meta.dart';
import 'package:quiver/core.dart' show hash2;
import '../android/android_sdk.dart';
import '../artifacts.dart';
......@@ -16,9 +13,9 @@ import '../compile.dart';
import '../dart/package_map.dart';
import '../globals.dart';
import '../ios/mac.dart';
import '../version.dart';
import 'context.dart';
import 'file_system.dart';
import 'fingerprint.dart';
import 'process.dart';
GenSnapshot get genSnapshot => context[GenSnapshot];
......@@ -66,97 +63,6 @@ class GenSnapshot {
}
}
/// A fingerprint for a set of build input files and properties.
///
/// This class can be used during build actions to compute a fingerprint of the
/// build action inputs, and if unchanged from the previous build, skip the
/// build step. This assumes that build outputs are strictly a product of the
/// fingerprint inputs.
class Fingerprint {
Fingerprint.fromBuildInputs(Map<String, String> properties, Iterable<String> inputPaths) {
final Iterable<File> files = inputPaths.map(fs.file);
final Iterable<File> missingInputs = files.where((File file) => !file.existsSync());
if (missingInputs.isNotEmpty)
throw new ArgumentError('Missing input files:\n' + missingInputs.join('\n'));
_checksums = <String, String>{};
for (File file in files) {
final List<int> bytes = file.readAsBytesSync();
_checksums[file.path] = md5.convert(bytes).toString();
}
_properties = <String, String>{}..addAll(properties);
}
/// Creates a Fingerprint from serialized JSON.
///
/// Throws [ArgumentError], if there is a version mismatch between the
/// serializing framework and this framework.
Fingerprint.fromJson(String jsonData) {
final Map<String, dynamic> content = json.decode(jsonData);
final String version = content['version'];
if (version != FlutterVersion.instance.frameworkRevision)
throw new ArgumentError('Incompatible fingerprint version: $version');
_checksums = content['files'] ?? <String, String>{};
_properties = content['properties'] ?? <String, String>{};
}
Map<String, String> _checksums;
Map<String, String> _properties;
String toJson() => json.encode(<String, dynamic>{
'version': FlutterVersion.instance.frameworkRevision,
'properties': _properties,
'files': _checksums,
});
@override
bool operator==(dynamic other) {
if (identical(other, this))
return true;
if (other.runtimeType != runtimeType)
return false;
final Fingerprint typedOther = other;
return _equalMaps(typedOther._checksums, _checksums)
&& _equalMaps(typedOther._properties, _properties);
}
bool _equalMaps(Map<String, String> a, Map<String, String> b) {
return a.length == b.length
&& a.keys.every((String key) => a[key] == b[key]);
}
@override
// Ignore map entries here to avoid becoming inconsistent with equals
// due to differences in map entry order.
int get hashCode => hash2(_properties.length, _checksums.length);
}
final RegExp _separatorExpr = new RegExp(r'([^\\]) ');
final RegExp _escapeExpr = new RegExp(r'\\(.)');
/// Parses a VM snapshot dependency file.
///
/// Snapshot dependency files are a single line mapping the output snapshot to a
/// space-separated list of input files used to generate that output. Spaces and
/// backslashes are escaped with a backslash. e.g,
///
/// outfile : file1.dart fil\\e2.dart fil\ e3.dart
///
/// will return a set containing: 'file1.dart', 'fil\e2.dart', 'fil e3.dart'.
Future<Set<String>> readDepfile(String depfilePath) async {
// Depfile format:
// outfile1 outfile2 : file1.dart file2.dart file3.dart
final String contents = await fs.file(depfilePath).readAsString();
final String dependencies = contents.split(': ')[1];
return dependencies
.replaceAllMapped(_separatorExpr, (Match match) => '${match.group(1)}\n')
.split('\n')
.map((String path) => path.replaceAllMapped(_escapeExpr, (Match match) => match.group(1)).trim())
.where((String path) => path.isNotEmpty)
.toSet();
}
/// Dart snapshot builder.
///
/// Builds Dart snapshots in one of three modes:
......@@ -186,24 +92,37 @@ class ScriptSnapshotter {
mainPath,
];
final String fingerprintPath = '$depfilePath.fingerprint';
final Set<String> outputPaths = <String>[snapshotPath].toSet();
if (!await _isBuildRequired(snapshotType, outputPaths, depfilePath, mainPath, fingerprintPath)) {
final Fingerprinter fingerprinter = new Fingerprinter(
fingerprintPath: '$depfilePath.fingerprint',
paths: <String>[
mainPath,
snapshotPath,
vmSnapshotData,
isolateSnapshotData,
],
properties: <String, String>{
'buildMode': snapshotType.mode.toString(),
'targetPlatform': snapshotType.platform?.toString() ?? '',
'entryPoint': mainPath,
},
depfilePaths: <String>[depfilePath],
);
if (await fingerprinter.doesFingerprintMatch()) {
printTrace('Skipping script snapshot build. Fingerprints match.');
return 0;
}
// Build the snapshot.
final int exitCode = await genSnapshot.run(
snapshotType: snapshotType,
packagesPath: packagesPath,
depfilePath: depfilePath,
additionalArgs: args,
snapshotType: snapshotType,
packagesPath: packagesPath,
depfilePath: depfilePath,
additionalArgs: args,
);
if (exitCode != 0)
return exitCode;
await _writeFingerprint(snapshotType, outputPaths, depfilePath, mainPath, fingerprintPath);
await fingerprinter.writeFingerprint();
return exitCode;
}
}
......@@ -311,9 +230,20 @@ class AOTSnapshotter {
}
// If inputs and outputs have not changed since last run, skip the build.
final String fingerprintPath = '$depfilePath.fingerprint';
final SnapshotType snapshotType = new SnapshotType(platform, buildMode);
if (!await _isBuildRequired(snapshotType, outputPaths, depfilePath, mainPath, fingerprintPath)) {
final Fingerprinter fingerprinter = new Fingerprinter(
fingerprintPath: '$depfilePath.fingerprint',
paths: <String>[mainPath]..addAll(inputPaths)..addAll(outputPaths),
properties: <String, String>{
'buildMode': buildMode.toString(),
'targetPlatform': platform.toString(),
'entryPoint': mainPath,
'dart2': previewDart2.toString(),
'sharedLib': compileToSharedLibrary.toString(),
'extraGenSnapshotOptions': extraGenSnapshotOptions.join(' '),
},
depfilePaths: <String>[depfilePath],
);
if (await fingerprinter.doesFingerprintMatch()) {
printTrace('Skipping AOT snapshot build. Fingerprint match.');
return 0;
}
......@@ -346,7 +276,7 @@ class AOTSnapshotter {
}
// Compute and record build fingerprint.
await _writeFingerprint(snapshotType, outputPaths, depfilePath, mainPath, fingerprintPath);
await fingerprinter.writeFingerprint();
return 0;
}
......@@ -454,48 +384,3 @@ class AOTSnapshotter {
return fs.path.dirname(fs.path.fromUri(packageMap.map[package]));
}
}
Future<bool> _isBuildRequired(SnapshotType type, Set<String> outputPaths, String depfilePath, String mainPath, String fingerprintPath) async {
final File fingerprintFile = fs.file(fingerprintPath);
final List<String> requiredFiles = <String>[fingerprintPath, depfilePath]..addAll(outputPaths);
if (!requiredFiles.every(fs.isFileSync))
return true;
try {
if (fingerprintFile.existsSync()) {
final Fingerprint oldFingerprint = new Fingerprint.fromJson(await fingerprintFile.readAsString());
final Set<String> inputFilePaths = await readDepfile(depfilePath)..add(mainPath)..addAll(outputPaths);
final Fingerprint newFingerprint = createFingerprint(type, mainPath, inputFilePaths);
return oldFingerprint != newFingerprint;
}
} catch (e) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Rebuilding snapshot due to fingerprint check error: $e');
}
return true;
}
Future<Null> _writeFingerprint(SnapshotType type, Set<String> outputPaths, String depfilePath, String mainPath, String fingerprintPath) async {
try {
final Set<String> inputFilePaths = await readDepfile(depfilePath)
..add(mainPath)
..addAll(outputPaths);
final Fingerprint fingerprint = createFingerprint(type, mainPath, inputFilePaths);
await fs.file(fingerprintPath).writeAsString(fingerprint.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printStatus('Error during snapshot fingerprinting: $e\n$s');
}
}
Fingerprint createFingerprint(SnapshotType type, String mainPath, Iterable<String> inputFilePaths) {
final Map<String, String> properties = <String, String>{
'buildMode': type.mode.toString(),
'targetPlatform': type.platform?.toString() ?? '',
'entryPoint': mainPath,
};
final List<String> pathsWithSnapshotData = inputFilePaths.toList()
..add(artifacts.getArtifactPath(Artifact.vmSnapshotData))
..add(artifacts.getArtifactPath(Artifact.isolateSnapshotData));
return new Fingerprint.fromBuildInputs(properties, pathsWithSnapshotData);
}
packages/flutter_tools/lib/src/base/fingerprint.dart 0 → 100644
View file @ 66c7b6a9
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show json;
import 'package:crypto/crypto.dart' show md5;
import 'package:meta/meta.dart';
import 'package:quiver/core.dart' show hash2;
import '../globals.dart';
import '../version.dart';
import 'file_system.dart';
/// A tool that can be used to compute, compare, and write [Fingerprint]s for a
/// set of input files and associated build settings.
///
/// This class can be used during build actions to compute a fingerprint of the
/// build action inputs and options, and if unchanged from the previous build,
/// skip the build step. This assumes that build outputs are strictly a product
/// of the fingerprint inputs.
class Fingerprinter {
Fingerprinter({
@required this.fingerprintPath,
@required Iterable<String> paths,
@required Map<String, String> properties,
Iterable<String> depfilePaths: const <String>[]
}) : _paths = paths.toList(),
_properties = new Map<String, String>.from(properties),
_depfilePaths = depfilePaths.toList(),
assert(fingerprintPath != null),
assert(paths != null && paths.every((String path) => path != null)),
assert(properties != null),
assert(depfilePaths != null && depfilePaths.every((String path) => path != null));
final String fingerprintPath;
final List<String> _paths;
final Map<String, String> _properties;
final List<String> _depfilePaths;
Future<Fingerprint> buildFingerprint() async {
final List<String> paths = await _getPaths();
return new Fingerprint.fromBuildInputs(_properties, paths);
}
Future<bool> doesFingerprintMatch() async {
final File fingerprintFile = fs.file(fingerprintPath);
if (!fingerprintFile.existsSync())
return false;
if (!_depfilePaths.every(fs.isFileSync))
return false;
final List<String> paths = await _getPaths();
if (!paths.every(fs.isFileSync))
return false;
try {
final Fingerprint oldFingerprint = new Fingerprint.fromJson(await fingerprintFile.readAsString());
final Fingerprint newFingerprint = await buildFingerprint();
return oldFingerprint == newFingerprint;
} catch (e) {
// Log exception and continue, fingerprinting is only a performance improvement.
printTrace('Fingerprint check error: $e');
}
return false;
}
Future<void> writeFingerprint() async {
try {
final Fingerprint fingerprint = await buildFingerprint();
return fs.file(fingerprintPath).writeAsStringSync(fingerprint.toJson());
} catch (e) {
// Log exception and continue, fingerprinting is only a performance improvement.
printTrace('Fingerprint write error: $e');
}
}
Future<List<String>> _getPaths() async {
final Set<String> paths = _paths.toSet();
for (String depfilePath in _depfilePaths)
paths.addAll(await readDepfile(depfilePath));
return paths.toList()..sort();
}
}
/// A fingerprint that uniquely identifies a set of build input files and
/// properties.
///
/// See [Fingerprinter].
class Fingerprint {
Fingerprint.fromBuildInputs(Map<String, String> properties, Iterable<String> inputPaths) {
final Iterable<File> files = inputPaths.map(fs.file);
final Iterable<File> missingInputs = files.where((File file) => !file.existsSync());
if (missingInputs.isNotEmpty)
throw new ArgumentError('Missing input files:\n' + missingInputs.join('\n'));
_checksums = <String, String>{};
for (File file in files) {
final List<int> bytes = file.readAsBytesSync();
_checksums[file.path] = md5.convert(bytes).toString();
}
_properties = <String, String>{}..addAll(properties);
}
/// Creates a Fingerprint from serialized JSON.
///
/// Throws [ArgumentError], if there is a version mismatch between the
/// serializing framework and this framework.
Fingerprint.fromJson(String jsonData) {
final Map<String, dynamic> content = json.decode(jsonData);
final String version = content['version'];
if (version != FlutterVersion.instance.frameworkRevision)
throw new ArgumentError('Incompatible fingerprint version: $version');
_checksums = content['files'] ?? <String, String>{};
_properties = content['properties'] ?? <String, String>{};
}
Map<String, String> _checksums;
Map<String, String> _properties;
String toJson() => json.encode(<String, dynamic>{
'version': FlutterVersion.instance.frameworkRevision,
'properties': _properties,
'files': _checksums,
});
@override
bool operator==(dynamic other) {
if (identical(other, this))
return true;
if (other.runtimeType != runtimeType)
return false;
final Fingerprint typedOther = other;
return _equalMaps(typedOther._checksums, _checksums)
&& _equalMaps(typedOther._properties, _properties);
}
bool _equalMaps(Map<String, String> a, Map<String, String> b) {
return a.length == b.length
&& a.keys.every((String key) => a[key] == b[key]);
}
@override
// Ignore map entries here to avoid becoming inconsistent with equals
// due to differences in map entry order.
int get hashCode => hash2(_properties.length, _checksums.length);
}
final RegExp _separatorExpr = new RegExp(r'([^\\]) ');
final RegExp _escapeExpr = new RegExp(r'\\(.)');
/// Parses a VM snapshot dependency file.
///
/// Snapshot dependency files are a single line mapping the output snapshot to a
/// space-separated list of input files used to generate that output. Spaces and
/// backslashes are escaped with a backslash. e.g,
///
/// outfile : file1.dart fil\\e2.dart fil\ e3.dart
///
/// will return a set containing: 'file1.dart', 'fil\e2.dart', 'fil e3.dart'.
Future<Set<String>> readDepfile(String depfilePath) async {
// Depfile format:
// outfile1 outfile2 : file1.dart file2.dart file3.dart
final String contents = await fs.file(depfilePath).readAsString();
final String dependencies = contents.split(': ')[1];
return dependencies
.replaceAllMapped(_separatorExpr, (Match match) => '${match.group(1)}\n')
.split('\n')
.map((String path) => path.replaceAllMapped(_escapeExpr, (Match match) => match.group(1)).trim())
.where((String path) => path.isNotEmpty)
.toSet();
}
packages/flutter_tools/lib/src/bundle.dart
View file @ 66c7b6a9
......@@ -9,6 +9,7 @@ import 'asset.dart';
import 'base/build.dart';
import 'base/common.dart';
import 'base/file_system.dart';
import 'base/fingerprint.dart';
import 'build_info.dart';
import 'compile.dart';
import 'dart/package_map.dart';
......@@ -70,36 +71,20 @@ Future<void> build({
DevFSContent kernelContent;
if (!precompiledSnapshot && previewDart2) {
final File fingerprintFile = fs.file('$depfilePath.fingerprint');
final List<String> inputPaths = <String>[
mainPath,
];
bool needBuild = true;
final List<File> fingerprintFiles = <File>[fingerprintFile, fs.file(depfilePath)]
..addAll(inputPaths.map(fs.file));
Future<Fingerprint> makeFingerprint() async {
final Set<String> compilerInputPaths = await readDepfile(depfilePath)
..add(mainPath);
final Map<String, String> properties = <String, String>{
final Fingerprinter fingerprinter = new Fingerprinter(
fingerprintPath: '$depfilePath.fingerprint',
paths: <String>[mainPath],
properties: <String, String>{
'entryPoint': mainPath,
'trackWidgetCreation': trackWidgetCreation.toString(),
};
return new Fingerprint.fromBuildInputs(properties, compilerInputPaths);
}
},
depfilePaths: <String>[depfilePath],
);
if (fingerprintFiles.every((File file) => file.existsSync())) {
try {
final String json = await fingerprintFile.readAsString();
final Fingerprint oldFingerprint = new Fingerprint.fromJson(json);
if (oldFingerprint == await makeFingerprint()) {
needBuild = false;
printStatus('Skipping compilation. Fingerprint match.');
}
} catch (e) {
printTrace('Rebuilding kernel file due to fingerprint check error: $e');
}
if (await fingerprinter.doesFingerprintMatch()) {
needBuild = false;
printStatus('Skipping compilation. Fingerprint match.');
}
String kernelBinaryFilename;
......@@ -121,13 +106,7 @@ Future<void> build({
throwToolExit('Compiler failed on $mainPath');
}
// Compute and record build fingerprint.
try {
final Fingerprint fingerprint = await makeFingerprint();
await fingerprintFile.writeAsString(fingerprint.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printStatus('Error during compilation output fingerprinting: $e\n$s');
}
await fingerprinter.writeFingerprint();
} else {
kernelBinaryFilename = applicationKernelFilePath;
}
......
packages/flutter_tools/test/base/fingerprint_test.dart 0 → 100644
View file @ 66c7b6a9
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab