Skip to content

F
Front-End
Commit 27d3e8a4 authored by Chris Bracken's avatar Chris Bracken Committed by GitHub
Browse files
Options

Extract snapshotting logic to Snapshotter class (#11820) 

Extract a Snapshotter class that can be shared between FLX snapshotting,
AOT snapshotting, and assembly AOT snapshotting. Allows for better
testability of snapshotting logic.

* Extracts script snapshotting used in FLX build.
* Adds tests for snapshot checksumming, build invalidation/skipping.

Remaining work: disentangle + extract AOT snapshotting and Assembly AOT
snapshotting logic from build_aot.dart.
parent 8d5fe6d4
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 377 additions and 84 deletions
packages/flutter_tools/lib/src/base/build.dart
View file @ 27d3e8a4
......@@ -6,11 +6,54 @@ import 'dart:async';
import 'dart:convert' show JSON;
import 'package:crypto/crypto.dart' show md5;
import 'package:meta/meta.dart';
import 'package:quiver/core.dart' show hash3;
import '../artifacts.dart';
import '../build_info.dart';
import '../globals.dart';
import '../version.dart';
import 'context.dart';
import 'file_system.dart';
import 'process.dart';
GenSnapshot get genSnapshot => context.putIfAbsent(GenSnapshot, () => const GenSnapshot());
/// A snapshot build configuration.
class SnapshotType {
const SnapshotType(this.platform, this.mode);
final TargetPlatform platform;
final BuildMode mode;
}
/// Interface to the gen_snapshot command-line tool.
class GenSnapshot {
const GenSnapshot();
Future<int> run({
@required SnapshotType snapshotType,
@required String packagesPath,
@required String depfilePath,
Iterable<String> additionalArgs: const <String>[],
}) {
final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
final List<String> args = <String>[
'--assert_initializer',
'--await_is_keyword',
'--causal_async_stacks',
'--vm_snapshot_data=$vmSnapshotData',
'--isolate_snapshot_data=$isolateSnapshotData',
'--packages=$packagesPath',
'--dependencies=$depfilePath',
'--print_snapshot_sizes',
]..addAll(additionalArgs);
final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, snapshotType.platform, snapshotType.mode);
return runCommandAndStreamOutput(<String>[snapshotterPath]..addAll(args));
}
}
/// A collection of checksums for a set of input files.
///
......@@ -19,14 +62,14 @@ import 'file_system.dart';
/// build step. This assumes that build outputs are strictly a product of the
/// input files.
class Checksum {
Checksum.fromFiles(BuildMode buildMode, TargetPlatform targetPlatform, Set<String> inputPaths) {
Checksum.fromFiles(SnapshotType type, Set<String> inputPaths) {
final Iterable<File> files = inputPaths.map(fs.file);
final Iterable<File> missingInputs = files.where((File file) => !file.existsSync());
if (missingInputs.isNotEmpty)
throw new ArgumentError('Missing input files:\n' + missingInputs.join('\n'));
_buildMode = buildMode.toString();
_targetPlatform = targetPlatform?.toString() ?? '';
_buildMode = type.mode.toString();
_targetPlatform = type.platform?.toString() ?? '';
_checksums = <String, String>{};
for (File file in files) {
final List<int> bytes = file.readAsBytesSync();
......@@ -109,3 +152,111 @@ Future<Set<String>> readDepfile(String depfilePath) async {
.where((String path) => path.isNotEmpty)
.toSet();
}
/// Dart snapshot builder.
///
/// Builds Dart snapshots in one of three modes:
/// * Script snapshot: architecture-independent snapshot of a Dart script
/// and core libraries.
/// * AOT snapshot: architecture-specific ahead-of-time compiled snapshot
/// suitable for loading with `mmap`.
/// * Assembly AOT snapshot: architecture-specific ahead-of-time compile to
/// assembly suitable for compilation as a static or dynamic library.
class Snapshotter {
/// Builds an architecture-independent snapshot of the specified script.
Future<int> buildScriptSnapshot({
@required String mainPath,
@required String snapshotPath,
@required String depfilePath,
@required String packagesPath
}) async {
const SnapshotType type = const SnapshotType(null, BuildMode.debug);
final List<String> args = <String>[
'--snapshot_kind=script',
'--script_snapshot=$snapshotPath',
mainPath,
];
final String checksumsPath = '$depfilePath.checksums';
final int exitCode = await _build(
snapshotType: type,
outputSnapshotPath: snapshotPath,
packagesPath: packagesPath,
snapshotArgs: args,
depfilePath: depfilePath,
mainPath: mainPath,
checksumsPath: checksumsPath,
);
if (exitCode != 0)
return exitCode;
await _writeChecksum(type, snapshotPath, depfilePath, mainPath, checksumsPath);
return exitCode;
}
/// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script.
Future<Null> buildAotSnapshot() async {
throw new UnimplementedError('AOT snapshotting not yet implemented');
}
Future<int> _build({
@required SnapshotType snapshotType,
@required List<String> snapshotArgs,
@required String outputSnapshotPath,
@required String packagesPath,
@required String depfilePath,
@required String mainPath,
@required String checksumsPath,
}) async {
if (!await _isBuildRequired(snapshotType, outputSnapshotPath, depfilePath, mainPath, checksumsPath)) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
}
// Build the snapshot.
final int exitCode = await genSnapshot.run(
snapshotType: snapshotType,
packagesPath: packagesPath,
depfilePath: depfilePath,
additionalArgs: snapshotArgs,
);
if (exitCode != 0)
return exitCode;
_writeChecksum(snapshotType, outputSnapshotPath, depfilePath, mainPath, checksumsPath);
return 0;
}
Future<bool> _isBuildRequired(SnapshotType type, String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
final File checksumFile = fs.file(checksumsPath);
final File outputSnapshotFile = fs.file(outputSnapshotPath);
final File depfile = fs.file(depfilePath);
if (!outputSnapshotFile.existsSync() || !depfile.existsSync() || !checksumFile.existsSync())
return true;
try {
if (checksumFile.existsSync()) {
final Checksum oldChecksum = new Checksum.fromJson(await checksumFile.readAsString());
final Set<String> checksumPaths = await readDepfile(depfilePath)
..addAll(<String>[outputSnapshotPath, mainPath]);
final Checksum newChecksum = new Checksum.fromFiles(type, checksumPaths);
return oldChecksum != newChecksum;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum output: $e\n$s');
}
return true;
}
Future<Null> _writeChecksum(SnapshotType type, String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
try {
final Set<String> checksumPaths = await readDepfile(depfilePath)
..addAll(<String>[outputSnapshotPath, mainPath]);
final Checksum checksum = new Checksum.fromFiles(type, checksumPaths);
await fs.file(checksumsPath).writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
print('Error during snapshot checksum output: $e\n$s');
}
}
}
packages/flutter_tools/lib/src/commands/build_aot.dart
View file @ 27d3e8a4
......@@ -108,6 +108,7 @@ Future<String> buildAotSnapshot(
}
}
// TODO(cbracken): split AOT and Assembly AOT snapshotting logic and migrate to Snapshotter class.
Future<String> _buildAotSnapshot(
String mainPath,
TargetPlatform platform,
......@@ -267,6 +268,7 @@ Future<String> _buildAotSnapshot(
genSnapshotCmd.add(mainPath);
final SnapshotType snapshotType = new SnapshotType(platform, buildMode);
final File checksumFile = fs.file('$dependencies.checksum');
final List<File> checksumFiles = <File>[checksumFile, fs.file(dependencies)]
..addAll(inputPaths.map(fs.file))
......@@ -278,7 +280,7 @@ Future<String> _buildAotSnapshot(
final Set<String> snapshotInputPaths = await readDepfile(dependencies)
..add(mainPath)
..addAll(outputPaths);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
final Checksum newChecksum = new Checksum.fromFiles(snapshotType, snapshotInputPaths);
if (oldChecksum == newChecksum) {
printStatus('Skipping AOT snapshot build. Checksums match.');
return outputPath;
......@@ -356,7 +358,7 @@ Future<String> _buildAotSnapshot(
final Set<String> snapshotInputPaths = await readDepfile(dependencies)
..add(mainPath)
..addAll(outputPaths);
final Checksum checksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
final Checksum checksum = new Checksum.fromFiles(snapshotType, snapshotInputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
......
packages/flutter_tools/lib/src/flx.dart
View file @ 27d3e8a4
......@@ -4,14 +4,10 @@
import 'dart:async';
import 'package:meta/meta.dart' show required;
import 'artifacts.dart';
import 'asset.dart';
import 'base/build.dart';
import 'base/common.dart';
import 'base/file_system.dart';
import 'base/process.dart';
import 'build_info.dart';
import 'dart/package_map.dart';
import 'devfs.dart';
......@@ -30,75 +26,6 @@ const String _kKernelKey = 'kernel_blob.bin';
const String _kSnapshotKey = 'snapshot_blob.bin';
const String _kDylibKey = 'libapp.so';
Future<int> _createSnapshot({
@required String mainPath,
@required String snapshotPath,
@required String depfilePath,
@required String packages
}) async {
assert(mainPath != null);
assert(snapshotPath != null);
assert(depfilePath != null);
assert(packages != null);
final BuildMode buildMode = BuildMode.debug;
final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, null, buildMode);
final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
final List<String> args = <String>[
snapshotterPath,
'--snapshot_kind=script',
'--vm_snapshot_data=$vmSnapshotData',
'--isolate_snapshot_data=$isolateSnapshotData',
'--packages=$packages',
'--script_snapshot=$snapshotPath',
'--dependencies=$depfilePath',
mainPath,
];
// Write the depfile path to disk.
fs.file(depfilePath).parent.childFile('gen_snapshot.d').writeAsString('$depfilePath: $snapshotterPath\n');
final File checksumFile = fs.file('$depfilePath.checksums');
final File snapshotFile = fs.file(snapshotPath);
final File depfile = fs.file(depfilePath);
if (snapshotFile.existsSync() && depfile.existsSync() && checksumFile.existsSync()) {
try {
final String json = await checksumFile.readAsString();
final Checksum oldChecksum = new Checksum.fromJson(json);
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, null, inputPaths);
if (oldChecksum == newChecksum) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum check: $e\n$s');
}
}
// Build the snapshot.
final int exitCode = await runCommandAndStreamOutput(args);
if (exitCode != 0)
return exitCode;
// Compute and record input file checksums.
try {
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum checksum = new Checksum.fromFiles(buildMode, null, inputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum output: $e\n$s');
}
return 0;
}
Future<Null> build({
String mainPath: defaultMainPath,
String manifestPath: defaultManifestPath,
......@@ -124,11 +51,12 @@ Future<Null> build({
// In a precompiled snapshot, the instruction buffer contains script
// content equivalents
final int result = await _createSnapshot(
final Snapshotter snapshotter = new Snapshotter();
final int result = await snapshotter.buildScriptSnapshot(
mainPath: mainPath,
snapshotPath: snapshotPath,
depfilePath: depfilePath,
packages: packagesPath
packagesPath: packagesPath,
);
if (result != 0)
throwToolExit('Failed to run the Flutter compiler. Exit code: $result', exitCode: result);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab