Skip to content

F
Front-End
Commit 27d3e8a4 authored by Chris Bracken's avatar Chris Bracken Committed by GitHub
Browse files
Options

Extract snapshotting logic to Snapshotter class (#11820) 

Extract a Snapshotter class that can be shared between FLX snapshotting,
AOT snapshotting, and assembly AOT snapshotting. Allows for better
testability of snapshotting logic.

* Extracts script snapshotting used in FLX build.
* Adds tests for snapshot checksumming, build invalidation/skipping.

Remaining work: disentangle + extract AOT snapshotting and Assembly AOT
snapshotting logic from build_aot.dart.
parent 8d5fe6d4
Hide whitespace changes
Inline Side-by-side
Showing with 377 additions and 84 deletions
packages/flutter_tools/lib/src/base/build.dart
View file @ 27d3e8a4
......@@ -6,11 +6,54 @@ import 'dart:async';
import 'dart:convert' show JSON;
import 'package:crypto/crypto.dart' show md5;
import 'package:meta/meta.dart';
import 'package:quiver/core.dart' show hash3;
import '../artifacts.dart';
import '../build_info.dart';
import '../globals.dart';
import '../version.dart';
import 'context.dart';
import 'file_system.dart';
import 'process.dart';
GenSnapshot get genSnapshot => context.putIfAbsent(GenSnapshot, () => const GenSnapshot());
/// A snapshot build configuration.
class SnapshotType {
const SnapshotType(this.platform, this.mode);
final TargetPlatform platform;
final BuildMode mode;
}
/// Interface to the gen_snapshot command-line tool.
class GenSnapshot {
const GenSnapshot();
Future<int> run({
@required SnapshotType snapshotType,
@required String packagesPath,
@required String depfilePath,
Iterable<String> additionalArgs: const <String>[],
}) {
final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
final List<String> args = <String>[
'--assert_initializer',
'--await_is_keyword',
'--causal_async_stacks',
'--vm_snapshot_data=$vmSnapshotData',
'--isolate_snapshot_data=$isolateSnapshotData',
'--packages=$packagesPath',
'--dependencies=$depfilePath',
'--print_snapshot_sizes',
]..addAll(additionalArgs);
final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, snapshotType.platform, snapshotType.mode);
return runCommandAndStreamOutput(<String>[snapshotterPath]..addAll(args));
}
}
/// A collection of checksums for a set of input files.
///
......@@ -19,14 +62,14 @@ import 'file_system.dart';
/// build step. This assumes that build outputs are strictly a product of the
/// input files.
class Checksum {
Checksum.fromFiles(BuildMode buildMode, TargetPlatform targetPlatform, Set<String> inputPaths) {
Checksum.fromFiles(SnapshotType type, Set<String> inputPaths) {
final Iterable<File> files = inputPaths.map(fs.file);
final Iterable<File> missingInputs = files.where((File file) => !file.existsSync());
if (missingInputs.isNotEmpty)
throw new ArgumentError('Missing input files:\n' + missingInputs.join('\n'));
_buildMode = buildMode.toString();
_targetPlatform = targetPlatform?.toString() ?? '';
_buildMode = type.mode.toString();
_targetPlatform = type.platform?.toString() ?? '';
_checksums = <String, String>{};
for (File file in files) {
final List<int> bytes = file.readAsBytesSync();
......@@ -109,3 +152,111 @@ Future<Set<String>> readDepfile(String depfilePath) async {
.where((String path) => path.isNotEmpty)
.toSet();
}
/// Dart snapshot builder.
///
/// Builds Dart snapshots in one of three modes:
/// * Script snapshot: architecture-independent snapshot of a Dart script
/// and core libraries.
/// * AOT snapshot: architecture-specific ahead-of-time compiled snapshot
/// suitable for loading with `mmap`.
/// * Assembly AOT snapshot: architecture-specific ahead-of-time compile to
/// assembly suitable for compilation as a static or dynamic library.
class Snapshotter {
/// Builds an architecture-independent snapshot of the specified script.
Future<int> buildScriptSnapshot({
@required String mainPath,
@required String snapshotPath,
@required String depfilePath,
@required String packagesPath
}) async {
const SnapshotType type = const SnapshotType(null, BuildMode.debug);
final List<String> args = <String>[
'--snapshot_kind=script',
'--script_snapshot=$snapshotPath',
mainPath,
];
final String checksumsPath = '$depfilePath.checksums';
final int exitCode = await _build(
snapshotType: type,
outputSnapshotPath: snapshotPath,
packagesPath: packagesPath,
snapshotArgs: args,
depfilePath: depfilePath,
mainPath: mainPath,
checksumsPath: checksumsPath,
);
if (exitCode != 0)
return exitCode;
await _writeChecksum(type, snapshotPath, depfilePath, mainPath, checksumsPath);
return exitCode;
}
/// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script.
Future<Null> buildAotSnapshot() async {
throw new UnimplementedError('AOT snapshotting not yet implemented');
}
Future<int> _build({
@required SnapshotType snapshotType,
@required List<String> snapshotArgs,
@required String outputSnapshotPath,
@required String packagesPath,
@required String depfilePath,
@required String mainPath,
@required String checksumsPath,
}) async {
if (!await _isBuildRequired(snapshotType, outputSnapshotPath, depfilePath, mainPath, checksumsPath)) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
}
// Build the snapshot.
final int exitCode = await genSnapshot.run(
snapshotType: snapshotType,
packagesPath: packagesPath,
depfilePath: depfilePath,
additionalArgs: snapshotArgs,
);
if (exitCode != 0)
return exitCode;
_writeChecksum(snapshotType, outputSnapshotPath, depfilePath, mainPath, checksumsPath);
return 0;
}
Future<bool> _isBuildRequired(SnapshotType type, String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
final File checksumFile = fs.file(checksumsPath);
final File outputSnapshotFile = fs.file(outputSnapshotPath);
final File depfile = fs.file(depfilePath);
if (!outputSnapshotFile.existsSync() || !depfile.existsSync() || !checksumFile.existsSync())
return true;
try {
if (checksumFile.existsSync()) {
final Checksum oldChecksum = new Checksum.fromJson(await checksumFile.readAsString());
final Set<String> checksumPaths = await readDepfile(depfilePath)
..addAll(<String>[outputSnapshotPath, mainPath]);
final Checksum newChecksum = new Checksum.fromFiles(type, checksumPaths);
return oldChecksum != newChecksum;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum output: $e\n$s');
}
return true;
}
Future<Null> _writeChecksum(SnapshotType type, String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
try {
final Set<String> checksumPaths = await readDepfile(depfilePath)
..addAll(<String>[outputSnapshotPath, mainPath]);
final Checksum checksum = new Checksum.fromFiles(type, checksumPaths);
await fs.file(checksumsPath).writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
print('Error during snapshot checksum output: $e\n$s');
}
}
}
packages/flutter_tools/lib/src/commands/build_aot.dart
View file @ 27d3e8a4
......@@ -108,6 +108,7 @@ Future<String> buildAotSnapshot(
}
}
// TODO(cbracken): split AOT and Assembly AOT snapshotting logic and migrate to Snapshotter class.
Future<String> _buildAotSnapshot(
String mainPath,
TargetPlatform platform,
......@@ -267,6 +268,7 @@ Future<String> _buildAotSnapshot(
genSnapshotCmd.add(mainPath);
final SnapshotType snapshotType = new SnapshotType(platform, buildMode);
final File checksumFile = fs.file('$dependencies.checksum');
final List<File> checksumFiles = <File>[checksumFile, fs.file(dependencies)]
..addAll(inputPaths.map(fs.file))
......@@ -278,7 +280,7 @@ Future<String> _buildAotSnapshot(
final Set<String> snapshotInputPaths = await readDepfile(dependencies)
..add(mainPath)
..addAll(outputPaths);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
final Checksum newChecksum = new Checksum.fromFiles(snapshotType, snapshotInputPaths);
if (oldChecksum == newChecksum) {
printStatus('Skipping AOT snapshot build. Checksums match.');
return outputPath;
......@@ -356,7 +358,7 @@ Future<String> _buildAotSnapshot(
final Set<String> snapshotInputPaths = await readDepfile(dependencies)
..add(mainPath)
..addAll(outputPaths);
final Checksum checksum = new Checksum.fromFiles(buildMode, platform, snapshotInputPaths);
final Checksum checksum = new Checksum.fromFiles(snapshotType, snapshotInputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
......
packages/flutter_tools/lib/src/flx.dart
View file @ 27d3e8a4
......@@ -4,14 +4,10 @@
import 'dart:async';
import 'package:meta/meta.dart' show required;
import 'artifacts.dart';
import 'asset.dart';
import 'base/build.dart';
import 'base/common.dart';
import 'base/file_system.dart';
import 'base/process.dart';
import 'build_info.dart';
import 'dart/package_map.dart';
import 'devfs.dart';
......@@ -30,75 +26,6 @@ const String _kKernelKey = 'kernel_blob.bin';
const String _kSnapshotKey = 'snapshot_blob.bin';
const String _kDylibKey = 'libapp.so';
Future<int> _createSnapshot({
@required String mainPath,
@required String snapshotPath,
@required String depfilePath,
@required String packages
}) async {
assert(mainPath != null);
assert(snapshotPath != null);
assert(depfilePath != null);
assert(packages != null);
final BuildMode buildMode = BuildMode.debug;
final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, null, buildMode);
final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
final List<String> args = <String>[
snapshotterPath,
'--snapshot_kind=script',
'--vm_snapshot_data=$vmSnapshotData',
'--isolate_snapshot_data=$isolateSnapshotData',
'--packages=$packages',
'--script_snapshot=$snapshotPath',
'--dependencies=$depfilePath',
mainPath,
];
// Write the depfile path to disk.
fs.file(depfilePath).parent.childFile('gen_snapshot.d').writeAsString('$depfilePath: $snapshotterPath\n');
final File checksumFile = fs.file('$depfilePath.checksums');
final File snapshotFile = fs.file(snapshotPath);
final File depfile = fs.file(depfilePath);
if (snapshotFile.existsSync() && depfile.existsSync() && checksumFile.existsSync()) {
try {
final String json = await checksumFile.readAsString();
final Checksum oldChecksum = new Checksum.fromJson(json);
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum newChecksum = new Checksum.fromFiles(buildMode, null, inputPaths);
if (oldChecksum == newChecksum) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum check: $e\n$s');
}
}
// Build the snapshot.
final int exitCode = await runCommandAndStreamOutput(args);
if (exitCode != 0)
return exitCode;
// Compute and record input file checksums.
try {
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum checksum = new Checksum.fromFiles(buildMode, null, inputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum output: $e\n$s');
}
return 0;
}
Future<Null> build({
String mainPath: defaultMainPath,
String manifestPath: defaultManifestPath,
......@@ -124,11 +51,12 @@ Future<Null> build({
// In a precompiled snapshot, the instruction buffer contains script
// content equivalents
final int result = await _createSnapshot(
final Snapshotter snapshotter = new Snapshotter();
final int result = await snapshotter.buildScriptSnapshot(
mainPath: mainPath,
snapshotPath: snapshotPath,
depfilePath: depfilePath,
packages: packagesPath
packagesPath: packagesPath,
);
if (result != 0)
throwToolExit('Failed to run the Flutter compiler. Exit code: $result', exitCode: result);
......
packages/flutter_tools/test/base/build_test.dart
View file @ 27d3e8a4
......@@ -2,11 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:convert' show JSON;
import 'package:file/memory.dart';
import 'package:flutter_tools/src/build_info.dart';
import 'package:flutter_tools/src/base/build.dart';
import 'package:flutter_tools/src/base/context.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/version.dart';
import 'package:mockito/mockito.dart';
......@@ -16,6 +19,41 @@ import '../src/context.dart';
class MockFlutterVersion extends Mock implements FlutterVersion {}
class _FakeGenSnapshot implements GenSnapshot {
_FakeGenSnapshot({
this.succeed: true,
this.snapshotPath: 'output.snapshot',
this.snapshotContent: '',
this.depfilePath: 'output.snapshot.d',
this.depfileContent: 'output.snapshot.d : main.dart',
});
final bool succeed;
final String snapshotPath;
final String snapshotContent;
final String depfilePath;
final String depfileContent;
int _callCount = 0;
int get callCount => _callCount;
@override
Future<int> run({
SnapshotType snapshotType,
String packagesPath,
String depfilePath,
Iterable<String> additionalArgs,
}) async {
_callCount += 1;
if (!succeed)
return 1;
await fs.file(snapshotPath).writeAsString(snapshotContent);
await fs.file(depfilePath).writeAsString(depfileContent);
return 0;
}
}
void main() {
group('Checksum', () {
MockFlutterVersion mockVersion;
......@@ -35,24 +73,27 @@ void main() {
testUsingContext('throws if any input file does not exist', () async {
await fs.file('a.dart').create();
const SnapshotType snapshotType = const SnapshotType(TargetPlatform.ios, BuildMode.debug);
expect(
() => new Checksum.fromFiles(BuildMode.debug, TargetPlatform.ios, <String>['a.dart', 'b.dart'].toSet()),
() => new Checksum.fromFiles(snapshotType, <String>['a.dart', 'b.dart'].toSet()),
throwsA(anything),
);
}, overrides: <Type, Generator>{ FileSystem: () => fs });
testUsingContext('throws if any build mode is null', () async {
await fs.file('a.dart').create();
const SnapshotType snapshotType = const SnapshotType(TargetPlatform.ios, null);
expect(
() => new Checksum.fromFiles(null, TargetPlatform.ios, <String>['a.dart', 'b.dart'].toSet()),
() => new Checksum.fromFiles(snapshotType, <String>['a.dart', 'b.dart'].toSet()),
throwsA(anything),
);
}, overrides: <Type, Generator>{ FileSystem: () => fs });
testUsingContext('does not throw if any target platform is null', () async {
await fs.file('a.dart').create();
const SnapshotType snapshotType = const SnapshotType(null, BuildMode.debug);
expect(
new Checksum.fromFiles(BuildMode.debug, null, <String>['a.dart'].toSet()),
new Checksum.fromFiles(snapshotType, <String>['a.dart'].toSet()),
isNotNull,
);
}, overrides: <Type, Generator>{ FileSystem: () => fs });
......@@ -60,7 +101,8 @@ void main() {
testUsingContext('populates checksums for valid files', () async {
await fs.file('a.dart').writeAsString('This is a');
await fs.file('b.dart').writeAsString('This is b');
final Checksum checksum = new Checksum.fromFiles(BuildMode.debug, TargetPlatform.ios, <String>['a.dart', 'b.dart'].toSet());
const SnapshotType snapshotType = const SnapshotType(TargetPlatform.ios, BuildMode.debug);
final Checksum checksum = new Checksum.fromFiles(snapshotType, <String>['a.dart', 'b.dart'].toSet());
final Map<String, dynamic> json = JSON.decode(checksum.toJson());
expect(json, hasLength(4));
......@@ -187,4 +229,174 @@ void main() {
]));
}, overrides: <Type, Generator>{ FileSystem: () => fs });
});
group('Snapshotter', () {
const String kVersion = '123456abcdef';
_FakeGenSnapshot genSnapshot;
MemoryFileSystem fs;
MockFlutterVersion mockVersion;
Snapshotter snapshotter;
setUp(() {
fs = new MemoryFileSystem();
genSnapshot = new _FakeGenSnapshot();
mockVersion = new MockFlutterVersion();
snapshotter = new Snapshotter();
when(mockVersion.frameworkRevision).thenReturn(kVersion);
});
testUsingContext('builds snapshot and checksums when no checksums are present', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot').create();
await fs.file('output.snapshot.d').writeAsString('snapshot : main.dart');
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json['files'], hasLength(2));
expect(json['files']['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['files']['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
FlutterVersion: () => mockVersion,
GenSnapshot: () => genSnapshot,
});
testUsingContext('builds snapshot and checksums when checksums differ', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot').create();
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'version': '$kVersion',
'buildMode': BuildMode.debug.toString(),
'files': <String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'deadbeef01234567890abcdef0123456',
},
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json['files'], hasLength(2));
expect(json['files']['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['files']['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
FlutterVersion: () => mockVersion,
GenSnapshot: () => genSnapshot,
});
testUsingContext('builds snapshot and checksums when checksums match but previous snapshot not present', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'version': '$kVersion',
'buildMode': BuildMode.debug.toString(),
'files': <String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
},
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json['files'], hasLength(2));
expect(json['files']['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['files']['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
FlutterVersion: () => mockVersion,
GenSnapshot: () => genSnapshot,
});
testUsingContext('builds snapshot and checksums when main entry point changes', () async {
final _FakeGenSnapshot genSnapshot = new _FakeGenSnapshot(
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
depfileContent: 'output.snapshot : other.dart',
);
context.setVariable(GenSnapshot, genSnapshot);
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('other.dart').writeAsString('void main() { print("Kanpai ima kimi wa jinsei no ookina ookina butai ni tachi"); }');
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'version': '$kVersion',
'buildMode': BuildMode.debug.toString(),
'targetPlatform': '',
'files': <String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
},
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'other.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json['files'], hasLength(2));
expect(json['files']['other.dart'], '3238d0ae341339b1731d3c2e195ad177');
expect(json['files']['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
FlutterVersion: () => mockVersion,
});
testUsingContext('skips snapshot when checksums match and previous snapshot is present', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot').create();
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'version': '$kVersion',
'buildMode': BuildMode.debug.toString(),
'targetPlatform': '',
'files': <String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
},
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 0);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json['files'], hasLength(2));
expect(json['files']['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['files']['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
FlutterVersion: () => mockVersion,
GenSnapshot: () => genSnapshot,
});
});
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab