Skip to content

F
Front-End
Unverified Commit df3505c1 authored by Jonah Williams's avatar Jonah Williams Committed by GitHub
Browse files
Options

Improve performance of build APK (~50%) by running gen_snapshot concurrently (#44534)

parent 6e5769d4
Hide whitespace changes
Inline Side-by-side
Showing with 330 additions and 64 deletions
dev/devicelab/bin/tasks/gradle_plugin_fat_apk_test.dart
View file @ df3505c1
......@@ -111,8 +111,8 @@ Future<void> main() async {
// When the platform-target isn't specified, we generate the snapshots
// for arm and arm64.
final List<String> targetPlatforms = <String>[
'android-arm',
'android-arm64',
'arm64-v8a',
'armeabi-v7a',
];
for (final String targetPlatform in targetPlatforms) {
final String androidArmSnapshotPath = path.join(
......
dev/devicelab/bin/tasks/module_test.dart
View file @ df3505c1
......@@ -216,12 +216,12 @@ Future<void> main() async {
}
final String analyticsOutput = analyticsOutputFile.readAsStringSync();
if (!analyticsOutput.contains('cd24: android-arm64')
if (!analyticsOutput.contains('cd24: android')
|| !analyticsOutput.contains('cd25: true')
|| !analyticsOutput.contains('viewName: assemble')) {
return TaskResult.failure(
'Building outer app produced the following analytics: "$analyticsOutput"'
'but not the expected strings: "cd24: android-arm64", "cd25: true" and '
'but not the expected strings: "cd24: android", "cd25: true" and '
'"viewName: assemble"'
);
}
......
dev/devicelab/lib/framework/apk_utils.dart
View file @ df3505c1
......@@ -396,7 +396,7 @@ Future<ProcessResult> _resultOfGradleTask({String workingDirectory, String task,
String validateSnapshotDependency(FlutterProject project, String expectedTarget) {
final File snapshotBlob = File(
path.join(project.rootPath, 'build', 'app', 'intermediates',
'flutter', 'debug', 'android-arm', 'flutter_build.d'));
'flutter', 'debug', 'flutter_build.d'));
assert(snapshotBlob.existsSync());
final String contentSnapshot = snapshotBlob.readAsStringSync();
......
packages/flutter_tools/gradle/flutter.gradle
View file @ df3505c1
......@@ -544,38 +544,36 @@ class FlutterPlugin implements Plugin<Project> {
}
}
String variantBuildMode = buildModeFor(variant.buildType)
List<FlutterTask> compileTasks = targetPlatforms.collect { targetArch ->
String taskName = toCammelCase(["compile", FLUTTER_BUILD_PREFIX, variant.name, targetArch.replace('android-', '')])
project.tasks.create(name: taskName, type: FlutterTask) {
flutterRoot this.flutterRoot
flutterExecutable this.flutterExecutable
buildMode variantBuildMode
localEngine this.localEngine
localEngineSrcPath this.localEngineSrcPath
abi PLATFORM_ARCH_MAP[targetArch]
targetPath target
verbose isVerbose()
fileSystemRoots fileSystemRootsValue
fileSystemScheme fileSystemSchemeValue
trackWidgetCreation trackWidgetCreationValue
targetPlatform targetArch
sourceDir project.file(project.flutter.source)
intermediateDir project.file("${project.buildDir}/${AndroidProject.FD_INTERMEDIATES}/flutter/${variant.name}/${targetArch}")
extraFrontEndOptions extraFrontEndOptionsValue
extraGenSnapshotOptions extraGenSnapshotOptionsValue
}
String taskName = toCammelCase(["compile", FLUTTER_BUILD_PREFIX, variant.name])
FlutterTask compileTask = project.tasks.create(name: taskName, type: FlutterTask) {
flutterRoot this.flutterRoot
flutterExecutable this.flutterExecutable
buildMode variantBuildMode
localEngine this.localEngine
localEngineSrcPath this.localEngineSrcPath
targetPath target
verbose isVerbose()
fileSystemRoots fileSystemRootsValue
fileSystemScheme fileSystemSchemeValue
trackWidgetCreation trackWidgetCreationValue
targetPlatformValues = targetPlatforms
sourceDir project.file(project.flutter.source)
intermediateDir project.file("${project.buildDir}/${AndroidProject.FD_INTERMEDIATES}/flutter/${variant.name}/")
extraFrontEndOptions extraFrontEndOptionsValue
extraGenSnapshotOptions extraGenSnapshotOptionsValue
}
File libJar = project.file("${project.buildDir}/${AndroidProject.FD_INTERMEDIATES}/flutter/${variant.name}/libs.jar")
Task packFlutterAppAotTask = project.tasks.create(name: "packLibs${FLUTTER_BUILD_PREFIX}${variant.name.capitalize()}", type: Jar) {
destinationDir libJar.parentFile
archiveName libJar.name
dependsOn compileTasks
compileTasks.each { compileTask ->
from(compileTask.intermediateDir) {
include '*.so'
dependsOn compileTask
targetPlatforms.each { targetPlatform ->
String abi = PLATFORM_ARCH_MAP[targetPlatform]
from("${compileTask.intermediateDir}/${abi}") {
include "*.so"
// Move `app.so` to `lib/<abi>/libapp.so`
rename { String filename ->
return "lib/${compileTask.abi}/lib${filename}"
return "lib/${abi}/lib${filename}"
}
}
}
......@@ -594,11 +592,8 @@ class FlutterPlugin implements Plugin<Project> {
name: "copyFlutterAssets${variant.name.capitalize()}",
type: Copy,
) {
dependsOn compileTasks
compileTasks.each { flutterTask ->
// Add flutter_assets.
with flutterTask.assets
}
dependsOn compileTask
with compileTask.assets
if (isUsedAsSubproject) {
dependsOn packageAssets
dependsOn cleanPackageAssets
......@@ -687,9 +682,7 @@ abstract class BaseFlutterTask extends DefaultTask {
@Input
Boolean trackWidgetCreation
@Optional @Input
String targetPlatform
@Input
String abi
List<String> targetPlatformValues
File sourceDir
File intermediateDir
@Optional @Input
......@@ -713,6 +706,16 @@ abstract class BaseFlutterTask extends DefaultTask {
intermediateDir.mkdirs()
// Compute the rule name for flutter assemble. To speed up builds that contain
// multiple ABIs, the target name is used to communicate which ones are required
// rather than the TargetPlatform. This allows multiple builds to share the same
// cache.
String[] ruleNames;
if (buildMode == "debug") {
ruleNames = ["debug_android_application"]
} else {
ruleNames = targetPlatformValues.collect { "android_aot_bundle_${buildMode}_$it" }
}
project.exec {
executable flutterExecutable.absolutePath
workingDir sourceDir
......@@ -729,7 +732,7 @@ abstract class BaseFlutterTask extends DefaultTask {
args "--depfile", "${intermediateDir}/flutter_build.d"
args "--output", "${intermediateDir}"
args "-dTargetFile=${targetPath}"
args "-dTargetPlatform=${targetPlatform}"
args "-dTargetPlatform=android"
args "-dBuildMode=${buildMode}"
if (extraFrontEndOptions != null) {
args "-dExtraFrontEndOptions=${extraFrontEndOptions}"
......@@ -737,15 +740,7 @@ abstract class BaseFlutterTask extends DefaultTask {
if (extraGenSnapshotOptions != null) {
args "-dExtraGenSnapshotOptions=${extraGenSnapshotOptions}"
}
if (buildMode == "profile") {
args "profile_android_application"
}
if (buildMode == "release") {
args "release_android_application"
}
if (buildMode == "debug") {
args "debug_android_application"
}
args ruleNames
}
}
}
......@@ -768,7 +763,9 @@ class FlutterTask extends BaseFlutterTask {
from "${intermediateDir}"
if (buildMode == 'release' || buildMode == 'profile') {
include "app.so"
targetPlatformValues.each {
include "${PLATFORM_ARCH_MAP[targetArch]}/app.so"
}
}
}
}
......
packages/flutter_tools/lib/src/application_package.dart
View file @ df3505c1
......@@ -36,6 +36,7 @@ class ApplicationPackageFactory {
File applicationBinary,
}) async {
switch (platform) {
case TargetPlatform.android:
case TargetPlatform.android_arm:
case TargetPlatform.android_arm64:
case TargetPlatform.android_x64:
......@@ -415,6 +416,7 @@ class ApplicationPackageStore {
Future<ApplicationPackage> getPackageForPlatform(TargetPlatform platform) async {
switch (platform) {
case TargetPlatform.android:
case TargetPlatform.android_arm:
case TargetPlatform.android_arm64:
case TargetPlatform.android_x64:
......
packages/flutter_tools/lib/src/artifacts.dart
View file @ df3505c1
......@@ -393,6 +393,9 @@ class CachedArtifacts extends Artifacts {
assert(mode != null, 'Need to specify a build mode for platform $platform.');
final String suffix = mode != BuildMode.debug ? '-${snakeCase(getModeName(mode), '-')}' : '';
return fs.path.join(engineDir, platformName + suffix);
case TargetPlatform.android:
assert(false, 'cannot use TargetPlatform.android to look up artifacts');
return null;
}
assert(false, 'Invalid platform $platform.');
return null;
......
packages/flutter_tools/lib/src/base/build.dart
View file @ df3505c1
......@@ -158,11 +158,11 @@ class AOTSnapshotter {
genSnapshotArgs.add(mainPath);
// Verify that all required inputs exist.
// TODO(jonahwilliams): fully remove input checks once all callers are
// using assemble.
final Iterable<String> missingInputs = inputPaths.where((String p) => !fs.isFileSync(p));
if (missingInputs.isNotEmpty) {
printError('Missing input files: $missingInputs from $inputPaths');
return 1;
printTrace('Missing input files: $missingInputs from $inputPaths');
}
final SnapshotType snapshotType = SnapshotType(platform, buildMode);
......@@ -197,6 +197,7 @@ class AOTSnapshotter {
// Write path to gen_snapshot, since snapshots have to be re-generated when we roll
// the Dart SDK.
// TODO(jonahwilliams): remove when all callers are using assemble.
final String genSnapshotPath = GenSnapshot.getSnapshotterPath(snapshotType);
outputDir.childFile('gen_snapshot.d').writeAsStringSync('gen_snapshot.d: $genSnapshotPath\n');
......
packages/flutter_tools/lib/src/build_info.dart
View file @ df3505c1
......@@ -260,7 +260,8 @@ String validatedBuildNameForPlatform(TargetPlatform targetPlatform, String build
}
return tmpBuildName;
}
if (targetPlatform == TargetPlatform.android_arm ||
if (targetPlatform == TargetPlatform.android ||
targetPlatform == TargetPlatform.android_arm ||
targetPlatform == TargetPlatform.android_arm64 ||
targetPlatform == TargetPlatform.android_x64 ||
targetPlatform == TargetPlatform.android_x86) {
......@@ -306,10 +307,7 @@ String getNameForHostPlatform(HostPlatform platform) {
}
enum TargetPlatform {
android_arm,
android_arm64,
android_x64,
android_x86,
android,
ios,
darwin_x64,
linux_x64,
......@@ -318,6 +316,14 @@ enum TargetPlatform {
fuchsia_x64,
tester,
web_javascript,
// The arch specific android target platforms are soft-depreacted.
// Instead of using TargetPlatform as a combination arch + platform
// the code will be updated to carry arch information in [DarwinArch]
// and [AndroidArch].
android_arm,
android_arm64,
android_x64,
android_x86,
}
/// iOS and macOS target device architecture.
......@@ -329,6 +335,7 @@ enum DarwinArch {
x86_64,
}
// TODO(jonahwilliams): replace all android TargetPlatform usage with AndroidArch.
enum AndroidArch {
armeabi_v7a,
arm64_v8a,
......@@ -391,6 +398,8 @@ String getNameForTargetPlatform(TargetPlatform platform) {
return 'flutter-tester';
case TargetPlatform.web_javascript:
return 'web-javascript';
case TargetPlatform.android:
return 'android';
}
assert(false);
return null;
......@@ -398,6 +407,8 @@ String getNameForTargetPlatform(TargetPlatform platform) {
TargetPlatform getTargetPlatformForName(String platform) {
switch (platform) {
case 'android':
return TargetPlatform.android;
case 'android-arm':
return TargetPlatform.android_arm;
case 'android-arm64':
......
packages/flutter_tools/lib/src/build_system/targets/android.dart
View file @ df3505c1
......@@ -3,6 +3,7 @@
// found in the LICENSE file.
import '../../artifacts.dart';
import '../../base/build.dart';
import '../../base/file_system.dart';
import '../../build_info.dart';
import '../../globals.dart';
......@@ -124,3 +125,150 @@ class ReleaseAndroidApplication extends CopyFlutterAotBundle {
AotAndroidAssetBundle(),
];
}
/// Generate an ELF binary from a dart kernel file in release mode.
///
/// This rule implementation outputs the generated so to a unique location
/// based on the Android ABI. This allows concurrent invocations of gen_snapshot
/// to run simultaneously.
///
/// The name of an instance of this rule would be 'android_aot_profile_android-x64'
/// and is relied upon by flutter.gradle to match the correct rule.
///
/// It will produce an 'app.so` in the build directory under a folder named with
/// the matching Android ABI.
class AndroidAot extends AotElfBase {
/// Create an [AndroidAot] implementation for a given [targetPlatform] and [buildMode].
const AndroidAot(this.targetPlatform, this.buildMode);
/// The name of the produced Android ABI.
String get _androidAbiName {
return getNameForAndroidArch(
getAndroidArchForName(getNameForTargetPlatform(targetPlatform)));
}
@override
String get name => 'android_aot_${getNameForBuildMode(buildMode)}_'
'${getNameForTargetPlatform(targetPlatform)}';
/// The specific Android ABI we are building for.
final TargetPlatform targetPlatform;
/// The selected build mode.
///
/// This is restricted to [BuildMode.profile] or [BuildMode.relese].
final BuildMode buildMode;
@override
List<Source> get inputs => <Source>[
const Source.pattern('{FLUTTER_ROOT}/packages/flutter_tools/lib/src/build_system/targets/dart.dart'),
const Source.pattern('{BUILD_DIR}/app.dill'),
const Source.pattern('{PROJECT_DIR}/.packages'),
const Source.artifact(Artifact.engineDartBinary),
const Source.artifact(Artifact.skyEnginePath),
Source.artifact(Artifact.genSnapshot,
mode: buildMode,
platform: targetPlatform,
),
];
@override
List<Source> get outputs => <Source>[
Source.pattern('{BUILD_DIR}/$_androidAbiName/app.so'),
];
@override
List<Target> get dependencies => const <Target>[
KernelSnapshot(),
];
@override
Future<void> build(Environment environment) async {
final AOTSnapshotter snapshotter = AOTSnapshotter(reportTimings: false);
final Directory output = environment.buildDir.childDirectory(_androidAbiName);
if (environment.defines[kBuildMode] == null) {
throw MissingDefineException(kBuildMode, 'aot_elf');
}
if (!output.existsSync()) {
output.createSync(recursive: true);
}
final BuildMode buildMode = getBuildModeForName(environment.defines[kBuildMode]);
final int snapshotExitCode = await snapshotter.build(
platform: targetPlatform,
buildMode: buildMode,
mainPath: environment.buildDir.childFile('app.dill').path,
packagesPath: environment.projectDir.childFile('.packages').path,
outputPath: output.path,
bitcode: false,
);
if (snapshotExitCode != 0) {
throw Exception('AOT snapshotter exited with code $snapshotExitCode');
}
}
}
// AndroidAot instances used by the bundle rules below.
const Target androidArmProfile = AndroidAot(TargetPlatform.android_arm, BuildMode.profile);
const Target androidArm64Profile = AndroidAot(TargetPlatform.android_arm64, BuildMode.profile);
const Target androidx64Profile = AndroidAot(TargetPlatform.android_x64, BuildMode.profile);
const Target androidArmRelease = AndroidAot(TargetPlatform.android_arm, BuildMode.release);
const Target androidArm64Release = AndroidAot(TargetPlatform.android_arm64, BuildMode.release);
const Target androidx64Release = AndroidAot(TargetPlatform.android_x64, BuildMode.release);
/// A rule paired with [AndroidAot] that copies the produced so files into the output directory.
class AndroidAotBundle extends Target {
/// Create an [AndroidAotBundle] implementation for a given [targetPlatform] and [buildMode].
const AndroidAotBundle(this.dependency);
/// The [AndroidAot] instance this bundle rule depends on.
final AndroidAot dependency;
/// The name of the produced Android ABI.
String get _androidAbiName {
return getNameForAndroidArch(
getAndroidArchForName(getNameForTargetPlatform(dependency.targetPlatform)));
}
@override
String get name => 'android_aot_bundle_${getNameForBuildMode(dependency.buildMode)}_'
'${getNameForTargetPlatform(dependency.targetPlatform)}';
@override
List<Source> get inputs => <Source>[
Source.pattern('{BUILD_DIR}/$_androidAbiName/app.so'),
];
// flutter.gradle has been updated to correctly consume it.
@override
List<Source> get outputs => <Source>[
Source.pattern('{OUTPUT_DIR}/$_androidAbiName/app.so'),
];
@override
List<Target> get dependencies => <Target>[
dependency,
const AotAndroidAssetBundle(),
];
@override
Future<void> build(Environment environment) async {
final File outputFile = environment.buildDir
.childDirectory(_androidAbiName)
.childFile('app.so');
final Directory outputDirectory = environment.outputDir
.childDirectory(_androidAbiName);
if (!outputDirectory.existsSync()) {
outputDirectory.createSync(recursive: true);
}
outputFile.copySync(outputDirectory.childFile('app.so').path);
}
}
// AndroidBundleAot instances.
const Target androidArmProfileBundle = AndroidAotBundle(androidArmProfile);
const Target androidArm64ProfileBundle = AndroidAotBundle(androidArm64Profile);
const Target androidx64ProfileBundle = AndroidAotBundle(androidx64Profile);
const Target androidArmReleaseBundle = AndroidAotBundle(androidArmRelease);
const Target androidArm64ReleaseBundle = AndroidAotBundle(androidArm64Release);
const Target androidx64ReleaseBundle = AndroidAotBundle(androidx64Release);
packages/flutter_tools/lib/src/commands/assemble.dart
View file @ df3505c1
......@@ -43,6 +43,13 @@ const List<Target> _kDefaultTargets = <Target>[
ReleaseCopyFlutterAotBundle(),
ProfileCopyFlutterAotBundle(),
CopyFlutterBundle(),
// Android ABI specific AOT rules.
androidArmProfileBundle,
androidArm64ProfileBundle,
androidx64ProfileBundle,
androidArmReleaseBundle,
androidArm64ReleaseBundle,
androidx64ReleaseBundle,
];
/// Assemble provides a low level API to interact with the flutter tool build
......@@ -99,18 +106,25 @@ class AssembleCommand extends FlutterCommand {
return const <CustomDimensions, String>{};
}
/// The target we are building.
Target get target {
/// The target(s) we are building.
List<Target> get targets {
if (argResults.rest.isEmpty) {
throwToolExit('missing target name for flutter assemble.');
}
final String name = argResults.rest.first;
final Target result = _kDefaultTargets
.firstWhere((Target target) => target.name == name, orElse: () => null);
if (result == null) {
final Map<String, Target> targetMap = <String, Target>{
for (Target target in _kDefaultTargets)
target.name: target
};
final List<Target> results = <Target>[
for (String targetName in argResults.rest)
if (targetMap.containsKey(targetName))
targetMap[targetName]
];
if (results.isEmpty) {
throwToolExit('No target named "$name" defined.');
}
return result;
return results;
}
/// The environmental configuration for a build invocation.
......@@ -151,8 +165,12 @@ class AssembleCommand extends FlutterCommand {
@override
Future<FlutterCommandResult> runCommand() async {
final List<Target> targets = this.targets;
final Target target = targets.length == 1 ? targets.single : _CompositeTarget(targets);
final BuildResult result = await buildSystem.build(target, environment, buildSystemConfig: BuildSystemConfig(
resourcePoolSize: argResults.wasParsed('resource-pool-size') ? int.parse(stringArg('resource-pool-size')) : null,
resourcePoolSize: argResults.wasParsed('resource-pool-size')
? int.tryParse(argResults['resource-pool-size'])
: null,
));
if (!result.success) {
for (ExceptionMeasurement measurement in result.exceptions.values) {
......@@ -197,3 +215,22 @@ void writeListIfChanged(List<File> files, String path) {
file.writeAsStringSync(newContents);
}
}
class _CompositeTarget extends Target {
_CompositeTarget(this.dependencies);
@override
final List<Target> dependencies;
@override
String get name => '_composite';
@override
Future<void> build(Environment environment) async { }
@override
List<Source> get inputs => <Source>[];
@override
List<Source> get outputs => <Source>[];
}
packages/flutter_tools/lib/src/runner/flutter_command.dart
View file @ df3505c1
......@@ -766,6 +766,7 @@ mixin TargetPlatformBasedDevelopmentArtifacts on FlutterCommand {
// if none is supported
DevelopmentArtifact _artifactFromTargetPlatform(TargetPlatform targetPlatform) {
switch (targetPlatform) {
case TargetPlatform.android:
case TargetPlatform.android_arm:
case TargetPlatform.android_arm64:
case TargetPlatform.android_x64:
......
packages/flutter_tools/test/general.shard/build_system/targets/android_test.dart
View file @ df3505c1
......@@ -2,12 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_tools/src/base/build.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/build_info.dart';
import 'package:flutter_tools/src/build_system/build_system.dart';
import 'package:flutter_tools/src/build_system/targets/android.dart';
import 'package:flutter_tools/src/build_system/targets/dart.dart';
import 'package:flutter_tools/src/cache.dart';
import 'package:mockito/mockito.dart';
import '../../../src/common.dart';
import '../../../src/testbed.dart';
......@@ -84,4 +86,68 @@ void main() {
expect(fs.file(fs.path.join('out', 'app.so')).existsSync(), true);
});
testbed.test('AndroidAot can build provided target platform', () async {
final Environment environment = Environment(
outputDir: fs.directory('out')..createSync(),
projectDir: fs.currentDirectory,
buildDir: fs.currentDirectory,
defines: <String, String>{
kBuildMode: 'release',
}
);
when(genSnapshot.run(
snapshotType: anyNamed('snapshotType'),
darwinArch: anyNamed('darwinArch'),
additionalArgs: anyNamed('additionalArgs'),
)).thenAnswer((Invocation invocation) async {
return 0;
});
environment.buildDir.createSync(recursive: true);
environment.buildDir.childFile('app.dill').createSync();
environment.projectDir.childFile('.packages')
.writeAsStringSync('sky_engine:file:///\n');
const AndroidAot androidAot = AndroidAot(TargetPlatform.android_arm64, BuildMode.release);
await androidAot.build(environment);
final SnapshotType snapshotType = verify(genSnapshot.run(
snapshotType: captureAnyNamed('snapshotType'),
darwinArch: anyNamed('darwinArch'),
additionalArgs: anyNamed('additionalArgs')
)).captured.single;
expect(snapshotType.platform, TargetPlatform.android_arm64);
expect(snapshotType.mode, BuildMode.release);
}, overrides: <Type, Generator>{
GenSnapshot: () => MockGenSnapshot(),
});
testbed.test('android aot bundle copies so from abi directory', () async {
final Environment environment = Environment(
outputDir: fs.directory('out')..createSync(),
projectDir: fs.currentDirectory,
buildDir: fs.currentDirectory,
defines: <String, String>{
kBuildMode: 'release',
}
);
environment.buildDir.createSync(recursive: true);
const AndroidAot androidAot = AndroidAot(TargetPlatform.android_arm64, BuildMode.release);
const AndroidAotBundle androidAotBundle = AndroidAotBundle(androidAot);
// Create required files.
environment.buildDir
.childDirectory('arm64-v8a')
.childFile('app.so')
.createSync(recursive: true);
await androidAotBundle.build(environment);
expect(environment.outputDir
.childDirectory('arm64-v8a')
.childFile('app.so').existsSync(), true);
});
}
class MockGenSnapshot extends Mock implements GenSnapshot {}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab