Unverified Commit 8ebb8d4f authored by Daco Harkes's avatar Daco Harkes Committed by GitHub

Speed up native assets target (#134523)

Speeds up the native assets target in the backend by

1. changing other targets `gen_dart_plugin_registrant` and
`release_unpack_ios` to do async I/O,
2. not reparsing the package config, and
3. not calling `dart pub deps --json` for 0 or 1 packages (fixed
package:native_assets_builder).

* https://github.com/flutter/flutter/issues/134427

```
           [   +2 ms] native_assets: Starting due to {}
           [   +2 ms] Skipping target: gen_localizations
           [   +1 ms] gen_dart_plugin_registrant: Starting due to {InvalidatedReasonKind.inputChanged: The following inputs have updated contents: /Users/dacoharkes/flt/engine/flutter/examples/hello_world/.dart_tool/package_config_subset}
           [  +33 ms] gen_dart_plugin_registrant: Complete
           [ +107 ms] release_unpack_ios: Complete
           [  +60 ms] Writing native_assets.yaml.
           [   +7 ms] Writing /Users/dacoharkes/flt/engine/flutter/examples/hello_world/.dart_tool/flutter_build/be2692bbfbc0b9a27fcd2422d52354c6/native_assets.yaml done.
           [        ] native_assets: Complete
```

->

```
           [   +4 ms] native_assets: Starting due to {}
           [        ] Skipping target: gen_localizations
           [   +1 ms] gen_dart_plugin_registrant: Starting due to {InvalidatedReasonKind.inputChanged: The following inputs have updated contents: /Users/dacoharkes/flt/engine/flutter/examples/hello_world/.dart_tool/package_config_subset}
           [  +31 ms] Writing native_assets.yaml.
           [   +8 ms] Writing /Users/dacoharkes/flt/engine/flutter/examples/hello_world/.dart_tool/flutter_build/f9451a65a465bfab70d004e21d6cc1d6/native_assets.yaml done.
           [   +1 ms] native_assets: Complete
```

## Pre-launch Checklist

- [x] I read the [Contributor Guide] and followed the process outlined
there for submitting PRs.
- [x] I read the [Tree Hygiene] wiki page, which explains my
responsibilities.
- [x] I read and followed the [Flutter Style Guide], including [Features
we expect every widget to implement].
- [x] I signed the [CLA].
- [x] I listed at least one issue that this PR fixes in the description
above.
- [x] I updated/added relevant documentation (doc comments with `///`).
- [ ] I added new tests to check the change I am making, or this PR is
[test-exempt].
- [x] All existing and new tests are passing.


<!-- Links -->
[Contributor Guide]:
https://github.com/flutter/flutter/wiki/Tree-hygiene#overview
[Tree Hygiene]: https://github.com/flutter/flutter/wiki/Tree-hygiene
[test-exempt]:
https://github.com/flutter/flutter/wiki/Tree-hygiene#tests
[Flutter Style Guide]:
https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo
[Features we expect every widget to implement]:
https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#features-we-expect-every-widget-to-implement
[CLA]: https://cla.developers.google.com/
[flutter/tests]: https://github.com/flutter/tests
[breaking change policy]:
https://github.com/flutter/flutter/wiki/Tree-hygiene#handling-breaking-changes
[Discord]: https://github.com/flutter/flutter/wiki/Chat
parent 72b69f94
......@@ -287,21 +287,21 @@ abstract class UnpackIOS extends Target {
if (archs == null) {
throw MissingDefineException(kIosArchs, name);
}
_copyFramework(environment, sdkRoot);
await _copyFramework(environment, sdkRoot);
final File frameworkBinary = environment.outputDir.childDirectory('Flutter.framework').childFile('Flutter');
final String frameworkBinaryPath = frameworkBinary.path;
if (!frameworkBinary.existsSync()) {
if (!await frameworkBinary.exists()) {
throw Exception('Binary $frameworkBinaryPath does not exist, cannot thin');
}
_thinFramework(environment, frameworkBinaryPath, archs);
await _thinFramework(environment, frameworkBinaryPath, archs);
if (buildMode == BuildMode.release) {
_bitcodeStripFramework(environment, frameworkBinaryPath);
await _bitcodeStripFramework(environment, frameworkBinaryPath);
}
await _signFramework(environment, frameworkBinary, buildMode);
}
void _copyFramework(Environment environment, String sdkRoot) {
Future<void> _copyFramework(Environment environment, String sdkRoot) async {
final EnvironmentType? environmentType = environmentTypeFromSdkroot(sdkRoot, environment.fileSystem);
final String basePath = environment.artifacts.getArtifactPath(
Artifact.flutterFramework,
......@@ -310,7 +310,7 @@ abstract class UnpackIOS extends Target {
environmentType: environmentType,
);
final ProcessResult result = environment.processManager.runSync(<String>[
final ProcessResult result = await environment.processManager.run(<String>[
'rsync',
'-av',
'--delete',
......@@ -328,16 +328,20 @@ abstract class UnpackIOS extends Target {
}
/// Destructively thin Flutter.framework to include only the specified architectures.
void _thinFramework(Environment environment, String frameworkBinaryPath, String archs) {
Future<void> _thinFramework(
Environment environment,
String frameworkBinaryPath,
String archs,
) async {
final List<String> archList = archs.split(' ').toList();
final ProcessResult infoResult = environment.processManager.runSync(<String>[
final ProcessResult infoResult = await environment.processManager.run(<String>[
'lipo',
'-info',
frameworkBinaryPath,
]);
final String lipoInfo = infoResult.stdout as String;
final ProcessResult verifyResult = environment.processManager.runSync(<String>[
final ProcessResult verifyResult = await environment.processManager.run(<String>[
'lipo',
frameworkBinaryPath,
'-verify_arch',
......@@ -355,7 +359,7 @@ abstract class UnpackIOS extends Target {
}
// Thin in-place.
final ProcessResult extractResult = environment.processManager.runSync(<String>[
final ProcessResult extractResult = await environment.processManager.run(<String>[
'lipo',
'-output',
frameworkBinaryPath,
......@@ -374,8 +378,11 @@ abstract class UnpackIOS extends Target {
/// Destructively strip bitcode from the framework. This can be removed
/// when the framework is no longer built with bitcode.
void _bitcodeStripFramework(Environment environment, String frameworkBinaryPath) {
final ProcessResult stripResult = environment.processManager.runSync(<String>[
Future<void> _bitcodeStripFramework(
Environment environment,
String frameworkBinaryPath,
) async {
final ProcessResult stripResult = await environment.processManager.run(<String>[
'xcrun',
'bitcode_strip',
frameworkBinaryPath,
......
......@@ -80,7 +80,7 @@ abstract class UnpackMacOS extends Target {
if (!frameworkBinary.existsSync()) {
throw Exception('Binary $frameworkBinaryPath does not exist, cannot thin');
}
_thinFramework(environment, frameworkBinaryPath);
await _thinFramework(environment, frameworkBinaryPath);
}
static const List<String> _copyDenylist = <String>['entitlements.txt', 'without_entitlements.txt'];
......@@ -96,17 +96,21 @@ abstract class UnpackMacOS extends Target {
}
}
void _thinFramework(Environment environment, String frameworkBinaryPath) {
Future<void> _thinFramework(
Environment environment,
String frameworkBinaryPath,
) async {
final String archs = environment.defines[kDarwinArchs] ?? 'x86_64 arm64';
final List<String> archList = archs.split(' ').toList();
final ProcessResult infoResult = environment.processManager.runSync(<String>[
final ProcessResult infoResult =
await environment.processManager.run(<String>[
'lipo',
'-info',
frameworkBinaryPath,
]);
final String lipoInfo = infoResult.stdout as String;
final ProcessResult verifyResult = environment.processManager.runSync(<String>[
final ProcessResult verifyResult = await environment.processManager.run(<String>[
'lipo',
frameworkBinaryPath,
'-verify_arch',
......
......@@ -4,11 +4,13 @@
import 'package:meta/meta.dart';
import 'package:native_assets_cli/native_assets_cli.dart' show Asset;
import 'package:package_config/package_config_types.dart';
import '../../base/common.dart';
import '../../base/file_system.dart';
import '../../base/platform.dart';
import '../../build_info.dart';
import '../../dart/package_map.dart';
import '../../ios/native_assets.dart';
import '../../macos/native_assets.dart';
import '../../macos/xcode.dart';
......@@ -52,7 +54,21 @@ class NativeAssets extends Target {
final Uri projectUri = environment.projectDir.uri;
final FileSystem fileSystem = environment.fileSystem;
final NativeAssetsBuildRunner buildRunner = _buildRunner ?? NativeAssetsBuildRunnerImpl(projectUri, fileSystem, environment.logger);
final File packagesFile = fileSystem
.directory(projectUri)
.childDirectory('.dart_tool')
.childFile('package_config.json');
final PackageConfig packageConfig = await loadPackageConfigWithLogging(
packagesFile,
logger: environment.logger,
);
final NativeAssetsBuildRunner buildRunner = _buildRunner ??
NativeAssetsBuildRunnerImpl(
projectUri,
packageConfig,
fileSystem,
environment.logger,
);
final List<Uri> dependencies;
switch (targetPlatform) {
......
......@@ -26,14 +26,20 @@ import 'platform_plugins.dart';
import 'plugins.dart';
import 'project.dart';
void _renderTemplateToFile(String template, Object? context, File file, TemplateRenderer templateRenderer) {
Future<void> _renderTemplateToFile(
String template,
Object? context,
File file,
TemplateRenderer templateRenderer,
) async {
final String renderedTemplate = templateRenderer
.renderString(template, context);
file.createSync(recursive: true);
file.writeAsStringSync(renderedTemplate);
await file.create(recursive: true);
await file.writeAsString(renderedTemplate);
}
Plugin? _pluginFromPackage(String name, Uri packageRoot, Set<String> appDependencies, {FileSystem? fileSystem}) {
Future<Plugin?> _pluginFromPackage(String name, Uri packageRoot, Set<String> appDependencies,
{FileSystem? fileSystem}) async {
final FileSystem fs = fileSystem ?? globals.fs;
final File pubspecFile = fs.file(packageRoot.resolve('pubspec.yaml'));
if (!pubspecFile.existsSync()) {
......@@ -42,7 +48,7 @@ Plugin? _pluginFromPackage(String name, Uri packageRoot, Set<String> appDependen
Object? pubspec;
try {
pubspec = loadYaml(pubspecFile.readAsStringSync());
pubspec = loadYaml(await pubspecFile.readAsString());
} on YamlException catch (err) {
globals.printTrace('Failed to parse plugin manifest for $name: $err');
// Do nothing, potentially not a plugin.
......@@ -85,7 +91,7 @@ Future<List<Plugin>> findPlugins(FlutterProject project, { bool throwOnError = t
);
for (final Package package in packageConfig.packages) {
final Uri packageRoot = package.packageUriRoot.resolve('..');
final Plugin? plugin = _pluginFromPackage(
final Plugin? plugin = await _pluginFromPackage(
package.name,
packageRoot,
project.manifest.dependencies,
......@@ -445,7 +451,7 @@ Future<void> _writeAndroidPluginRegistrant(FlutterProject project, List<Plugin>
templateContent = _androidPluginRegistryTemplateOldEmbedding;
}
globals.printTrace('Generating $registryPath');
_renderTemplateToFile(
await _renderTemplateToFile(
templateContent,
templateContext,
globals.fs.file(registryPath),
......@@ -774,20 +780,20 @@ Future<void> _writeIOSPluginRegistrant(FlutterProject project, List<Plugin> plug
};
if (project.isModule) {
final Directory registryDirectory = project.ios.pluginRegistrantHost;
_renderTemplateToFile(
await _renderTemplateToFile(
_pluginRegistrantPodspecTemplate,
context,
registryDirectory.childFile('FlutterPluginRegistrant.podspec'),
globals.templateRenderer,
);
}
_renderTemplateToFile(
await _renderTemplateToFile(
_objcPluginRegistryHeaderTemplate,
context,
project.ios.pluginRegistrantHeader,
globals.templateRenderer,
);
_renderTemplateToFile(
await _renderTemplateToFile(
_objcPluginRegistryImplementationTemplate,
context,
project.ios.pluginRegistrantImplementation,
......@@ -829,13 +835,13 @@ Future<void> _writeLinuxPluginFiles(FlutterProject project, List<Plugin> plugins
}
Future<void> _writeLinuxPluginRegistrant(Directory destination, Map<String, Object> templateContext) async {
_renderTemplateToFile(
await _renderTemplateToFile(
_linuxPluginRegistryHeaderTemplate,
templateContext,
destination.childFile('generated_plugin_registrant.h'),
globals.templateRenderer,
);
_renderTemplateToFile(
await _renderTemplateToFile(
_linuxPluginRegistryImplementationTemplate,
templateContext,
destination.childFile('generated_plugin_registrant.cc'),
......@@ -844,7 +850,7 @@ Future<void> _writeLinuxPluginRegistrant(Directory destination, Map<String, Obje
}
Future<void> _writePluginCmakefile(File destinationFile, Map<String, Object> templateContext, TemplateRenderer templateRenderer) async {
_renderTemplateToFile(
await _renderTemplateToFile(
_pluginCmakefileTemplate,
templateContext,
destinationFile,
......@@ -860,7 +866,7 @@ Future<void> _writeMacOSPluginRegistrant(FlutterProject project, List<Plugin> pl
'framework': 'FlutterMacOS',
'methodChannelPlugins': macosMethodChannelPlugins,
};
_renderTemplateToFile(
await _renderTemplateToFile(
_swiftPluginRegistryTemplate,
context,
project.macos.managedDirectory.childFile('GeneratedPluginRegistrant.swift'),
......@@ -931,13 +937,13 @@ Future<void> writeWindowsPluginFiles(FlutterProject project, List<Plugin> plugin
}
Future<void> _writeCppPluginRegistrant(Directory destination, Map<String, Object> templateContext, TemplateRenderer templateRenderer) async {
_renderTemplateToFile(
await _renderTemplateToFile(
_cppPluginRegistryHeaderTemplate,
templateContext,
destination.childFile('generated_plugin_registrant.h'),
templateRenderer,
);
_renderTemplateToFile(
await _renderTemplateToFile(
_cppPluginRegistryImplementationTemplate,
templateContext,
destination.childFile('generated_plugin_registrant.cc'),
......@@ -955,7 +961,7 @@ Future<void> _writeWebPluginRegistrant(FlutterProject project, List<Plugin> plug
final String template = webPlugins.isEmpty ? _noopDartPluginRegistryTemplate : _dartPluginRegistryTemplate;
_renderTemplateToFile(
await _renderTemplateToFile(
template,
context,
pluginFile,
......@@ -1411,8 +1417,8 @@ Future<void> generateMainDartWithPluginRegistrant(
final File newMainDart = rootProject.dartPluginRegistrant;
if (resolutions.isEmpty) {
try {
if (newMainDart.existsSync()) {
newMainDart.deleteSync();
if (await newMainDart.exists()) {
await newMainDart.delete();
}
} on FileSystemException catch (error) {
globals.printWarning(
......@@ -1428,7 +1434,7 @@ Future<void> generateMainDartWithPluginRegistrant(
(templateContext[resolution.platform] as List<Object?>?)?.add(resolution.toMap());
}
try {
_renderTemplateToFile(
await _renderTemplateToFile(
_dartPluginRegistryForNonWebTemplate,
templateContext,
newMainDart,
......
......@@ -5,7 +5,8 @@
// Logic for native assets shared between all host OSes.
import 'package:logging/logging.dart' as logging;
import 'package:native_assets_builder/native_assets_builder.dart' as native_assets_builder;
import 'package:native_assets_builder/native_assets_builder.dart' hide NativeAssetsBuildRunner;
import 'package:native_assets_builder/native_assets_builder.dart' as native_assets_builder show NativeAssetsBuildRunner;
import 'package:native_assets_cli/native_assets_cli.dart';
import 'package:package_config/package_config_types.dart';
......@@ -37,7 +38,7 @@ abstract class NativeAssetsBuildRunner {
Future<List<Package>> packagesWithNativeAssets();
/// Runs all [packagesWithNativeAssets] `build.dart` in dry run.
Future<native_assets_builder.DryRunResult> dryRun({
Future<DryRunResult> dryRun({
required bool includeParentEnvironment,
required LinkModePreference linkModePreference,
required OS targetOs,
......@@ -45,7 +46,7 @@ abstract class NativeAssetsBuildRunner {
});
/// Runs all [packagesWithNativeAssets] `build.dart`.
Future<native_assets_builder.BuildResult> build({
Future<BuildResult> build({
required bool includeParentEnvironment,
required BuildMode buildMode,
required LinkModePreference linkModePreference,
......@@ -62,9 +63,15 @@ abstract class NativeAssetsBuildRunner {
/// Uses `package:native_assets_builder` for its implementation.
class NativeAssetsBuildRunnerImpl implements NativeAssetsBuildRunner {
NativeAssetsBuildRunnerImpl(this.projectUri, this.fileSystem, this.logger);
NativeAssetsBuildRunnerImpl(
this.projectUri,
this.packageConfig,
this.fileSystem,
this.logger,
);
final Uri projectUri;
final PackageConfig packageConfig;
final FileSystem fileSystem;
final Logger logger;
......@@ -90,8 +97,6 @@ class NativeAssetsBuildRunnerImpl implements NativeAssetsBuildRunner {
dartExecutable: _dartExecutable,
);
native_assets_builder.PackageLayout? _packageLayout;
@override
Future<bool> hasPackageConfig() {
final File packageConfigJson = fileSystem
......@@ -103,27 +108,35 @@ class NativeAssetsBuildRunnerImpl implements NativeAssetsBuildRunner {
@override
Future<List<Package>> packagesWithNativeAssets() async {
_packageLayout ??= await native_assets_builder.PackageLayout.fromRootPackageRoot(projectUri);
return _packageLayout!.packagesWithNativeAssets;
final PackageLayout packageLayout = PackageLayout.fromPackageConfig(
packageConfig,
projectUri.resolve('.dart_tool/package_config.json'),
);
return packageLayout.packagesWithNativeAssets;
}
@override
Future<native_assets_builder.DryRunResult> dryRun({
Future<DryRunResult> dryRun({
required bool includeParentEnvironment,
required LinkModePreference linkModePreference,
required OS targetOs,
required Uri workingDirectory,
}) {
final PackageLayout packageLayout = PackageLayout.fromPackageConfig(
packageConfig,
projectUri.resolve('.dart_tool/package_config.json'),
);
return _buildRunner.dryRun(
includeParentEnvironment: includeParentEnvironment,
linkModePreference: linkModePreference,
targetOs: targetOs,
workingDirectory: workingDirectory,
packageLayout: packageLayout,
);
}
@override
Future<native_assets_builder.BuildResult> build({
Future<BuildResult> build({
required bool includeParentEnvironment,
required BuildMode buildMode,
required LinkModePreference linkModePreference,
......@@ -133,6 +146,10 @@ class NativeAssetsBuildRunnerImpl implements NativeAssetsBuildRunner {
int? targetAndroidNdkApi,
IOSSdk? targetIOSSdk,
}) {
final PackageLayout packageLayout = PackageLayout.fromPackageConfig(
packageConfig,
projectUri.resolve('.dart_tool/package_config.json'),
);
return _buildRunner.build(
buildMode: buildMode,
cCompilerConfig: cCompilerConfig,
......@@ -142,6 +159,7 @@ class NativeAssetsBuildRunnerImpl implements NativeAssetsBuildRunner {
targetAndroidNdkApi: targetAndroidNdkApi,
targetIOSSdk: targetIOSSdk,
workingDirectory: workingDirectory,
packageLayout: packageLayout,
);
}
......
......@@ -366,7 +366,12 @@ class HotRunner extends ResidentRunner {
await _calculateTargetPlatform();
final Uri projectUri = Uri.directory(projectRootPath);
_buildRunner ??= NativeAssetsBuildRunnerImpl(projectUri, fileSystem, globals.logger);
_buildRunner ??= NativeAssetsBuildRunnerImpl(
projectUri,
debuggingOptions.buildInfo.packageConfig,
fileSystem,
globals.logger,
);
final Uri? nativeAssetsYaml = await dryRunNativeAssets(
projectUri: projectUri,
fileSystem: fileSystem,
......
......@@ -168,7 +168,12 @@ class TestCompiler {
Uri? nativeAssetsYaml;
final Uri projectUri = FlutterProject.current().directory.uri;
final NativeAssetsBuildRunner buildRunner = NativeAssetsBuildRunnerImpl(projectUri, globals.fs, globals.logger);
final NativeAssetsBuildRunner buildRunner = NativeAssetsBuildRunnerImpl(
projectUri,
buildInfo.packageConfig,
globals.fs,
globals.logger,
);
if (globals.platform.isMacOS) {
(nativeAssetsYaml, _) = await buildNativeAssetsMacOS(
buildMode: BuildMode.debug,
......
......@@ -52,7 +52,7 @@ dependencies:
cli_config: 0.1.1
graphs: 2.3.1
native_assets_builder: 0.2.0
native_assets_builder: 0.2.3
native_assets_cli: 0.2.0
# We depend on very specific internal implementation details of the
......@@ -112,4 +112,4 @@ dartdoc:
# Exclude this package from the hosted API docs.
nodoc: true
# PUBSPEC CHECKSUM: 284b
# PUBSPEC CHECKSUM: 7f4e
......@@ -55,11 +55,15 @@ void main() {
});
testUsingContext('NativeAssets throws error if missing ios archs', () async {
await createPackageConfig(iosEnvironment);
iosEnvironment.defines.remove(kIosArchs);
expect(const NativeAssets().build(iosEnvironment), throwsA(isA<MissingDefineException>()));
});
testUsingContext('NativeAssets throws error if missing sdk root', () async {
await createPackageConfig(iosEnvironment);
iosEnvironment.defines.remove(kSdkRoot);
expect(const NativeAssets().build(iosEnvironment), throwsA(isA<MissingDefineException>()));
});
......@@ -78,6 +82,8 @@ void main() {
),
},
() async {
await createPackageConfig(iosEnvironment);
final NativeAssetsBuildRunner buildRunner = FakeNativeAssetsBuildRunner();
await NativeAssets(buildRunner: buildRunner).build(iosEnvironment);
......@@ -95,6 +101,8 @@ void main() {
FeatureFlags: () => TestFeatureFlags(isNativeAssetsEnabled: true),
},
() async {
await createPackageConfig(iosEnvironment);
final NativeAssetsBuildRunner buildRunner = FakeNativeAssetsBuildRunner(
buildResult: FakeNativeAssetsBuilderResult(assets: <native_assets_cli.Asset>[
native_assets_cli.Asset(
......@@ -138,3 +146,11 @@ void main() {
},
);
}
Future<void> createPackageConfig(Environment iosEnvironment) async {
final File packageConfig = iosEnvironment.projectDir
.childDirectory('.dart_tool')
.childFile('package_config.json');
await packageConfig.parent.create();
await packageConfig.create();
}
......@@ -10,6 +10,7 @@ import 'package:flutter_tools/src/base/logger.dart';
import 'package:flutter_tools/src/base/platform.dart';
import 'package:flutter_tools/src/build_info.dart';
import 'package:flutter_tools/src/build_system/build_system.dart';
import 'package:flutter_tools/src/dart/package_map.dart';
import 'package:flutter_tools/src/features.dart';
import 'package:flutter_tools/src/globals.dart' as globals;
import 'package:flutter_tools/src/macos/native_assets.dart';
......@@ -373,7 +374,22 @@ InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault
return;
}
final NativeAssetsBuildRunner runner = NativeAssetsBuildRunnerImpl(projectUri, fileSystem, logger);
final File packagesFile = fileSystem
.directory(projectUri)
.childDirectory('.dart_tool')
.childFile('package_config.json');
await packagesFile.parent.create();
await packagesFile.create();
final PackageConfig packageConfig = await loadPackageConfigWithLogging(
packagesFile,
logger: environment.logger,
);
final NativeAssetsBuildRunner runner = NativeAssetsBuildRunnerImpl(
projectUri,
packageConfig,
fileSystem,
logger,
);
final CCompilerConfig result = await runner.cCompilerConfig;
expect(
result.cc,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment