// Copyright (c) 2016 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:convert' show json; import 'dart:io'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import '../framework/adb.dart'; import '../framework/framework.dart'; import '../framework/ios.dart'; import '../framework/utils.dart'; TaskFunction createComplexLayoutScrollPerfTest() { return PerfTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', 'test_driver/scroll_perf.dart', 'complex_layout_scroll_perf', ).run; } TaskFunction createTilesScrollPerfTest() { return PerfTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', 'test_driver/scroll_perf.dart', 'tiles_scroll_perf', ).run; } TaskFunction createCullOpacityPerfTest() { return PerfTest( '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', 'test_driver/cull_opacity_perf.dart', 'cull_opacity_perf', ).run; } TaskFunction createFlutterGalleryStartupTest() { return StartupTest( '${flutterDirectory.path}/examples/flutter_gallery', ).run; } TaskFunction createComplexLayoutStartupTest() { return StartupTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', ).run; } TaskFunction createFlutterGalleryCompileTest() { return CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run; } TaskFunction createHelloWorldCompileTest() { return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run; } TaskFunction createComplexLayoutCompileTest() { return CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run; } TaskFunction createFlutterViewStartupTest() { return StartupTest( '${flutterDirectory.path}/examples/flutter_view', reportMetrics: false, ).run; } TaskFunction createPlatformViewStartupTest() { return StartupTest( '${flutterDirectory.path}/examples/platform_view', reportMetrics: false, ).run; } TaskFunction createBasicMaterialCompileTest() { return () async { const String sampleAppName = 'sample_flutter_app'; final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName'); rmTree(sampleDir); await inDirectory<void>(Directory.systemTemp, () async { await flutter('create', options: <String>['--template=app', sampleAppName]); }); if (!(await sampleDir.exists())) throw 'Failed to create default Flutter app in ${sampleDir.path}'; return CompileTest(sampleDir.path).run(); }; } /// Measure application startup performance. class StartupTest { const StartupTest(this.testDirectory, { this.reportMetrics = true }); final String testDirectory; final bool reportMetrics; Future<TaskResult> run() async { return await inDirectory<TaskResult>(testDirectory, () async { final String deviceId = (await devices.workingDevice).deviceId; await flutter('packages', options: <String>['get']); if (deviceOperatingSystem == DeviceOperatingSystem.ios) await prepareProvisioningCertificates(testDirectory); await flutter('run', options: <String>[ '--verbose', '--profile', '--trace-startup', '-d', deviceId, ]); final Map<String, dynamic> data = json.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync()); if (!reportMetrics) return TaskResult.success(data); return TaskResult.success(data, benchmarkScoreKeys: <String>[ 'timeToFirstFrameMicros', ]); }); } } /// Measures application runtime performance, specifically per-frame /// performance. class PerfTest { const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName); final String testDirectory; final String testTarget; final String timelineFileName; Future<TaskResult> run() { return inDirectory<TaskResult>(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); final String deviceId = device.deviceId; await flutter('packages', options: <String>['get']); if (deviceOperatingSystem == DeviceOperatingSystem.ios) await prepareProvisioningCertificates(testDirectory); await flutter('drive', options: <String>[ '-v', '--profile', '--trace-startup', // Enables "endless" timeline event buffering. '-t', testTarget, '-d', deviceId, ]); final Map<String, dynamic> data = json.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync()); if (data['frame_count'] < 5) { return TaskResult.failure( 'Timeline contains too few frames: ${data['frame_count']}. Possibly ' 'trace events are not being captured.', ); } return TaskResult.success(data, benchmarkScoreKeys: <String>[ 'average_frame_build_time_millis', 'worst_frame_build_time_millis', 'missed_frame_build_budget_count', '90th_percentile_frame_build_time_millis', '99th_percentile_frame_build_time_millis', 'average_frame_rasterizer_time_millis', 'worst_frame_rasterizer_time_millis', 'missed_frame_rasterizer_budget_count', '90th_percentile_frame_rasterizer_time_millis', '99th_percentile_frame_rasterizer_time_millis', ]); }); } } /// Measures how long it takes to compile a Flutter app and how big the compiled /// code is. class CompileTest { const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false }); final String testDirectory; final bool reportPackageContentSizes; Future<TaskResult> run() async { return await inDirectory<TaskResult>(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); await flutter('packages', options: <String>['get']); final Map<String, dynamic> metrics = <String, dynamic>{} ..addAll(await _compileAot()) ..addAll(await _compileApp(reportPackageContentSizes: reportPackageContentSizes)) ..addAll(await _compileDebug()); return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList()); }); } static Future<Map<String, dynamic>> _compileAot() async { // Generate blobs instead of assembly. await flutter('clean'); final Stopwatch watch = Stopwatch()..start(); final List<String> options = <String>[ 'aot', '-v', '--extra-gen-snapshot-options=--print_snapshot_sizes', '--release', '--no-pub', '--target-platform', ]; switch (deviceOperatingSystem) { case DeviceOperatingSystem.ios: options.add('ios'); break; case DeviceOperatingSystem.android: options.add('android-arm'); break; } setLocalEngineOptionIfNecessary(options); final String compileLog = await evalFlutter('build', options: options); watch.stop(); final RegExp metricExpression = RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)'); final Map<String, dynamic> metrics = <String, dynamic>{}; for (Match m in metricExpression.allMatches(compileLog)) { metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2)); } if (metrics.length != _kSdkNameToMetricNameMapping.length) { throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.'; } metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds; return metrics; } static Future<Map<String, dynamic>> _compileApp({ bool reportPackageContentSizes = false }) async { await flutter('clean'); final Stopwatch watch = Stopwatch(); int releaseSizeInBytes; final List<String> options = <String>['--release']; setLocalEngineOptionIfNecessary(options); final Map<String, dynamic> metrics = <String, dynamic>{}; switch (deviceOperatingSystem) { case DeviceOperatingSystem.ios: options.insert(0, 'ios'); await prepareProvisioningCertificates(cwd); watch.start(); await flutter('build', options: options); watch.stop(); final String appPath = '$cwd/build/ios/Release-iphoneos/Runner.app/'; // IPAs are created manually, https://flutter.io/ios-release/ await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]); releaseSizeInBytes = await file('$cwd/build/app.ipa').length(); if (reportPackageContentSizes) metrics.addAll(await getSizesFromIosApp(appPath)); break; case DeviceOperatingSystem.android: options.insert(0, 'apk'); watch.start(); await flutter('build', options: options); watch.stop(); String apkPath = '$cwd/build/app/outputs/apk/app.apk'; File apk = file(apkPath); if (!apk.existsSync()) { // Pre Android SDK 26 path apkPath = '$cwd/build/app/outputs/apk/app-release.apk'; apk = file(apkPath); } releaseSizeInBytes = apk.lengthSync(); if (reportPackageContentSizes) metrics.addAll(await getSizesFromApk(apkPath)); break; } metrics.addAll(<String, dynamic>{ 'release_full_compile_millis': watch.elapsedMilliseconds, 'release_size_bytes': releaseSizeInBytes, }); return metrics; } static Future<Map<String, dynamic>> _compileDebug() async { await flutter('clean'); final Stopwatch watch = Stopwatch(); final List<String> options = <String>['--debug']; setLocalEngineOptionIfNecessary(options); switch (deviceOperatingSystem) { case DeviceOperatingSystem.ios: options.insert(0, 'ios'); await prepareProvisioningCertificates(cwd); break; case DeviceOperatingSystem.android: options.insert(0, 'apk'); break; } watch.start(); await flutter('build', options: options); watch.stop(); return <String, dynamic>{ 'debug_full_compile_millis': watch.elapsedMilliseconds, }; } static const Map<String, String> _kSdkNameToMetricNameMapping = <String, String> { 'VMIsolate': 'aot_snapshot_size_vmisolate', 'Isolate': 'aot_snapshot_size_isolate', 'ReadOnlyData': 'aot_snapshot_size_rodata', 'Instructions': 'aot_snapshot_size_instructions', 'Total': 'aot_snapshot_size_total', }; static String _sdkNameToMetricName(String sdkName) { if (!_kSdkNameToMetricNameMapping.containsKey(sdkName)) throw 'Unrecognized SDK snapshot metric name: $sdkName'; return _kSdkNameToMetricNameMapping[sdkName]; } static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async { // Thin the binary to only contain one architecture. final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh'); await exec(xcodeBackend, <String>['thin'], environment: <String, String>{ 'ARCHS': 'arm64', 'WRAPPER_NAME': path.basename(appPath), 'TARGET_BUILD_DIR': path.dirname(appPath), }); final File appFramework = File(path.join(appPath, 'Frameworks', 'App.framework', 'App')); final File flutterFramework = File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter')); return <String, dynamic>{ 'app_framework_uncompressed_bytes': await appFramework.length(), 'flutter_framework_uncompressed_bytes': await flutterFramework.length(), }; } static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async { final String output = await eval('unzip', <String>['-v', apkPath]); final List<String> lines = output.split('\n'); final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{}; // First three lines are header, last two lines are footer. for (int i = 3; i < lines.length - 2; i++) { final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]); fileToMetadata[entry.path] = entry; } final _UnzipListEntry icudtl = fileToMetadata['assets/flutter_shared/icudtl.dat']; final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so']; final _UnzipListEntry isolateSnapshotData = fileToMetadata['assets/isolate_snapshot_data']; final _UnzipListEntry isolateSnapshotInstr = fileToMetadata['assets/isolate_snapshot_instr']; final _UnzipListEntry vmSnapshotData = fileToMetadata['assets/vm_snapshot_data']; final _UnzipListEntry vmSnapshotInstr = fileToMetadata['assets/vm_snapshot_instr']; final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/LICENSE']; return <String, dynamic>{ 'icudtl_uncompressed_bytes': icudtl.uncompressedSize, 'icudtl_compressed_bytes': icudtl.compressedSize, 'libflutter_uncompressed_bytes': libflutter.uncompressedSize, 'libflutter_compressed_bytes': libflutter.compressedSize, 'snapshot_uncompressed_bytes': isolateSnapshotData.uncompressedSize + isolateSnapshotInstr.uncompressedSize + vmSnapshotData.uncompressedSize + vmSnapshotInstr.uncompressedSize, 'snapshot_compressed_bytes': isolateSnapshotData.compressedSize + isolateSnapshotInstr.compressedSize + vmSnapshotData.compressedSize + vmSnapshotInstr.compressedSize, 'license_uncompressed_bytes': license.uncompressedSize, 'license_compressed_bytes': license.compressedSize, }; } } /// Measure application memory usage. class MemoryTest { MemoryTest(this.project, this.test, this.package); final String project; final String test; final String package; /// Completes when the log line specified in the last call to /// [prepareForNextMessage] is seen by `adb logcat`. Future<void> get receivedNextMessage => _receivedNextMessage?.future; Completer<void> _receivedNextMessage; String _nextMessage; /// Prepares the [receivedNextMessage] future such that it will complete /// when `adb logcat` sees a log line with the given `message`. void prepareForNextMessage(String message) { _nextMessage = message; _receivedNextMessage = Completer<void>(); } int get iterationCount => 15; Device get device => _device; Device _device; Future<TaskResult> run() { return inDirectory<TaskResult>(project, () async { // This test currently only works on Android, because device.logcat, // device.getMemoryStats, etc, aren't implemented for iOS. _device = await devices.workingDevice; await device.unlock(); await flutter('packages', options: <String>['get']); if (deviceOperatingSystem == DeviceOperatingSystem.ios) await prepareProvisioningCertificates(project); final StreamSubscription<String> adb = device.logcat.listen( (String data) { if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ====')) _receivedNextMessage.complete(); }, ); for (int iteration = 0; iteration < iterationCount; iteration += 1) { print('running memory test iteration $iteration...'); _startMemoryUsage = null; await useMemory(); assert(_startMemoryUsage != null); assert(_startMemory.length == iteration + 1); assert(_endMemory.length == iteration + 1); assert(_diffMemory.length == iteration + 1); print('terminating...'); await device.stop(package); await Future<void>.delayed(const Duration(milliseconds: 10)); } await adb.cancel(); final ListStatistics startMemoryStatistics = ListStatistics(_startMemory); final ListStatistics endMemoryStatistics = ListStatistics(_endMemory); final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory); final Map<String, dynamic> memoryUsage = <String, dynamic>{}; memoryUsage.addAll(startMemoryStatistics.asMap('start')); memoryUsage.addAll(endMemoryStatistics.asMap('end')); memoryUsage.addAll(diffMemoryStatistics.asMap('diff')); _device = null; _startMemory.clear(); _endMemory.clear(); _diffMemory.clear(); return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList()); }); } /// Starts the app specified by [test] on the [device]. /// /// The [run] method will terminate it by its package name ([package]). Future<void> launchApp() async { prepareForNextMessage('READY'); print('launching $project$test on device...'); await flutter('run', options: <String>[ '--verbose', '--release', '--no-resident', '-d', device.deviceId, test, ]); print('awaiting "ready" message...'); await receivedNextMessage; } /// To change the behaviour of the test, override this. /// /// Make sure to call recordStart() and recordEnd() once each in that order. /// /// By default it just launches the app, records memory usage, taps the device, /// awaits a DONE notification, and records memory usage again. Future<void> useMemory() async { await launchApp(); await recordStart(); prepareForNextMessage('DONE'); print('tapping device...'); await device.tap(100, 100); print('awaiting "done" message...'); await receivedNextMessage; await recordEnd(); } final List<int> _startMemory = <int>[]; final List<int> _endMemory = <int>[]; final List<int> _diffMemory = <int>[]; Map<String, dynamic> _startMemoryUsage; @protected Future<void> recordStart() async { assert(_startMemoryUsage == null); print('snapshotting memory usage...'); _startMemoryUsage = await device.getMemoryStats(package); } @protected Future<void> recordEnd() async { assert(_startMemoryUsage != null); print('snapshotting memory usage...'); final Map<String, dynamic> endMemoryUsage = await device.getMemoryStats(package); _startMemory.add(_startMemoryUsage['total_kb']); _endMemory.add(endMemoryUsage['total_kb']); _diffMemory.add(endMemoryUsage['total_kb'] - _startMemoryUsage['total_kb']); } } /// Holds simple statistics of an odd-lengthed list of integers. class ListStatistics { factory ListStatistics(Iterable<int> data) { assert(data.isNotEmpty); assert(data.length % 2 == 1); final List<int> sortedData = data.toList()..sort(); return ListStatistics._( sortedData.first, sortedData.last, sortedData[(sortedData.length - 1) ~/ 2], ); } const ListStatistics._(this.min, this.max, this.median); final int min; final int max; final int median; Map<String, int> asMap(String prefix) { return <String, int>{ '$prefix-min': min, '$prefix-max': max, '$prefix-median': median, }; } } class _UnzipListEntry { factory _UnzipListEntry.fromLine(String line) { final List<String> data = line.trim().split(RegExp('\\s+')); assert(data.length == 8); return _UnzipListEntry._( uncompressedSize: int.parse(data[0]), compressedSize: int.parse(data[2]), path: data[7], ); } _UnzipListEntry._({ @required this.uncompressedSize, @required this.compressedSize, @required this.path, }) : assert(uncompressedSize != null), assert(compressedSize != null), assert(compressedSize <= uncompressedSize), assert(path != null); final int uncompressedSize; final int compressedSize; final String path; }