perf_tests.dart 19.3 KB
Newer Older
1 2 3 4 5
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6
import 'dart:convert' show json;
7
import 'dart:io';
8

9
import 'package:meta/meta.dart';
10
import 'package:path/path.dart' as path;
11

12 13
import '../framework/adb.dart';
import '../framework/framework.dart';
14
import '../framework/ios.dart';
15 16
import '../framework/utils.dart';

17
TaskFunction createComplexLayoutScrollPerfTest() {
18
  return PerfTest(
19 20 21
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
22
  ).run;
23 24
}

25
TaskFunction createTilesScrollPerfTest() {
26
  return PerfTest(
27 28 29 30 31 32
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'tiles_scroll_perf',
  ).run;
}

33 34 35 36 37 38 39 40
TaskFunction createHomeScrollPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'test_driver/scroll_perf.dart',
    'home_scroll_perf',
  ).run;
}

41 42 43 44 45 46 47 48
TaskFunction createCullOpacityPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cull_opacity_perf.dart',
    'cull_opacity_perf',
  ).run;
}

49 50 51 52 53 54 55 56
TaskFunction createCubicBezierPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cubic_bezier_perf.dart',
    'cubic_bezier_perf',
  ).run;
}

57
TaskFunction createFlutterGalleryStartupTest() {
58
  return StartupTest(
59
    '${flutterDirectory.path}/examples/flutter_gallery',
60
  ).run;
61 62
}

63
TaskFunction createComplexLayoutStartupTest() {
64
  return StartupTest(
65
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
66
  ).run;
67 68
}

69
TaskFunction createFlutterGalleryCompileTest() {
70
  return CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run;
71 72
}

73
TaskFunction createHelloWorldCompileTest() {
74
  return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
75 76
}

77
TaskFunction createComplexLayoutCompileTest() {
78
  return CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
79 80
}

81
TaskFunction createFlutterViewStartupTest() {
82
  return StartupTest(
83 84
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
85 86 87
  ).run;
}

88
TaskFunction createPlatformViewStartupTest() {
89
  return StartupTest(
90 91 92 93 94
    '${flutterDirectory.path}/examples/platform_view',
    reportMetrics: false,
  ).run;
}

95 96 97 98 99
TaskFunction createBasicMaterialCompileTest() {
  return () async {
    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

100
    rmTree(sampleDir);
101

102
    await inDirectory<void>(Directory.systemTemp, () async {
103
      await flutter('create', options: <String>['--template=app', sampleAppName]);
104 105 106 107 108
    });

    if (!(await sampleDir.exists()))
      throw 'Failed to create default Flutter app in ${sampleDir.path}';

109
    return CompileTest(sampleDir.path).run();
110
  };
111 112
}

113

114 115
/// Measure application startup performance.
class StartupTest {
116
  const StartupTest(this.testDirectory, { this.reportMetrics = true });
117 118

  final String testDirectory;
119
  final bool reportMetrics;
120

121
  Future<TaskResult> run() async {
122
    return await inDirectory<TaskResult>(testDirectory, () async {
123
      final String deviceId = (await devices.workingDevice).deviceId;
124
      await flutter('packages', options: <String>['get']);
125

126
      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
127
        await prepareProvisioningCertificates(testDirectory);
128 129

      await flutter('run', options: <String>[
130
        '--verbose',
131 132 133 134
        '--profile',
        '--trace-startup',
        '-d',
        deviceId,
135
      ]);
136
      final Map<String, dynamic> data = json.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
137

138
      if (!reportMetrics)
139
        return TaskResult.success(data);
140

141
      return TaskResult.success(data, benchmarkScoreKeys: <String>[
142 143 144 145 146 147 148 149 150
        'timeToFirstFrameMicros',
      ]);
    });
  }
}

/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
151
  const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName);
152 153 154 155 156

  final String testDirectory;
  final String testTarget;
  final String timelineFileName;

157
  Future<TaskResult> run() {
158
    return inDirectory<TaskResult>(testDirectory, () async {
159
      final Device device = await devices.workingDevice;
160
      await device.unlock();
161
      final String deviceId = device.deviceId;
162
      await flutter('packages', options: <String>['get']);
163

164
      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
165
        await prepareProvisioningCertificates(testDirectory);
166 167 168 169 170 171 172 173 174 175

      await flutter('drive', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // Enables "endless" timeline event buffering.
        '-t',
        testTarget,
        '-d',
        deviceId,
      ]);
176
      final Map<String, dynamic> data = json.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
177 178

      if (data['frame_count'] < 5) {
179
        return TaskResult.failure(
180 181 182 183 184
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

185
      return TaskResult.success(data, benchmarkScoreKeys: <String>[
186 187 188
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        'missed_frame_build_budget_count',
189 190
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
191 192
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
193
        'missed_frame_rasterizer_budget_count',
194 195
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
196 197 198 199 200
      ]);
    });
  }
}

201
/// Measures how long it takes to compile a Flutter app and how big the compiled
202
/// code is.
203
class CompileTest {
204
  const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });
205 206

  final String testDirectory;
207
  final bool reportPackageContentSizes;
208

209
  Future<TaskResult> run() async {
210
    return await inDirectory<TaskResult>(testDirectory, () async {
211
      final Device device = await devices.workingDevice;
212
      await device.unlock();
213
      await flutter('packages', options: <String>['get']);
214

215
      final Map<String, dynamic> metrics = <String, dynamic>{}
216
        ..addAll(await _compileAot())
217
        ..addAll(await _compileApp(reportPackageContentSizes: reportPackageContentSizes))
218
        ..addAll(await _compileDebug());
219

220
      return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
221 222
    });
  }
223

224
  static Future<Map<String, dynamic>> _compileAot() async {
Ian Hickson's avatar
Ian Hickson committed
225
    // Generate blobs instead of assembly.
226
    await flutter('clean');
227
    final Stopwatch watch = Stopwatch()..start();
228
    final List<String> options = <String>[
229 230
      'aot',
      '-v',
231
      '--extra-gen-snapshot-options=--print_snapshot_sizes',
232 233
      '--release',
      '--no-pub',
Ian Hickson's avatar
Ian Hickson committed
234
      '--target-platform',
235
    ];
Ian Hickson's avatar
Ian Hickson committed
236 237 238 239 240 241 242 243
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.add('ios');
        break;
      case DeviceOperatingSystem.android:
        options.add('android-arm');
        break;
    }
244
    setLocalEngineOptionIfNecessary(options);
245
    final String compileLog = await evalFlutter('build', options: options);
246 247
    watch.stop();

248
    final RegExp metricExpression = RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
249 250 251 252
    final Map<String, dynamic> metrics = <String, dynamic>{};
    for (Match m in metricExpression.allMatches(compileLog)) {
      metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2));
    }
253 254 255
    if (metrics.length != _kSdkNameToMetricNameMapping.length) {
      throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.';
    }
256
    metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds;
257 258 259 260

    return metrics;
  }

261
  static Future<Map<String, dynamic>> _compileApp({ bool reportPackageContentSizes = false }) async {
262
    await flutter('clean');
263
    final Stopwatch watch = Stopwatch();
264 265
    int releaseSizeInBytes;
    final List<String> options = <String>['--release'];
266
    setLocalEngineOptionIfNecessary(options);
267 268
    final Map<String, dynamic> metrics = <String, dynamic>{};

Ian Hickson's avatar
Ian Hickson committed
269 270 271 272 273 274 275
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        await prepareProvisioningCertificates(cwd);
        watch.start();
        await flutter('build', options: options);
        watch.stop();
276
        final String appPath =  '$cwd/build/ios/Release-iphoneos/Runner.app/';
277
        // IPAs are created manually, https://flutter.dev/ios-release/
278
        await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
Ian Hickson's avatar
Ian Hickson committed
279
        releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
280 281
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromIosApp(appPath));
Ian Hickson's avatar
Ian Hickson committed
282 283 284 285 286 287
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
        watch.start();
        await flutter('build', options: options);
        watch.stop();
288 289
        String apkPath = '$cwd/build/app/outputs/apk/app.apk';
        File apk = file(apkPath);
290 291
        if (!apk.existsSync()) {
          // Pre Android SDK 26 path
292 293
          apkPath = '$cwd/build/app/outputs/apk/app-release.apk';
          apk = file(apkPath);
294 295
        }
        releaseSizeInBytes = apk.lengthSync();
296 297
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromApk(apkPath));
Ian Hickson's avatar
Ian Hickson committed
298
        break;
299
    }
300

301
    metrics.addAll(<String, dynamic>{
302 303
      'release_full_compile_millis': watch.elapsedMilliseconds,
      'release_size_bytes': releaseSizeInBytes,
304 305 306
    });

    return metrics;
307 308
  }

309
  static Future<Map<String, dynamic>> _compileDebug() async {
310
    await flutter('clean');
311
    final Stopwatch watch = Stopwatch();
Ian Hickson's avatar
Ian Hickson committed
312
    final List<String> options = <String>['--debug'];
313
    setLocalEngineOptionIfNecessary(options);
Ian Hickson's avatar
Ian Hickson committed
314 315 316 317 318 319 320 321
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        await prepareProvisioningCertificates(cwd);
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
        break;
322
    }
Ian Hickson's avatar
Ian Hickson committed
323 324 325
    watch.start();
    await flutter('build', options: options);
    watch.stop();
326 327

    return <String, dynamic>{
328
      'debug_full_compile_millis': watch.elapsedMilliseconds,
329 330 331
    };
  }

332
  static const Map<String, String> _kSdkNameToMetricNameMapping = <String, String> {
333 334 335 336 337 338 339
    'VMIsolate': 'aot_snapshot_size_vmisolate',
    'Isolate': 'aot_snapshot_size_isolate',
    'ReadOnlyData': 'aot_snapshot_size_rodata',
    'Instructions': 'aot_snapshot_size_instructions',
    'Total': 'aot_snapshot_size_total',
  };

340 341
  static String _sdkNameToMetricName(String sdkName) {

342
    if (!_kSdkNameToMetricNameMapping.containsKey(sdkName))
343 344
      throw 'Unrecognized SDK snapshot metric name: $sdkName';

345
    return _kSdkNameToMetricNameMapping[sdkName];
346
  }
347

348 349
  static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
    // Thin the binary to only contain one architecture.
350
    final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
351 352
    await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
      'ARCHS': 'arm64',
353 354
      'WRAPPER_NAME': path.basename(appPath),
      'TARGET_BUILD_DIR': path.dirname(appPath),
355 356
    });

357 358
    final File appFramework = File(path.join(appPath, 'Frameworks', 'App.framework', 'App'));
    final File flutterFramework = File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
359 360 361 362 363 364 365 366

    return <String, dynamic>{
      'app_framework_uncompressed_bytes': await appFramework.length(),
      'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
    };
  }


367 368 369 370 371 372 373
  static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
    final  String output = await eval('unzip', <String>['-v', apkPath]);
    final List<String> lines = output.split('\n');
    final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

    // First three lines are header, last two lines are footer.
    for (int i = 3; i < lines.length - 2; i++) {
374
      final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]);
375 376 377 378 379 380 381 382
      fileToMetadata[entry.path] = entry;
    }

    final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
    final _UnzipListEntry isolateSnapshotData = fileToMetadata['assets/isolate_snapshot_data'];
    final _UnzipListEntry isolateSnapshotInstr = fileToMetadata['assets/isolate_snapshot_instr'];
    final _UnzipListEntry vmSnapshotData = fileToMetadata['assets/vm_snapshot_data'];
    final _UnzipListEntry vmSnapshotInstr = fileToMetadata['assets/vm_snapshot_instr'];
383
    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/LICENSE'];
384 385 386 387 388 389 390 391 392 393 394 395

    return <String, dynamic>{
      'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
      'libflutter_compressed_bytes': libflutter.compressedSize,
      'snapshot_uncompressed_bytes': isolateSnapshotData.uncompressedSize +
          isolateSnapshotInstr.uncompressedSize +
          vmSnapshotData.uncompressedSize +
          vmSnapshotInstr.uncompressedSize,
      'snapshot_compressed_bytes': isolateSnapshotData.compressedSize +
          isolateSnapshotInstr.compressedSize +
          vmSnapshotData.compressedSize +
          vmSnapshotInstr.compressedSize,
396 397
      'license_uncompressed_bytes': license.uncompressedSize,
      'license_compressed_bytes': license.compressedSize,
398 399
    };
  }
400
}
401

402
/// Measure application memory usage.
403
class MemoryTest {
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
  MemoryTest(this.project, this.test, this.package);

  final String project;
  final String test;
  final String package;

  /// Completes when the log line specified in the last call to
  /// [prepareForNextMessage] is seen by `adb logcat`.
  Future<void> get receivedNextMessage => _receivedNextMessage?.future;
  Completer<void> _receivedNextMessage;
  String _nextMessage;

  /// Prepares the [receivedNextMessage] future such that it will complete
  /// when `adb logcat` sees a log line with the given `message`.
  void prepareForNextMessage(String message) {
    _nextMessage = message;
420
    _receivedNextMessage = Completer<void>();
421
  }
422

423
  int get iterationCount => 10;
424

425 426
  Device get device => _device;
  Device _device;
427

428
  Future<TaskResult> run() {
429
    return inDirectory<TaskResult>(project, () async {
430 431 432 433
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
434 435 436
      await device.unlock();
      await flutter('packages', options: <String>['get']);

437
      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
        await prepareProvisioningCertificates(project);

      final StreamSubscription<String> adb = device.logcat.listen(
        (String data) {
          if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ===='))
            _receivedNextMessage.complete();
        },
      );

      for (int iteration = 0; iteration < iterationCount; iteration += 1) {
        print('running memory test iteration $iteration...');
        _startMemoryUsage = null;
        await useMemory();
        assert(_startMemoryUsage != null);
        assert(_startMemory.length == iteration + 1);
        assert(_endMemory.length == iteration + 1);
        assert(_diffMemory.length == iteration + 1);
        print('terminating...');
        await device.stop(package);
457
        await Future<void>.delayed(const Duration(milliseconds: 10));
458
      }
459

460 461
      await adb.cancel();

462 463 464
      final ListStatistics startMemoryStatistics = ListStatistics(_startMemory);
      final ListStatistics endMemoryStatistics = ListStatistics(_endMemory);
      final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory);
465 466 467 468 469

      final Map<String, dynamic> memoryUsage = <String, dynamic>{};
      memoryUsage.addAll(startMemoryStatistics.asMap('start'));
      memoryUsage.addAll(endMemoryStatistics.asMap('end'));
      memoryUsage.addAll(diffMemoryStatistics.asMap('diff'));
470

471 472 473 474 475
      _device = null;
      _startMemory.clear();
      _endMemory.clear();
      _diffMemory.clear();

476
      return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
477 478
    });
  }
479

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
  /// Starts the app specified by [test] on the [device].
  ///
  /// The [run] method will terminate it by its package name ([package]).
  Future<void> launchApp() async {
    prepareForNextMessage('READY');
    print('launching $project$test on device...');
    await flutter('run', options: <String>[
      '--verbose',
      '--release',
      '--no-resident',
      '-d', device.deviceId,
      test,
    ]);
    print('awaiting "ready" message...');
    await receivedNextMessage;
  }
496

497
  /// To change the behavior of the test, override this.
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
  ///
  /// Make sure to call recordStart() and recordEnd() once each in that order.
  ///
  /// By default it just launches the app, records memory usage, taps the device,
  /// awaits a DONE notification, and records memory usage again.
  Future<void> useMemory() async {
    await launchApp();
    await recordStart();

    prepareForNextMessage('DONE');
    print('tapping device...');
    await device.tap(100, 100);
    print('awaiting "done" message...');
    await receivedNextMessage;

    await recordEnd();
  }
515

516 517 518
  final List<int> _startMemory = <int>[];
  final List<int> _endMemory = <int>[];
  final List<int> _diffMemory = <int>[];
519

520
  Map<String, dynamic> _startMemoryUsage;
521

522 523 524 525 526 527
  @protected
  Future<void> recordStart() async {
    assert(_startMemoryUsage == null);
    print('snapshotting memory usage...');
    _startMemoryUsage = await device.getMemoryStats(package);
  }
528

529 530 531 532 533 534 535 536 537 538
  @protected
  Future<void> recordEnd() async {
    assert(_startMemoryUsage != null);
    print('snapshotting memory usage...');
    final Map<String, dynamic> endMemoryUsage = await device.getMemoryStats(package);
    _startMemory.add(_startMemoryUsage['total_kb']);
    _endMemory.add(endMemoryUsage['total_kb']);
    _diffMemory.add(endMemoryUsage['total_kb'] - _startMemoryUsage['total_kb']);
  }
}
539

540 541 542 543 544 545
/// Holds simple statistics of an odd-lengthed list of integers.
class ListStatistics {
  factory ListStatistics(Iterable<int> data) {
    assert(data.isNotEmpty);
    assert(data.length % 2 == 1);
    final List<int> sortedData = data.toList()..sort();
546
    return ListStatistics._(
547 548 549 550 551
      sortedData.first,
      sortedData.last,
      sortedData[(sortedData.length - 1) ~/ 2],
    );
  }
552

553
  const ListStatistics._(this.min, this.max, this.median);
554

555 556 557 558 559 560 561 562 563 564
  final int min;
  final int max;
  final int median;

  Map<String, int> asMap(String prefix) {
    return <String, int>{
      '$prefix-min': min,
      '$prefix-max': max,
      '$prefix-median': median,
    };
565 566
  }
}
567 568 569

class _UnzipListEntry {
  factory _UnzipListEntry.fromLine(String line) {
570
    final List<String> data = line.trim().split(RegExp('\\s+'));
571
    assert(data.length == 8);
572
    return _UnzipListEntry._(
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
      uncompressedSize:  int.parse(data[0]),
      compressedSize: int.parse(data[2]),
      path: data[7],
    );
  }

  _UnzipListEntry._({
    @required this.uncompressedSize,
    @required this.compressedSize,
    @required this.path,
  }) : assert(uncompressedSize != null),
       assert(compressedSize != null),
       assert(compressedSize <= uncompressedSize),
       assert(path != null);

  final int uncompressedSize;
  final int compressedSize;
  final String path;
}