perf_tests.dart 25.1 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6
import 'dart:convert' show json;
7
import 'dart:io';
8

9
import 'package:meta/meta.dart';
10
import 'package:path/path.dart' as path;
11

12 13
import '../framework/adb.dart';
import '../framework/framework.dart';
14
import '../framework/ios.dart';
15 16
import '../framework/utils.dart';

17
TaskFunction createComplexLayoutScrollPerfTest() {
18
  return PerfTest(
19 20 21
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
22
  ).run;
23 24
}

25
TaskFunction createTilesScrollPerfTest() {
26
  return PerfTest(
27 28 29 30 31 32
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'tiles_scroll_perf',
  ).run;
}

33 34 35 36 37 38 39 40
TaskFunction createHomeScrollPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'test_driver/scroll_perf.dart',
    'home_scroll_perf',
  ).run;
}

41 42 43 44 45 46 47 48
TaskFunction createCullOpacityPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cull_opacity_perf.dart',
    'cull_opacity_perf',
  ).run;
}

49 50 51 52 53 54
TaskFunction createCubicBezierPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cubic_bezier_perf.dart',
    'cubic_bezier_perf',
  ).run;
55 56
}

57
TaskFunction createBackdropFilterPerfTest({bool needsMeasureCpuGpu = false}) {
58 59 60 61
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/backdrop_filter_perf.dart',
    'backdrop_filter_perf',
62 63 64 65
    needsMeasureCpuGPu: needsMeasureCpuGpu,
  ).run;
}

66 67 68 69 70 71 72 73 74
TaskFunction createPostBackdropFilterPerfTest({bool needsMeasureCpuGpu = false}) {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/post_backdrop_filter_perf.dart',
    'post_backdrop_filter_perf',
    needsMeasureCpuGPu: needsMeasureCpuGpu,
  ).run;
}

75 76 77 78 79 80
TaskFunction createSimpleAnimationPerfTest({bool needsMeasureCpuGpu = false}) {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/simple_animation_perf.dart',
    'simple_animation_perf',
    needsMeasureCpuGPu: needsMeasureCpuGpu,
81
  ).run;
82 83
}

84 85 86 87 88 89 90 91
TaskFunction createPictureCachePerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/picture_cache_perf.dart',
    'picture_cache_perf',
  ).run;
}

92
TaskFunction createFlutterGalleryStartupTest() {
93
  return StartupTest(
94
    '${flutterDirectory.path}/examples/flutter_gallery',
95
  ).run;
96 97
}

98
TaskFunction createComplexLayoutStartupTest() {
99
  return StartupTest(
100
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
101
  ).run;
102 103
}

104 105 106 107 108 109 110
TaskFunction createHelloWorldStartupTest() {
  return StartupTest(
    '${flutterDirectory.path}/examples/hello_world',
    reportMetrics: false,
  ).run;
}

111
TaskFunction createFlutterGalleryCompileTest() {
112
  return CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run;
113 114
}

115
TaskFunction createHelloWorldCompileTest() {
116
  return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
117 118
}

119 120 121 122
TaskFunction createWebCompileTest() {
  return const WebCompileTest().run;
}

123
TaskFunction createComplexLayoutCompileTest() {
124
  return CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
125 126
}

127
TaskFunction createFlutterViewStartupTest() {
128
  return StartupTest(
129 130
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
131 132 133
  ).run;
}

134
TaskFunction createPlatformViewStartupTest() {
135
  return StartupTest(
136 137 138 139 140
    '${flutterDirectory.path}/examples/platform_view',
    reportMetrics: false,
  ).run;
}

141 142 143 144 145
TaskFunction createBasicMaterialCompileTest() {
  return () async {
    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

146
    rmTree(sampleDir);
147

148
    await inDirectory<void>(Directory.systemTemp, () async {
149
      await flutter('create', options: <String>['--template=app', sampleAppName]);
150 151
    });

152
    if (!sampleDir.existsSync())
153 154
      throw 'Failed to create default Flutter app in ${sampleDir.path}';

155
    return CompileTest(sampleDir.path).run();
156
  };
157 158
}

159 160 161 162 163 164 165 166
TaskFunction createTextfieldPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/textfield_perf.dart',
    'textfield_perf',
  ).run;
}

167

168 169
/// Measure application startup performance.
class StartupTest {
170
  const StartupTest(this.testDirectory, { this.reportMetrics = true });
171 172

  final String testDirectory;
173
  final bool reportMetrics;
174

175
  Future<TaskResult> run() async {
176
    return await inDirectory<TaskResult>(testDirectory, () async {
177
      final String deviceId = (await devices.workingDevice).deviceId;
178
      await flutter('packages', options: <String>['get']);
179 180

      await flutter('run', options: <String>[
181
        '--verbose',
182 183 184 185
        '--profile',
        '--trace-startup',
        '-d',
        deviceId,
186
      ]);
187 188 189
      final Map<String, dynamic> data = json.decode(
        file('$testDirectory/build/start_up_info.json').readAsStringSync(),
      ) as Map<String, dynamic>;
190

191
      if (!reportMetrics)
192
        return TaskResult.success(data);
193

194
      return TaskResult.success(data, benchmarkScoreKeys: <String>[
195
        'timeToFirstFrameMicros',
196
        'timeToFirstFrameRasterizedMicros',
197 198 199 200 201 202 203 204
      ]);
    });
  }
}

/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
205 206 207 208 209
  const PerfTest(
      this.testDirectory,
      this.testTarget,
      this.timelineFileName,
      {this.needsMeasureCpuGPu = false});
210 211 212 213 214

  final String testDirectory;
  final String testTarget;
  final String timelineFileName;

215 216
  final bool needsMeasureCpuGPu;

217
  Future<TaskResult> run() {
218
    return inDirectory<TaskResult>(testDirectory, () async {
219
      final Device device = await devices.workingDevice;
220
      await device.unlock();
221
      final String deviceId = device.deviceId;
222
      await flutter('packages', options: <String>['get']);
223 224 225 226 227 228 229 230 231 232

      await flutter('drive', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // Enables "endless" timeline event buffering.
        '-t',
        testTarget,
        '-d',
        deviceId,
      ]);
233 234 235
      final Map<String, dynamic> data = json.decode(
        file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync(),
      ) as Map<String, dynamic>;
236

237
      if (data['frame_count'] as int < 5) {
238
        return TaskResult.failure(
239 240 241 242 243
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

244 245 246 247 248 249
      if (needsMeasureCpuGPu) {
        await inDirectory<void>('$testDirectory/build', () async {
          data.addAll(await measureIosCpuGpu(deviceId: deviceId));
        });
      }

250
      return TaskResult.success(data, benchmarkScoreKeys: <String>[
251 252 253
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        'missed_frame_build_budget_count',
254 255
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
256 257
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
258
        'missed_frame_rasterizer_budget_count',
259 260
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
261 262
        if (needsMeasureCpuGPu) 'cpu_percentage',
        if (needsMeasureCpuGPu) 'gpu_percentage',
263 264 265 266 267
      ]);
    });
  }
}

268 269 270 271 272 273 274 275 276 277 278 279 280 281
/// Measures how long it takes to compile a Flutter app to JavaScript and how
/// big the compiled code is.
class WebCompileTest {
  const WebCompileTest();

  Future<TaskResult> run() async {
    final Map<String, Object> metrics = <String, Object>{};
    await inDirectory<TaskResult>('${flutterDirectory.path}/examples/hello_world', () async {
      await flutter('packages', options: <String>['get']);
      await evalFlutter('build', options: <String>[
        'web',
        '-v',
        '--release',
        '--no-pub',
282 283 284
      ], environment: <String, String>{
        'FLUTTER_WEB': 'true',
      });
285 286 287 288 289 290 291 292 293 294 295
      final String output = '${flutterDirectory.path}/examples/hello_world/build/web/main.dart.js';
      await _measureSize('hello_world', output, metrics);
      return null;
    });
    await inDirectory<TaskResult>('${flutterDirectory.path}/examples/flutter_gallery', () async {
      await flutter('packages', options: <String>['get']);
      await evalFlutter('build', options: <String>[
        'web',
        '-v',
        '--release',
        '--no-pub',
296 297 298
      ], environment: <String, String>{
        'FLUTTER_WEB': 'true',
      });
299 300 301 302 303 304 305 306 307 308
      final String output = '${flutterDirectory.path}/examples/flutter_gallery/build/web/main.dart.js';
      await _measureSize('flutter_gallery', output, metrics);
      return null;
    });
    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

    rmTree(sampleDir);

    await inDirectory<void>(Directory.systemTemp, () async {
309
      await flutter('create', options: <String>['--template=app', sampleAppName], environment: <String, String>{
310 311
          'FLUTTER_WEB': 'true',
        });
312 313 314 315 316 317 318
      await inDirectory(sampleDir, () async {
        await flutter('packages', options: <String>['get']);
        await evalFlutter('build', options: <String>[
          'web',
          '-v',
          '--release',
          '--no-pub',
319 320 321
        ], environment: <String, String>{
          'FLUTTER_WEB': 'true',
        });
322 323 324 325 326 327 328 329 330 331
        await _measureSize('basic_material_app', path.join(sampleDir.path, 'build/web/main.dart.js'), metrics);
      });
    });
    return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
  }

  static Future<void> _measureSize(String metric, String output, Map<String, Object> metrics) async {
    final ProcessResult result = await Process.run('du', <String>['-k', output]);
    await Process.run('gzip',<String>['-k', '9', output]);
    final ProcessResult resultGzip = await Process.run('du', <String>['-k', output + '.gz']);
332 333
    metrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String);
    metrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String);
334 335 336 337 338 339 340
  }

  static int _parseDu(String source) {
    return int.parse(source.split(RegExp(r'\s+')).first.trim());
  }
}

341
/// Measures how long it takes to compile a Flutter app and how big the compiled
342
/// code is.
343
class CompileTest {
344
  const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });
345 346

  final String testDirectory;
347
  final bool reportPackageContentSizes;
348

349
  Future<TaskResult> run() async {
350
    return await inDirectory<TaskResult>(testDirectory, () async {
351
      final Device device = await devices.workingDevice;
352
      await device.unlock();
353
      await flutter('packages', options: <String>['get']);
354

355 356 357 358 359
      final Map<String, dynamic> metrics = <String, dynamic>{
        ...await _compileAot(),
        ...await _compileApp(reportPackageContentSizes: reportPackageContentSizes),
        ...await _compileDebug(),
      };
360

361
      return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
362 363
    });
  }
364

365
  static Future<Map<String, dynamic>> _compileAot() async {
366
    await flutter('clean');
367
    final Stopwatch watch = Stopwatch()..start();
368
    final List<String> options = <String>[
369 370
      'aot',
      '-v',
371
      '--extra-gen-snapshot-options=--print_snapshot_sizes',
372 373
      '--release',
      '--no-pub',
Ian Hickson's avatar
Ian Hickson committed
374
      '--target-platform',
375
    ];
Ian Hickson's avatar
Ian Hickson committed
376 377 378 379 380 381 382 383
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.add('ios');
        break;
      case DeviceOperatingSystem.android:
        options.add('android-arm');
        break;
    }
384
    final String compileLog = await evalFlutter('build', options: options);
385 386
    watch.stop();

387
    final RegExp metricExpression = RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
388 389 390 391
    final Map<String, dynamic> metrics = <String, dynamic>{};
    for (Match m in metricExpression.allMatches(compileLog)) {
      metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2));
    }
392 393 394
    if (metrics.length != _kSdkNameToMetricNameMapping.length) {
      throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.';
    }
395
    metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds;
396 397 398 399

    return metrics;
  }

400
  static Future<Map<String, dynamic>> _compileApp({ bool reportPackageContentSizes = false }) async {
401
    await flutter('clean');
402
    final Stopwatch watch = Stopwatch();
403 404
    int releaseSizeInBytes;
    final List<String> options = <String>['--release'];
405 406
    final Map<String, dynamic> metrics = <String, dynamic>{};

Ian Hickson's avatar
Ian Hickson committed
407 408 409 410 411 412
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        watch.start();
        await flutter('build', options: options);
        watch.stop();
413
        final String appPath =  '$cwd/build/ios/Release-iphoneos/Runner.app/';
414
        // IPAs are created manually, https://flutter.dev/ios-release/
415
        await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
Ian Hickson's avatar
Ian Hickson committed
416
        releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
417 418
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromIosApp(appPath));
Ian Hickson's avatar
Ian Hickson committed
419 420 421
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
422
        options.add('--target-platform=android-arm');
Ian Hickson's avatar
Ian Hickson committed
423 424 425
        watch.start();
        await flutter('build', options: options);
        watch.stop();
426 427
        String apkPath = '$cwd/build/app/outputs/apk/app.apk';
        File apk = file(apkPath);
428 429
        if (!apk.existsSync()) {
          // Pre Android SDK 26 path
430 431
          apkPath = '$cwd/build/app/outputs/apk/app-release.apk';
          apk = file(apkPath);
432 433
        }
        releaseSizeInBytes = apk.lengthSync();
434 435
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromApk(apkPath));
Ian Hickson's avatar
Ian Hickson committed
436
        break;
437
    }
438

439
    metrics.addAll(<String, dynamic>{
440 441
      'release_full_compile_millis': watch.elapsedMilliseconds,
      'release_size_bytes': releaseSizeInBytes,
442 443 444
    });

    return metrics;
445 446
  }

447
  static Future<Map<String, dynamic>> _compileDebug() async {
448
    await flutter('clean');
449
    final Stopwatch watch = Stopwatch();
Ian Hickson's avatar
Ian Hickson committed
450 451 452 453 454 455 456
    final List<String> options = <String>['--debug'];
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
457
        options.add('--target-platform=android-arm');
Ian Hickson's avatar
Ian Hickson committed
458
        break;
459
    }
Ian Hickson's avatar
Ian Hickson committed
460 461 462
    watch.start();
    await flutter('build', options: options);
    watch.stop();
463 464

    return <String, dynamic>{
465
      'debug_full_compile_millis': watch.elapsedMilliseconds,
466 467 468
    };
  }

469
  static const Map<String, String> _kSdkNameToMetricNameMapping = <String, String> {
470 471 472 473 474 475 476
    'VMIsolate': 'aot_snapshot_size_vmisolate',
    'Isolate': 'aot_snapshot_size_isolate',
    'ReadOnlyData': 'aot_snapshot_size_rodata',
    'Instructions': 'aot_snapshot_size_instructions',
    'Total': 'aot_snapshot_size_total',
  };

477 478
  static String _sdkNameToMetricName(String sdkName) {

479
    if (!_kSdkNameToMetricNameMapping.containsKey(sdkName))
480 481
      throw 'Unrecognized SDK snapshot metric name: $sdkName';

482
    return _kSdkNameToMetricNameMapping[sdkName];
483
  }
484

485 486
  static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
    // Thin the binary to only contain one architecture.
487
    final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
488 489
    await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
      'ARCHS': 'arm64',
490 491
      'WRAPPER_NAME': path.basename(appPath),
      'TARGET_BUILD_DIR': path.dirname(appPath),
492 493
    });

494 495
    final File appFramework = File(path.join(appPath, 'Frameworks', 'App.framework', 'App'));
    final File flutterFramework = File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
496 497 498 499 500 501 502 503

    return <String, dynamic>{
      'app_framework_uncompressed_bytes': await appFramework.length(),
      'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
    };
  }


504 505 506 507 508 509 510
  static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
    final  String output = await eval('unzip', <String>['-v', apkPath]);
    final List<String> lines = output.split('\n');
    final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

    // First three lines are header, last two lines are footer.
    for (int i = 3; i < lines.length - 2; i++) {
511
      final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]);
512 513 514 515
      fileToMetadata[entry.path] = entry;
    }

    final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
516
    final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so'];
517
    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/LICENSE'];
518 519 520 521

    return <String, dynamic>{
      'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
      'libflutter_compressed_bytes': libflutter.compressedSize,
522 523
      'libapp_uncompressed_bytes': libapp.uncompressedSize,
      'libapp_compressed_bytes': libapp.compressedSize,
524 525
      'license_uncompressed_bytes': license.uncompressedSize,
      'license_compressed_bytes': license.compressedSize,
526 527
    };
  }
528
}
529

530
/// Measure application memory usage.
531
class MemoryTest {
532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
  MemoryTest(this.project, this.test, this.package);

  final String project;
  final String test;
  final String package;

  /// Completes when the log line specified in the last call to
  /// [prepareForNextMessage] is seen by `adb logcat`.
  Future<void> get receivedNextMessage => _receivedNextMessage?.future;
  Completer<void> _receivedNextMessage;
  String _nextMessage;

  /// Prepares the [receivedNextMessage] future such that it will complete
  /// when `adb logcat` sees a log line with the given `message`.
  void prepareForNextMessage(String message) {
    _nextMessage = message;
548
    _receivedNextMessage = Completer<void>();
549
  }
550

551
  int get iterationCount => 10;
552

553 554
  Device get device => _device;
  Device _device;
555

556
  Future<TaskResult> run() {
557
    return inDirectory<TaskResult>(project, () async {
558 559 560 561
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
562 563 564
      await device.unlock();
      await flutter('packages', options: <String>['get']);

565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
      final StreamSubscription<String> adb = device.logcat.listen(
        (String data) {
          if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ===='))
            _receivedNextMessage.complete();
        },
      );

      for (int iteration = 0; iteration < iterationCount; iteration += 1) {
        print('running memory test iteration $iteration...');
        _startMemoryUsage = null;
        await useMemory();
        assert(_startMemoryUsage != null);
        assert(_startMemory.length == iteration + 1);
        assert(_endMemory.length == iteration + 1);
        assert(_diffMemory.length == iteration + 1);
        print('terminating...');
        await device.stop(package);
582
        await Future<void>.delayed(const Duration(milliseconds: 10));
583
      }
584

585 586
      await adb.cancel();

587 588 589
      final ListStatistics startMemoryStatistics = ListStatistics(_startMemory);
      final ListStatistics endMemoryStatistics = ListStatistics(_endMemory);
      final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory);
590

591 592 593 594 595
      final Map<String, dynamic> memoryUsage = <String, dynamic>{
        ...startMemoryStatistics.asMap('start'),
        ...endMemoryStatistics.asMap('end'),
        ...diffMemoryStatistics.asMap('diff'),
      };
596

597 598 599 600 601
      _device = null;
      _startMemory.clear();
      _endMemory.clear();
      _diffMemory.clear();

602
      return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
603 604
    });
  }
605

606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
  /// Starts the app specified by [test] on the [device].
  ///
  /// The [run] method will terminate it by its package name ([package]).
  Future<void> launchApp() async {
    prepareForNextMessage('READY');
    print('launching $project$test on device...');
    await flutter('run', options: <String>[
      '--verbose',
      '--release',
      '--no-resident',
      '-d', device.deviceId,
      test,
    ]);
    print('awaiting "ready" message...');
    await receivedNextMessage;
  }
622

623
  /// To change the behavior of the test, override this.
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
  ///
  /// Make sure to call recordStart() and recordEnd() once each in that order.
  ///
  /// By default it just launches the app, records memory usage, taps the device,
  /// awaits a DONE notification, and records memory usage again.
  Future<void> useMemory() async {
    await launchApp();
    await recordStart();

    prepareForNextMessage('DONE');
    print('tapping device...');
    await device.tap(100, 100);
    print('awaiting "done" message...');
    await receivedNextMessage;

    await recordEnd();
  }
641

642 643 644
  final List<int> _startMemory = <int>[];
  final List<int> _endMemory = <int>[];
  final List<int> _diffMemory = <int>[];
645

646
  Map<String, dynamic> _startMemoryUsage;
647

648 649 650 651 652 653
  @protected
  Future<void> recordStart() async {
    assert(_startMemoryUsage == null);
    print('snapshotting memory usage...');
    _startMemoryUsage = await device.getMemoryStats(package);
  }
654

655 656 657 658 659
  @protected
  Future<void> recordEnd() async {
    assert(_startMemoryUsage != null);
    print('snapshotting memory usage...');
    final Map<String, dynamic> endMemoryUsage = await device.getMemoryStats(package);
660 661 662
    _startMemory.add(_startMemoryUsage['total_kb'] as int);
    _endMemory.add(endMemoryUsage['total_kb'] as int);
    _diffMemory.add((endMemoryUsage['total_kb'] as int) - (_startMemoryUsage['total_kb'] as int));
663 664
  }
}
665

666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
enum ReportedDurationTestFlavor {
  debug, profile, release
}

String _reportedDurationTestToString(ReportedDurationTestFlavor flavor) {
  switch (flavor) {
    case ReportedDurationTestFlavor.debug:
      return 'debug';
    case ReportedDurationTestFlavor.profile:
      return 'profile';
    case ReportedDurationTestFlavor.release:
      return 'release';
  }
  throw ArgumentError('Unexpected value for enum $flavor');
}

682
class ReportedDurationTest {
683
  ReportedDurationTest(this.flavor, this.project, this.test, this.package, this.durationPattern);
684

685
  final ReportedDurationTestFlavor flavor;
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
  final String project;
  final String test;
  final String package;
  final RegExp durationPattern;

  final Completer<int> durationCompleter = Completer<int>();

  int get iterationCount => 10;

  Device get device => _device;
  Device _device;

  Future<TaskResult> run() {
    return inDirectory<TaskResult>(project, () async {
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
      await device.unlock();
      await flutter('packages', options: <String>['get']);

      final StreamSubscription<String> adb = device.logcat.listen(
        (String data) {
          if (durationPattern.hasMatch(data))
            durationCompleter.complete(int.parse(durationPattern.firstMatch(data).group(1)));
        },
      );
      print('launching $project$test on device...');
      await flutter('run', options: <String>[
        '--verbose',
716
        '--${_reportedDurationTestToString(flavor)}',
717 718 719 720 721 722 723 724 725 726 727 728 729
        '--no-resident',
        '-d', device.deviceId,
        test,
      ]);

      final int duration = await durationCompleter.future;
      print('terminating...');
      await device.stop(package);
      await adb.cancel();

      _device = null;

      final Map<String, dynamic> reportedDuration = <String, dynamic>{
730
        'duration': duration,
731 732 733 734 735 736 737 738
      };
      _device = null;

      return TaskResult.success(reportedDuration, benchmarkScoreKeys: reportedDuration.keys.toList());
    });
  }
}

739 740 741 742 743 744
/// Holds simple statistics of an odd-lengthed list of integers.
class ListStatistics {
  factory ListStatistics(Iterable<int> data) {
    assert(data.isNotEmpty);
    assert(data.length % 2 == 1);
    final List<int> sortedData = data.toList()..sort();
745
    return ListStatistics._(
746 747 748 749 750
      sortedData.first,
      sortedData.last,
      sortedData[(sortedData.length - 1) ~/ 2],
    );
  }
751

752
  const ListStatistics._(this.min, this.max, this.median);
753

754 755 756 757 758 759 760 761 762 763
  final int min;
  final int max;
  final int median;

  Map<String, int> asMap(String prefix) {
    return <String, int>{
      '$prefix-min': min,
      '$prefix-max': max,
      '$prefix-median': median,
    };
764 765
  }
}
766 767 768

class _UnzipListEntry {
  factory _UnzipListEntry.fromLine(String line) {
769
    final List<String> data = line.trim().split(RegExp('\\s+'));
770
    assert(data.length == 8);
771
    return _UnzipListEntry._(
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
      uncompressedSize:  int.parse(data[0]),
      compressedSize: int.parse(data[2]),
      path: data[7],
    );
  }

  _UnzipListEntry._({
    @required this.uncompressedSize,
    @required this.compressedSize,
    @required this.path,
  }) : assert(uncompressedSize != null),
       assert(compressedSize != null),
       assert(compressedSize <= uncompressedSize),
       assert(path != null);

  final int uncompressedSize;
  final int compressedSize;
  final String path;
}