perf_tests.dart 33.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6
import 'dart:convert' show LineSplitter, json, utf8;
7
import 'dart:io';
8
import 'dart:math' as math;
9

10
import 'package:flutter_devicelab/tasks/track_widget_creation_enabled_task.dart';
11
import 'package:meta/meta.dart';
12
import 'package:path/path.dart' as path;
13

14 15
import '../framework/adb.dart';
import '../framework/framework.dart';
16
import '../framework/ios.dart';
17 18
import '../framework/utils.dart';

19
TaskFunction createComplexLayoutScrollPerfTest() {
20
  return PerfTest(
21 22 23
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
24
  ).run;
25 26
}

27
TaskFunction createTilesScrollPerfTest() {
28
  return PerfTest(
29 30 31 32 33 34
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'tiles_scroll_perf',
  ).run;
}

35
TaskFunction createUiKitViewScrollPerfTest() {
36 37
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
38 39 40 41 42 43 44 45 46 47
    'test_driver/uikit_view_scroll_perf.dart',
    'platform_views_scroll_perf',
    testDriver: 'test_driver/scroll_perf_test.dart',
  ).run;
}

TaskFunction createAndroidTextureScrollPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
    'test_driver/android_texture_scroll_perf.dart',
48
    'platform_views_scroll_perf',
49 50 51 52 53 54 55 56
    testDriver: 'test_driver/scroll_perf_test.dart',
  ).run;
}

TaskFunction createAndroidViewScrollPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
    'test_driver/android_view_scroll_perf.dart',
57
    'platform_views_scroll_perf',
58
    testDriver: 'test_driver/scroll_perf_test.dart',
59 60 61
  ).run;
}

62 63
TaskFunction createHomeScrollPerfTest() {
  return PerfTest(
64
    '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
65 66 67 68 69
    'test_driver/scroll_perf.dart',
    'home_scroll_perf',
  ).run;
}

70 71 72 73 74 75 76 77
TaskFunction createCullOpacityPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cull_opacity_perf.dart',
    'cull_opacity_perf',
  ).run;
}

78 79 80 81 82 83
TaskFunction createCubicBezierPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cubic_bezier_perf.dart',
    'cubic_bezier_perf',
  ).run;
84 85
}

86 87 88 89 90 91 92 93
TaskFunction createCubicBezierPerfSkSLWarmupTest() {
  return PerfTestWithSkSL(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/cubic_bezier_perf.dart',
    'cubic_bezier_perf',
  ).run;
}

94
TaskFunction createBackdropFilterPerfTest({bool needsMeasureCpuGpu = false}) {
95 96 97 98
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/backdrop_filter_perf.dart',
    'backdrop_filter_perf',
99
    needsMeasureCpuGpu: needsMeasureCpuGpu,
100 101 102
  ).run;
}

103 104 105 106 107
TaskFunction createPostBackdropFilterPerfTest({bool needsMeasureCpuGpu = false}) {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/post_backdrop_filter_perf.dart',
    'post_backdrop_filter_perf',
108
    needsMeasureCpuGpu: needsMeasureCpuGpu,
109 110 111
  ).run;
}

112 113 114 115 116
TaskFunction createSimpleAnimationPerfTest({bool needsMeasureCpuGpu = false}) {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/simple_animation_perf.dart',
    'simple_animation_perf',
117
    needsMeasureCpuGpu: needsMeasureCpuGpu,
118
  ).run;
119 120
}

Dan Field's avatar
Dan Field committed
121 122 123 124 125
TaskFunction createAnimatedPlaceholderPerfTest({bool needsMeasureCpuGpu = false}) {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/animated_placeholder_perf.dart',
    'animated_placeholder_perf',
126
    needsMeasureCpuGpu: needsMeasureCpuGpu,
Dan Field's avatar
Dan Field committed
127 128 129
  ).run;
}

130 131 132 133 134 135 136 137
TaskFunction createPictureCachePerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/picture_cache_perf.dart',
    'picture_cache_perf',
  ).run;
}

138
TaskFunction createFlutterGalleryStartupTest() {
139
  return StartupTest(
140
    '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
141
  ).run;
142 143
}

144
TaskFunction createComplexLayoutStartupTest() {
145
  return StartupTest(
146
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
147
  ).run;
148 149
}

150 151 152 153 154 155 156
TaskFunction createHelloWorldStartupTest() {
  return StartupTest(
    '${flutterDirectory.path}/examples/hello_world',
    reportMetrics: false,
  ).run;
}

157
TaskFunction createFlutterGalleryCompileTest() {
158
  return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run;
159 160
}

161
TaskFunction createHelloWorldCompileTest() {
162
  return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
163 164
}

165 166 167 168
TaskFunction createWebCompileTest() {
  return const WebCompileTest().run;
}

169
TaskFunction createComplexLayoutCompileTest() {
170
  return CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
171 172
}

173
TaskFunction createFlutterViewStartupTest() {
174
  return StartupTest(
175 176
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
177 178 179
  ).run;
}

180
TaskFunction createPlatformViewStartupTest() {
181
  return StartupTest(
182 183 184 185 186
    '${flutterDirectory.path}/examples/platform_view',
    reportMetrics: false,
  ).run;
}

187 188 189 190 191
TaskFunction createBasicMaterialCompileTest() {
  return () async {
    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

192
    rmTree(sampleDir);
193

194
    await inDirectory<void>(Directory.systemTemp, () async {
195
      await flutter('create', options: <String>['--template=app', sampleAppName]);
196 197
    });

198
    if (!sampleDir.existsSync())
199 200
      throw 'Failed to create default Flutter app in ${sampleDir.path}';

201
    return CompileTest(sampleDir.path).run();
202
  };
203 204
}

205 206 207 208 209 210 211 212
TaskFunction createTextfieldPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/textfield_perf.dart',
    'textfield_perf',
  ).run;
}

213 214 215 216 217 218 219 220
TaskFunction createColorFilterAndFadePerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/color_filter_and_fade_perf.dart',
    'color_filter_and_fade_perf',
  ).run;
}

221 222 223 224 225 226 227 228
TaskFunction createFadingChildAnimationPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/fading_child_animation_perf.dart',
    'fading_child_animation_perf',
  ).run;
}

229 230 231 232 233 234 235 236
TaskFunction createImageFilteredTransformAnimationPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/imagefiltered_transform_animation_perf.dart',
    'imagefiltered_transform_animation_perf',
  ).run;
}

237 238
/// Measure application startup performance.
class StartupTest {
239
  const StartupTest(this.testDirectory, { this.reportMetrics = true });
240 241

  final String testDirectory;
242
  final bool reportMetrics;
243

244
  Future<TaskResult> run() async {
245
    return await inDirectory<TaskResult>(testDirectory, () async {
246
      final String deviceId = (await devices.workingDevice).deviceId;
247
      await flutter('packages', options: <String>['get']);
248 249

      await flutter('run', options: <String>[
250
        '--verbose',
251 252 253 254
        '--profile',
        '--trace-startup',
        '-d',
        deviceId,
255
      ]);
256 257 258
      final Map<String, dynamic> data = json.decode(
        file('$testDirectory/build/start_up_info.json').readAsStringSync(),
      ) as Map<String, dynamic>;
259

260
      if (!reportMetrics)
261
        return TaskResult.success(data);
262

263
      return TaskResult.success(data, benchmarkScoreKeys: <String>[
264
        'timeToFirstFrameMicros',
265
        'timeToFirstFrameRasterizedMicros',
266 267 268 269 270 271 272 273
      ]);
    });
  }
}

/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
274
  const PerfTest(
275 276 277
    this.testDirectory,
    this.testTarget,
    this.timelineFileName, {
278
    this.needsMeasureCpuGpu = false,
279 280 281 282
    this.testDriver,
  });

  /// The directory where the app under test is defined.
283
  final String testDirectory;
284
  /// The main entry-point file of the application, as run on the device.
285
  final String testTarget;
286
  // The prefix name of the filename such as `<timelineFileName>.timeline_summary.json`.
287
  final String timelineFileName;
288 289 290
  /// The test file to run on the host.
  final String testDriver;
  /// Whether to collect CPU and GPU metrics.
291
  final bool needsMeasureCpuGpu;
292

293
  Future<TaskResult> run() {
294 295 296 297 298 299 300 301 302 303
    return internalRun();
  }

  @protected
  Future<TaskResult> internalRun({
      bool keepRunning = false,
      bool cacheSkSL = false,
      String existingApp,
      String writeSkslFileName,
  }) {
304
    return inDirectory<TaskResult>(testDirectory, () async {
305
      final Device device = await devices.workingDevice;
306
      await device.unlock();
307
      final String deviceId = device.deviceId;
308
      await flutter('packages', options: <String>['get']);
309 310 311 312 313 314 315

      await flutter('drive', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // Enables "endless" timeline event buffering.
        '-t',
        testTarget,
316
        if (testDriver != null)
317 318 319 320 321 322 323
          ...<String>['--driver', testDriver],
        if (existingApp != null)
          ...<String>['--use-existing-app', existingApp],
        if (writeSkslFileName != null)
          ...<String>['--write-sksl-on-exit', writeSkslFileName],
        if (cacheSkSL) '--cache-sksl',
        if (keepRunning) '--keep-app-running',
324 325 326
        '-d',
        deviceId,
      ]);
327 328 329
      final Map<String, dynamic> data = json.decode(
        file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync(),
      ) as Map<String, dynamic>;
330

331
      if (data['frame_count'] as int < 5) {
332
        return TaskResult.failure(
333 334 335 336 337
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

338
      if (needsMeasureCpuGpu) {
339 340 341 342 343
        await inDirectory<void>('$testDirectory/build', () async {
          data.addAll(await measureIosCpuGpu(deviceId: deviceId));
        });
      }

344
      return TaskResult.success(data, benchmarkScoreKeys: <String>[
345 346
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
347 348
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
349 350
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
351 352
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
353 354 355
        'average_vsync_transitions_missed',
        '90th_percentile_vsync_transitions_missed',
        '99th_percentile_vsync_transitions_missed',
356 357
        if (needsMeasureCpuGpu) 'cpu_percentage',
        if (needsMeasureCpuGpu) 'gpu_percentage',
358 359 360 361 362
      ]);
    });
  }
}

363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
class PerfTestWithSkSL extends PerfTest {
  PerfTestWithSkSL(
    String testDirectory,
    String testTarget,
    String timelineFileName, {
    bool needsMeasureCpuGpu = false,
    String testDriver,
  }) : super(
    testDirectory,
    testTarget,
    timelineFileName,
    needsMeasureCpuGpu: needsMeasureCpuGpu,
    testDriver: testDriver,
  );

  @override
  Future<TaskResult> run() async {
    return inDirectory<TaskResult>(testDirectory, () async {
      // Some initializations
      _device = await devices.workingDevice;
      _flutterPath = path.join(flutterDirectory.path, 'bin', 'flutter');

      // Prepare the SkSL by running the driver test.
      await _generateSkSL();

      // Build the app with SkSL artifacts and run that app
      final String observatoryUri = await _buildAndRun();

      // Attach to the running app and run the final driver test to get metrics.
      final TaskResult result = await internalRun(
        existingApp: observatoryUri,
      );

      _runProcess.kill();
      await _runProcess.exitCode;

      return result;
    });
  }

  Future<void> _generateSkSL() async {
    await super.internalRun(
      keepRunning: true,
      cacheSkSL: true,
      writeSkslFileName: _skslJsonFileName,
    );
  }

  // Return the VMService URI.
  Future<String> _buildAndRun() async {
    await flutter('build', options: <String>[
      // TODO(liyuqian): also supports iOS once https://github.com/flutter/flutter/issues/53115 is fully closed.
      'apk',
      '--profile',
      '--bundle-sksl-path', _skslJsonFileName,
      '-t', testTarget,
    ]);

    if (File(_vmserviceFileName).existsSync()) {
      File(_vmserviceFileName).deleteSync();
    }

    _runProcess = await startProcess(
      _flutterPath,
      <String>[
        'run',
        '--verbose',
        '--profile',
        '-d', _device.deviceId,
        '-t', testTarget,
        '--endless-trace-buffer',
        '--use-application-binary',
        '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk',
        '--vmservice-out-file', _vmserviceFileName,
      ],
    );

    final Stream<List<int>> broadcastOut = _runProcess.stdout.asBroadcastStream();
    _forwardStream(broadcastOut, 'run stdout');
    _forwardStream(_runProcess.stderr, 'run stderr');

    final File file = await waitForFile(_vmserviceFileName);
    return file.readAsStringSync();
  }

  String get _skslJsonFileName => '$testDirectory/flutter_01.sksl.json';
  String get _vmserviceFileName => '$testDirectory/$_kVmserviceOutFileName';

  Stream<String> _transform(Stream<List<int>> stream) =>
      stream.transform<String>(utf8.decoder).transform<String>(const LineSplitter());

  void _forwardStream(Stream<List<int>> stream, String label) {
    _transform(stream).listen((String line) {
      print('$label: $line');
    });
  }

  String _flutterPath;
  Device _device;
  Process _runProcess;

  static const String _kVmserviceOutFileName = 'vmservice.out';
}

467 468 469 470 471 472 473
/// Measures how long it takes to compile a Flutter app to JavaScript and how
/// big the compiled code is.
class WebCompileTest {
  const WebCompileTest();

  Future<TaskResult> run() async {
    final Map<String, Object> metrics = <String, Object>{};
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490

    metrics.addAll(await runSingleBuildTest(
      directory: '${flutterDirectory.path}/examples/hello_world',
      metric: 'hello_world',
    ));

    metrics.addAll(await runSingleBuildTest(
      directory: '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
      metric: 'flutter_gallery',
    ));

    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

    rmTree(sampleDir);

    await inDirectory<void>(Directory.systemTemp, () async {
491
      await flutter('create', options: <String>['--template=app', sampleAppName]);
492
    });
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509

    metrics.addAll(await runSingleBuildTest(
      directory: sampleDir.path,
      metric: 'basic_material_app',
    ));

    return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
  }

  /// Run a single web compile test and return its metrics.
  ///
  /// Run a single web compile test for the app under [directory], and store
  /// its metrics with prefix [metric].
  static Future<Map<String, int>> runSingleBuildTest({String directory, String metric, bool measureBuildTime = false}) {
    return inDirectory<Map<String, int>>(directory, () async {
      final Map<String, int> metrics = <String, int>{};

510
      await flutter('packages', options: <String>['get']);
511 512
      final Stopwatch watch = measureBuildTime ? Stopwatch() : null;
      watch?.start();
513 514 515 516 517
      await evalFlutter('build', options: <String>[
        'web',
        '-v',
        '--release',
        '--no-pub',
518
      ]);
519 520 521
      watch?.stop();
      final String outputFileName = path.join(directory, 'build/web/main.dart.js');
      metrics.addAll(await getSize(outputFileName, metric: metric));
522

523 524 525
      if (measureBuildTime) {
        metrics['${metric}_dart2js_millis'] = watch.elapsedMilliseconds;
      }
526

527
      return metrics;
528 529 530
    });
  }

531 532 533 534 535 536 537 538 539 540 541 542
  /// Obtains the size and gzipped size of a file given by [fileName].
  static Future<Map<String, int>> getSize(String fileName, {String metric}) async {
    final Map<String, int> sizeMetrics = <String, int>{};

    final ProcessResult result = await Process.run('du', <String>['-k', fileName]);
    sizeMetrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String);

    await Process.run('gzip',<String>['-k', '9', fileName]);
    final ProcessResult resultGzip = await Process.run('du', <String>['-k', fileName + '.gz']);
    sizeMetrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String);

    return sizeMetrics;
543 544 545 546 547 548 549
  }

  static int _parseDu(String source) {
    return int.parse(source.split(RegExp(r'\s+')).first.trim());
  }
}

550
/// Measures how long it takes to compile a Flutter app and how big the compiled
551
/// code is.
552
class CompileTest {
553
  const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });
554 555

  final String testDirectory;
556
  final bool reportPackageContentSizes;
557

558
  Future<TaskResult> run() async {
559
    return await inDirectory<TaskResult>(testDirectory, () async {
560
      final Device device = await devices.workingDevice;
561
      await device.unlock();
562
      await flutter('packages', options: <String>['get']);
563

564 565 566 567
      final Map<String, dynamic> metrics = <String, dynamic>{
        ...await _compileApp(reportPackageContentSizes: reportPackageContentSizes),
        ...await _compileDebug(),
      };
568

569
      return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
570 571
    });
  }
572

573
  static Future<Map<String, dynamic>> _compileApp({ bool reportPackageContentSizes = false }) async {
574
    await flutter('clean');
575
    final Stopwatch watch = Stopwatch();
576 577
    int releaseSizeInBytes;
    final List<String> options = <String>['--release'];
578 579
    final Map<String, dynamic> metrics = <String, dynamic>{};

Ian Hickson's avatar
Ian Hickson committed
580 581 582
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
583 584
        options.add('--tree-shake-icons');
        options.add('--split-debug-info=infos/');
Ian Hickson's avatar
Ian Hickson committed
585 586 587
        watch.start();
        await flutter('build', options: options);
        watch.stop();
588 589 590 591 592 593 594 595 596
        final Directory appBuildDirectory = dir(path.join(cwd, 'build/ios/Release-iphoneos'));
        final Directory appBundle = appBuildDirectory
            .listSync()
            .whereType<Directory>()
            .singleWhere((Directory directory) => path.extension(directory.path) == '.app', orElse: () => null);
        if (appBundle == null) {
          throw 'Failed to find app bundle in ${appBuildDirectory.path}';
        }
        final String appPath =  appBundle.path;
597
        // IPAs are created manually, https://flutter.dev/ios-release/
598
        await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
Ian Hickson's avatar
Ian Hickson committed
599
        releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
600 601
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromIosApp(appPath));
Ian Hickson's avatar
Ian Hickson committed
602 603 604
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
605
        options.add('--target-platform=android-arm');
606
        options.add('--tree-shake-icons');
607
        options.add('--split-debug-info=infos/');
Ian Hickson's avatar
Ian Hickson committed
608 609 610
        watch.start();
        await flutter('build', options: options);
        watch.stop();
611 612
        final String apkPath = '$cwd/build/app/outputs/flutter-apk/app-release.apk';
        final File apk = file(apkPath);
613
        releaseSizeInBytes = apk.lengthSync();
614 615
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromApk(apkPath));
Ian Hickson's avatar
Ian Hickson committed
616
        break;
617 618
      case DeviceOperatingSystem.fuchsia:
        throw Exception('Unsupported option for Fuchsia devices');
619
    }
620

621
    metrics.addAll(<String, dynamic>{
622 623
      'release_full_compile_millis': watch.elapsedMilliseconds,
      'release_size_bytes': releaseSizeInBytes,
624 625 626
    });

    return metrics;
627 628
  }

629
  static Future<Map<String, dynamic>> _compileDebug() async {
630
    await flutter('clean');
631
    final Stopwatch watch = Stopwatch();
Ian Hickson's avatar
Ian Hickson committed
632 633 634 635 636 637 638
    final List<String> options = <String>['--debug'];
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
639
        options.add('--target-platform=android-arm');
Ian Hickson's avatar
Ian Hickson committed
640
        break;
641 642
      case DeviceOperatingSystem.fuchsia:
        throw Exception('Unsupported option for Fuchsia devices');
643
    }
Ian Hickson's avatar
Ian Hickson committed
644 645 646
    watch.start();
    await flutter('build', options: options);
    watch.stop();
647 648

    return <String, dynamic>{
649
      'debug_full_compile_millis': watch.elapsedMilliseconds,
650 651 652
    };
  }

653 654
  static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
    // Thin the binary to only contain one architecture.
655
    final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
656 657
    await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
      'ARCHS': 'arm64',
658 659
      'WRAPPER_NAME': path.basename(appPath),
      'TARGET_BUILD_DIR': path.dirname(appPath),
660 661
    });

662 663
    final File appFramework = File(path.join(appPath, 'Frameworks', 'App.framework', 'App'));
    final File flutterFramework = File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
664 665 666 667 668 669 670

    return <String, dynamic>{
      'app_framework_uncompressed_bytes': await appFramework.length(),
      'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
    };
  }

671 672 673 674 675 676 677
  static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
    final  String output = await eval('unzip', <String>['-v', apkPath]);
    final List<String> lines = output.split('\n');
    final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

    // First three lines are header, last two lines are footer.
    for (int i = 3; i < lines.length - 2; i++) {
678
      final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]);
679 680 681 682
      fileToMetadata[entry.path] = entry;
    }

    final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
683
    final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so'];
684
    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES'];
685 686 687 688

    return <String, dynamic>{
      'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
      'libflutter_compressed_bytes': libflutter.compressedSize,
689 690
      'libapp_uncompressed_bytes': libapp.uncompressedSize,
      'libapp_compressed_bytes': libapp.compressedSize,
691 692
      'license_uncompressed_bytes': license.uncompressedSize,
      'license_compressed_bytes': license.compressedSize,
693 694
    };
  }
695
}
696

697
/// Measure application memory usage.
698
class MemoryTest {
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
  MemoryTest(this.project, this.test, this.package);

  final String project;
  final String test;
  final String package;

  /// Completes when the log line specified in the last call to
  /// [prepareForNextMessage] is seen by `adb logcat`.
  Future<void> get receivedNextMessage => _receivedNextMessage?.future;
  Completer<void> _receivedNextMessage;
  String _nextMessage;

  /// Prepares the [receivedNextMessage] future such that it will complete
  /// when `adb logcat` sees a log line with the given `message`.
  void prepareForNextMessage(String message) {
    _nextMessage = message;
715
    _receivedNextMessage = Completer<void>();
716
  }
717

718
  int get iterationCount => 10;
719

720 721
  Device get device => _device;
  Device _device;
722

723
  Future<TaskResult> run() {
724
    return inDirectory<TaskResult>(project, () async {
725 726 727 728
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
729 730 731
      await device.unlock();
      await flutter('packages', options: <String>['get']);

732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
      final StreamSubscription<String> adb = device.logcat.listen(
        (String data) {
          if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ===='))
            _receivedNextMessage.complete();
        },
      );

      for (int iteration = 0; iteration < iterationCount; iteration += 1) {
        print('running memory test iteration $iteration...');
        _startMemoryUsage = null;
        await useMemory();
        assert(_startMemoryUsage != null);
        assert(_startMemory.length == iteration + 1);
        assert(_endMemory.length == iteration + 1);
        assert(_diffMemory.length == iteration + 1);
        print('terminating...');
        await device.stop(package);
749
        await Future<void>.delayed(const Duration(milliseconds: 10));
750
      }
751

752 753
      await adb.cancel();

754 755 756
      final ListStatistics startMemoryStatistics = ListStatistics(_startMemory);
      final ListStatistics endMemoryStatistics = ListStatistics(_endMemory);
      final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory);
757

758 759 760 761 762
      final Map<String, dynamic> memoryUsage = <String, dynamic>{
        ...startMemoryStatistics.asMap('start'),
        ...endMemoryStatistics.asMap('end'),
        ...diffMemoryStatistics.asMap('diff'),
      };
763

764 765 766 767 768
      _device = null;
      _startMemory.clear();
      _endMemory.clear();
      _diffMemory.clear();

769
      return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
770 771
    });
  }
772

773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
  /// Starts the app specified by [test] on the [device].
  ///
  /// The [run] method will terminate it by its package name ([package]).
  Future<void> launchApp() async {
    prepareForNextMessage('READY');
    print('launching $project$test on device...');
    await flutter('run', options: <String>[
      '--verbose',
      '--release',
      '--no-resident',
      '-d', device.deviceId,
      test,
    ]);
    print('awaiting "ready" message...');
    await receivedNextMessage;
  }
789

790
  /// To change the behavior of the test, override this.
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
  ///
  /// Make sure to call recordStart() and recordEnd() once each in that order.
  ///
  /// By default it just launches the app, records memory usage, taps the device,
  /// awaits a DONE notification, and records memory usage again.
  Future<void> useMemory() async {
    await launchApp();
    await recordStart();

    prepareForNextMessage('DONE');
    print('tapping device...');
    await device.tap(100, 100);
    print('awaiting "done" message...');
    await receivedNextMessage;

    await recordEnd();
  }
808

809 810 811
  final List<int> _startMemory = <int>[];
  final List<int> _endMemory = <int>[];
  final List<int> _diffMemory = <int>[];
812

813
  Map<String, dynamic> _startMemoryUsage;
814

815 816 817 818 819 820
  @protected
  Future<void> recordStart() async {
    assert(_startMemoryUsage == null);
    print('snapshotting memory usage...');
    _startMemoryUsage = await device.getMemoryStats(package);
  }
821

822 823 824 825 826
  @protected
  Future<void> recordEnd() async {
    assert(_startMemoryUsage != null);
    print('snapshotting memory usage...');
    final Map<String, dynamic> endMemoryUsage = await device.getMemoryStats(package);
827 828 829
    _startMemory.add(_startMemoryUsage['total_kb'] as int);
    _endMemory.add(endMemoryUsage['total_kb'] as int);
    _diffMemory.add((endMemoryUsage['total_kb'] as int) - (_startMemoryUsage['total_kb'] as int));
830 831
  }
}
832

833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
class DevToolsMemoryTest {
  DevToolsMemoryTest(this.project, this.driverTest);

  final String project;
  final String driverTest;

  Future<TaskResult> run() {
    return inDirectory<TaskResult>(project, () async {
      _device = await devices.workingDevice;
      await _device.unlock();
      await flutter('packages', options: <String>['get']);

      await _launchApp();
      if (_observatoryUri == null) {
        return  TaskResult.failure('Observatory URI not found.');
      }

      await _launchDevTools();

      await flutter(
        'drive',
        options: <String>[
          '--use-existing-app', _observatoryUri,
          '-d', _device.deviceId,
          '--profile',
          driverTest,
        ],
      );

      _devToolsProcess.kill();
      await _devToolsProcess.exitCode;

      _runProcess.kill();
      await _runProcess.exitCode;

      final Map<String, dynamic> data = json.decode(
        file('$project/$_kJsonFileName').readAsStringSync(),
      ) as Map<String, dynamic>;
      final List<dynamic> samples = data['samples']['data'] as List<dynamic>;
      int maxRss = 0;
      int maxAdbTotal = 0;
      for (final dynamic sample in samples) {
        maxRss = math.max(maxRss, sample['rss'] as int);
        if (sample['adb_memoryInfo'] != null) {
          maxAdbTotal = math.max(maxAdbTotal, sample['adb_memoryInfo']['Total'] as int);
        }
      }
      return TaskResult.success(
          <String, dynamic>{'maxRss': maxRss, 'maxAdbTotal': maxAdbTotal},
          benchmarkScoreKeys: <String>['maxRss', 'maxAdbTotal'],
      );
    });
  }

  Future<void> _launchApp() async {
    print('launching $project$driverTest on device...');
    final String flutterPath = path.join(flutterDirectory.path, 'bin', 'flutter');
    _runProcess = await startProcess(
      flutterPath,
      <String>[
        'run',
        '--verbose',
        '--profile',
        '-d', _device.deviceId,
        driverTest,
      ],
    );

    // Listen for Observatory URI and forward stdout/stderr
    final Completer<String> observatoryUri = Completer<String>();
    _runProcess.stdout
        .transform<String>(utf8.decoder)
        .transform<String>(const LineSplitter())
        .listen((String line) {
          print('run stdout: $line');
          final RegExpMatch match = RegExp(r'An Observatory debugger and profiler on .+ is available at: ((http|//)[a-zA-Z0-9:/=_\-\.\[\]]+)').firstMatch(line);
          if (match != null) {
            observatoryUri.complete(match[1]);
            _observatoryUri = match[1];
          }
        }, onDone: () { observatoryUri.complete(null); });
    _forwardStream(_runProcess.stderr, 'run stderr');

    _observatoryUri = await observatoryUri.future;
  }

  Future<void> _launchDevTools() async {
920
    await exec(pubBin, <String>[
921 922 923 924 925 926
      'global',
      'activate',
      'devtools',
      '0.2.5',
    ]);
    _devToolsProcess = await startProcess(
927
      pubBin,
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956
      <String>[
        'global',
        'run',
        'devtools',
        '--vm-uri', _observatoryUri,
        '--profile-memory', _kJsonFileName,
      ],
    );
    _forwardStream(_devToolsProcess.stdout, 'devtools stdout');
    _forwardStream(_devToolsProcess.stderr, 'devtools stderr');
  }

  void _forwardStream(Stream<List<int>> stream, String label) {
    stream
        .transform<String>(utf8.decoder)
        .transform<String>(const LineSplitter())
        .listen((String line) {
          print('$label: $line');
        });
  }

  Device _device;
  String _observatoryUri;
  Process _runProcess;
  Process _devToolsProcess;

  static const String _kJsonFileName = 'devtools_memory.json';
}

957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972
enum ReportedDurationTestFlavor {
  debug, profile, release
}

String _reportedDurationTestToString(ReportedDurationTestFlavor flavor) {
  switch (flavor) {
    case ReportedDurationTestFlavor.debug:
      return 'debug';
    case ReportedDurationTestFlavor.profile:
      return 'profile';
    case ReportedDurationTestFlavor.release:
      return 'release';
  }
  throw ArgumentError('Unexpected value for enum $flavor');
}

973
class ReportedDurationTest {
974
  ReportedDurationTest(this.flavor, this.project, this.test, this.package, this.durationPattern);
975

976
  final ReportedDurationTestFlavor flavor;
977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
  final String project;
  final String test;
  final String package;
  final RegExp durationPattern;

  final Completer<int> durationCompleter = Completer<int>();

  int get iterationCount => 10;

  Device get device => _device;
  Device _device;

  Future<TaskResult> run() {
    return inDirectory<TaskResult>(project, () async {
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
      await device.unlock();
      await flutter('packages', options: <String>['get']);

      final StreamSubscription<String> adb = device.logcat.listen(
        (String data) {
          if (durationPattern.hasMatch(data))
            durationCompleter.complete(int.parse(durationPattern.firstMatch(data).group(1)));
        },
      );
      print('launching $project$test on device...');
      await flutter('run', options: <String>[
        '--verbose',
1007
        '--no-fast-start',
1008
        '--${_reportedDurationTestToString(flavor)}',
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
        '--no-resident',
        '-d', device.deviceId,
        test,
      ]);

      final int duration = await durationCompleter.future;
      print('terminating...');
      await device.stop(package);
      await adb.cancel();

      _device = null;

      final Map<String, dynamic> reportedDuration = <String, dynamic>{
1022
        'duration': duration,
1023 1024 1025 1026 1027 1028 1029 1030
      };
      _device = null;

      return TaskResult.success(reportedDuration, benchmarkScoreKeys: reportedDuration.keys.toList());
    });
  }
}

1031 1032 1033 1034 1035 1036
/// Holds simple statistics of an odd-lengthed list of integers.
class ListStatistics {
  factory ListStatistics(Iterable<int> data) {
    assert(data.isNotEmpty);
    assert(data.length % 2 == 1);
    final List<int> sortedData = data.toList()..sort();
1037
    return ListStatistics._(
1038 1039 1040 1041 1042
      sortedData.first,
      sortedData.last,
      sortedData[(sortedData.length - 1) ~/ 2],
    );
  }
1043

1044
  const ListStatistics._(this.min, this.max, this.median);
1045

1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
  final int min;
  final int max;
  final int median;

  Map<String, int> asMap(String prefix) {
    return <String, int>{
      '$prefix-min': min,
      '$prefix-max': max,
      '$prefix-median': median,
    };
1056 1057
  }
}
1058 1059 1060

class _UnzipListEntry {
  factory _UnzipListEntry.fromLine(String line) {
1061
    final List<String> data = line.trim().split(RegExp(r'\s+'));
1062
    assert(data.length == 8);
1063
    return _UnzipListEntry._(
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
      uncompressedSize:  int.parse(data[0]),
      compressedSize: int.parse(data[2]),
      path: data[7],
    );
  }

  _UnzipListEntry._({
    @required this.uncompressedSize,
    @required this.compressedSize,
    @required this.path,
  }) : assert(uncompressedSize != null),
       assert(compressedSize != null),
       assert(compressedSize <= uncompressedSize),
       assert(path != null);

  final int uncompressedSize;
  final int compressedSize;
  final String path;
}