perf_tests.dart 55.6 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6
import 'dart:convert' show LineSplitter, json, utf8;
7
import 'dart:io';
8
import 'dart:math' as math;
9

10 11
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
12

13
import '../common.dart';
14 15 16 17 18
import '../framework/devices.dart';
import '../framework/framework.dart';
import '../framework/host_agent.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
19

20 21 22
/// Must match flutter_driver/lib/src/common.dart.
///
/// Redefined here to avoid taking a dependency on flutter_driver.
Dan Field's avatar
Dan Field committed
23 24 25
String _testOutputDirectory(String testDirectory) {
  return Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '$testDirectory/build';
}
26

27
TaskFunction createComplexLayoutScrollPerfTest({bool measureCpuGpu = true}) {
28
  return PerfTest(
29 30 31
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
32
    measureCpuGpu: measureCpuGpu,
33
  ).run;
34 35
}

36
TaskFunction createTilesScrollPerfTest() {
37
  return PerfTest(
38 39 40 41 42 43
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'tiles_scroll_perf',
  ).run;
}

44
TaskFunction createUiKitViewScrollPerfTest() {
45 46
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
47 48 49
    'test_driver/uikit_view_scroll_perf.dart',
    'platform_views_scroll_perf',
    testDriver: 'test_driver/scroll_perf_test.dart',
50
    needsFullTimeline: false,
51 52 53 54 55 56
  ).run;
}

TaskFunction createAndroidTextureScrollPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
57
    'test_driver/android_view_scroll_perf.dart',
58
    'platform_views_scroll_perf',
59 60 61 62 63 64
    testDriver: 'test_driver/scroll_perf_test.dart',
  ).run;
}

TaskFunction createAndroidViewScrollPerfTest() {
  return PerfTest(
65
    '${flutterDirectory.path}/dev/benchmarks/platform_views_layout_hybrid_composition',
66
    'test_driver/android_view_scroll_perf.dart',
67
    'platform_views_scroll_perf_hybrid_composition',
68
    testDriver: 'test_driver/scroll_perf_test.dart',
69 70 71
  ).run;
}

72 73
TaskFunction createHomeScrollPerfTest() {
  return PerfTest(
74
    '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
75 76 77 78 79
    'test_driver/scroll_perf.dart',
    'home_scroll_perf',
  ).run;
}

80 81 82
TaskFunction createCullOpacityPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
83
    'test_driver/run_app.dart',
84
    'cull_opacity_perf',
85
    testDriver: 'test_driver/cull_opacity_perf_test.dart',
86 87 88
  ).run;
}

89
TaskFunction createCullOpacityPerfE2ETest() {
90
  return PerfTest.e2e(
91 92 93 94 95
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/cull_opacity_perf_e2e.dart',
  ).run;
}

96 97 98
TaskFunction createCubicBezierPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
99
    'test_driver/run_app.dart',
100
    'cubic_bezier_perf',
101
    testDriver: 'test_driver/cubic_bezier_perf_test.dart',
102
  ).run;
103 104
}

105 106 107 108 109 110 111 112 113 114 115 116 117 118
TaskFunction createCubicBezierPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/cubic_bezier_perf_e2e.dart',
  ).run;
}

TaskFunction createCubicBezierPerfSkSlWarmupE2ETest() {
  return PerfTestWithSkSL.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/cubic_bezier_perf_e2e.dart',
  ).run;
}

119 120 121
TaskFunction createCubicBezierPerfSkSLWarmupTest() {
  return PerfTestWithSkSL(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
122
    'test_driver/run_app.dart',
123
    'cubic_bezier_perf',
124
    testDriver: 'test_driver/cubic_bezier_perf_test.dart',
125 126 127
  ).run;
}

128 129 130 131 132 133 134 135
TaskFunction createFlutterGalleryTransitionsPerfSkSLWarmupTest() {
  return PerfTestWithSkSL(
    '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
    'test_driver/transitions_perf.dart',
    'transitions',
  ).run;
}

136 137 138 139 140 141 142 143
TaskFunction createFlutterGalleryTransitionsPerfSkSLWarmupE2ETest() {
  return PerfTestWithSkSL.e2e(
    '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
    'test_driver/transitions_perf_e2e.dart',
    testDriver: 'test_driver/transitions_perf_e2e_test.dart',
  ).run;
}

144
TaskFunction createBackdropFilterPerfTest({bool measureCpuGpu = true}) {
145 146
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
147
    'test_driver/run_app.dart',
148
    'backdrop_filter_perf',
149
    measureCpuGpu: measureCpuGpu,
150
    testDriver: 'test_driver/backdrop_filter_perf_test.dart',
151
    saveTraceFile: true,
152 153 154
  ).run;
}

155 156 157 158 159 160 161 162 163 164 165
TaskFunction createAnimationWithMicrotasksPerfTest({bool measureCpuGpu = true}) {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/run_app.dart',
    'animation_with_microtasks_perf',
    measureCpuGpu: measureCpuGpu,
    testDriver: 'test_driver/animation_with_microtasks_perf_test.dart',
    saveTraceFile: true,
  ).run;
}

166 167 168 169 170 171 172
TaskFunction createBackdropFilterPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/backdrop_filter_perf_e2e.dart',
  ).run;
}

173
TaskFunction createPostBackdropFilterPerfTest({bool measureCpuGpu = true}) {
174 175
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
176
    'test_driver/run_app.dart',
177
    'post_backdrop_filter_perf',
178
    measureCpuGpu: measureCpuGpu,
179
    testDriver: 'test_driver/post_backdrop_filter_perf_test.dart',
180
    saveTraceFile: true,
181 182 183
  ).run;
}

184
TaskFunction createSimpleAnimationPerfTest({bool measureCpuGpu = true}) {
185 186
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
187
    'test_driver/run_app.dart',
188
    'simple_animation_perf',
189
    measureCpuGpu: measureCpuGpu,
190
    testDriver: 'test_driver/simple_animation_perf_test.dart',
191
    saveTraceFile: true,
192
  ).run;
193 194
}

195 196 197 198 199 200 201
TaskFunction createAnimatedPlaceholderPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/animated_placeholder_perf_e2e.dart',
  ).run;
}

202 203 204
TaskFunction createPictureCachePerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
205
    'test_driver/run_app.dart',
206
    'picture_cache_perf',
207
    testDriver: 'test_driver/picture_cache_perf_test.dart',
208 209 210
  ).run;
}

211
TaskFunction createPictureCachePerfE2ETest() {
212
  return PerfTest.e2e(
213 214 215 216 217
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/picture_cache_perf_e2e.dart',
  ).run;
}

218
TaskFunction createFlutterGalleryStartupTest({String target = 'lib/main.dart'}) {
219
  return StartupTest(
220
    '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
221
    target: target,
222
  ).run;
223 224
}

225
TaskFunction createComplexLayoutStartupTest() {
226
  return StartupTest(
227
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
228
  ).run;
229 230
}

231
TaskFunction createFlutterGalleryCompileTest() {
232
  return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run;
233 234
}

235
TaskFunction createHelloWorldCompileTest() {
236
  return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
237 238
}

239 240 241 242
TaskFunction createWebCompileTest() {
  return const WebCompileTest().run;
}

243
TaskFunction createComplexLayoutCompileTest() {
244
  return CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
245 246
}

247
TaskFunction createFlutterViewStartupTest() {
248
  return StartupTest(
249 250
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
251 252 253
  ).run;
}

254
TaskFunction createPlatformViewStartupTest() {
255
  return StartupTest(
256 257 258 259 260
    '${flutterDirectory.path}/examples/platform_view',
    reportMetrics: false,
  ).run;
}

261 262 263 264 265
TaskFunction createBasicMaterialCompileTest() {
  return () async {
    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

266
    rmTree(sampleDir);
267

268
    await inDirectory<void>(Directory.systemTemp, () async {
269
      await flutter('create', options: <String>['--template=app', sampleAppName]);
270 271
    });

272
    if (!sampleDir.existsSync())
273 274
      throw 'Failed to create default Flutter app in ${sampleDir.path}';

275
    return CompileTest(sampleDir.path).run();
276
  };
277 278
}

279 280 281
TaskFunction createTextfieldPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
282
    'test_driver/run_app.dart',
283
    'textfield_perf',
284
    testDriver: 'test_driver/textfield_perf_test.dart',
285 286 287
  ).run;
}

288 289 290 291 292 293 294
TaskFunction createTextfieldPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/textfield_perf_e2e.dart',
  ).run;
}

295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
TaskFunction createStackSizeTest() {
  final String testDirectory =
      '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';
  const String testTarget = 'test_driver/run_app.dart';
  const String testDriver = 'test_driver/stack_size_perf_test.dart';
  return () {
    return inDirectory<TaskResult>(testDirectory, () async {
      final Device device = await devices.workingDevice;
      await device.unlock();
      final String deviceId = device.deviceId;
      await flutter('packages', options: <String>['get']);

      await flutter('drive', options: <String>[
        '--no-android-gradle-daemon',
        '-v',
        '--verbose-system-logs',
        '--profile',
        '-t', testTarget,
        '--driver', testDriver,
        '-d',
        deviceId,
      ]);
      final Map<String, dynamic> data = json.decode(
Dan Field's avatar
Dan Field committed
318
        file('${_testOutputDirectory(testDirectory)}/stack_size.json').readAsStringSync(),
319 320 321 322 323 324 325 326 327 328 329 330 331
      ) as Map<String, dynamic>;

      final Map<String, dynamic> result = <String, dynamic>{
        'stack_size_per_nesting_level': data['stack_size'],
      };
      return TaskResult.success(
        result,
        benchmarkScoreKeys: result.keys.toList(),
      );
    });
  };
}

332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
TaskFunction createFullscreenTextfieldPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test_driver/run_app.dart',
    'fullscreen_textfield_perf',
    testDriver: 'test_driver/fullscreen_textfield_perf_test.dart',
  ).run;
}

TaskFunction createFullscreenTextfieldPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/fullscreen_textfield_perf_e2e.dart',
  ).run;
}

348 349 350
TaskFunction createColorFilterAndFadePerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
351
    'test_driver/run_app.dart',
352
    'color_filter_and_fade_perf',
353
    testDriver: 'test_driver/color_filter_and_fade_perf_test.dart',
354
    saveTraceFile: true,
355 356 357
  ).run;
}

358 359 360 361 362 363 364
TaskFunction createColorFilterAndFadePerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/color_filter_and_fade_perf_e2e.dart',
  ).run;
}

365 366 367
TaskFunction createFadingChildAnimationPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
368
    'test_driver/run_app.dart',
369
    'fading_child_animation_perf',
370
    testDriver: 'test_driver/fading_child_animation_perf_test.dart',
371
    saveTraceFile: true,
372 373 374
  ).run;
}

375 376 377
TaskFunction createImageFilteredTransformAnimationPerfTest() {
  return PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
378
    'test_driver/run_app.dart',
379
    'imagefiltered_transform_animation_perf',
380
    testDriver: 'test_driver/imagefiltered_transform_animation_perf_test.dart',
381
    saveTraceFile: true,
382 383 384
  ).run;
}

385
TaskFunction createsMultiWidgetConstructPerfE2ETest() {
386
  return PerfTest.e2e(
387 388 389 390 391
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/multi_widget_construction_perf_e2e.dart',
  ).run;
}

Yuqian Li's avatar
Yuqian Li committed
392 393 394 395 396 397 398 399 400 401 402 403
TaskFunction createsScrollSmoothnessPerfTest() {
  final String testDirectory =
      '${flutterDirectory.path}/dev/benchmarks/complex_layout';
  const String testTarget = 'test/measure_scroll_smoothness.dart';
  return () {
    return inDirectory<TaskResult>(testDirectory, () async {
      final Device device = await devices.workingDevice;
      await device.unlock();
      final String deviceId = device.deviceId;
      await flutter('packages', options: <String>['get']);

      await flutter('drive', options: <String>[
404
        '--no-android-gradle-daemon',
Yuqian Li's avatar
Yuqian Li committed
405 406 407 408 409 410 411 412
        '-v',
        '--verbose-system-logs',
        '--profile',
        '-t', testTarget,
        '-d',
        deviceId,
      ]);
      final Map<String, dynamic> data = json.decode(
Dan Field's avatar
Dan Field committed
413
        file('${_testOutputDirectory(testDirectory)}/scroll_smoothness_test.json').readAsStringSync(),
Yuqian Li's avatar
Yuqian Li committed
414 415 416 417 418
      ) as Map<String, dynamic>;

      final Map<String, dynamic> result = <String, dynamic>{};
      void addResult(dynamic data, String suffix) {
        assert(data is Map<String, dynamic>);
419 420 421 422 423 424 425 426 427
        if (data is Map<String, dynamic>) {
          const List<String> metricKeys = <String>[
            'janky_count',
            'average_abs_jerk',
            'dropped_frame_count',
          ];
          for (final String key in metricKeys) {
            result[key + suffix] = data[key];
          }
Yuqian Li's avatar
Yuqian Li committed
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
        }
      }
      addResult(data['resample on with 90Hz input'], '_with_resampler_90Hz');
      addResult(data['resample on with 59Hz input'], '_with_resampler_59Hz');
      addResult(data['resample off with 90Hz input'], '_without_resampler_90Hz');
      addResult(data['resample off with 59Hz input'], '_without_resampler_59Hz');

      return TaskResult.success(
        result,
        benchmarkScoreKeys: result.keys.toList(),
      );
    });
  };
}

443 444 445 446 447 448 449 450 451 452 453 454
TaskFunction createFramePolicyIntegrationTest() {
  final String testDirectory =
      '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';
  const String testTarget = 'test/frame_policy.dart';
  return () {
    return inDirectory<TaskResult>(testDirectory, () async {
      final Device device = await devices.workingDevice;
      await device.unlock();
      final String deviceId = device.deviceId;
      await flutter('packages', options: <String>['get']);

      await flutter('drive', options: <String>[
455
        '--no-android-gradle-daemon',
456 457 458 459 460 461 462 463
        '-v',
        '--verbose-system-logs',
        '--profile',
        '-t', testTarget,
        '-d',
        deviceId,
      ]);
      final Map<String, dynamic> data = json.decode(
Dan Field's avatar
Dan Field committed
464
        file('${_testOutputDirectory(testDirectory)}/frame_policy_event_delay.json').readAsStringSync(),
465 466 467
      ) as Map<String, dynamic>;
      final Map<String, dynamic> fullLiveData = data['fullyLive'] as Map<String, dynamic>;
      final Map<String, dynamic> benchmarkLiveData = data['benchmarkLive'] as Map<String, dynamic>;
468
      final Map<String, dynamic> dataFormatted = <String, dynamic>{
469 470 471 472 473 474 475 476 477 478 479
        'average_delay_fullyLive_millis':
          fullLiveData['average_delay_millis'],
        'average_delay_benchmarkLive_millis':
          benchmarkLiveData['average_delay_millis'],
        '90th_percentile_delay_fullyLive_millis':
          fullLiveData['90th_percentile_delay_millis'],
        '90th_percentile_delay_benchmarkLive_millis':
          benchmarkLiveData['90th_percentile_delay_millis'],
      };

      return TaskResult.success(
480 481
        dataFormatted,
        benchmarkScoreKeys: dataFormatted.keys.toList(),
482 483 484 485 486
      );
    });
  };
}

487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
TaskFunction createOpacityPeepholeOneRectPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/opacity_peephole_one_rect_perf_e2e.dart',
  ).run;
}

TaskFunction createOpacityPeepholeColOfRowsPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/opacity_peephole_col_of_rows_perf_e2e.dart',
  ).run;
}

TaskFunction createOpacityPeepholeOpacityOfGridPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/opacity_peephole_opacity_of_grid_perf_e2e.dart',
  ).run;
}

TaskFunction createOpacityPeepholeGridOfOpacityPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/opacity_peephole_grid_of_opacity_perf_e2e.dart',
  ).run;
}

TaskFunction createOpacityPeepholeFadeTransitionTextPerfE2ETest() {
  return PerfTest.e2e(
    '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
    'test/opacity_peephole_fade_transition_text_perf_e2e.dart',
  ).run;
}

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
Map<String, dynamic> _average(List<Map<String, dynamic>> results, int iterations) {
  final Map<String, dynamic> tally = <String, dynamic>{};
  for (final Map<String, dynamic> item in results) {
    item.forEach((String key, dynamic value) {
      if (tally.containsKey(key)) {
        tally[key] = (tally[key] as int) + (value as int);
      } else {
        tally[key] = value;
      }
    });
  }
  tally.forEach((String key, dynamic value) {
    tally[key] = (value as int) ~/ iterations;
  });
  return tally;
}

539 540
/// Measure application startup performance.
class StartupTest {
541
  const StartupTest(this.testDirectory, { this.reportMetrics = true, this.target = 'lib/main.dart' });
542 543

  final String testDirectory;
544
  final bool reportMetrics;
545
  final String target;
546

547
  Future<TaskResult> run() async {
548
    return inDirectory<TaskResult>(testDirectory, () async {
549 550
      final Device device = await devices.workingDevice;
      const int iterations = 5;
551
      final List<Map<String, dynamic>> results = <Map<String, dynamic>>[];
552 553

      section('Building application');
554
      String? applicationBinaryPath;
555 556 557 558 559 560 561
      switch (deviceOperatingSystem) {
        case DeviceOperatingSystem.android:
          await flutter('build', options: <String>[
            'apk',
            '-v',
            '--profile',
            '--target-platform=android-arm,android-arm64',
562
            '--target=$target',
563 564 565
          ]);
          applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
          break;
566 567 568 569 570 571
        case DeviceOperatingSystem.androidArm:
          await flutter('build', options: <String>[
            'apk',
            '-v',
            '--profile',
            '--target-platform=android-arm',
572
            '--target=$target',
573 574 575
          ]);
          applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
          break;
576 577 578 579 580 581
        case DeviceOperatingSystem.androidArm64:
          await flutter('build', options: <String>[
            'apk',
            '-v',
            '--profile',
            '--target-platform=android-arm64',
582
            '--target=$target',
583 584 585
          ]);
          applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
          break;
586 587 588 589 590
        case DeviceOperatingSystem.ios:
          await flutter('build', options: <String>[
            'ios',
             '-v',
            '--profile',
591
            '--target=$target',
592 593 594 595 596 597 598 599
          ]);
          applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos');
          break;
        case DeviceOperatingSystem.fuchsia:
        case DeviceOperatingSystem.fake:
          break;
      }

600 601
      const int maxFailures = 3;
      int currentFailures = 0;
602
      for (int i = 0; i < iterations; i += 1) {
603
        final int result = await flutter('run', options: <String>[
604
          '--no-android-gradle-daemon',
605
          '--no-publish-port',
606 607 608
          '--verbose',
          '--profile',
          '--trace-startup',
609
          '--target=$target',
610
          '-d',
611 612 613
          device.deviceId,
          if (applicationBinaryPath != null)
            '--use-application-binary=$applicationBinaryPath',
614 615 616
         ], canFail: true);
        if (result == 0) {
          final Map<String, dynamic> data = json.decode(
Dan Field's avatar
Dan Field committed
617
            file('${_testOutputDirectory(testDirectory)}/start_up_info.json').readAsStringSync(),
618 619 620 621
          ) as Map<String, dynamic>;
          results.add(data);
        } else {
          currentFailures += 1;
622 623 624 625 626 627 628
          if (hostAgent.dumpDirectory != null) {
            await flutter(
              'screenshot',
              options: <String>[
                '-d',
                device.deviceId,
                '--out',
629
                hostAgent.dumpDirectory!
630 631 632 633 634 635
                    .childFile('screenshot_startup_failure_$currentFailures.png')
                    .path,
              ],
              canFail: true,
            );
          }
636 637 638 639 640
          i -= 1;
          if (currentFailures == maxFailures) {
            return TaskResult.failure('Application failed to start $maxFailures times');
          }
        }
641 642 643 644 645 646

        await flutter('install', options: <String>[
          '--uninstall-only',
          '-d',
          device.deviceId,
        ]);
647 648 649
      }

      final Map<String, dynamic> averageResults = _average(results, iterations);
650

651
      if (!reportMetrics)
652
        return TaskResult.success(averageResults);
653

654
      return TaskResult.success(averageResults, benchmarkScoreKeys: <String>[
655
        'timeToFirstFrameMicros',
656
        'timeToFirstFrameRasterizedMicros',
657 658 659 660 661
      ]);
    });
  }
}

662 663 664 665 666 667 668 669 670 671 672
/// A one-off test to verify that devtools starts in profile mode.
class DevtoolsStartupTest {
  const DevtoolsStartupTest(this.testDirectory);

  final String testDirectory;

  Future<TaskResult> run() async {
    return inDirectory<TaskResult>(testDirectory, () async {
      final Device device = await devices.workingDevice;

      section('Building application');
673
      String? applicationBinaryPath;
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
      switch (deviceOperatingSystem) {
        case DeviceOperatingSystem.android:
          await flutter('build', options: <String>[
            'apk',
            '-v',
            '--profile',
            '--target-platform=android-arm,android-arm64',
          ]);
          applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
          break;
        case DeviceOperatingSystem.androidArm:
          await flutter('build', options: <String>[
            'apk',
            '-v',
            '--profile',
            '--target-platform=android-arm',
          ]);
          applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
          break;
        case DeviceOperatingSystem.androidArm64:
          await flutter('build', options: <String>[
            'apk',
            '-v',
            '--profile',
            '--target-platform=android-arm64',
          ]);
          applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
          break;
        case DeviceOperatingSystem.ios:
          await flutter('build', options: <String>[
            'ios',
             '-v',
            '--profile',
          ]);
          applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos');
          break;
        case DeviceOperatingSystem.fuchsia:
        case DeviceOperatingSystem.fake:
          break;
      }

      final Process process = await startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), <String>[
        'run',
        '--no-android-gradle-daemon',
        '--no-publish-port',
        '--verbose',
        '--profile',
        '-d',
        device.deviceId,
        if (applicationBinaryPath != null)
          '--use-application-binary=$applicationBinaryPath',
       ]);
      final Completer<void> completer = Completer<void>();
      bool sawLine = false;
      process.stdout
        .transform(utf8.decoder)
        .transform(const LineSplitter())
        .listen((String line) {
          print('[STDOUT]: $line');
        // Wait for devtools output.
        if (line.contains('The Flutter DevTools debugger and profiler')) {
          sawLine = true;
          completer.complete();
        }
      });
      bool didExit = false;
      unawaited(process.exitCode.whenComplete(() {
        didExit = true;
      }));
      await Future.any(<Future<void>>[completer.future, Future<void>.delayed(const Duration(minutes: 5)), process.exitCode]);
      if (!didExit) {
        process.stdin.writeln('q');
        await process.exitCode;
      }

      await flutter('install', options: <String>[
        '--uninstall-only',
        '-d',
        device.deviceId,
      ]);

      if (sawLine)
        return TaskResult.success(null, benchmarkScoreKeys: <String>[]);
      return TaskResult.failure('Did not see line "The Flutter DevTools debugger and profiler" in output');
    });
  }
}

762 763 764
/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
765
  const PerfTest(
766 767 768
    this.testDirectory,
    this.testTarget,
    this.timelineFileName, {
769
    this.measureCpuGpu = true,
770
    this.measureMemory = true,
771
    this.saveTraceFile = false,
772
    this.testDriver,
773 774
    this.needsFullTimeline = true,
    this.benchmarkScoreKeys,
775
    this.dartDefine = '',
776
    String? resultFilename,
777 778 779 780 781
  }): _resultFilename = resultFilename;

  const PerfTest.e2e(
    this.testDirectory,
    this.testTarget, {
782 783
    this.measureCpuGpu = false,
    this.measureMemory = false,
784 785 786 787 788 789
    this.testDriver =  'test_driver/e2e_test.dart',
    this.needsFullTimeline = false,
    this.benchmarkScoreKeys = _kCommonScoreKeys,
    this.dartDefine = '',
    String resultFilename = 'e2e_perf_summary',
  }) : saveTraceFile = false, timelineFileName = null, _resultFilename = resultFilename;
790 791

  /// The directory where the app under test is defined.
792
  final String testDirectory;
793
  /// The main entry-point file of the application, as run on the device.
794
  final String testTarget;
795
  // The prefix name of the filename such as `<timelineFileName>.timeline_summary.json`.
796
  final String? timelineFileName;
797
  String get traceFilename => '$timelineFileName.timeline';
798
  String get resultFilename => _resultFilename ?? '$timelineFileName.timeline_summary';
799
  final String? _resultFilename;
800
  /// The test file to run on the host.
801
  final String? testDriver;
802
  /// Whether to collect CPU and GPU metrics.
803
  final bool measureCpuGpu;
804 805
  /// Whether to collect memory metrics.
  final bool measureMemory;
806 807
  /// Whether to collect full timeline, meaning if `--trace-startup` flag is needed.
  final bool needsFullTimeline;
808 809
  /// Whether to save the trace timeline file `*.timeline.json`.
  final bool saveTraceFile;
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826

  /// The keys of the values that need to be reported.
  ///
  /// If it's `null`, then report:
  /// ```Dart
  /// <String>[
  ///   'average_frame_build_time_millis',
  ///   'worst_frame_build_time_millis',
  ///   '90th_percentile_frame_build_time_millis',
  ///   '99th_percentile_frame_build_time_millis',
  ///   'average_frame_rasterizer_time_millis',
  ///   'worst_frame_rasterizer_time_millis',
  ///   '90th_percentile_frame_rasterizer_time_millis',
  ///   '99th_percentile_frame_rasterizer_time_millis',
  ///   'average_vsync_transitions_missed',
  ///   '90th_percentile_vsync_transitions_missed',
  ///   '99th_percentile_vsync_transitions_missed',
827 828
  ///   if (measureCpuGpu) 'average_cpu_usage',
  ///   if (measureCpuGpu) 'average_gpu_usage',
829 830
  /// ]
  /// ```
831
  final List<String>? benchmarkScoreKeys;
832

833 834 835
  /// Additional flags for `--dart-define` to control the test
  final String dartDefine;

836
  Future<TaskResult> run() {
837 838 839 840 841 842
    return internalRun();
  }

  @protected
  Future<TaskResult> internalRun({
      bool cacheSkSL = false,
843 844
      String? existingApp,
      String? writeSkslFileName,
845
  }) {
846
    return inDirectory<TaskResult>(testDirectory, () async {
847
      final Device device = await devices.workingDevice;
848
      await device.unlock();
849
      final String deviceId = device.deviceId;
850 851

      await flutter('drive', options: <String>[
852 853 854 855
        if (localEngine != null)
          ...<String>['--local-engine', localEngine!],
        if (localEngineSrcPath != null)
          ...<String>['--local-engine-src-path', localEngineSrcPath!],
856
        '--no-dds',
857
        '--no-android-gradle-daemon',
858
        '-v',
859
        '--verbose-system-logs',
860
        '--profile',
861 862
        if (needsFullTimeline)
          '--trace-startup', // Enables "endless" timeline event buffering.
863
        '-t', testTarget,
864
        if (testDriver != null)
865
          ...<String>['--driver', testDriver!],
866 867 868 869 870
        if (existingApp != null)
          ...<String>['--use-existing-app', existingApp],
        if (writeSkslFileName != null)
          ...<String>['--write-sksl-on-exit', writeSkslFileName],
        if (cacheSkSL) '--cache-sksl',
871 872
        if (dartDefine.isNotEmpty)
          ...<String>['--dart-define', dartDefine],
873 874 875
        '-d',
        deviceId,
      ]);
876
      final Map<String, dynamic> data = json.decode(
Dan Field's avatar
Dan Field committed
877
        file('${_testOutputDirectory(testDirectory)}/$resultFilename.json').readAsStringSync(),
878
      ) as Map<String, dynamic>;
879

880
      if (data['frame_count'] as int < 5) {
881
        return TaskResult.failure(
882 883 884 885 886
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

887 888 889
      // TODO(liyuqian): Remove isAndroid restriction once
      // https://github.com/flutter/flutter/issues/61567 is fixed.
      final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
890 891
      return TaskResult.success(
        data,
892 893
        detailFiles: <String>[
          if (saveTraceFile)
Dan Field's avatar
Dan Field committed
894
            '${_testOutputDirectory(testDirectory)}/$traceFilename.json',
895
        ],
896
        benchmarkScoreKeys: benchmarkScoreKeys ?? <String>[
897
          ..._kCommonScoreKeys,
898 899 900
          'average_vsync_transitions_missed',
          '90th_percentile_vsync_transitions_missed',
          '99th_percentile_vsync_transitions_missed',
901
          if (measureCpuGpu && !isAndroid) ...<String>[
902 903 904
            // See https://github.com/flutter/flutter/issues/68888
            if (data['average_cpu_usage'] != null) 'average_cpu_usage',
            if (data['average_gpu_usage'] != null) 'average_gpu_usage',
905 906
          ],
          if (measureMemory && !isAndroid) ...<String>[
907 908
            // See https://github.com/flutter/flutter/issues/68888
            if (data['average_memory_usage'] != null) 'average_memory_usage',
909 910
            if (data['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
            if (data['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
911
          ],
912 913
        ],
      );
914 915 916 917
    });
  }
}

918 919 920 921 922 923 924 925 926
const List<String> _kCommonScoreKeys = <String>[
  'average_frame_build_time_millis',
  'worst_frame_build_time_millis',
  '90th_percentile_frame_build_time_millis',
  '99th_percentile_frame_build_time_millis',
  'average_frame_rasterizer_time_millis',
  'worst_frame_rasterizer_time_millis',
  '90th_percentile_frame_rasterizer_time_millis',
  '99th_percentile_frame_rasterizer_time_millis',
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
  'average_layer_cache_count',
  '90th_percentile_layer_cache_count',
  '99th_percentile_layer_cache_count',
  'worst_layer_cache_count',
  'average_layer_cache_memory',
  '90th_percentile_layer_cache_memory',
  '99th_percentile_layer_cache_memory',
  'worst_layer_cache_memory',
  'average_picture_cache_count',
  '90th_percentile_picture_cache_count',
  '99th_percentile_picture_cache_count',
  'worst_picture_cache_count',
  'average_picture_cache_memory',
  '90th_percentile_picture_cache_memory',
  '99th_percentile_picture_cache_memory',
  'worst_picture_cache_memory',
943 944
  'new_gen_gc_count',
  'old_gen_gc_count',
945
];
946

947 948 949 950 951
class PerfTestWithSkSL extends PerfTest {
  PerfTestWithSkSL(
    String testDirectory,
    String testTarget,
    String timelineFileName, {
952
    bool measureCpuGpu = false,
953
    String? testDriver,
954
    bool needsFullTimeline = true,
955
    List<String>? benchmarkScoreKeys,
956 957 958 959
  }) : super(
    testDirectory,
    testTarget,
    timelineFileName,
960
    measureCpuGpu: measureCpuGpu,
961
    testDriver: testDriver,
962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
    needsFullTimeline: needsFullTimeline,
    benchmarkScoreKeys: benchmarkScoreKeys,
  );


  PerfTestWithSkSL.e2e(
    String testDirectory,
    String testTarget, {
    String testDriver =  'test_driver/e2e_test.dart',
    String resultFilename = 'e2e_perf_summary',
  }) : super.e2e(
    testDirectory,
    testTarget,
    testDriver: testDriver,
    needsFullTimeline: false,
    resultFilename: resultFilename,
978 979 980 981 982 983 984 985 986 987 988 989 990
  );

  @override
  Future<TaskResult> run() async {
    return inDirectory<TaskResult>(testDirectory, () async {
      // Some initializations
      _device = await devices.workingDevice;
      _flutterPath = path.join(flutterDirectory.path, 'bin', 'flutter');

      // Prepare the SkSL by running the driver test.
      await _generateSkSL();

      // Build the app with SkSL artifacts and run that app
991
      final String observatoryUri = await _runApp(skslPath: _skslJsonFileName);
992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005

      // Attach to the running app and run the final driver test to get metrics.
      final TaskResult result = await internalRun(
        existingApp: observatoryUri,
      );

      _runProcess.kill();
      await _runProcess.exitCode;

      return result;
    });
  }

  Future<void> _generateSkSL() async {
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
    // `flutter drive` without `flutter run`, and `flutter drive --existing-app`
    // with `flutter run` may generate different SkSLs. Hence we run both
    // versions to generate as many SkSLs as possible.
    //
    // 1st, `flutter drive --existing-app` with `flutter run`. The
    // `--write-sksl-on-exit` option doesn't seem to be compatible with
    // `flutter drive --existing-app` as it will complain web socket connection
    // issues.
    final String observatoryUri = await _runApp(cacheSkSL: true);
    await super.internalRun(cacheSkSL: true, existingApp: observatoryUri);
    _runProcess.kill();
    await _runProcess.exitCode;

    // 2nd, `flutter drive` without `flutter run`. The --no-build option ensures
    // that we won't remove the SkSLs generated earlier.
1021 1022 1023 1024 1025 1026
    await super.internalRun(
      cacheSkSL: true,
      writeSkslFileName: _skslJsonFileName,
    );
  }

1027
  Future<String> _runApp({String? appBinary, bool cacheSkSL = false, String? skslPath}) async {
1028 1029 1030 1031 1032 1033 1034 1035
    if (File(_vmserviceFileName).existsSync()) {
      File(_vmserviceFileName).deleteSync();
    }

    _runProcess = await startProcess(
      _flutterPath,
      <String>[
        'run',
1036 1037 1038 1039
        if (localEngine != null)
          ...<String>['--local-engine', localEngine!],
        if (localEngineSrcPath != null)
          ...<String>['--local-engine-src-path', localEngineSrcPath!],
1040
        '--no-dds',
1041 1042
        if (deviceOperatingSystem == DeviceOperatingSystem.ios)
          ...<String>[
1043
            '--device-timeout', '5',
1044
          ],
1045
        '--verbose',
1046
        '--verbose-system-logs',
1047
        '--purge-persistent-cache',
1048
        '--no-publish-port',
1049
        '--profile',
1050
        if (skslPath != null) '--bundle-sksl-path=$skslPath',
1051
        if (cacheSkSL) '--cache-sksl',
1052 1053 1054
        '-d', _device.deviceId,
        '-t', testTarget,
        '--endless-trace-buffer',
1055
        if (appBinary != null) ...<String>['--use-application-binary', _appBinary],
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
        '--vmservice-out-file', _vmserviceFileName,
      ],
    );

    final Stream<List<int>> broadcastOut = _runProcess.stdout.asBroadcastStream();
    _forwardStream(broadcastOut, 'run stdout');
    _forwardStream(_runProcess.stderr, 'run stderr');

    final File file = await waitForFile(_vmserviceFileName);
    return file.readAsStringSync();
  }

  String get _skslJsonFileName => '$testDirectory/flutter_01.sksl.json';
  String get _vmserviceFileName => '$testDirectory/$_kVmserviceOutFileName';

1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
  bool get _isAndroid => deviceOperatingSystem == DeviceOperatingSystem.android;

  String get _appBinary {
    if (_isAndroid) {
      return '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
    }
    for (final FileSystemEntity entry in Directory('$testDirectory/build/ios/iphoneos/').listSync()) {
      if (entry.path.endsWith('.app')) {
        return entry.path;
      }
    }
    throw 'No app found.';
  }

1085 1086 1087 1088 1089 1090 1091 1092 1093
  Stream<String> _transform(Stream<List<int>> stream) =>
      stream.transform<String>(utf8.decoder).transform<String>(const LineSplitter());

  void _forwardStream(Stream<List<int>> stream, String label) {
    _transform(stream).listen((String line) {
      print('$label: $line');
    });
  }

1094 1095 1096
  late String _flutterPath;
  late Device _device;
  late Process _runProcess;
1097 1098 1099 1100

  static const String _kVmserviceOutFileName = 'vmservice.out';
}

1101 1102 1103 1104 1105 1106 1107
/// Measures how long it takes to compile a Flutter app to JavaScript and how
/// big the compiled code is.
class WebCompileTest {
  const WebCompileTest();

  Future<TaskResult> run() async {
    final Map<String, Object> metrics = <String, Object>{};
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124

    metrics.addAll(await runSingleBuildTest(
      directory: '${flutterDirectory.path}/examples/hello_world',
      metric: 'hello_world',
    ));

    metrics.addAll(await runSingleBuildTest(
      directory: '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
      metric: 'flutter_gallery',
    ));

    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

    rmTree(sampleDir);

    await inDirectory<void>(Directory.systemTemp, () async {
1125
      await flutter('create', options: <String>['--template=app', sampleAppName]);
1126
    });
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139

    metrics.addAll(await runSingleBuildTest(
      directory: sampleDir.path,
      metric: 'basic_material_app',
    ));

    return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
  }

  /// Run a single web compile test and return its metrics.
  ///
  /// Run a single web compile test for the app under [directory], and store
  /// its metrics with prefix [metric].
1140 1141 1142 1143 1144
  static Future<Map<String, int>> runSingleBuildTest({
    required String directory,
    required String metric,
    bool measureBuildTime = false,
  }) {
1145 1146 1147
    return inDirectory<Map<String, int>>(directory, () async {
      final Map<String, int> metrics = <String, int>{};

1148
      await flutter('packages', options: <String>['get']);
1149
      final Stopwatch? watch = measureBuildTime ? Stopwatch() : null;
1150
      watch?.start();
1151 1152 1153 1154 1155
      await evalFlutter('build', options: <String>[
        'web',
        '-v',
        '--release',
        '--no-pub',
1156
      ]);
1157 1158 1159
      watch?.stop();
      final String outputFileName = path.join(directory, 'build/web/main.dart.js');
      metrics.addAll(await getSize(outputFileName, metric: metric));
1160

1161
      if (measureBuildTime) {
1162
        metrics['${metric}_dart2js_millis'] = watch!.elapsedMilliseconds;
1163
      }
1164

1165
      return metrics;
1166 1167 1168
    });
  }

1169
  /// Obtains the size and gzipped size of a file given by [fileName].
1170
  static Future<Map<String, int>> getSize(String fileName, {required String metric}) async {
1171 1172 1173 1174 1175 1176
    final Map<String, int> sizeMetrics = <String, int>{};

    final ProcessResult result = await Process.run('du', <String>['-k', fileName]);
    sizeMetrics['${metric}_dart2js_size'] = _parseDu(result.stdout as String);

    await Process.run('gzip',<String>['-k', '9', fileName]);
1177
    final ProcessResult resultGzip = await Process.run('du', <String>['-k', '$fileName.gz']);
1178 1179 1180
    sizeMetrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout as String);

    return sizeMetrics;
1181 1182 1183 1184 1185 1186 1187
  }

  static int _parseDu(String source) {
    return int.parse(source.split(RegExp(r'\s+')).first.trim());
  }
}

1188
/// Measures how long it takes to compile a Flutter app and how big the compiled
1189
/// code is.
1190
class CompileTest {
1191
  const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });
1192 1193

  final String testDirectory;
1194
  final bool reportPackageContentSizes;
1195

1196
  Future<TaskResult> run() async {
1197
    return inDirectory<TaskResult>(testDirectory, () async {
1198
      final Device device = await devices.workingDevice;
1199
      await device.unlock();
1200
      await flutter('packages', options: <String>['get']);
1201

1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
      final Map<String, dynamic> compileRelease = await _compileApp(reportPackageContentSizes: reportPackageContentSizes);
      final Map<String, dynamic> compileDebug = await _compileDebug(
        clean: true,
        metricKey: 'debug_full_compile_millis',
      );
      // Build again without cleaning, should be faster.
      final Map<String, dynamic> compileSecondDebug = await _compileDebug(
        clean: false,
        metricKey: 'debug_second_compile_millis',
      );

1213
      final Map<String, dynamic> metrics = <String, dynamic>{
1214 1215 1216
        ...compileRelease,
        ...compileDebug,
        ...compileSecondDebug,
1217
      };
1218

1219
      return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
1220 1221
    });
  }
1222

1223
  static Future<Map<String, dynamic>> _compileApp({ bool reportPackageContentSizes = false }) async {
1224
    await flutter('clean');
1225
    final Stopwatch watch = Stopwatch();
1226 1227
    int releaseSizeInBytes;
    final List<String> options = <String>['--release'];
1228 1229
    final Map<String, dynamic> metrics = <String, dynamic>{};

Ian Hickson's avatar
Ian Hickson committed
1230 1231 1232
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
1233 1234
        options.add('--tree-shake-icons');
        options.add('--split-debug-info=infos/');
Ian Hickson's avatar
Ian Hickson committed
1235 1236 1237
        watch.start();
        await flutter('build', options: options);
        watch.stop();
1238
        final Directory appBuildDirectory = dir(path.join(cwd, 'build/ios/Release-iphoneos'));
1239
        final Directory? appBundle = appBuildDirectory
1240
            .listSync()
1241 1242 1243 1244
            .whereType<Directory?>()
            .singleWhere((Directory? directory) =>
              directory != null && path.extension(directory.path) == '.app',
              orElse: () => null);
1245 1246 1247 1248
        if (appBundle == null) {
          throw 'Failed to find app bundle in ${appBuildDirectory.path}';
        }
        final String appPath =  appBundle.path;
1249
        // IPAs are created manually, https://flutter.dev/ios-release/
1250
        await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
Ian Hickson's avatar
Ian Hickson committed
1251
        releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
1252 1253
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromIosApp(appPath));
Ian Hickson's avatar
Ian Hickson committed
1254 1255
        break;
      case DeviceOperatingSystem.android:
1256
      case DeviceOperatingSystem.androidArm:
Ian Hickson's avatar
Ian Hickson committed
1257
        options.insert(0, 'apk');
1258
        options.add('--target-platform=android-arm');
1259
        options.add('--tree-shake-icons');
1260
        options.add('--split-debug-info=infos/');
Ian Hickson's avatar
Ian Hickson committed
1261 1262 1263
        watch.start();
        await flutter('build', options: options);
        watch.stop();
1264 1265
        final String apkPath = '$cwd/build/app/outputs/flutter-apk/app-release.apk';
        final File apk = file(apkPath);
1266
        releaseSizeInBytes = apk.lengthSync();
1267 1268
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromApk(apkPath));
Ian Hickson's avatar
Ian Hickson committed
1269
        break;
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
      case DeviceOperatingSystem.androidArm64:
        options.insert(0, 'apk');
        options.add('--target-platform=android-arm64');
        options.add('--tree-shake-icons');
        options.add('--split-debug-info=infos/');
        watch.start();
        await flutter('build', options: options);
        watch.stop();
        final String apkPath = '$cwd/build/app/outputs/flutter-apk/app-release.apk';
        final File apk = file(apkPath);
        releaseSizeInBytes = apk.lengthSync();
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromApk(apkPath));
        break;
1284 1285
      case DeviceOperatingSystem.fuchsia:
        throw Exception('Unsupported option for Fuchsia devices');
1286 1287
      case DeviceOperatingSystem.fake:
        throw Exception('Unsupported option for fake devices');
1288
    }
1289

1290
    metrics.addAll(<String, dynamic>{
1291 1292
      'release_full_compile_millis': watch.elapsedMilliseconds,
      'release_size_bytes': releaseSizeInBytes,
1293 1294 1295
    });

    return metrics;
1296 1297
  }

1298
  static Future<Map<String, dynamic>> _compileDebug({
1299 1300
    required bool clean,
    required String metricKey,
1301 1302 1303 1304
  }) async {
    if (clean) {
      await flutter('clean');
    }
1305
    final Stopwatch watch = Stopwatch();
Ian Hickson's avatar
Ian Hickson committed
1306 1307 1308 1309 1310 1311
    final List<String> options = <String>['--debug'];
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        break;
      case DeviceOperatingSystem.android:
1312
      case DeviceOperatingSystem.androidArm:
Ian Hickson's avatar
Ian Hickson committed
1313
        options.insert(0, 'apk');
1314
        options.add('--target-platform=android-arm');
Ian Hickson's avatar
Ian Hickson committed
1315
        break;
1316 1317 1318 1319
      case DeviceOperatingSystem.androidArm64:
        options.insert(0, 'apk');
        options.add('--target-platform=android-arm64');
        break;
1320 1321
      case DeviceOperatingSystem.fuchsia:
        throw Exception('Unsupported option for Fuchsia devices');
1322 1323
      case DeviceOperatingSystem.fake:
        throw Exception('Unsupported option for fake devices');
1324
    }
Ian Hickson's avatar
Ian Hickson committed
1325 1326 1327
    watch.start();
    await flutter('build', options: options);
    watch.stop();
1328 1329

    return <String, dynamic>{
1330
      metricKey: watch.elapsedMilliseconds,
1331 1332 1333
    };
  }

1334 1335
  static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
    // Thin the binary to only contain one architecture.
1336
    final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
1337 1338
    await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
      'ARCHS': 'arm64',
1339 1340
      'WRAPPER_NAME': path.basename(appPath),
      'TARGET_BUILD_DIR': path.dirname(appPath),
1341 1342
    });

1343 1344
    final File appFramework = File(path.join(appPath, 'Frameworks', 'App.framework', 'App'));
    final File flutterFramework = File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
1345 1346 1347 1348 1349 1350 1351

    return <String, dynamic>{
      'app_framework_uncompressed_bytes': await appFramework.length(),
      'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
    };
  }

1352 1353 1354 1355 1356 1357 1358
  static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
    final  String output = await eval('unzip', <String>['-v', apkPath]);
    final List<String> lines = output.split('\n');
    final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

    // First three lines are header, last two lines are footer.
    for (int i = 3; i < lines.length - 2; i++) {
1359
      final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]);
1360 1361 1362
      fileToMetadata[entry.path] = entry;
    }

1363 1364 1365
    final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so']!;
    final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so']!;
    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES.Z']!;
1366 1367 1368 1369

    return <String, dynamic>{
      'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
      'libflutter_compressed_bytes': libflutter.compressedSize,
1370 1371
      'libapp_uncompressed_bytes': libapp.uncompressedSize,
      'libapp_compressed_bytes': libapp.compressedSize,
1372 1373
      'license_uncompressed_bytes': license.uncompressedSize,
      'license_compressed_bytes': license.compressedSize,
1374 1375
    };
  }
1376
}
1377

1378
/// Measure application memory usage.
1379
class MemoryTest {
1380 1381 1382 1383 1384 1385 1386 1387
  MemoryTest(this.project, this.test, this.package);

  final String project;
  final String test;
  final String package;

  /// Completes when the log line specified in the last call to
  /// [prepareForNextMessage] is seen by `adb logcat`.
1388 1389 1390
  Future<void>? get receivedNextMessage => _receivedNextMessage?.future;
  Completer<void>? _receivedNextMessage;
  String? _nextMessage;
1391 1392 1393 1394 1395

  /// Prepares the [receivedNextMessage] future such that it will complete
  /// when `adb logcat` sees a log line with the given `message`.
  void prepareForNextMessage(String message) {
    _nextMessage = message;
1396
    _receivedNextMessage = Completer<void>();
1397
  }
1398

1399
  int get iterationCount => 10;
1400

1401 1402
  Device? get device => _device;
  Device? _device;
1403

1404
  Future<TaskResult> run() {
1405
    return inDirectory<TaskResult>(project, () async {
1406 1407 1408 1409
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
1410
      await device!.unlock();
1411 1412
      await flutter('packages', options: <String>['get']);

1413
      final StreamSubscription<String> adb = device!.logcat.listen(
1414 1415
        (String data) {
          if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ===='))
1416
            _receivedNextMessage?.complete();
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
        },
      );

      for (int iteration = 0; iteration < iterationCount; iteration += 1) {
        print('running memory test iteration $iteration...');
        _startMemoryUsage = null;
        await useMemory();
        assert(_startMemoryUsage != null);
        assert(_startMemory.length == iteration + 1);
        assert(_endMemory.length == iteration + 1);
        assert(_diffMemory.length == iteration + 1);
        print('terminating...');
1429
        await device!.stop(package);
1430
        await Future<void>.delayed(const Duration(milliseconds: 10));
1431
      }
1432

1433
      await adb.cancel();
1434
      await flutter('install', options: <String>['--uninstall-only', '-d', device!.deviceId]);
1435

1436 1437 1438
      final ListStatistics startMemoryStatistics = ListStatistics(_startMemory);
      final ListStatistics endMemoryStatistics = ListStatistics(_endMemory);
      final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory);
1439

1440 1441 1442 1443 1444
      final Map<String, dynamic> memoryUsage = <String, dynamic>{
        ...startMemoryStatistics.asMap('start'),
        ...endMemoryStatistics.asMap('end'),
        ...diffMemoryStatistics.asMap('diff'),
      };
1445

1446 1447 1448 1449 1450
      _device = null;
      _startMemory.clear();
      _endMemory.clear();
      _diffMemory.clear();

1451
      return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
1452 1453
    });
  }
1454

1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
  /// Starts the app specified by [test] on the [device].
  ///
  /// The [run] method will terminate it by its package name ([package]).
  Future<void> launchApp() async {
    prepareForNextMessage('READY');
    print('launching $project$test on device...');
    await flutter('run', options: <String>[
      '--verbose',
      '--release',
      '--no-resident',
1465
      '-d', device!.deviceId,
1466 1467 1468 1469 1470
      test,
    ]);
    print('awaiting "ready" message...');
    await receivedNextMessage;
  }
1471

1472
  /// To change the behavior of the test, override this.
1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
  ///
  /// Make sure to call recordStart() and recordEnd() once each in that order.
  ///
  /// By default it just launches the app, records memory usage, taps the device,
  /// awaits a DONE notification, and records memory usage again.
  Future<void> useMemory() async {
    await launchApp();
    await recordStart();

    prepareForNextMessage('DONE');
    print('tapping device...');
1484
    await device!.tap(100, 100);
1485 1486 1487 1488 1489
    print('awaiting "done" message...');
    await receivedNextMessage;

    await recordEnd();
  }
1490

1491 1492 1493
  final List<int> _startMemory = <int>[];
  final List<int> _endMemory = <int>[];
  final List<int> _diffMemory = <int>[];
1494

1495
  Map<String, dynamic>? _startMemoryUsage;
1496

1497 1498 1499 1500
  @protected
  Future<void> recordStart() async {
    assert(_startMemoryUsage == null);
    print('snapshotting memory usage...');
1501
    _startMemoryUsage = await device!.getMemoryStats(package);
1502
  }
1503

1504 1505 1506 1507
  @protected
  Future<void> recordEnd() async {
    assert(_startMemoryUsage != null);
    print('snapshotting memory usage...');
1508 1509
    final Map<String, dynamic> endMemoryUsage = await device!.getMemoryStats(package);
    _startMemory.add(_startMemoryUsage!['total_kb'] as int);
1510
    _endMemory.add(endMemoryUsage['total_kb'] as int);
1511
    _diffMemory.add((endMemoryUsage['total_kb'] as int) - (_startMemoryUsage!['total_kb'] as int));
1512 1513
  }
}
1514

1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530
class DevToolsMemoryTest {
  DevToolsMemoryTest(this.project, this.driverTest);

  final String project;
  final String driverTest;

  Future<TaskResult> run() {
    return inDirectory<TaskResult>(project, () async {
      _device = await devices.workingDevice;
      await _device.unlock();

      await flutter(
        'drive',
        options: <String>[
          '-d', _device.deviceId,
          '--profile',
1531 1532 1533
          '--profile-memory', _kJsonFileName,
          '--no-publish-port',
          '-v',
1534 1535 1536 1537 1538 1539 1540
          driverTest,
        ],
      );

      final Map<String, dynamic> data = json.decode(
        file('$project/$_kJsonFileName').readAsStringSync(),
      ) as Map<String, dynamic>;
1541
      final List<dynamic> samples = (data['samples'] as Map<String, dynamic>)['data'] as List<dynamic>;
1542 1543
      int maxRss = 0;
      int maxAdbTotal = 0;
1544
      for (final Map<String, dynamic> sample in samples.cast<Map<String, dynamic>>()) {
1545 1546 1547
        if (sample['rss'] != null) {
          maxRss = math.max(maxRss, sample['rss'] as int);
        }
1548
        if (sample['adb_memoryInfo'] != null) {
1549
          maxAdbTotal = math.max(maxAdbTotal, (sample['adb_memoryInfo'] as Map<String, dynamic>)['Total'] as int);
1550 1551
        }
      }
1552

1553
      return TaskResult.success(
1554 1555
        <String, dynamic>{'maxRss': maxRss, 'maxAdbTotal': maxAdbTotal},
        benchmarkScoreKeys: <String>['maxRss', 'maxAdbTotal'],
1556 1557 1558 1559
      );
    });
  }

1560
  late Device _device;
1561 1562 1563 1564

  static const String _kJsonFileName = 'devtools_memory.json';
}

1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
enum ReportedDurationTestFlavor {
  debug, profile, release
}

String _reportedDurationTestToString(ReportedDurationTestFlavor flavor) {
  switch (flavor) {
    case ReportedDurationTestFlavor.debug:
      return 'debug';
    case ReportedDurationTestFlavor.profile:
      return 'profile';
    case ReportedDurationTestFlavor.release:
      return 'release';
  }
}

1580
class ReportedDurationTest {
1581
  ReportedDurationTest(this.flavor, this.project, this.test, this.package, this.durationPattern);
1582

1583
  final ReportedDurationTestFlavor flavor;
1584 1585 1586 1587 1588 1589 1590 1591 1592
  final String project;
  final String test;
  final String package;
  final RegExp durationPattern;

  final Completer<int> durationCompleter = Completer<int>();

  int get iterationCount => 10;

1593 1594
  Device? get device => _device;
  Device? _device;
1595 1596 1597 1598 1599 1600 1601

  Future<TaskResult> run() {
    return inDirectory<TaskResult>(project, () async {
      // This test currently only works on Android, because device.logcat,
      // device.getMemoryStats, etc, aren't implemented for iOS.

      _device = await devices.workingDevice;
1602
      await device!.unlock();
1603 1604
      await flutter('packages', options: <String>['get']);

1605
      final StreamSubscription<String> adb = device!.logcat.listen(
1606 1607
        (String data) {
          if (durationPattern.hasMatch(data))
1608
            durationCompleter.complete(int.parse(durationPattern.firstMatch(data)!.group(1)!));
1609 1610 1611 1612 1613
        },
      );
      print('launching $project$test on device...');
      await flutter('run', options: <String>[
        '--verbose',
1614
        '--no-publish-port',
1615
        '--no-fast-start',
1616
        '--${_reportedDurationTestToString(flavor)}',
1617
        '--no-resident',
1618
        '-d', device!.deviceId,
1619 1620 1621 1622 1623
        test,
      ]);

      final int duration = await durationCompleter.future;
      print('terminating...');
1624
      await device!.stop(package);
1625 1626 1627 1628 1629
      await adb.cancel();

      _device = null;

      final Map<String, dynamic> reportedDuration = <String, dynamic>{
1630
        'duration': duration,
1631 1632 1633 1634 1635 1636 1637 1638
      };
      _device = null;

      return TaskResult.success(reportedDuration, benchmarkScoreKeys: reportedDuration.keys.toList());
    });
  }
}

1639 1640 1641 1642
/// Holds simple statistics of an odd-lengthed list of integers.
class ListStatistics {
  factory ListStatistics(Iterable<int> data) {
    assert(data.isNotEmpty);
1643
    assert(data.length.isOdd);
1644
    final List<int> sortedData = data.toList()..sort();
1645
    return ListStatistics._(
1646 1647 1648 1649 1650
      sortedData.first,
      sortedData.last,
      sortedData[(sortedData.length - 1) ~/ 2],
    );
  }
1651

1652
  const ListStatistics._(this.min, this.max, this.median);
1653

1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
  final int min;
  final int max;
  final int median;

  Map<String, int> asMap(String prefix) {
    return <String, int>{
      '$prefix-min': min,
      '$prefix-max': max,
      '$prefix-median': median,
    };
1664 1665
  }
}
1666 1667 1668

class _UnzipListEntry {
  factory _UnzipListEntry.fromLine(String line) {
1669
    final List<String> data = line.trim().split(RegExp(r'\s+'));
1670
    assert(data.length == 8);
1671
    return _UnzipListEntry._(
1672 1673 1674 1675 1676 1677 1678
      uncompressedSize:  int.parse(data[0]),
      compressedSize: int.parse(data[2]),
      path: data[7],
    );
  }

  _UnzipListEntry._({
1679 1680 1681
    required this.uncompressedSize,
    required this.compressedSize,
    required this.path,
1682 1683 1684 1685 1686 1687 1688 1689 1690
  }) : assert(uncompressedSize != null),
       assert(compressedSize != null),
       assert(compressedSize <= uncompressedSize),
       assert(path != null);

  final int uncompressedSize;
  final int compressedSize;
  final String path;
}
1691

1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704
/// Wait for up to 400 seconds for the file to appear.
Future<File> waitForFile(String path) async {
  for (int i = 0; i < 20; i += 1) {
    final File file = File(path);
    print('looking for ${file.path}');
    if (file.existsSync()) {
      return file;
    }
    await Future<void>.delayed(const Duration(seconds: 20));
  }
  throw StateError('Did not find vmservice out file after 400 seconds');
}

1705
String? _findIosAppInBuildDirectory(String searchDirectory) {
1706 1707 1708 1709 1710 1711 1712
  for (final FileSystemEntity entity in Directory(searchDirectory).listSync()) {
    if (entity.path.endsWith('.app')) {
      return entity.path;
    }
  }
  return null;
}