perf_tests.dart 11.6 KB
Newer Older
1 2 3 4 5 6 7 8 9
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert' show JSON;

import '../framework/adb.dart';
import '../framework/framework.dart';
10
import '../framework/ios.dart';
11 12
import '../framework/utils.dart';

13
TaskFunction createComplexLayoutScrollPerfTest() {
14 15 16 17 18 19 20
  return new PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
  );
}

21
TaskFunction createComplexLayoutScrollMemoryTest() {
22 23 24
  return new MemoryTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'com.yourcompany.complexLayout',
25
    testTarget: 'test_driver/scroll_perf.dart',
26 27 28
  );
}

29
TaskFunction createFlutterGalleryStartupTest() {
30 31 32 33 34
  return new StartupTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
  );
}

35
TaskFunction createComplexLayoutStartupTest() {
36 37 38 39 40 41 42 43 44 45 46 47 48
  return new StartupTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
  );
}

TaskFunction createFlutterGalleryBuildTest() {
  return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery');
}

TaskFunction createComplexLayoutBuildTest() {
  return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout');
}

49 50 51
TaskFunction createHelloWorldMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/examples/hello_world',
52
    'io.flutter.examples.hello_world',
53 54 55
  );
}

56 57 58 59 60 61 62 63
TaskFunction createGalleryNavigationMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'io.flutter.examples.gallery',
    testTarget: 'test_driver/memory_nav.dart',
  );
}

64 65 66 67
TaskFunction createGalleryBackButtonMemoryTest() {
  return new AndroidBackButtonMemoryTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'io.flutter.examples.gallery',
68
    'io.flutter.examples.gallery.MainActivity',
69 70 71
  );
}

72 73 74 75 76 77 78
TaskFunction createFlutterViewStartupTest() {
  return new StartupTest(
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
  );
}

79 80
/// Measure application startup performance.
class StartupTest {
81
  static const Duration _startupTimeout = const Duration(minutes: 5);
82

83
  const StartupTest(this.testDirectory, { this.reportMetrics: true });
84 85

  final String testDirectory;
86
  final bool reportMetrics;
87 88 89

  Future<TaskResult> call() async {
    return await inDirectory(testDirectory, () async {
90
      final String deviceId = (await devices.workingDevice).deviceId;
91
      await flutter('packages', options: <String>['get']);
92

93
      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
94
        await prepareProvisioningCertificates(testDirectory);
95 96

      await flutter('run', options: <String>[
97
        '--verbose',
98 99 100 101 102
        '--profile',
        '--trace-startup',
        '-d',
        deviceId,
      ]).timeout(_startupTimeout);
103
      final Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
104

105 106
      if (!reportMetrics)
        return new TaskResult.success(data);
107

108 109 110 111 112 113 114 115 116 117 118
      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
        'timeToFirstFrameMicros',
      ]);
    });
  }
}

/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {

119
  const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName);
120 121 122 123 124 125 126

  final String testDirectory;
  final String testTarget;
  final String timelineFileName;

  Future<TaskResult> call() {
    return inDirectory(testDirectory, () async {
127
      final Device device = await devices.workingDevice;
128
      await device.unlock();
129
      final String deviceId = device.deviceId;
130
      await flutter('packages', options: <String>['get']);
131

132
      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
133
        await prepareProvisioningCertificates(testDirectory);
134 135 136 137 138 139 140 141 142 143

      await flutter('drive', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // Enables "endless" timeline event buffering.
        '-t',
        testTarget,
        '-d',
        deviceId,
      ]);
144
      final Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
145 146 147 148 149 150 151 152

      if (data['frame_count'] < 5) {
        return new TaskResult.failure(
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

153 154 155 156
      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        'missed_frame_build_budget_count',
157 158 159
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        'missed_frame_rasterizer_budget_count',
160 161 162 163 164
      ]);
    });
  }
}

165 166
/// Measures how long it takes to build a Flutter app and how big the compiled
/// code is.
167 168
class BuildTest {

169
  const BuildTest(this.testDirectory);
170 171 172 173 174

  final String testDirectory;

  Future<TaskResult> call() async {
    return await inDirectory(testDirectory, () async {
175
      final Device device = await devices.workingDevice;
176
      await device.unlock();
177
      await flutter('packages', options: <String>['get']);
178

179 180
      final Map<String, dynamic> aotResults = await _buildAot();
      final Map<String, dynamic> debugResults = await _buildDebug();
181

182 183 184
      final Map<String, dynamic> metrics = <String, dynamic>{}
        ..addAll(aotResults)
        ..addAll(debugResults);
185

186
      return new TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
187 188
    });
  }
189

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
  static Future<Map<String, dynamic>> _buildAot() async {
    await flutter('build', options: <String>['clean']);
    final Stopwatch watch = new Stopwatch()..start();
    final String buildLog = await evalFlutter('build', options: <String>[
      'aot',
      '-v',
      '--release',
      '--no-pub',
      '--target-platform', 'android-arm'  // Generate blobs instead of assembly.
    ]);
    watch.stop();

    final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
    final Map<String, dynamic> metrics = new Map<String, dynamic>.fromIterable(
      metricExpression.allMatches(buildLog),
      key: (Match m) => _sdkNameToMetricName(m.group(1)),
      value: (Match m) => int.parse(m.group(2)),
    );
    metrics['aot_snapshot_build_millis'] = watch.elapsedMilliseconds;

    return metrics;
  }

  static Future<Map<String, dynamic>> _buildDebug() async {
    await flutter('build', options: <String>['clean']);

    final Stopwatch watch = new Stopwatch();
    if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      await prepareProvisioningCertificates(cwd);
      watch.start();
      await flutter('build', options: <String>['ios', '--debug']);
      watch.stop();
    } else {
      watch.start();
      await flutter('build', options: <String>['apk', '--debug']);
      watch.stop();
    }

    return <String, dynamic>{
      'debug_full_build_millis': watch.elapsedMilliseconds,
    };
  }

233 234 235 236 237 238 239 240 241 242 243 244 245 246
  static String _sdkNameToMetricName(String sdkName) {
    const Map<String, String> kSdkNameToMetricNameMapping = const <String, String> {
      'VMIsolate': 'aot_snapshot_size_vmisolate',
      'Isolate': 'aot_snapshot_size_isolate',
      'ReadOnlyData': 'aot_snapshot_size_rodata',
      'Instructions': 'aot_snapshot_size_instructions',
      'Total': 'aot_snapshot_size_total',
    };

    if (!kSdkNameToMetricNameMapping.containsKey(sdkName))
      throw 'Unrecognized SDK snapshot metric name: $sdkName';

    return kSdkNameToMetricNameMapping[sdkName];
  }
247
}
248

249
/// Measure application memory usage.
250
class MemoryTest {
251
  const MemoryTest(this.testDirectory, this.packageName, { this.testTarget });
252 253 254 255

  final String testDirectory;
  final String packageName;

256 257 258 259 260
  /// Path to a flutter driver script that will run after starting the app.
  ///
  /// If not specified, then the test will start the app, gather statistics, and then exit.
  final String testTarget;

261 262
  Future<TaskResult> call() {
    return inDirectory(testDirectory, () async {
263
      final Device device = await devices.workingDevice;
264
      await device.unlock();
265
      final String deviceId = device.deviceId;
266 267
      await flutter('packages', options: <String>['get']);

268
      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
269
        await prepareProvisioningCertificates(testDirectory);
270

271
      final int observatoryPort = await findAvailablePort();
272

273
      final List<String> runOptions = <String>[
274 275 276 277 278
        '-v',
        '--profile',
        '--trace-startup', // wait for the first frame to render
        '-d',
        deviceId,
279 280
        '--observatory-port',
        observatoryPort.toString(),
281 282 283 284
      ];
      if (testTarget != null)
        runOptions.addAll(<String>['-t', testTarget]);
      await flutter('run', options: runOptions);
285

286
      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);
287

288
      final Map<String, dynamic> data = <String, dynamic>{
289 290 291
         'start_total_kb': startData['total_kb'],
      };

292 293 294 295 296 297 298
      if (testTarget != null) {
        await flutter('drive', options: <String>[
          '-v',
          '-t',
          testTarget,
          '-d',
          deviceId,
299 300
          '--use-existing-app=http://localhost:$observatoryPort',
        ]);
301

302
        final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
303 304 305 306
        data['end_total_kb'] = endData['total_kb'];
        data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
      }

307 308
      await device.stop(packageName);

309
      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
310 311 312
    });
  }
}
313 314 315 316

/// Measure application memory usage after pausing and resuming the app
/// with the Android back button.
class AndroidBackButtonMemoryTest {
317 318
  const AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName);

319 320
  final String testDirectory;
  final String packageName;
321
  final String activityName;
322 323 324 325 326 327 328

  Future<TaskResult> call() {
    return inDirectory(testDirectory, () async {
      if (deviceOperatingSystem != DeviceOperatingSystem.android) {
        throw 'This test is only supported on Android';
      }

329
      final AndroidDevice device = await devices.workingDevice;
330
      await device.unlock();
331
      final String deviceId = device.deviceId;
332 333 334 335 336 337 338 339 340 341
      await flutter('packages', options: <String>['get']);

      await flutter('run', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // wait for the first frame to render
        '-d',
        deviceId,
      ]);

342
      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);
343

344
      final Map<String, dynamic> data = <String, dynamic>{
345 346 347 348 349
         'start_total_kb': startData['total_kb'],
      };

      // Perform a series of back button suspend and resume cycles.
      for (int i = 0; i < 10; i++) {
350
        await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
351
        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
352
        final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
353 354 355
        print(output);
        if (output.contains('Error'))
          return new TaskResult.failure('unable to launch activity');
356
        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
357 358
      }

359
      final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
360 361 362 363 364 365 366 367 368
      data['end_total_kb'] = endData['total_kb'];
      data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];

      await device.stop(packageName);

      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
    });
  }
}