perf_tests.dart 12.4 KB
Newer Older
1 2 3 4 5 6 7 8 9
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert' show JSON;

import '../framework/adb.dart';
import '../framework/framework.dart';
10
import '../framework/ios.dart';
11 12
import '../framework/utils.dart';

13
TaskFunction createComplexLayoutScrollPerfTest() {
14 15 16 17 18 19 20
  return new PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
  );
}

21
TaskFunction createComplexLayoutScrollMemoryTest() {
22 23 24
  return new MemoryTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'com.yourcompany.complexLayout',
25
    testTarget: 'test_driver/scroll_perf.dart',
26 27 28
  );
}

29
TaskFunction createFlutterGalleryStartupTest() {
30 31 32 33 34
  return new StartupTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
  );
}

35
TaskFunction createComplexLayoutStartupTest() {
36 37 38 39 40 41 42 43 44 45 46 47 48
  return new StartupTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
  );
}

TaskFunction createFlutterGalleryBuildTest() {
  return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery');
}

TaskFunction createComplexLayoutBuildTest() {
  return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout');
}

49 50 51
TaskFunction createHelloWorldMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/examples/hello_world',
52
    'io.flutter.examples.hello_world',
53 54 55
  );
}

56 57 58 59 60 61 62 63
TaskFunction createGalleryNavigationMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'io.flutter.examples.gallery',
    testTarget: 'test_driver/memory_nav.dart',
  );
}

64 65 66 67
TaskFunction createGalleryBackButtonMemoryTest() {
  return new AndroidBackButtonMemoryTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'io.flutter.examples.gallery',
68
    'io.flutter.examples.gallery.MainActivity',
69 70 71
  );
}

72 73 74 75
TaskFunction createFlutterViewStartupTest() {
  return new StartupTest(
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
76 77 78
      // This project has a non-standard CocoaPods Podfile. Run pod install
      // before building the project.
      runPodInstall: true,
79 80 81
  );
}

82 83
/// Measure application startup performance.
class StartupTest {
84
  static const Duration _startupTimeout = const Duration(minutes: 5);
85

86
  const StartupTest(this.testDirectory, { this.reportMetrics: true, this.runPodInstall: false });
87 88

  final String testDirectory;
89
  final bool reportMetrics;
90 91 92 93
  /// Used to trigger a `pod install` when the project has a custom Podfile and
  /// flutter build ios won't automatically run `pod install` via the managed
  /// plugin system.
  final bool runPodInstall;
94 95 96

  Future<TaskResult> call() async {
    return await inDirectory(testDirectory, () async {
97
      final String deviceId = (await devices.workingDevice).deviceId;
98
      await flutter('packages', options: <String>['get']);
99

100
      if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
101 102
        if (runPodInstall)
          await runPodInstallForCustomPodfile(testDirectory);
103
        await prepareProvisioningCertificates(testDirectory);
104 105 106 107 108
        // This causes an Xcode project to be created.
        await flutter('build', options: <String>['ios', '--profile']);
      }

      await flutter('run', options: <String>[
109
        '--verbose',
110 111 112 113 114
        '--profile',
        '--trace-startup',
        '-d',
        deviceId,
      ]).timeout(_startupTimeout);
115
      final Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
116

117 118
      if (!reportMetrics)
        return new TaskResult.success(data);
119

120 121 122 123 124 125 126 127 128 129 130
      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
        'timeToFirstFrameMicros',
      ]);
    });
  }
}

/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {

131
  const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName);
132 133 134 135 136 137 138

  final String testDirectory;
  final String testTarget;
  final String timelineFileName;

  Future<TaskResult> call() {
    return inDirectory(testDirectory, () async {
139
      final Device device = await devices.workingDevice;
140
      await device.unlock();
141
      final String deviceId = device.deviceId;
142
      await flutter('packages', options: <String>['get']);
143

144
      if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
145
        await prepareProvisioningCertificates(testDirectory);
146 147 148 149 150 151 152 153 154 155 156 157 158
        // This causes an Xcode project to be created.
        await flutter('build', options: <String>['ios', '--profile']);
      }

      await flutter('drive', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // Enables "endless" timeline event buffering.
        '-t',
        testTarget,
        '-d',
        deviceId,
      ]);
159
      final Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
160 161 162 163 164 165 166 167

      if (data['frame_count'] < 5) {
        return new TaskResult.failure(
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

168 169 170 171
      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        'missed_frame_build_budget_count',
172 173 174
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        'missed_frame_rasterizer_budget_count',
175 176 177 178 179
      ]);
    });
  }
}

180 181
/// Measures how long it takes to build a Flutter app and how big the compiled
/// code is.
182 183
class BuildTest {

184
  const BuildTest(this.testDirectory);
185 186 187 188 189

  final String testDirectory;

  Future<TaskResult> call() async {
    return await inDirectory(testDirectory, () async {
190
      final Device device = await devices.workingDevice;
191
      await device.unlock();
192
      await flutter('packages', options: <String>['get']);
193

194 195
      final Map<String, dynamic> aotResults = await _buildAot();
      final Map<String, dynamic> debugResults = await _buildDebug();
196

197 198 199
      final Map<String, dynamic> metrics = <String, dynamic>{}
        ..addAll(aotResults)
        ..addAll(debugResults);
200

201
      return new TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
202 203
    });
  }
204

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
  static Future<Map<String, dynamic>> _buildAot() async {
    await flutter('build', options: <String>['clean']);
    final Stopwatch watch = new Stopwatch()..start();
    final String buildLog = await evalFlutter('build', options: <String>[
      'aot',
      '-v',
      '--release',
      '--no-pub',
      '--target-platform', 'android-arm'  // Generate blobs instead of assembly.
    ]);
    watch.stop();

    final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
    final Map<String, dynamic> metrics = new Map<String, dynamic>.fromIterable(
      metricExpression.allMatches(buildLog),
      key: (Match m) => _sdkNameToMetricName(m.group(1)),
      value: (Match m) => int.parse(m.group(2)),
    );
    metrics['aot_snapshot_build_millis'] = watch.elapsedMilliseconds;

    return metrics;
  }

  static Future<Map<String, dynamic>> _buildDebug() async {
    await flutter('build', options: <String>['clean']);

    final Stopwatch watch = new Stopwatch();
    if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      await prepareProvisioningCertificates(cwd);
      watch.start();
      await flutter('build', options: <String>['ios', '--debug']);
      watch.stop();
    } else {
      watch.start();
      await flutter('build', options: <String>['apk', '--debug']);
      watch.stop();
    }

    return <String, dynamic>{
      'debug_full_build_millis': watch.elapsedMilliseconds,
    };
  }

248 249 250 251 252 253 254 255 256 257 258 259 260 261
  static String _sdkNameToMetricName(String sdkName) {
    const Map<String, String> kSdkNameToMetricNameMapping = const <String, String> {
      'VMIsolate': 'aot_snapshot_size_vmisolate',
      'Isolate': 'aot_snapshot_size_isolate',
      'ReadOnlyData': 'aot_snapshot_size_rodata',
      'Instructions': 'aot_snapshot_size_instructions',
      'Total': 'aot_snapshot_size_total',
    };

    if (!kSdkNameToMetricNameMapping.containsKey(sdkName))
      throw 'Unrecognized SDK snapshot metric name: $sdkName';

    return kSdkNameToMetricNameMapping[sdkName];
  }
262
}
263

264
/// Measure application memory usage.
265
class MemoryTest {
266
  const MemoryTest(this.testDirectory, this.packageName, { this.testTarget });
267 268 269 270

  final String testDirectory;
  final String packageName;

271 272 273 274 275
  /// Path to a flutter driver script that will run after starting the app.
  ///
  /// If not specified, then the test will start the app, gather statistics, and then exit.
  final String testTarget;

276 277
  Future<TaskResult> call() {
    return inDirectory(testDirectory, () async {
278
      final Device device = await devices.workingDevice;
279
      await device.unlock();
280
      final String deviceId = device.deviceId;
281 282
      await flutter('packages', options: <String>['get']);

283
      if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
284
        await prepareProvisioningCertificates(testDirectory);
285 286 287 288
        // This causes an Xcode project to be created.
        await flutter('build', options: <String>['ios', '--profile']);
      }

289
      final int observatoryPort = await findAvailablePort();
290

291
      final List<String> runOptions = <String>[
292 293 294 295 296
        '-v',
        '--profile',
        '--trace-startup', // wait for the first frame to render
        '-d',
        deviceId,
297 298
        '--observatory-port',
        observatoryPort.toString(),
299 300 301 302
      ];
      if (testTarget != null)
        runOptions.addAll(<String>['-t', testTarget]);
      await flutter('run', options: runOptions);
303

304
      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);
305

306
      final Map<String, dynamic> data = <String, dynamic>{
307 308 309
         'start_total_kb': startData['total_kb'],
      };

310 311 312 313 314 315 316
      if (testTarget != null) {
        await flutter('drive', options: <String>[
          '-v',
          '-t',
          testTarget,
          '-d',
          deviceId,
317 318
          '--use-existing-app=http://localhost:$observatoryPort',
        ]);
319

320
        final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
321 322 323 324
        data['end_total_kb'] = endData['total_kb'];
        data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
      }

325 326
      await device.stop(packageName);

327
      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
328 329 330
    });
  }
}
331 332 333 334

/// Measure application memory usage after pausing and resuming the app
/// with the Android back button.
class AndroidBackButtonMemoryTest {
335 336
  const AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName);

337 338
  final String testDirectory;
  final String packageName;
339
  final String activityName;
340 341 342 343 344 345 346

  Future<TaskResult> call() {
    return inDirectory(testDirectory, () async {
      if (deviceOperatingSystem != DeviceOperatingSystem.android) {
        throw 'This test is only supported on Android';
      }

347
      final AndroidDevice device = await devices.workingDevice;
348
      await device.unlock();
349
      final String deviceId = device.deviceId;
350 351 352 353 354 355 356 357 358 359
      await flutter('packages', options: <String>['get']);

      await flutter('run', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // wait for the first frame to render
        '-d',
        deviceId,
      ]);

360
      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);
361

362
      final Map<String, dynamic> data = <String, dynamic>{
363 364 365 366 367
         'start_total_kb': startData['total_kb'],
      };

      // Perform a series of back button suspend and resume cycles.
      for (int i = 0; i < 10; i++) {
368
        await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
369
        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
370
        final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
371 372 373
        print(output);
        if (output.contains('Error'))
          return new TaskResult.failure('unable to launch activity');
374
        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
375 376
      }

377
      final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
378 379 380 381 382 383 384 385 386
      data['end_total_kb'] = endData['total_kb'];
      data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];

      await device.stop(packageName);

      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
    });
  }
}