perf_tests.dart

// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert' show json;
import 'dart:io';

import 'package:meta/meta.dart';
import 'package:path/path.dart' as p;

import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/ios.dart';
import '../framework/utils.dart';

TaskFunction createComplexLayoutScrollPerfTest() {
  return new PerfTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'test_driver/scroll_perf.dart',
    'complex_layout_scroll_perf',
  ).run;
}

TaskFunction createComplexLayoutScrollMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
    'com.yourcompany.complexLayout',
    testTarget: 'test_driver/scroll_perf.dart',
  ).run;
}

TaskFunction createFlutterGalleryStartupTest() {
  return new StartupTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
  ).run;
}

TaskFunction createComplexLayoutStartupTest() {
  return new StartupTest(
    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
  ).run;
}

TaskFunction createFlutterGalleryCompileTest() {
  return new CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run;
}

TaskFunction createHelloWorldCompileTest() {
  return new CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
}

TaskFunction createComplexLayoutCompileTest() {
  return new CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
}

TaskFunction createHelloWorldMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/examples/hello_world',
    'io.flutter.examples.hello_world',
  ).run;
}

TaskFunction createGalleryNavigationMemoryTest() {
  return new MemoryTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'io.flutter.demo.gallery',
    testTarget: 'test_driver/memory_nav.dart',
  ).run;
}

TaskFunction createGalleryBackButtonMemoryTest() {
  return new AndroidBackButtonMemoryTest(
    '${flutterDirectory.path}/examples/flutter_gallery',
    'io.flutter.demo.gallery',
    'io.flutter.demo.gallery.MainActivity',
  ).run;
}

TaskFunction createFlutterViewStartupTest() {
  return new StartupTest(
      '${flutterDirectory.path}/examples/flutter_view',
      reportMetrics: false,
  ).run;
}

TaskFunction createPlatformViewStartupTest() {
  return new StartupTest(
    '${flutterDirectory.path}/examples/platform_view',
    reportMetrics: false,
  ).run;
}

TaskFunction createBasicMaterialCompileTest() {
  return () async {
    const String sampleAppName = 'sample_flutter_app';
    final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

    if (await sampleDir.exists())
      rmTree(sampleDir);

    await inDirectory(Directory.systemTemp, () async {
      await flutter('create', options: <String>[sampleAppName]);
    });

    if (!(await sampleDir.exists()))
      throw 'Failed to create default Flutter app in ${sampleDir.path}';

    return new CompileTest(sampleDir.path).run();
  };
}


/// Measure application startup performance.
class StartupTest {
  static const Duration _startupTimeout = const Duration(minutes: 5);

  const StartupTest(this.testDirectory, { this.reportMetrics = true });

  final String testDirectory;
  final bool reportMetrics;

  Future<TaskResult> run() async {
    return await inDirectory(testDirectory, () async {
      final String deviceId = (await devices.workingDevice).deviceId;
      await flutter('packages', options: <String>['get']);

      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
        await prepareProvisioningCertificates(testDirectory);

      await flutter('run', options: <String>[
        '--verbose',
        '--profile',
        '--trace-startup',
        '-d',
        deviceId,
      ]).timeout(_startupTimeout);
      final Map<String, dynamic> data = json.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());

      if (!reportMetrics)
        return new TaskResult.success(data);

      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
        'timeToFirstFrameMicros',
      ]);
    });
  }
}

/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
  const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName);

  final String testDirectory;
  final String testTarget;
  final String timelineFileName;

  Future<TaskResult> run() {
    return inDirectory(testDirectory, () async {
      final Device device = await devices.workingDevice;
      await device.unlock();
      final String deviceId = device.deviceId;
      await flutter('packages', options: <String>['get']);

      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
        await prepareProvisioningCertificates(testDirectory);

      await flutter('drive', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // Enables "endless" timeline event buffering.
        '-t',
        testTarget,
        '-d',
        deviceId,
      ]);
      final Map<String, dynamic> data = json.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());

      if (data['frame_count'] < 5) {
        return new TaskResult.failure(
          'Timeline contains too few frames: ${data['frame_count']}. Possibly '
          'trace events are not being captured.',
        );
      }

      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        'missed_frame_build_budget_count',
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
      ]);
    });
  }
}

/// Measures how long it takes to compile a Flutter app and how big the compiled
/// code is.
class CompileTest {
  const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });

  final String testDirectory;
  final bool reportPackageContentSizes;

  Future<TaskResult> run() async {
    return await inDirectory(testDirectory, () async {
      final Device device = await devices.workingDevice;
      await device.unlock();
      await flutter('packages', options: <String>['get']);

      final Map<String, dynamic> metrics = <String, dynamic>{}
        ..addAll(await _compileAot())
        ..addAll(await _compileApp(reportPackageContentSizes: reportPackageContentSizes))
        ..addAll(await _compileDebug())
        ..addAll(_suffix(await _compileAot(previewDart2: false), '__dart1'))
        ..addAll(_suffix(await _compileApp(previewDart2: false), '__dart1'))
        ..addAll(_suffix(await _compileDebug(previewDart2: false), '__dart1'));

      return new TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
    });
  }

  static Map<String, dynamic> _suffix(Map<String, dynamic> map, String suffix) {
    return new Map<String, dynamic>.fromIterables(
      map.keys.map<String>((String key) => '$key$suffix'),
      map.values,
    );
  }

  static Future<Map<String, dynamic>> _compileAot({ bool previewDart2 = true }) async {
    // Generate blobs instead of assembly.
    await flutter('clean');
    final Stopwatch watch = new Stopwatch()..start();
    final List<String> options = <String>[
      'aot',
      '-v',
      '--extra-gen-snapshot-options=--print_snapshot_sizes',
      '--release',
      '--no-pub',
      '--target-platform',
    ];
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.add('ios');
        break;
      case DeviceOperatingSystem.android:
        options.add('android-arm');
        break;
    }
    if (previewDart2)
      options.add('--preview-dart-2');
    else
      options.add('--no-preview-dart-2');
    setLocalEngineOptionIfNecessary(options);
    final String compileLog = await evalFlutter('build', options: options);
    watch.stop();

    final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
    final Map<String, dynamic> metrics = <String, dynamic>{};
    for (Match m in metricExpression.allMatches(compileLog)) {
      metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2));
    }
    if (metrics.length != _kSdkNameToMetricNameMapping.length) {
      throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.';
    }
    metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds;

    return metrics;
  }

  static Future<Map<String, dynamic>> _compileApp({ bool previewDart2 = true, bool reportPackageContentSizes = false }) async {
    await flutter('clean');
    final Stopwatch watch = new Stopwatch();
    int releaseSizeInBytes;
    final List<String> options = <String>['--release'];
    if (previewDart2)
      options.add('--preview-dart-2');
    else
      options.add('--no-preview-dart-2');
    setLocalEngineOptionIfNecessary(options);
    final Map<String, dynamic> metrics = <String, dynamic>{};

    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        await prepareProvisioningCertificates(cwd);
        watch.start();
        await flutter('build', options: options);
        watch.stop();
        final String appPath =  '$cwd/build/ios/Release-iphoneos/Runner.app/';
        // IPAs are created manually, https://flutter.io/ios-release/
        await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
        releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromIosApp(appPath));
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
        watch.start();
        await flutter('build', options: options);
        watch.stop();
        String apkPath = '$cwd/build/app/outputs/apk/app.apk';
        File apk = file(apkPath);
        if (!apk.existsSync()) {
          // Pre Android SDK 26 path
          apkPath = '$cwd/build/app/outputs/apk/app-release.apk';
          apk = file(apkPath);
        }
        releaseSizeInBytes = apk.lengthSync();
        if (reportPackageContentSizes)
          metrics.addAll(await getSizesFromApk(apkPath));
        break;
    }

    metrics.addAll(<String, dynamic>{
      'release_full_compile_millis': watch.elapsedMilliseconds,
      'release_size_bytes': releaseSizeInBytes,
    });

    return metrics;
  }

  static Future<Map<String, dynamic>> _compileDebug({ bool previewDart2 = true }) async {
    await flutter('clean');
    final Stopwatch watch = new Stopwatch();
    final List<String> options = <String>['--debug'];
    if (previewDart2)
      options.add('--preview-dart-2');
    else
      options.add('--no-preview-dart-2');
    setLocalEngineOptionIfNecessary(options);
    switch (deviceOperatingSystem) {
      case DeviceOperatingSystem.ios:
        options.insert(0, 'ios');
        await prepareProvisioningCertificates(cwd);
        break;
      case DeviceOperatingSystem.android:
        options.insert(0, 'apk');
        break;
    }
    watch.start();
    await flutter('build', options: options);
    watch.stop();

    return <String, dynamic>{
      'debug_full_compile_millis': watch.elapsedMilliseconds,
    };
  }

  static const Map<String, String> _kSdkNameToMetricNameMapping = const <String, String> {
    'VMIsolate': 'aot_snapshot_size_vmisolate',
    'Isolate': 'aot_snapshot_size_isolate',
    'ReadOnlyData': 'aot_snapshot_size_rodata',
    'Instructions': 'aot_snapshot_size_instructions',
    'Total': 'aot_snapshot_size_total',
  };

  static String _sdkNameToMetricName(String sdkName) {

    if (!_kSdkNameToMetricNameMapping.containsKey(sdkName))
      throw 'Unrecognized SDK snapshot metric name: $sdkName';

    return _kSdkNameToMetricNameMapping[sdkName];
  }

  static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
    // Thin the binary to only contain one architecture.
    final String xcodeBackend = p.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
    await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
      'ARCHS': 'arm64',
      'WRAPPER_NAME': p.basename(appPath),
      'TARGET_BUILD_DIR': p.dirname(appPath),
    });

    final File appFramework = new File(p.join(appPath, 'Frameworks', 'App.framework', 'App'));
    final File flutterFramework = new File(p.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));

    return <String, dynamic>{
      'app_framework_uncompressed_bytes': await appFramework.length(),
      'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
    };
  }


  static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
    final  String output = await eval('unzip', <String>['-v', apkPath]);
    final List<String> lines = output.split('\n');
    final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

    // First three lines are header, last two lines are footer.
    for (int i = 3; i < lines.length - 2; i++) {
      final _UnzipListEntry entry = new _UnzipListEntry.fromLine(lines[i]);
      fileToMetadata[entry.path] = entry;
    }

    final _UnzipListEntry icudtl = fileToMetadata['assets/flutter_shared/icudtl.dat'];
    final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
    final _UnzipListEntry isolateSnapshotData = fileToMetadata['assets/isolate_snapshot_data'];
    final _UnzipListEntry isolateSnapshotInstr = fileToMetadata['assets/isolate_snapshot_instr'];
    final _UnzipListEntry vmSnapshotData = fileToMetadata['assets/vm_snapshot_data'];
    final _UnzipListEntry vmSnapshotInstr = fileToMetadata['assets/vm_snapshot_instr'];

    return <String, dynamic>{
      'icudtl_uncompressed_bytes': icudtl.uncompressedSize,
      'icudtl_compressed_bytes': icudtl.compressedSize,
      'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
      'libflutter_compressed_bytes': libflutter.compressedSize,
      'snapshot_uncompressed_bytes': isolateSnapshotData.uncompressedSize +
          isolateSnapshotInstr.uncompressedSize +
          vmSnapshotData.uncompressedSize +
          vmSnapshotInstr.uncompressedSize,
      'snapshot_compressed_bytes': isolateSnapshotData.compressedSize +
          isolateSnapshotInstr.compressedSize +
          vmSnapshotData.compressedSize +
          vmSnapshotInstr.compressedSize,
    };
  }
}

/// Measure application memory usage.
class MemoryTest {
  const MemoryTest(this.testDirectory, this.packageName, { this.testTarget });

  final String testDirectory;
  final String packageName;

  /// Path to a flutter driver script that will run after starting the app.
  ///
  /// If not specified, then the test will start the app, gather statistics, and then exit.
  final String testTarget;

  Future<TaskResult> run() {
    return inDirectory(testDirectory, () async {
      final Device device = await devices.workingDevice;
      await device.unlock();
      final String deviceId = device.deviceId;
      await flutter('packages', options: <String>['get']);

      if (deviceOperatingSystem == DeviceOperatingSystem.ios)
        await prepareProvisioningCertificates(testDirectory);

      final List<String> runOptions = <String>[
        '-v',
        '--profile',
        '--trace-startup', // wait for the first frame to render
        '-d',
        deviceId,
        '--observatory-port',
        '0',
      ];
      if (testTarget != null)
        runOptions.addAll(<String>['-t', testTarget]);
      final String output = await evalFlutter('run', options: runOptions);
      final int observatoryPort = parseServicePort(output, prefix: 'Successfully connected to service protocol: ', multiLine: true);
      if (observatoryPort == null)
        throw new Exception('Could not find observatory port in "flutter run" output.');

      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);

      final Map<String, dynamic> data = <String, dynamic>{
         'start_total_kb': startData['total_kb'],
      };

      if (testTarget != null) {
        await flutter('drive', options: <String>[
          '-v',
          '-t',
          testTarget,
          '-d',
          deviceId,
          '--use-existing-app=http://localhost:$observatoryPort',
        ]);

        final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
        data['end_total_kb'] = endData['total_kb'];
        data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
      }

      await device.stop(packageName);

      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
    });
  }
}

/// Measure application memory usage after pausing and resuming the app
/// with the Android back button.
class AndroidBackButtonMemoryTest {
  const AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName);

  final String testDirectory;
  final String packageName;
  final String activityName;

  Future<TaskResult> run() {
    return inDirectory(testDirectory, () async {
      if (deviceOperatingSystem != DeviceOperatingSystem.android) {
        throw 'This test is only supported on Android';
      }

      final AndroidDevice device = await devices.workingDevice;
      await device.unlock();
      final String deviceId = device.deviceId;
      await flutter('packages', options: <String>['get']);

      await flutter('run', options: <String>[
        '-v',
        '--profile',
        '--trace-startup', // wait for the first frame to render
        '-d',
        deviceId,
      ]);

      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);

      final Map<String, dynamic> data = <String, dynamic>{
         'start_total_kb': startData['total_kb'],
      };

      // Perform a series of back button suspend and resume cycles.
      for (int i = 0; i < 10; i++) {
        await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
        final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
        print(output);
        if (output.contains('Error'))
          return new TaskResult.failure('unable to launch activity');
        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
      }

      final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
      data['end_total_kb'] = endData['total_kb'];
      data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];

      await device.stop(packageName);

      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
    });
  }
}

class _UnzipListEntry {
  factory _UnzipListEntry.fromLine(String line) {
    final List<String> data = line.trim().split(new RegExp('\\s+'));
    assert(data.length == 8);
    return new _UnzipListEntry._(
      uncompressedSize:  int.parse(data[0]),
      compressedSize: int.parse(data[2]),
      path: data[7],
    );
  }

  _UnzipListEntry._({
    @required this.uncompressedSize,
    @required this.compressedSize,
    @required this.path,
  }) : assert(uncompressedSize != null),
       assert(compressedSize != null),
       assert(compressedSize <= uncompressedSize),
       assert(path != null);

  final int uncompressedSize;
  final int compressedSize;
  final String path;
}