microbenchmarks.dart 5.64 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as path;

import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
13
import 'package:flutter_devicelab/framework/ios.dart';
14 15
import 'package:flutter_devicelab/framework/utils.dart';

16
/// The maximum amount of time a single microbenchmark is allowed to take.
17
const Duration _kBenchmarkTimeout = const Duration(minutes: 10);
18

19 20 21 22
/// Creates a device lab task that runs benchmarks in
/// `dev/benchmarks/microbenchmarks` reports results to the dashboard.
TaskFunction createMicrobenchmarkTask() {
  return () async {
23
    final Device device = await devices.workingDevice;
24 25
    await device.unlock();

26
    Future<Map<String, double>> _runMicrobench(String benchmarkPath, {bool previewDart2: true}) async {
27 28 29 30 31
      Future<Map<String, double>> _run() async {
        print('Running $benchmarkPath');
        final Directory appDir = dir(
            path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
        final Process flutterProcess = await inDirectory(appDir, () async {
32
          if (deviceOperatingSystem == DeviceOperatingSystem.ios)
33
            await prepareProvisioningCertificates(appDir.path);
34 35 36 37 38 39 40 41 42
          final List<String> options = <String>[
            '-v',
            // --release doesn't work on iOS due to code signing issues
            '--profile',
            '-d',
            device.deviceId,
          ];
          if (previewDart2)
            options.add('--preview-dart-2');
43 44
          else
            options.add('--no-preview-dart-2');
45
          setLocalEngineOptionIfNecessary(options);
46
          options.add(benchmarkPath);
47
          return await _startFlutter(
48
            options: options,
49 50 51
            canFail: false,
          );
        });
52

53 54 55
        return await _readJsonResults(flutterProcess);
      }
      return _run().timeout(_kBenchmarkTimeout);
56 57
    }

58
    final Map<String, double> allResults = <String, double>{};
59 60 61
    allResults.addAll(await _runMicrobench('lib/stocks/layout_bench.dart'));
    allResults.addAll(await _runMicrobench('lib/stocks/build_bench.dart'));
    allResults.addAll(await _runMicrobench('lib/gestures/velocity_tracker_bench.dart'));
62
    allResults.addAll(await _runMicrobench('lib/stocks/animation_bench.dart'));
63

64 65
    // Run micro-benchmarks once again in --no-preview-dart-2 mode.
    // Append "_dart1" suffix to the result keys to distinguish them from
66 67
    // the original results.

68
    void addDart1Results(Map<String, double> benchmarkResults) {
69
      benchmarkResults.forEach((String key, double result) {
70
        allResults[key + '_dart1'] = result;
71 72 73
      });
    }

74 75 76 77 78 79 80 81 82 83
    addDart1Results(await _runMicrobench(
        'lib/stocks/layout_bench.dart', previewDart2: false));
    addDart1Results(await _runMicrobench(
        'lib/stocks/layout_bench.dart', previewDart2: false));
    addDart1Results(await _runMicrobench(
        'lib/stocks/build_bench.dart', previewDart2: false));
    addDart1Results(await _runMicrobench(
        'lib/gestures/velocity_tracker_bench.dart', previewDart2: false));
    addDart1Results(await _runMicrobench(
        'lib/stocks/animation_bench.dart', previewDart2: false));
84 85 86 87
    return new TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
  };
}

88
Future<Process> _startFlutter({
89
  String command: 'run',
90 91 92 93
  List<String> options: const <String>[],
  bool canFail: false,
  Map<String, String> environment,
}) {
94
  final List<String> args = <String>['run']..addAll(options);
95
  return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, environment: environment);
96 97 98 99 100 101
}

Future<Map<String, double>> _readJsonResults(Process process) {
  // IMPORTANT: keep these values in sync with dev/benchmarks/microbenchmarks/lib/common.dart
  const String jsonStart = '================ RESULTS ================';
  const String jsonEnd = '================ FORMATTED ==============';
102
  const String jsonPrefix = ':::JSON:::';
103
  bool jsonStarted = false;
104 105
  final StringBuffer jsonBuf = new StringBuffer();
  final Completer<Map<String, double>> completer = new Completer<Map<String, double>>();
106 107 108 109 110 111 112

  final StreamSubscription<String> stderrSub = process.stderr
      .transform(const Utf8Decoder())
      .transform(const LineSplitter())
      .listen((String line) {
        stderr.writeln('[STDERR] $line');
      });
113

114 115
  bool processWasKilledIntentionally = false;
  final StreamSubscription<String> stdoutSub = process.stdout
116 117 118 119 120 121 122 123 124 125 126 127
      .transform(const Utf8Decoder())
      .transform(const LineSplitter())
      .listen((String line) {
    print(line);

    if (line.contains(jsonStart)) {
      jsonStarted = true;
      return;
    }

    if (line.contains(jsonEnd)) {
      jsonStarted = false;
128
      processWasKilledIntentionally = true;
129 130
      // ignore: deprecated_member_use
      process.kill(ProcessSignal.SIGINT); // flutter run doesn't quit automatically
131 132 133 134 135 136
      final String jsonOutput = jsonBuf.toString();
      try {
        completer.complete(json.decode(jsonOutput));
      } catch (ex) {
        completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput');
      }
137 138 139
      return;
    }

140
    if (jsonStarted && line.contains(jsonPrefix))
141
      jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length));
142 143
  });

144
  process.exitCode.then<int>((int code) {
145 146 147
    stdoutSub.cancel();
    stderrSub.cancel();
    if (!processWasKilledIntentionally && code != 0) {
148 149 150 151
      completer.completeError('flutter run failed: exit code=$code');
    }
  });

152 153
  return completer.future;
}