microbenchmarks.dart 5.82 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as path;

import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/utils.dart';

/// Creates a device lab task that runs benchmarks in
/// `dev/benchmarks/microbenchmarks` reports results to the dashboard.
TaskFunction createMicrobenchmarkTask() {
  return () async {
19
    final Device device = await devices.workingDevice;
20 21
    await device.unlock();

22
    Future<Map<String, double>> _runMicrobench(String benchmarkPath) async {
23 24 25 26 27
      Future<Map<String, double>> _run() async {
        print('Running $benchmarkPath');
        final Directory appDir = dir(
            path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
        final Process flutterProcess = await inDirectory(appDir, () async {
28 29 30 31 32 33 34 35
          final List<String> options = <String>[
            '-v',
            // --release doesn't work on iOS due to code signing issues
            '--profile',
            '-d',
            device.deviceId,
          ];
          options.add(benchmarkPath);
36
          return await _startFlutter(
37
            options: options,
38 39 40
            canFail: false,
          );
        });
41

42 43
        return await _readJsonResults(flutterProcess);
      }
44
      return _run();
45 46
    }

47 48 49
    final Map<String, double> allResults = <String, double>{
      ...await _runMicrobench('lib/stocks/layout_bench.dart'),
      ...await _runMicrobench('lib/stocks/build_bench.dart'),
50
      ...await _runMicrobench('lib/geometry/matrix_utils_transform_bench.dart'),
51 52 53 54
      ...await _runMicrobench('lib/geometry/rrect_contains_bench.dart'),
      ...await _runMicrobench('lib/gestures/velocity_tracker_bench.dart'),
      ...await _runMicrobench('lib/gestures/gesture_detector_bench.dart'),
      ...await _runMicrobench('lib/stocks/animation_bench.dart'),
55 56
      ...await _runMicrobench('lib/language/sync_star_bench.dart'),
      ...await _runMicrobench('lib/language/sync_star_semantics_bench.dart'),
57
    };
58

59
    return TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
60 61 62
  };
}

63
Future<Process> _startFlutter({
64 65
  List<String> options = const <String>[],
  bool canFail = false,
66 67
  Map<String, String> environment,
}) {
68
  final List<String> args = flutterCommandArgs('run', options);
69
  return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, environment: environment);
70 71 72 73 74 75
}

Future<Map<String, double>> _readJsonResults(Process process) {
  // IMPORTANT: keep these values in sync with dev/benchmarks/microbenchmarks/lib/common.dart
  const String jsonStart = '================ RESULTS ================';
  const String jsonEnd = '================ FORMATTED ==============';
76
  const String jsonPrefix = ':::JSON:::';
77
  bool jsonStarted = false;
78 79
  final StringBuffer jsonBuf = StringBuffer();
  final Completer<Map<String, double>> completer = Completer<Map<String, double>>();
80 81

  final StreamSubscription<String> stderrSub = process.stderr
82 83
      .transform<String>(const Utf8Decoder())
      .transform<String>(const LineSplitter())
84 85 86
      .listen((String line) {
        stderr.writeln('[STDERR] $line');
      });
87

88
  bool processWasKilledIntentionally = false;
89
  bool resultsHaveBeenParsed = false;
90
  final StreamSubscription<String> stdoutSub = process.stdout
91 92
      .transform<String>(const Utf8Decoder())
      .transform<String>(const LineSplitter())
93
      .listen((String line) async {
94 95 96 97 98 99 100 101
    print(line);

    if (line.contains(jsonStart)) {
      jsonStarted = true;
      return;
    }

    if (line.contains(jsonEnd)) {
102 103 104
      final String jsonOutput = jsonBuf.toString();

      // If we end up here and have already parsed the results, it suggests that
Chris Bracken's avatar
Chris Bracken committed
105
      // we have received output from another test because our `flutter run`
106 107 108 109
      // process did not terminate correctly.
      // https://github.com/flutter/flutter/issues/19096#issuecomment-402756549
      if (resultsHaveBeenParsed) {
        throw 'Additional JSON was received after results has already been '
110 111 112 113 114 115
              'processed. This suggests the `flutter run` process may have lived '
              'past the end of our test and collected additional output from the '
              'next test.\n\n'
              'The JSON below contains all collected output, including both from '
              'the original test and what followed.\n\n'
              '$jsonOutput';
116 117
      }

118
      jsonStarted = false;
119
      processWasKilledIntentionally = true;
120
      resultsHaveBeenParsed = true;
121 122 123 124 125 126 127
      // Sending a SIGINT/SIGTERM to the process here isn't reliable because [process] is
      // the shell (flutter is a shell script) and doesn't pass the signal on.
      // Sending a `q` is an instruction to quit using the console runner.
      // See https://github.com/flutter/flutter/issues/19208
      process.stdin.write('q');
      await process.stdin.flush();
      // Also send a kill signal in case the `q` above didn't work.
128
      process.kill(ProcessSignal.sigint);
129
      try {
130
        completer.complete(Map<String, double>.from(json.decode(jsonOutput) as Map<String, dynamic>));
131 132 133
      } catch (ex) {
        completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput');
      }
134 135 136
      return;
    }

137
    if (jsonStarted && line.contains(jsonPrefix))
138
      jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length));
139 140
  });

141
  process.exitCode.then<void>((int code) async {
142 143 144 145
    await Future.wait<void>(<Future<void>>[
      stdoutSub.cancel(),
      stderrSub.cancel(),
    ]);
146
    if (!processWasKilledIntentionally && code != 0) {
147 148 149 150
      completer.completeError('flutter run failed: exit code=$code');
    }
  });

151 152
  return completer.future;
}