analysis.dart 4.01 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:io';

import 'package:path/path.dart' as path;

import '../framework/framework.dart';
import '../framework/utils.dart';

13 14 15 16 17 18
/// Run each benchmark this many times and compute average, min, max.
///
/// This must be small enough that we can do all the work in 15 minutes, the
/// devicelab deadline. Since there's four different analysis tasks, on average,
/// each can have 4 minutes. The tasks currently average a little more than a
/// minute, so that allows three runs per task.
19
const int _kRunsPerBenchmark = 3;
20

21 22
/// Path to the generated "mega gallery" app.
Directory get _megaGalleryDirectory => dir(path.join(Directory.systemTemp.path, 'mega_gallery'));
23

24
Future<TaskResult> analyzerBenchmarkTask() async {
25
  await inDirectory<void>(flutterDirectory, () async {
26 27 28 29
    rmTree(_megaGalleryDirectory);
    mkdirs(_megaGalleryDirectory);
    await dart(<String>['dev/tools/mega_gallery.dart', '--out=${_megaGalleryDirectory.path}']);
  });
30

31
  final Map<String, dynamic> data = <String, dynamic>{};
32 33 34 35
  data.addAll((await _run(_FlutterRepoBenchmark())).asMap('flutter_repo', 'batch'));
  data.addAll((await _run(_FlutterRepoBenchmark(watch: true))).asMap('flutter_repo', 'watch'));
  data.addAll((await _run(_MegaGalleryBenchmark())).asMap('mega_gallery', 'batch'));
  data.addAll((await _run(_MegaGalleryBenchmark(watch: true))).asMap('mega_gallery', 'watch'));
36

37
  return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
38 39
}

40 41
class _BenchmarkResult {
  const _BenchmarkResult(this.mean, this.min, this.max);
42

43
  final double mean; // seconds
44

45
  final double min; // seconds
46

47
  final double max; // seconds
48

49 50 51 52 53 54
  Map<String, dynamic> asMap(String benchmark, String mode) {
    return <String, dynamic>{
      '${benchmark}_$mode': mean,
      '${benchmark}_${mode}_minimum': min,
      '${benchmark}_${mode}_maximum': max,
    };
55 56 57
  }
}

58 59
abstract class _Benchmark {
  _Benchmark({ this.watch = false });
60

61
  final bool watch;
62

63
  String get title;
64

65
  Directory get directory;
66

67 68 69 70 71 72 73 74 75
  List<String> get options {
    final List<String> result = <String>[ '--benchmark' ];
    if (watch)
      result.add('--watch');
    return result;
  }

  Future<double> execute(int iteration, int targetIterations) async {
    section('Analyze $title ${watch ? 'with watcher' : ''} - ${iteration + 1} / $targetIterations');
76
    final Stopwatch stopwatch = Stopwatch();
77
    await inDirectory<void>(directory, () async {
78 79 80
      stopwatch.start();
      await flutter('analyze', options: options);
      stopwatch.stop();
81
    });
82
    return stopwatch.elapsedMicroseconds / (1000.0 * 1000.0);
83
  }
84
}
85

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
/// Times how long it takes to analyze the Flutter repository.
class _FlutterRepoBenchmark extends _Benchmark {
  _FlutterRepoBenchmark({ bool watch = false }) : super(watch: watch);

  @override
  String get title => 'Flutter repo';

  @override
  Directory get directory => flutterDirectory;

  @override
  List<String> get options {
    return super.options
      ..add('--flutter-repo');
  }
}

/// Times how long it takes to analyze the generated "mega_gallery" app.
class _MegaGalleryBenchmark extends _Benchmark {
  _MegaGalleryBenchmark({ bool watch = false }) : super(watch: watch);

  @override
  String get title => 'mega gallery';

  @override
  Directory get directory => _megaGalleryDirectory;
}

/// Runs `benchmark` several times and reports the results.
Future<_BenchmarkResult> _run(_Benchmark benchmark) async {
  final List<double> results = <double>[];
  for (int i = 0; i < _kRunsPerBenchmark; i += 1) {
118 119
    // Delete cached analysis results.
    rmTree(dir('${Platform.environment['HOME']}/.dartServer'));
120
    results.add(await benchmark.execute(i, _kRunsPerBenchmark));
121
  }
122 123 124 125 126
  results.sort();
  final double sum = results.fold<double>(
    0.0,
    (double previousValue, double element) => previousValue + element,
  );
127
  return _BenchmarkResult(sum / results.length, results.first, results.last);
128
}