analysis.dart 3.87 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io';

import 'package:path/path.dart' as path;

9
import '../framework/task_result.dart';
10 11
import '../framework/utils.dart';

12 13 14 15 16 17
/// Run each benchmark this many times and compute average, min, max.
///
/// This must be small enough that we can do all the work in 15 minutes, the
/// devicelab deadline. Since there's four different analysis tasks, on average,
/// each can have 4 minutes. The tasks currently average a little more than a
/// minute, so that allows three runs per task.
18
const int _kRunsPerBenchmark = 3;
19

20 21
/// Path to the generated "mega gallery" app.
Directory get _megaGalleryDirectory => dir(path.join(Directory.systemTemp.path, 'mega_gallery'));
22

23
Future<TaskResult> analyzerBenchmarkTask() async {
24
  await inDirectory<void>(flutterDirectory, () async {
25 26
    rmTree(_megaGalleryDirectory);
    mkdirs(_megaGalleryDirectory);
27
    await flutter('update-packages');
28 29
    await dart(<String>['dev/tools/mega_gallery.dart', '--out=${_megaGalleryDirectory.path}']);
  });
30

31 32 33 34 35 36
  final Map<String, dynamic> data = <String, dynamic>{
    ...(await _run(_FlutterRepoBenchmark())).asMap('flutter_repo', 'batch'),
    ...(await _run(_FlutterRepoBenchmark(watch: true))).asMap('flutter_repo', 'watch'),
    ...(await _run(_MegaGalleryBenchmark())).asMap('mega_gallery', 'batch'),
    ...(await _run(_MegaGalleryBenchmark(watch: true))).asMap('mega_gallery', 'watch'),
  };
37

38
  return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
39 40
}

41 42
class _BenchmarkResult {
  const _BenchmarkResult(this.mean, this.min, this.max);
43

44
  final double mean; // seconds
45

46
  final double min; // seconds
47

48
  final double max; // seconds
49

50 51 52 53 54 55
  Map<String, dynamic> asMap(String benchmark, String mode) {
    return <String, dynamic>{
      '${benchmark}_$mode': mean,
      '${benchmark}_${mode}_minimum': min,
      '${benchmark}_${mode}_maximum': max,
    };
56 57 58
  }
}

59
abstract class _Benchmark {
60
  _Benchmark({this.watch = false});
61

62
  final bool watch;
63

64
  String get title;
65

66
  Directory get directory;
67

68
  List<String> get options => <String>[
69 70 71
        '--benchmark',
        if (watch) '--watch',
      ];
72 73 74

  Future<double> execute(int iteration, int targetIterations) async {
    section('Analyze $title ${watch ? 'with watcher' : ''} - ${iteration + 1} / $targetIterations');
75
    final Stopwatch stopwatch = Stopwatch();
76
    await inDirectory<void>(directory, () async {
77 78 79
      stopwatch.start();
      await flutter('analyze', options: options);
      stopwatch.stop();
80
    });
81
    return stopwatch.elapsedMicroseconds / (1000.0 * 1000.0);
82
  }
83
}
84

85 86
/// Times how long it takes to analyze the Flutter repository.
class _FlutterRepoBenchmark extends _Benchmark {
87
  _FlutterRepoBenchmark({super.watch});
88 89 90 91 92 93 94 95 96

  @override
  String get title => 'Flutter repo';

  @override
  Directory get directory => flutterDirectory;

  @override
  List<String> get options {
97
    return super.options..add('--flutter-repo');
98 99 100 101 102
  }
}

/// Times how long it takes to analyze the generated "mega_gallery" app.
class _MegaGalleryBenchmark extends _Benchmark {
103
  _MegaGalleryBenchmark({super.watch});
104 105 106 107 108 109 110 111 112 113 114 115

  @override
  String get title => 'mega gallery';

  @override
  Directory get directory => _megaGalleryDirectory;
}

/// Runs `benchmark` several times and reports the results.
Future<_BenchmarkResult> _run(_Benchmark benchmark) async {
  final List<double> results = <double>[];
  for (int i = 0; i < _kRunsPerBenchmark; i += 1) {
116 117
    // Delete cached analysis results.
    rmTree(dir('${Platform.environment['HOME']}/.dartServer'));
118
    results.add(await benchmark.execute(i, _kRunsPerBenchmark));
119
  }
120 121 122 123 124
  results.sort();
  final double sum = results.fold<double>(
    0.0,
    (double previousValue, double element) => previousValue + element,
  );
125
  return _BenchmarkResult(sum / results.length, results.first, results.last);
126
}