1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:path/path.dart' as path;
import '../framework/task_result.dart';
import '../framework/utils.dart';
/// Run each benchmark this many times and compute average, min, max.
const int _kRunsPerBenchmark = 10;
Future<TaskResult> flutterToolStartupBenchmarkTask() async {
final Directory projectParentDirectory =
Directory.systemTemp.createTempSync('flutter_tool_startup_benchmark');
final Directory projectDirectory =
dir(path.join(projectParentDirectory.path, 'benchmark'));
await inDirectory<void>(flutterDirectory, () async {
await flutter('update-packages');
await flutter('create', options: <String>[projectDirectory.path]);
// Remove 'test' directory so we don't time the actual testing, but only the launching of the flutter tool
rmTree(dir(path.join(projectDirectory.path, 'test')));
});
final Map<String, dynamic> data = <String, dynamic>{
// `flutter test` in dir with no `test` folder.
...(await _Benchmark(
projectDirectory,
'test startup',
'test',
).run())
.asMap('flutter_tool_startup_test'),
// `flutter test -d foo_device` in dir with no `test` folder.
...(await _Benchmark(
projectDirectory,
'test startup with specified device',
'test',
options: <String>['-d', 'foo_device'],
).run())
.asMap('flutter_tool_startup_test_with_specified_device'),
// `flutter test -v` where no android sdk will be found (at least currently).
...(await _Benchmark(
projectDirectory,
'test startup no android sdk',
'test',
options: <String>['-v'],
environment: <String, String>{
'ANDROID_HOME': 'dummy value',
'ANDROID_SDK_ROOT': 'dummy value',
'PATH': pathWithoutWhereHits(<String>['adb', 'aapt']),
},
).run())
.asMap('flutter_tool_startup_test_no_android_sdk'),
// `flutter -h`.
...(await _Benchmark(
projectDirectory,
'help startup',
'-h',
).run())
.asMap('flutter_tool_startup_help'),
};
// Cleanup.
rmTree(projectParentDirectory);
return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
}
String pathWithoutWhereHits(List<String> whats) {
final String pathEnvironment = Platform.environment['PATH'] ?? '';
List<String> paths;
if (Platform.isWindows) {
paths = pathEnvironment.split(';');
} else {
paths = pathEnvironment.split(':');
}
// This isn't great but will probably work for our purposes.
final List<String> extensions = <String>['', '.exe', '.bat', '.com'];
final List<String> notFound = <String>[];
for (final String path in paths) {
bool found = false;
for (final String extension in extensions) {
for (final String what in whats) {
final File f = File('$path${Platform.pathSeparator}$what$extension');
if (f.existsSync()) {
found = true;
break;
}
}
if (found) {
break;
}
}
if (!found) {
notFound.add(path);
}
}
if (Platform.isWindows) {
return notFound.join(';');
} else {
return notFound.join(':');
}
}
class _BenchmarkResult {
const _BenchmarkResult(this.mean, this.min, this.max);
final int mean; // Milliseconds
final int min; // Milliseconds
final int max; // Milliseconds
Map<String, dynamic> asMap(String name) {
return <String, dynamic>{
name: mean,
'${name}_minimum': min,
'${name}_maximum': max,
};
}
}
class _Benchmark {
_Benchmark(this.directory, this.title, this.command,
{this.options = const <String>[], this.environment});
final Directory directory;
final String title;
final String command;
final List<String> options;
final Map<String, String>? environment;
Future<int> execute(int iteration, int targetIterations) async {
section('Benchmark $title - ${iteration + 1} / $targetIterations');
final Stopwatch stopwatch = Stopwatch();
await inDirectory<void>(directory, () async {
stopwatch.start();
// canFail is set to true, as e.g. `flutter test` in a dir with no `test`
// directory sets a non-zero return value.
await flutter(command,
options: options, canFail: true, environment: environment);
stopwatch.stop();
});
return stopwatch.elapsedMilliseconds;
}
/// Runs `benchmark` several times and reports the results.
Future<_BenchmarkResult> run() async {
final List<int> results = <int>[];
int sum = 0;
for (int i = 0; i < _kRunsPerBenchmark; i++) {
final int thisRuntime = await execute(i, _kRunsPerBenchmark);
results.add(thisRuntime);
sum += thisRuntime;
}
results.sort();
return _BenchmarkResult(sum ~/ results.length, results.first, results.last);
}
}