Unverified Commit a152c0f4 authored by Jonah Williams's avatar Jonah Williams Committed by GitHub

[devicelab] reland: reduce iterations of startup test, use application binary (#68532)

These devicelab tests are incredibly flaky. Currently they will build & install the same application 15 times. This causes the temp storage to fill up on android, and has a good chance of flaking on iOS due to install issues.

The change from 3 to 15 increased total test time for 2 -> 12 minutes on iOS, or roughly 5 more test equivalents for every single iOS test. Reduce the iteration count back to 5

Uninstall the app after each run so temp storage does not fill up.
parent ea3b08f3
......@@ -454,24 +454,58 @@ class StartupTest {
Future<TaskResult> run() async {
return await inDirectory<TaskResult>(testDirectory, () async {
final String deviceId = (await devices.workingDevice).deviceId;
await flutter('packages', options: <String>['get']);
const int iterations = 15;
final Device device = await devices.workingDevice;
const int iterations = 5;
final List<Map<String, dynamic>> results = <Map<String, dynamic>>[];
for (int i = 0; i < iterations; ++i) {
section('Building application');
String applicationBinaryPath;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm,android-arm64',
'--machine',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
break;
case DeviceOperatingSystem.ios:
await flutter('build', options: <String>[
'ios',
'-v',
'--profile',
'--machine',
]);
applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos');
break;
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.fake:
break;
}
for (int i = 0; i < iterations; i += 1) {
await flutter('run', options: <String>[
'--no-android-gradle-daemon',
'--verbose',
'--profile',
'--trace-startup',
'-d',
deviceId,
device.deviceId,
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath',
]);
final Map<String, dynamic> data = json.decode(
file('$testDirectory/build/start_up_info.json').readAsStringSync(),
) as Map<String, dynamic>;
results.add(data);
await flutter('install', options: <String>[
'--uninstall-only',
'-d',
device.deviceId,
]);
}
final Map<String, dynamic> averageResults = _average(results, iterations);
......@@ -1428,3 +1462,12 @@ class _UnzipListEntry {
final int compressedSize;
final String path;
}
String _findIosAppInBuildDirectory(String searchDirectory) {
for (final FileSystemEntity entity in Directory(searchDirectory).listSync()) {
if (entity.path.endsWith('.app')) {
return entity.path;
}
}
return null;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment