Skip to content

F
Front-End
Unverified Commit 33903c17 authored by Jonah Williams's avatar Jonah Williams Committed by GitHub
Browse files
Options

[devicelab] delete disabled or not useful benchmarks (#68107) 

- run_without_leak_tests: have been disabled for months
- build_benchmark: the tooling work here is mostly done and we're not tracking further improvements, free up more devicelab capacity
- system_debug_ios: does not work post iOS13
- mac_enable_twc: not adding more mac tests to devicelab
- hello_world_start_up: disabled

Any tests that we think will be valuable in the future can be resurrected from the git history.
parent 431c599b
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 461 deletions
dev/devicelab/bin/tasks/build_benchmark.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/build_benchmarks.dart';
Future<void> main() async {
await task(createAndroidBuildBenchmarkTask());
}
dev/devicelab/bin/tasks/build_benchmark_ios.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/build_benchmarks.dart';
Future<void> main() async {
await task(createIosBuildBenchmarkTask());
}
dev/devicelab/bin/tasks/hello_world__start_up.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createHelloWorldStartupTest());
}
dev/devicelab/bin/tasks/mac_enable_twc.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/track_widget_creation_enabled_task.dart';
/// Verify that twc can be enabled/disabled on macOS
Future<void> main() async {
await task(TrackWidgetCreationEnabledTask('macos').task);
}
dev/devicelab/bin/tasks/run_without_leak_linux.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/tasks/run_without_leak.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
}
dev/devicelab/bin/tasks/run_without_leak_mac.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/tasks/run_without_leak.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
}
dev/devicelab/bin/tasks/run_without_leak_win.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/tasks/run_without_leak.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
}
dev/devicelab/bin/tasks/system_debug_ios.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'service_extensions_test.dart';
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(() async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
final Directory testDirectory =
dir('${flutterDirectory.path}/dev/integration_tests/ui');
await inDirectory<void>(testDirectory, () async {
await flutter('packages', options: <String>['get']);
await checkNoWarningHostLaunch(deviceId);
await checkNoWarningXcodeLaunch(deviceId);
await checkWarningHomeScreenLaunch(deviceId);
});
return TaskResult.success(<String, dynamic>{});
});
}
const String expectedWarning =
'Launching a debug-mode app from the home screen may cause problems.';
// When a debug-mode app is launched from the host there should be no warnings.
Future<void> checkNoWarningHostLaunch(String deviceId) async {
final String output = await evalFlutter('drive', options: <String>[
'--debug',
'--verbose',
'--verbose-system-logs',
'-d',
deviceId,
'lib/empty.dart',
]);
expect(!output.contains(expectedWarning));
}
// When a debug-mode app is launched from Xcode there should be no warnings. The
// Xcode launch is simulated by keeping LLDB attached throughout the lifetime of
// the app.
Future<void> checkNoWarningXcodeLaunch(String deviceId) async {
await flutter('build',
options: <String>['ios', '--debug', '--verbose', 'lib/exit.dart']);
final String output = await eval(
'${flutterDirectory.path}/bin/cache/artifacts/ios-deploy/ios-deploy',
<String>[
'--bundle',
'build/ios/iphoneos/Runner.app',
'-d', // Actually start the app in LLDB, don't just install it.
'--noninteractive',
'--args',
'--verbose-logging',
]);
expect(output.contains('success') && !output.contains(expectedWarning));
}
// When a debug-mode app is launched from the home screen there should be a
// warning every ~100 launches. We lower the threshold from to 1 via
// "--verbose-system-logs" and simulate a home-screen-launch by setting an
// environment variable. The environment variable forces "flutter drive" to not
// pass a flag which it normally passes to debug-mode apps, imitating launchd,
// which doesn't pass any command-line flags.
Future<void> checkWarningHomeScreenLaunch(String deviceId) async {
final String output = await evalFlutter('drive', options: <String>[
'--debug',
'--verbose',
'--verbose-system-logs',
'-d',
deviceId,
'lib/empty.dart',
], environment: <String, String>{
'FLUTTER_TOOLS_DEBUG_WITHOUT_CHECKED_MODE': 'true',
});
expect(output.contains(expectedWarning));
}
dev/devicelab/bin/tasks/time_to_development_benchmark__android.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createTimeToDevelopmentCommand);
}
final Directory flutterGalleryDir =
dir(path.join(flutterDirectory.path, 'dev', 'integration_tests', 'flutter_gallery'));
final Directory editedFlutterGalleryDir =
dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery'));
Future<TaskResult> createTimeToDevelopmentCommand() async {
final Map<String, double> allResults = <String, double>{};
bool failed = false;
await inDirectory<void>(flutterDirectory, () async {
rmTree(editedFlutterGalleryDir);
mkdirs(editedFlutterGalleryDir);
recursiveCopy(flutterGalleryDir, editedFlutterGalleryDir);
await inDirectory<void>(editedFlutterGalleryDir, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final Stopwatch stopwatch = Stopwatch()..start();
final Process initialBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['run', '--debug', '-d', device.deviceId, '--no-resident',],
);
int exitCode = await initialBuild.exitCode;
if (exitCode != 0) {
failed = true;
return;
}
final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds;
stopwatch
..reset()
..start();
// Update a source file.
final File appDartSource = file(path.join(
editedFlutterGalleryDir.path,
'lib/gallery/app.dart',
));
appDartSource.writeAsStringSync(
appDartSource.readAsStringSync().replaceFirst(
"'Flutter Gallery'",
"'Updated Flutter Gallery'",
),
);
final Process secondBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['run', '--debug', '-d', device.deviceId, '--no-resident'],
environment: null,
);
exitCode = await secondBuild.exitCode;
if (exitCode != 0) {
failed = true;
return;
}
stopwatch.stop();
allResults['time_to_development'] = initialBuildMilliseconds.toDouble();
allResults['time_to_development_incremental'] =
stopwatch.elapsedMilliseconds.toDouble();
});
});
if (failed) {
return TaskResult.failure('Failed to build debug app');
}
return TaskResult.success(allResults,
benchmarkScoreKeys: allResults.keys.toList());
}
dev/devicelab/lib/tasks/build_benchmarks.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path;
final Directory helloWorldDir = dir(path.join(flutterDirectory.path, 'examples', 'hello_world'));
/// Creates a devicelab build benchmark for Android.
TaskFunction createAndroidBuildBenchmarkTask() {
return () async {
return createBuildCommand('apk');
};
}
/// Creates a devicelab build benchmark for iOS.
TaskFunction createIosBuildBenchmarkTask() {
return () async {
return createBuildCommand('ios');
};
}
Future<TaskResult> createBuildCommand(String buildKind) {
return inDirectory<TaskResult>(helloWorldDir, () async {
final Stopwatch stopwatch = Stopwatch()
..start();
final Process initialBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--debug'],
environment: null,
);
int exitCode = await initialBuild.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build debug app');
}
final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds;
stopwatch
..reset()
..start();
final Process secondBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--debug'],
environment: null,
);
exitCode = await secondBuild.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build debug app');
}
final int secondBuildMilliseconds = stopwatch.elapsedMilliseconds;
final Process newBuildConfig = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--profile'],
environment: null,
);
exitCode = await newBuildConfig.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build profile app');
}
stopwatch
..reset()
..start();
final Process thirdBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--debug'],
environment: null,
);
exitCode = await thirdBuild.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build debug app');
}
final int thirdBuildMilliseconds = stopwatch.elapsedMilliseconds;
stopwatch.stop();
final Map<String, double> allResults = <String, double>{};
allResults['first_build_debug_millis'] = initialBuildMilliseconds.toDouble();
allResults['second_build_debug_millis'] = secondBuildMilliseconds.toDouble();
allResults['after_config_change_build_debug_millis'] = thirdBuildMilliseconds.toDouble();
return TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
});
}
dev/devicelab/lib/tasks/perf_tests.dart
View file @ 33903c17
...@@ -218,13 +218,6 @@ TaskFunction createComplexLayoutStartupTest() { ...@@ -218,13 +218,6 @@ TaskFunction createComplexLayoutStartupTest() {
).run; ).run;
} }
TaskFunction createHelloWorldStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/hello_world',
reportMetrics: false,
).run;
}
TaskFunction createFlutterGalleryCompileTest() { TaskFunction createFlutterGalleryCompileTest() {
return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run; return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run;
} }
......
dev/devicelab/lib/tasks/run_without_leak.dart deleted 100644 → 0
View file @ 431c599b
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:path/path.dart' as path;
import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
TaskFunction createRunWithoutLeakTest(dynamic dir) {
return () async {
final Device device = await devices.workingDevice;
await device.unlock();
final List<String> options = <String>[
'-d', device.deviceId, '--verbose',
];
int exitCode;
await inDirectory<void>(dir, () async {
final Process process = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
flutterCommandArgs('run', options),
environment: null,
);
final Completer<void> stdoutDone = Completer<void>();
final Completer<void> stderrDone = Completer<void>();
process.stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
if (line.contains('] For a more detailed help message, press "h". To detach, press "d"; to quit, press "q"')) {
process.stdin.writeln('q');
}
print('stdout: $line');
}, onDone: () {
stdoutDone.complete();
});
process.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('stderr: $line');
}, onDone: () {
stderrDone.complete();
});
await Future.wait<void>(
<Future<void>>[stdoutDone.future, stderrDone.future]);
exitCode = await process.exitCode;
});
return exitCode == 0
? TaskResultCheckProcesses()
: TaskResult.failure('Failed to run $dir');
};
}
dev/devicelab/manifest.yaml
View file @ 33903c17
...@@ -389,12 +389,6 @@ tasks: ...@@ -389,12 +389,6 @@ tasks:
stage: devicelab stage: devicelab
required_agent_capabilities: ["mac/android"] required_agent_capabilities: ["mac/android"]
# hello_world__start_up:
# description: >
# Verifies that Hello World can start on an array of devices.
# stage: devicelab
# required_agent_capabilities: ["linux/android_esoteric"]
microbenchmarks: microbenchmarks:
description: > description: >
Runs benchmarks from dev/benchmarks/microbenchmarks. Runs benchmarks from dev/benchmarks/microbenchmarks.
...@@ -480,18 +474,6 @@ tasks: ...@@ -480,18 +474,6 @@ tasks:
stage: devicelab stage: devicelab
required_agent_capabilities: ["linux/android"] required_agent_capabilities: ["linux/android"]
build_benchmark:
description: >
Measures APK build performance across config changes.
stage: devicelab
required_agent_capabilities: ["linux/android"]
time_to_development_benchmark__android:
description: >
Measures time from flutter run until resident runner setup is complete.
stage: devicelab
required_agent_capabilities: ["linux/android"]
# iOS on-device tests # iOS on-device tests
ios_defines_test: ios_defines_test:
...@@ -644,25 +626,12 @@ tasks: ...@@ -644,25 +626,12 @@ tasks:
stage: devicelab_ios stage: devicelab_ios
required_agent_capabilities: ["mac/ios"] required_agent_capabilities: ["mac/ios"]
# TODO(fujino): does not pass on iOS13 https://github.com/flutter/flutter/issues/43029
# system_debug_ios:
# description: >
# Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps.
# stage: devicelab_ios
# required_agent_capabilities: ["mac/ios"]
macos_chrome_dev_mode: macos_chrome_dev_mode:
description: > description: >
Run flutter web on the devicelab and hot restart. Run flutter web on the devicelab and hot restart.
stage: devicelab_ios stage: devicelab_ios
required_agent_capabilities: ["mac/ios"] required_agent_capabilities: ["mac/ios"]
build_benchmark_ios:
description: >
Measures iOS build performance across config changes.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
simple_animation_perf_ios: simple_animation_perf_ios:
description: > description: >
Measure CPU/GPU usage percentages of a simple animation. Measure CPU/GPU usage percentages of a simple animation.
...@@ -702,14 +671,6 @@ tasks: ...@@ -702,14 +671,6 @@ tasks:
stage: devicelab stage: devicelab
required_agent_capabilities: ["mac/ios"] required_agent_capabilities: ["mac/ios"]
# TODO(jonahwilliams): investigate build failures on devicelab infra to re-enable.
# mac_enable_twc:
# description: >
# Verifies that track-widget-creation can be enabled and disabled.
# stage: devicelab_ios
# required_agent_capabilities: ["mac/ios"]
# flaky: true
ios_app_with_extensions_test: ios_app_with_extensions_test:
description: > description: >
Checks that an iOS app with extensions can be built for physical and simulated devices. Checks that an iOS app with extensions can be built for physical and simulated devices.
...@@ -742,12 +703,6 @@ tasks: ...@@ -742,12 +703,6 @@ tasks:
stage: devicelab_win stage: devicelab_win
required_agent_capabilities: ["windows/android"] required_agent_capabilities: ["windows/android"]
# run_without_leak_win:
# description: >
# Checks that `flutter run` does not leak dart.exe on Windows.
# stage: devicelab_win
# required_agent_capabilities: ["windows/android"]
# Tests running on Linux hosts # Tests running on Linux hosts
hot_mode_dev_cycle_linux__benchmark: hot_mode_dev_cycle_linux__benchmark:
...@@ -939,20 +894,6 @@ tasks: ...@@ -939,20 +894,6 @@ tasks:
stage: devicelab stage: devicelab
required_agent_capabilities: ["linux-vm"] required_agent_capabilities: ["linux-vm"]
# run_without_leak_linux:
# description: >
# Checks that `flutter run` does not leak dart on Linux.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
# flaky: true
# run_without_leak_mac:
# description: >
# Checks that `flutter run` does not leak dart on macOS.
# stage: devicelab
# required_agent_capabilities: ["mac/android"]
# flaky: true
# android_splash_screen_integration_test: # android_splash_screen_integration_test:
# description: > # description: >
# Runs end-to-end test of Flutter's Android splash behavior. # Runs end-to-end test of Flutter's Android splash behavior.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab