# Describes the tasks we run in the continuous integration (CI) environment. # # Cocoon[1] uses this file to generate a checklist of tasks to be performed for # every master commit. # # [1] github.com/flutter/cocoon # CI tasks. # # Each key in this dictionary is the unique name of a task, which also # corresponds to a file in the "bin/" directory that the task runner will run. # # Required properties: # # * description: free form string # describes what the task is about # * stage: one of "devicelab", "devicelab_ios", "devicelab_win" # tasks are grouped by stage so they appear next to each on the dashboard and have a distinct # icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not # run device tests on red Cirrus builds. # * required_agent_capabilities: a list of strings # list of capabilities a devicelab agent must have to be able to run this task. # # Optional properties: # # * flaky: boolean true or false # whether the task is considered flaky; the result of running a flaky task does not affect # the overall build status. Tests should be marked flaky when newly added, until they have # been proved for a few cycles. Tests that are actually flaky but not being actively worked # on should be hidden because they just cause confusion otherwise. # * on_luci: boolean true or false # whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the # devicelab to LUCI migration. And when the migration is complete, on_luci flag will be # removed. tasks: # Tests of compiling in a variety of modes complex_layout_ios__compile: description: > Collects various performance metrics of compiling the Complex Layout sample app for iOS from Mac. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] complex_layout_win__compile: description: > Collects various performance metrics of compiling the Complex Layout for Android from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] basic_material_app_ios__compile: description: > Collects various performance metrics of compiling the default app for iOS from Mac. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] basic_material_app_win__compile: description: > Collects various performance metrics of compiling the default app for Android from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] flutter_gallery_ios__compile: description: > Collects various performance metrics of compiling the Flutter Gallery for iOS from Mac. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_win__compile: description: > Collects various performance metrics of compiling the Flutter Gallery for Android from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] windows_chrome_dev_mode: description: > Run flutter web on the devicelab and hot restart. stage: devicelab_win required_agent_capabilities: ["windows/android"] # Android on-device tests complex_layout_scroll_perf__timeline_summary: description: > Measures the runtime performance of the Complex Layout sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] tiles_scroll_perf__timeline_summary: description: > Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] home_scroll_perf__timeline_summary: description: > Measures the runtime performance of scrolling the material page in the flutter_gallery app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] drive_perf_debug_warning: description: > Check that driver will print warnings when traceAction is called in debug mode. stage: devicelab required_agent_capabilities: ["mac/android"] cull_opacity_perf__timeline_summary: description: > Measures the runtime performance of culling opacity widgets on Android. stage: devicelab required_agent_capabilities: ["mac/android"] picture_cache_perf__timeline_summary: description: > Measures the runtime performance of raster caching many pictures on Android. stage: devicelab required_agent_capabilities: ["mac/android"] cubic_bezier_perf__timeline_summary: description: > Measures the runtime performance of cubic bezier animations on Android. stage: devicelab required_agent_capabilities: ["mac/android"] cubic_bezier_perf_sksl_warmup__timeline_summary: description: > Measures the runtime performance of cubic bezier animations on Android with SkSL shader warm-up. stage: devicelab required_agent_capabilities: ["mac/android"] flutter_gallery_sksl_warmup__transition_perf_e2e_ios32: description: > Measures the runtime performance of Flutter gallery transitions on iPhone4s with SkSL shader warm-up with e2e. stage: devicelab_ios required_agent_capabilities: ["mac/ios32"] flaky: true backdrop_filter_perf__timeline_summary: description: > Measures the runtime performance of backdrop filter blurs on Android. stage: devicelab required_agent_capabilities: ["mac/android"] textfield_perf__timeline_summary: description: > Measures the runtime performance of textfield on Android. stage: devicelab required_agent_capabilities: ["mac/android"] fullscreen_textfield_perf__timeline_summary: description: > Measures the runtime performance of large textfields with complex paragraph on Android. stage: devicelab required_agent_capabilities: ["mac/android"] color_filter_and_fade_perf__timeline_summary: description: > Measures the runtime performance of color filter with fade on Android. stage: devicelab required_agent_capabilities: ["mac/android"] fading_child_animation_perf__timeline_summary: description: > Measures the runtime performance of opacity filter with fade on Android. stage: devicelab required_agent_capabilities: ["mac/android"] imagefiltered_transform_animation_perf__timeline_summary: description: > Measures the runtime performance of imagefiltered widget with transform on Android. stage: devicelab required_agent_capabilities: ["mac/android"] flavors_test: description: > Checks that flavored builds work on Android. stage: devicelab required_agent_capabilities: ["mac/android"] channels_integration_test: description: > Checks that platform channels work on Android. stage: devicelab required_agent_capabilities: ["mac/android"] external_ui_integration_test: description: > Checks that external UIs work on Android. stage: devicelab required_agent_capabilities: ["mac/android"] embedded_android_views_integration_test: description: > Tests embedded Android views. stage: devicelab required_agent_capabilities: ["mac/android"] hybrid_android_views_integration_test: description: > Tests hybrid Android views. stage: devicelab required_agent_capabilities: ["mac/android"] android_semantics_integration_test: description: > Tests that the Android accessibility bridge produces correct semantics. stage: devicelab required_agent_capabilities: ["mac/android"] android_plugin_example_app_build_test: description: > Tests that the plugin example app can be built using the Flutter Gradle plugin. stage: devicelab required_agent_capabilities: ["mac/android"] run_release_test: description: > Checks that `flutter run --release` does not crash. stage: devicelab required_agent_capabilities: ["mac/android"] platform_interaction_test: description: > Checks platform interaction on Android. stage: devicelab required_agent_capabilities: ["mac/android"] platform_channel_sample_test: description: > Runs a driver test on the Platform Channel sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] platform_view__start_up: description: > Verifies that Platform View can be used from an Android project. stage: devicelab required_agent_capabilities: ["mac/android"] complex_layout__start_up: description: > Measures the startup time of the Complex Layout sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] hot_mode_dev_cycle__benchmark: description: > Measures the performance of Dart VM hot patching feature. stage: devicelab required_agent_capabilities: ["mac/android"] complex_layout_scroll_perf__memory: description: > Measures memory usage of the scroll performance test. stage: devicelab required_agent_capabilities: ["mac/android"] hello_world_android__compile: description: > Measures the APK size of Hello World. stage: devicelab required_agent_capabilities: ["mac/android"] hello_world__memory: description: > Measures starting memory usage of the simplest Flutter app. stage: devicelab required_agent_capabilities: ["mac/android"] microbenchmarks: description: > Runs benchmarks from dev/benchmarks/microbenchmarks. stage: devicelab required_agent_capabilities: ["mac/android"] flutter_view__start_up: description: > Verifies that Flutter View can be used from an Android project. stage: devicelab required_agent_capabilities: ["mac/android"] integration_ui_driver: description: > Runs end-to-end Flutter tests on Android. stage: devicelab required_agent_capabilities: ["mac/android"] integration_ui_keyboard_resize: description: > Runs end-to-end Flutter tests on Android. stage: devicelab required_agent_capabilities: [ "mac/android" ] integration_ui_screenshot: description: > Runs end-to-end Flutter tests on Android. stage: devicelab required_agent_capabilities: [ "mac/android" ] integration_ui_textfield: description: > Runs end-to-end Flutter tests on Android. stage: devicelab required_agent_capabilities: [ "mac/android" ] service_extensions_test: description: > Validates our service protocol extensions. stage: devicelab required_agent_capabilities: ["mac/android"] # iOS on-device tests ios_defines_test: description: > Builds a Framework with a --dart-define and verifies it can be used as a constant stage: devicelab_ios required_agent_capabilities: ["mac/ios"] ios_content_validation_test: description: > Builds an obfuscated app and verifies contents and structure stage: devicelab_ios required_agent_capabilities: ["mac/ios"] on_luci: true tiles_scroll_perf_ios__timeline_summary: description: > Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_views_scroll_perf_ios__timeline_summary: description: > Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_ios32__start_up: description: > Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S). stage: devicelab_ios required_agent_capabilities: ["mac/ios32"] flavors_test_ios: description: > Checks that flavored builds work on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] external_ui_integration_test_ios: description: > Checks that external UIs work on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] channels_integration_test_ios: description: > Checks that platform channels work on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_interaction_test_ios: description: > Checks platform interaction on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_channel_sample_test_ios: description: > Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_channel_sample_test_swift: description: > Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_view_ios__start_up: description: > Verifies that Platform View can be used from an iOS project. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] backdrop_filter_perf_ios__timeline_summary: description: > Measures the runtime performance of backdrop filter blurs on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] post_backdrop_filter_perf_ios__timeline_summary: description: > Measures the runtime performance of animations after a backdrop filter is removed on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] complex_layout_scroll_perf_ios__timeline_summary: description: > Measures the runtime performance of the Complex Layout sample app on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_ios__start_up: description: > Measures the startup time of the Flutter Gallery app on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] complex_layout_ios__start_up: description: > Measures the startup time of the Complex Layout sample app on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_ios__transition_perf: description: > Measures the performance of screen transitions in Flutter Gallery on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] hello_world_ios__compile: description: > Measures the IPA size of Hello World. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] microbenchmarks_ios: description: > Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6. stage: devicelab_ios #TODO(godofredoc): Remove ios/12 capability once # https://github.com/flutter/flutter/issues/49635 is fixed. required_agent_capabilities: ["mac/ios", "ios/12"] flutter_view_ios__start_up: description: > Verifies that Flutter View can be used from an iOS project. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] integration_ui_ios_driver: description: > Runs end-to-end Flutter tests on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] integration_ui_ios_keyboard_resize: description: > Runs end-to-end Flutter tests on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] integration_ui_ios_screenshot: description: > Runs end-to-end Flutter tests on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] integration_ui_ios_textfield: description: > Runs end-to-end Flutter tests on iPhone 6. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] ios_platform_view_tests: description: > Runs end-to-end tests with platform views in the scene. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] macos_chrome_dev_mode: description: > Run flutter web on the devicelab and hot restart. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] simple_animation_perf_ios: description: > Measure CPU/GPU usage percentages of a simple animation. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] smoke_catalina_start_up_ios: description: > A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test. stage: devicelab_ios required_agent_capabilities: ["mac-catalina/ios"] smoke_catalina_hot_mode_dev_cycle_ios__benchmark: description: > A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking. stage: devicelab_ios required_agent_capabilities: ["mac-catalina/ios"] smoke_catalina_start_up: description: > A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test. stage: devicelab required_agent_capabilities: ["mac-catalina/android"] smoke_catalina_hot_mode_dev_cycle__benchmark: description: > A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking. stage: devicelab required_agent_capabilities: ["mac-catalina/android"] # macOS target platform tests hot_mode_dev_cycle_macos_target__benchmark: description: > Checks the functionality and performance of hot reload on a macOS target platform stage: devicelab required_agent_capabilities: ["mac/ios"] ios_app_with_extensions_test: description: > Checks that an iOS app with extensions can be built for physical and simulated devices. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] # Tests running on Windows host flavors_test_win: description: > Checks that flavored builds work on Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] channels_integration_test_win: description: > Checks that platform channels work when app is launched from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] hot_mode_dev_cycle_win__benchmark: description: > Measures the performance of Dart VM hot patching feature on Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] # Tests running on Linux hosts flutter_gallery__transition_perf_e2e_ios32: description: > Measures the performance of screen transitions in Flutter Gallery on Android with e2e. stage: devicelab_ios required_agent_capabilities: ["mac/ios32"] flaky: true flutter_gallery__transition_perf_e2e_ios: description: > Measures the performance of screen transitions in Flutter Gallery on Android with e2e. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] new_gallery__transition_perf: description: > Measures the performance of screen transitions in the new Flutter Gallery on Android. stage: devicelab required_agent_capabilities: ["mac/android"] new_gallery_ios__transition_perf: description: > Measures the performance of screen transitions in the new Flutter Gallery on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] fast_scroll_large_images__memory: description: > Measures memory usage for scrolling through a list of large images. stage: devicelab required_agent_capabilities: ["mac/android"] large_image_changer_perf_ios: description: > Measures memory, cpu, and gpu usage when rotating through a series of large images. stage: devicelab required_agent_capabilities: ["mac/ios"] # android_splash_screen_integration_test: # description: > # Runs end-to-end test of Flutter's Android splash behavior. # stage: devicelab # required_agent_capabilities: ["linux/android"] # flaky: true