# Describes the tasks we run in the continuous integration (CI) environment. # # Cocoon[1] uses this file to generate a checklist of tasks to be performed for # every master commit. # # [1] github.com/flutter/cocoon # CI tasks. # # Each key in this dictionary is the unique name of a task, which also # corresponds to a file in the "bin/" directory that the task runner will run. # # Required properties: # # * description: free form string # describes what the task is about # * stage: one of "devicelab", "devicelab_ios", "devicelab_win" # tasks are grouped by stage so they appear next to each on the dashboard and have a distinct # icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not # run device tests on red Cirrus builds. # * required_agent_capabilities: a list of strings # list of capabilities a devicelab agent must have to be able to run this task. # # Optional properties: # # * flaky: boolean true or false # whether the task is considered flaky; the result of running a flaky task does not affect # the overall build status. Tests should be marked flaky when newly added, until they have # been proved for a few cycles. Tests that are actually flaky but not being actively worked # on should be hidden because they just cause confusion otherwise. # * timeout_in_minutes: integer # a custom task timeout, specified in minutes. tasks: # Tests of compiling in a variety of modes complex_layout_android__compile: description: > Collects various performance metrics of compiling the Complex Layout sample app for Android from Linux. stage: devicelab required_agent_capabilities: ["linux/android"] complex_layout_ios__compile: description: > Collects various performance metrics of compiling the Complex Layout sample app for iOS from Mac. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] complex_layout_win__compile: description: > Collects various performance metrics of compiling the Complex Layout for Android from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] basic_material_app_android__compile: description: > Collects various performance metrics of compiling the default app for Android from Linux. stage: devicelab required_agent_capabilities: ["linux/android"] basic_material_app_ios__compile: description: > Collects various performance metrics of compiling the default app for iOS from Mac. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] basic_material_app_win__compile: description: > Collects various performance metrics of compiling the default app for Android from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] codegen_integration_win: description: > Runs codegeneration and verifies that it can execute correctly. stage: devicelab_win required_agent_capabilities: ["windows/android"] codegen_integration_mac: description: > Runs codegeneration and verifies that it can execute correctly. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] codegen_integration_linux: description: > Runs codegeneration and verifies that it can execute correctly. stage: devicelab required_agent_capabilities: ["linux/android"] uncaught_image_error_linux: description: > Ensures that an error thrown into the zone can be caught by the ImageStream completer stage: devicelab flaky: true required_agent_capabilities: ["linux/android"] flutter_gallery_android__compile: description: > Collects various performance metrics of compiling the Flutter Gallery for Android from Linux. stage: devicelab required_agent_capabilities: ["linux/android"] flutter_gallery_ios__compile: description: > Collects various performance metrics of compiling the Flutter Gallery for iOS from Mac. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_win__compile: description: > Collects various performance metrics of compiling the Flutter Gallery for Android from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] # Android on-device tests complex_layout_scroll_perf__timeline_summary: description: > Measures the runtime performance of the Complex Layout sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] tiles_scroll_perf__timeline_summary: description: > Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] home_scroll_perf__timeline_summary: description: > Measures the runtime performance of scrolling the material page in the flutter_gallery app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] drive_perf_debug_warning: description: > Check that driver will print warnings when traceAction is called in debug mode. stage: devicelab required_agent_capabilities: ["mac/android"] flaky: true cull_opacity_perf__timeline_summary: description: > Measures the runtime performance of culling opacity widgets on Android. stage: devicelab required_agent_capabilities: ["mac/android"] cubic_bezier_perf__timeline_summary: description: > Measures the runtime performance of cubic bezier animations on Android. stage: devicelab required_agent_capabilities: ["mac/android"] flavors_test: description: > Checks that flavored builds work on Android. stage: devicelab required_agent_capabilities: ["mac/android"] channels_integration_test: description: > Checks that platform channels work on Android. stage: devicelab required_agent_capabilities: ["mac/android"] external_ui_integration_test: description: > Checks that external UIs work on Android. stage: devicelab required_agent_capabilities: ["mac/android"] embedded_android_views_integration_test: description: > Tests embedded Android views. stage: devicelab required_agent_capabilities: ["mac/android"] android_semantics_integration_test: description: > Tests that the Android accessibility bridge produces correct semantics. stage: devicelab required_agent_capabilities: ["mac/android"] run_release_test: description: > Checks that `flutter run --release` does not crash. stage: devicelab required_agent_capabilities: ["mac/android"] platform_interaction_test: description: > Checks platform interaction on Android. stage: devicelab required_agent_capabilities: ["mac/android"] platform_channel_sample_test: description: > Runs a driver test on the Platform Channel sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] platform_view__start_up: description: > Verifies that Platform View can be used from an Android project. stage: devicelab required_agent_capabilities: ["mac/android"] complex_layout__start_up: description: > Measures the startup time of the Complex Layout sample app on Android. stage: devicelab required_agent_capabilities: ["mac/android"] hot_mode_dev_cycle__benchmark: description: > Measures the performance of Dart VM hot patching feature. stage: devicelab required_agent_capabilities: ["mac/android"] complex_layout_scroll_perf__memory: description: > Measures memory usage of the scroll performance test. stage: devicelab required_agent_capabilities: ["mac/android"] hello_world_android__compile: description: > Measures the APK size of Hello World. stage: devicelab required_agent_capabilities: ["mac/android"] hello_world__memory: description: > Measures starting memory usage of the simplest Flutter app. stage: devicelab required_agent_capabilities: ["mac/android"] microbenchmarks: description: > Runs benchmarks from dev/benchmarks/microbenchmarks. stage: devicelab required_agent_capabilities: ["mac/android"] timeout_in_minutes: 30 flutter_view__start_up: description: > Verifies that Flutter View can be used from an Android project. stage: devicelab required_agent_capabilities: ["mac/android"] integration_ui: description: > Runs end-to-end Flutter tests on Android. stage: devicelab required_agent_capabilities: ["mac/android"] commands_test: description: > Runs tests of flutter run commands. stage: devicelab required_agent_capabilities: ["mac/android"] run_machine_concurrent_hot_reload: description: > Runs tests of concurrent hot reload commands via flutter run --machine. stage: devicelab required_agent_capabilities: ["mac/android"] service_extensions_test: description: > Validates our service protocol extensions. stage: devicelab required_agent_capabilities: ["mac/android"] android_sample_catalog_generator: description: > Builds sample catalog markdown pages and Android screenshots stage: devicelab required_agent_capabilities: ["mac/android"] complex_layout_semantics_perf: description: > Measures duration of building the initial semantics tree. stage: devicelab required_agent_capabilities: ["linux/android"] routing_test: description: > Verifies that `flutter drive --route` still works. No performance numbers. stage: devicelab required_agent_capabilities: ["linux/android"] # TODO(dnfield): Remove these once we're confident it's all working on cirrus # gradle_plugin_test: # description: > # Verifies that the Flutter Gradle plugin supports standard and custom Android build types. # stage: devicelab # required_agent_capabilities: ["linux/android"] # plugin_test: # description: > # Checks that the project template works and supports plugins. # stage: devicelab # required_agent_capabilities: ["linux/android"] # module_test: # description: > # Checks that the module project template works and supports add2app on Android. # stage: devicelab # required_agent_capabilities: ["linux/android"] flutter_gallery_instrumentation_test: description: > Same as flutter_gallery__transition_perf but uses Android instrumentation framework, and therefore does not require a host computer to run. This test can run on off-the-shelf infrastructures, such as Firebase Test Lab. stage: devicelab required_agent_capabilities: ["linux/android"] flutter_attach_test: description: > Tests the `flutter attach` command. stage: devicelab required_agent_capabilities: ["linux/android"] named_isolates_test: description: > Tests naming and attaching to specific isolates. stage: devicelab required_agent_capabilities: ["linux/android"] # TODO(dnfield): Remove these once we're confident it's all working on cirrus # flutter_create_offline_test_linux: # description: > # Tests the `flutter create --offline` command. # stage: devicelab # required_agent_capabilities: ["linux/android"] # flutter_create_offline_test_windows: # description: > # Tests the `flutter create --offline` command. # stage: devicelab # required_agent_capabilities: ["windows/android"] # flutter_create_offline_test_mac: # description: > # Tests the `flutter create --offline` command. # stage: devicelab # required_agent_capabilities: ["mac/android"] # iOS on-device tests flavors_test_ios: description: > Checks that flavored builds work on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] # TODO(dnfield): Remove these once we're confident it's all working on cirrus # plugin_test_ios: # description: > # Checks that the project template works and supports plugins on iOS. # stage: devicelab_ios # required_agent_capabilities: ["mac/ios"] # module_test_ios: # description: > # Checks that the module project template works and supports add2app on iOS. # stage: devicelab # required_agent_capabilities: ["mac/ios"] external_ui_integration_test_ios: description: > Checks that external UIs work on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] channels_integration_test_ios: description: > Checks that platform channels work on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_interaction_test_ios: description: > Checks platform interaction on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_channel_sample_test_ios: description: > Runs a driver test on the Platform Channel sample app on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] platform_view_ios__start_up: description: > Verifies that Platform View can be used from an iOS project. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] complex_layout_scroll_perf_ios__timeline_summary: description: > Measures the runtime performance of the Complex Layout sample app on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_ios__start_up: description: > Measures the startup time of the Flutter Gallery app on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] complex_layout_ios__start_up: description: > Measures the startup time of the Complex Layout sample app on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] flutter_gallery_ios__transition_perf: description: > Measures the performance of screen transitions in Flutter Gallery on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] hello_world_ios__compile: description: > Measures the IPA size of Hello World. stage: devicelab required_agent_capabilities: ["mac/ios"] microbenchmarks_ios: description: > Runs benchmarks from dev/benchmarks/microbenchmarks on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] timeout_in_minutes: 30 flutter_view_ios__start_up: description: > Verifies that Flutter View can be used from an iOS project. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] integration_ui_ios: description: > Runs end-to-end Flutter tests on iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] timeout_in_minutes: 20 flaky: true ios_sample_catalog_generator: description: > Builds sample catalog markdown pages and iOS screenshots stage: devicelab_ios required_agent_capabilities: ["mac/ios"] # Tests running on Windows host flavors_test_win: description: > Checks that flavored builds work on Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] channels_integration_test_win: description: > Checks that platform channels work when app is launched from Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] plugin_test_win: description: > Checks that the project template works and supports plugins on Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] hot_mode_dev_cycle_win__benchmark: description: > Measures the performance of Dart VM hot patching feature on Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] run_without_leak_win: description: > Checks that `flutter run` does not leak dart.exe on Windows. stage: devicelab_win required_agent_capabilities: ["windows/android"] flaky: true # Tests running on Linux hosts hot_mode_dev_cycle_linux__benchmark: description: > Measures the performance of Dart VM hot patching feature on a Linux host. stage: devicelab required_agent_capabilities: ["linux/android"] # TODO(dnfield): Remove these once we're confident it's all working on cirrus # dartdocs: # description: > # Tracks how many members are still lacking documentation. # stage: devicelab # required_agent_capabilities: ["linux/android"] flutter_test_performance: description: > Measures performance of running flutter test. stage: devicelab required_agent_capabilities: ["linux/android"] technical_debt__cost: description: > Estimates our technical debt (TODOs, analyzer ignores, etc). stage: devicelab required_agent_capabilities: ["linux/android"] flutter_gallery__start_up: description: > Measures the startup time of the Flutter Gallery app on Android. stage: devicelab required_agent_capabilities: ["linux/android"] flutter_gallery__transition_perf: description: > Measures the performance of screen transitions in Flutter Gallery on Android. stage: devicelab required_agent_capabilities: ["linux/android"] flutter_gallery__transition_perf_with_semantics: description: > Measures the delta in performance of screen transitions without and with semantics enabled. stage: devicelab required_agent_capabilities: ["linux/android"] flutter_gallery__memory_nav: description: > Measures memory usage after repeated navigation in Gallery. stage: devicelab required_agent_capabilities: ["linux/android"] flutter_gallery__back_button_memory: description: > Measures memory usage after Android app suspend and resume. stage: devicelab required_agent_capabilities: ["linux/android"] analyzer_benchmark: description: > Measures the speed of Dart analyzer. stage: devicelab required_agent_capabilities: ["linux/android"] run_without_leak_linux: description: > Checks that `flutter run` does not leak dart on Linux. stage: devicelab required_agent_capabilities: ["linux/android"] flaky: true flutter_gallery_ios32__start_up: description: > Measures the startup time of the Flutter Gallery app on 32-bit iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios32"] flutter_gallery_ios32__transition_perf: description: > Measures the performance of screen transitions in Flutter Gallery on 32-bit iOS. stage: devicelab_ios required_agent_capabilities: ["mac/ios32"] run_without_leak_mac: description: > Checks that `flutter run` does not leak dart on macOS. stage: devicelab required_agent_capabilities: ["mac/android"] flaky: true