manifest.yaml 29.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon

# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
Yegor's avatar
Yegor committed
13
# Required properties:
14
#
Yegor's avatar
Yegor committed
15 16 17 18 19
# * description: free form string
#       describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
#       tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
#       icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
20
#       run device tests on red Cirrus builds.
Yegor's avatar
Yegor committed
21 22
# * required_agent_capabilities: a list of strings
#       list of capabilities a devicelab agent must have to be able to run this task.
23
#
Yegor's avatar
Yegor committed
24 25 26 27
# Optional properties:
#
# * flaky: boolean true or false
#       whether the task is considered flaky; the result of running a flaky task does not affect
28 29 30
#       the overall build status. Tests should be marked flaky when newly added, until they have
#       been proved for a few cycles. Tests that are actually flaky but not being actively worked
#       on should be hidden because they just cause confusion otherwise.
31 32 33 34
# * on_luci: boolean true or false
#       whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the
#       devicelab to LUCI migration. And when the migration is complete, on_luci flag will be
#       removed.
35 36

tasks:
37
  # Tests of compiling in a variety of modes
38

39 40 41 42 43 44
  complex_layout_android__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout sample app for Android from Linux.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
45

46
  complex_layout_ios__compile:
47
    description: >
48 49 50
      Collects various performance metrics of compiling the Complex
      Layout sample app for iOS from Mac.
    stage: devicelab_ios
51
    required_agent_capabilities: ["mac/ios"]
52 53 54 55 56 57

  complex_layout_win__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout for Android from Windows.
    stage: devicelab_win
58
    required_agent_capabilities: ["windows/android"]
59 60 61 62 63

  basic_material_app_android__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Linux.
64
    stage: devicelab
65 66 67 68 69 70 71
    required_agent_capabilities: ["linux/android"]

  basic_material_app_ios__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for iOS from Mac.
    stage: devicelab_ios
72
    required_agent_capabilities: ["mac/ios"]
73 74 75 76 77 78

  basic_material_app_win__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Windows.
    stage: devicelab_win
79
    required_agent_capabilities: ["windows/android"]
80

81
  flutter_gallery_android__compile:
82
    description: >
83 84
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Linux.
85
    stage: devicelab
86 87 88 89 90 91 92
    required_agent_capabilities: ["linux/android"]

  flutter_gallery_ios__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for iOS from Mac.
    stage: devicelab_ios
93
    required_agent_capabilities: ["mac/ios"]
94 95 96 97 98 99

  flutter_gallery_win__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Windows.
    stage: devicelab_win
100
    required_agent_capabilities: ["windows/android"]
101

102 103 104 105 106
  windows_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]
107

108 109
  # Android on-device tests

Yuqian Li's avatar
Yuqian Li committed
110 111 112 113 114 115 116
  complex_layout_android__scroll_smoothness:
    description: >
      Measures the smoothness of scrolling of the Complex Layout sample app on
      Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

117 118 119 120 121
  complex_layout_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      Android.
    stage: devicelab
122
    required_agent_capabilities: ["mac/android"]
123

124 125 126 127 128 129
  tiles_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

130 131 132
  platform_views_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of platform views in the Complex Layout sample app on Android.
133
    stage: devicelab
134 135
    required_agent_capabilities: ["linux/android"]

136 137 138 139 140 141
  android_view_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of Android views in the platform view layout sample app on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

142 143 144 145 146 147 148
  home_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of scrolling the material page in the
      flutter_gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

149 150 151 152 153 154
  drive_perf_debug_warning:
    description: >
      Check that driver will print warnings when traceAction is called in debug mode.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

155 156 157 158 159 160
  cull_opacity_perf__timeline_summary:
    description: >
      Measures the runtime performance of culling opacity widgets on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

161 162 163 164 165 166 167
  cull_opacity_perf__e2e_summary:
    description: >
      Measures the runtime performance of culling opacity widgets on Android on
      E2E with self-driving test app.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

168 169 170 171 172 173
  multi_widget_construction_perf__timeline_summary:
    description: >
      Measures the runtime performance of constructing and destructing widgets on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

174 175 176 177 178 179
  multi_widget_construction_perf__e2e_summary:
    description: >
      Measures the runtime performance of constructing and destructing widgets on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

180 181 182 183 184 185
  frame_policy_delay_test_android:
    description: >
      Tests the effect of LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

186 187 188 189 190 191
  picture_cache_perf__timeline_summary:
    description: >
      Measures the runtime performance of raster caching many pictures on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

192 193 194 195 196 197
  picture_cache_perf__e2e_summary:
    description: >
      Measures the runtime performance of raster caching many pictures on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

198 199 200 201 202 203
  cubic_bezier_perf__timeline_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

204 205 206 207 208 209 210
  cubic_bezier_perf__e2e_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android
      using e2e.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

211 212 213 214 215 216 217
  cubic_bezier_perf_sksl_warmup__timeline_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android
      with SkSL shader warm-up.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

218 219 220 221 222 223 224
  cubic_bezier_perf_sksl_warmup__e2e_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android
      with SkSL shader warm-up using e2e.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

225 226 227 228 229 230
  flutter_gallery_sksl_warmup__transition_perf_e2e_ios32:
    description: >
      Measures the runtime performance of Flutter gallery transitions on iPhone4s
      with SkSL shader warm-up with e2e.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]
231
    flaky: true
232

233 234 235 236 237 238
  backdrop_filter_perf__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

239 240 241 242 243 244
  backdrop_filter_perf__e2e_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

245 246 247 248 249 250
  textfield_perf__timeline_summary:
    description: >
      Measures the runtime performance of textfield on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

251 252 253 254 255 256
  textfield_perf__e2e_summary:
    description: >
      Measures the runtime performance of textfield on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

257 258 259 260 261 262
  color_filter_and_fade_perf__timeline_summary:
    description: >
      Measures the runtime performance of color filter with fade on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

263 264 265 266 267 268
  color_filter_and_fade_perf__e2e_summary:
    description: >
      Measures the runtime performance of color filter with fade on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

269 270 271 272 273 274
  fading_child_animation_perf__timeline_summary:
    description: >
      Measures the runtime performance of opacity filter with fade on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

275 276 277 278 279 280
  imagefiltered_transform_animation_perf__timeline_summary:
    description: >
      Measures the runtime performance of imagefiltered widget with transform on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

281 282 283 284
  flavors_test:
    description: >
      Checks that flavored builds work on Android.
    stage: devicelab
285
    required_agent_capabilities: ["mac/android"]
286

287 288 289 290
  channels_integration_test:
    description: >
      Checks that platform channels work on Android.
    stage: devicelab
291
    required_agent_capabilities: ["mac/android"]
292

293 294 295 296
  external_ui_integration_test:
    description: >
      Checks that external UIs work on Android.
    stage: devicelab
297
    required_agent_capabilities: ["mac/android"]
298

299 300 301 302 303 304
  embedded_android_views_integration_test:
    description: >
      Tests embedded Android views.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

305 306 307 308 309 310
  hybrid_android_views_integration_test:
    description: >
      Tests hybrid Android views.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

311 312 313 314 315 316
  android_semantics_integration_test:
    description: >
      Tests that the Android accessibility bridge produces correct semantics.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

317 318 319 320 321 322
  run_release_test:
    description: >
      Checks that `flutter run --release` does not crash.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

323 324 325 326
  platform_interaction_test:
    description: >
      Checks platform interaction on Android.
    stage: devicelab
327
    required_agent_capabilities: ["mac/android"]
328

329
  platform_channel_sample_test:
330
    description: >
331
     Runs a driver test on the Platform Channel sample app on Android.
332
    stage: devicelab
333
    required_agent_capabilities: ["mac/android"]
334

335 336 337 338 339 340
  platform_view__start_up:
    description: >
      Verifies that Platform View can be used from an Android project.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

341 342 343 344
  complex_layout__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on Android.
    stage: devicelab
345
    required_agent_capabilities: ["mac/android"]
346

347 348 349 350
  hot_mode_dev_cycle__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature.
    stage: devicelab
351
    required_agent_capabilities: ["mac/android"]
352

353 354 355 356
  complex_layout_scroll_perf__memory:
    description: >
      Measures memory usage of the scroll performance test.
    stage: devicelab
357
    required_agent_capabilities: ["mac/android"]
358

359 360 361 362 363 364
  complex_layout_scroll_perf__devtools_memory:
    description: >
      Measures memory usage of the scroll performance test using DevTools.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

365 366 367 368 369 370
  hello_world_android__compile:
    description: >
      Measures the APK size of Hello World.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

371 372 373 374
  hello_world__memory:
    description: >
      Measures starting memory usage of the simplest Flutter app.
    stage: devicelab
375
    required_agent_capabilities: ["mac/android"]
376

377 378 379 380
  microbenchmarks:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks.
    stage: devicelab
381
    required_agent_capabilities: ["mac/android"]
382

383 384 385 386
  flutter_view__start_up:
    description: >
      Verifies that Flutter View can be used from an Android project.
    stage: devicelab
387
    required_agent_capabilities: ["mac/android"]
388

389
  integration_ui_driver:
390 391 392
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
393
    required_agent_capabilities: ["mac/android"]
394

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412
  integration_ui_keyboard_resize:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
    required_agent_capabilities: [ "mac/android" ]

  integration_ui_screenshot:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
    required_agent_capabilities: [ "mac/android" ]

  integration_ui_textfield:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
    required_agent_capabilities: [ "mac/android" ]

413 414 415 416
  service_extensions_test:
    description: >
      Validates our service protocol extensions.
    stage: devicelab
417
    required_agent_capabilities: ["mac/android"]
418

419 420 421 422 423
  android_defines_test:
    description: >
      Builds an APK with a --dart-define and verifies it can be used as a constant
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
424
    on_luci: true
425

426 427 428 429 430 431
  android_obfuscate_test:
    description: >
      Builds an obfuscated APK and verifies a dart identifier cannot be found
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

432 433 434 435 436
  complex_layout_semantics_perf:
    description: >
      Measures duration of building the initial semantics tree.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
437

438
  routing_test:
439 440 441 442
    description: >
      Verifies that `flutter drive --route` still works. No performance numbers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
443

444 445 446 447 448
  linux_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
449

450 451 452 453 454
  web_size__compile_test:
    description: >
      Measures the size of a dart2js bundle.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
455

456 457
  image_list_reported_duration:
    description: >
458 459 460 461 462 463 464
      Measures image loading performance on release (aot) build.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  image_list_jit_reported_duration:
    description: >
      Measures image loading performance on debug (jit) build.
465 466
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
467

468 469
  # iOS on-device tests

470 471 472
  ios_defines_test:
    description: >
      Builds a Framework with a --dart-define and verifies it can be used as a constant
473
    stage: devicelab_ios
474 475
    required_agent_capabilities: ["mac/ios"]

476
  ios_content_validation_test:
477
    description: >
478
      Builds an obfuscated app and verifies contents and structure
479
    stage: devicelab_ios
480
    required_agent_capabilities: ["mac/ios"]
481
    on_luci: true
482

483 484 485 486 487 488
  tiles_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

489 490 491 492 493 494
  platform_views_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

495 496 497 498 499 500
  flutter_gallery_ios32__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]

501 502
  flavors_test_ios:
    description: >
503
      Checks that flavored builds work on iPhone 6.
504
    stage: devicelab_ios
505
    required_agent_capabilities: ["mac/ios"]
506

507 508
  external_ui_integration_test_ios:
    description: >
509
      Checks that external UIs work on iPhone 6.
510
    stage: devicelab_ios
511
    required_agent_capabilities: ["mac/ios"]
512

513 514
  channels_integration_test_ios:
    description: >
515
      Checks that platform channels work on iPhone 6.
516
    stage: devicelab_ios
517
    required_agent_capabilities: ["mac/ios"]
518

519 520
  platform_interaction_test_ios:
    description: >
521
      Checks platform interaction on iPhone 6.
522
    stage: devicelab_ios
523
    required_agent_capabilities: ["mac/ios"]
524

525
  platform_channel_sample_test_ios:
526
    description: >
527
      Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
528
    stage: devicelab_ios
529
    required_agent_capabilities: ["mac/ios"]
530

531 532 533 534 535 536
  platform_channel_sample_test_swift:
    description: >
      Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

537 538 539 540 541 542
  platform_view_ios__start_up:
    description: >
      Verifies that Platform View can be used from an iOS project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

543 544 545 546 547 548
  backdrop_filter_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

549 550 551 552 553
  post_backdrop_filter_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of animations after a backdrop filter is removed on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
554

555 556 557 558 559
  complex_layout_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      iOS.
    stage: devicelab_ios
560
    required_agent_capabilities: ["mac/ios"]
561

562 563
  flutter_gallery_ios__start_up:
    description: >
564
      Measures the startup time of the Flutter Gallery app on iPhone 6.
565
    stage: devicelab_ios
566
    required_agent_capabilities: ["mac/ios"]
567

568 569
  complex_layout_ios__start_up:
    description: >
570
      Measures the startup time of the Complex Layout sample app on iPhone 6.
571
    stage: devicelab_ios
572
    required_agent_capabilities: ["mac/ios"]
573

574 575 576 577 578
  flutter_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      iOS.
    stage: devicelab_ios
579
    required_agent_capabilities: ["mac/ios"]
Yegor's avatar
Yegor committed
580

581 582 583
  hello_world_ios__compile:
    description: >
      Measures the IPA size of Hello World.
584
    stage: devicelab_ios
585 586
    required_agent_capabilities: ["mac/ios"]

587 588
  microbenchmarks_ios:
    description: >
589
      Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
590
    stage: devicelab_ios
591 592 593
    #TODO(godofredoc): Remove ios/12 capability once
    # https://github.com/flutter/flutter/issues/49635 is fixed.
    required_agent_capabilities: ["mac/ios", "ios/12"]
594

595 596 597 598
  flutter_view_ios__start_up:
    description: >
      Verifies that Flutter View can be used from an iOS project.
    stage: devicelab_ios
599
    required_agent_capabilities: ["mac/ios"]
600

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
  integration_ui_ios_driver:
    description: >
      Runs end-to-end Flutter tests on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  integration_ui_ios_keyboard_resize:
    description: >
      Runs end-to-end Flutter tests on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  integration_ui_ios_screenshot:
    description: >
      Runs end-to-end Flutter tests on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  integration_ui_ios_textfield:
620
    description: >
621
      Runs end-to-end Flutter tests on iPhone 6.
622
    stage: devicelab_ios
623
    required_agent_capabilities: ["mac/ios"]
624

625 626 627 628 629 630
  ios_platform_view_tests:
    description: >
      Runs end-to-end tests with platform views in the scene.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

631 632 633 634 635
  macos_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
636

637 638 639 640 641
  simple_animation_perf_ios:
    description: >
      Measure CPU/GPU usage percentages of a simple animation.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
642

643
  smoke_catalina_start_up_ios:
644 645 646 647 648
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]

649
  smoke_catalina_hot_mode_dev_cycle_ios__benchmark:
650 651 652 653 654
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]

655 656 657 658 659 660 661 662 663 664 665 666
  smoke_catalina_start_up:
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab
    required_agent_capabilities: ["mac-catalina/android"]

  smoke_catalina_hot_mode_dev_cycle__benchmark:
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab
    required_agent_capabilities: ["mac-catalina/android"]

667 668 669 670 671
  # macOS target platform tests
  hot_mode_dev_cycle_macos_target__benchmark:
    description: >
      Checks the functionality and performance of hot reload on a macOS target platform
    stage: devicelab
672
    required_agent_capabilities: ["mac/ios"]
673

674
  ios_app_with_extensions_test:
675
    description: >
676
      Checks that an iOS app with extensions can be built for physical and simulated devices.
677
    stage: devicelab_ios
678 679
    required_agent_capabilities: ["mac/ios"]

680 681
  # Tests running on Windows host

682 683 684 685
  flavors_test_win:
    description: >
      Checks that flavored builds work on Windows.
    stage: devicelab_win
686
    required_agent_capabilities: ["windows/android"]
687

688 689 690 691
  channels_integration_test_win:
    description: >
      Checks that platform channels work when app is launched from Windows.
    stage: devicelab_win
692
    required_agent_capabilities: ["windows/android"]
693

694 695 696 697 698 699
  plugin_test_win:
    description: >
      Checks that the project template works and supports plugins on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]

700 701 702 703
  hot_mode_dev_cycle_win__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on Windows.
    stage: devicelab_win
704
    required_agent_capabilities: ["windows/android"]
705 706 707 708 709 710 711 712 713

  # Tests running on Linux hosts

  hot_mode_dev_cycle_linux__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on a Linux host.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

714 715 716 717 718 719
  flutter_test_performance:
    description: >
      Measures performance of running flutter test.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

720 721 722 723
  dartdocs:
    description: >
      Tracks how many members are still lacking documentation.
    stage: devicelab
724
    required_agent_capabilities: ["linux-vm"]
725

726 727 728 729
  technical_debt__cost:
    description: >
      Estimates our technical debt (TODOs, analyzer ignores, etc).
    stage: devicelab
730
    required_agent_capabilities: ["linux-vm"]
731 732 733 734 735 736 737 738 739 740 741 742 743 744

  flutter_gallery__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

745 746 747 748 749 750 751
  flutter_gallery__transition_perf_e2e:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android with e2e.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

752 753 754 755 756 757
  flutter_gallery__transition_perf_e2e_ios32:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android with e2e.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]
758
    flaky: true
759 760 761 762 763

  flutter_gallery__transition_perf_e2e_ios:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android with e2e.
764
    stage: devicelab_ios
765 766
    required_agent_capabilities: ["mac/ios"]

767 768 769 770 771 772 773 774 775
  flutter_gallery__transition_perf_hybrid:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android where the page transitions are self-driven on device without host
      interventions, but the timeline events are still sent to host to be
      processed.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

776 777 778 779 780 781 782
  flutter_gallery_sksl_warmup__transition_perf:
    description: >
      Measures the runtime performance of Flutter gallery transitions on Android
      with SkSL shader warm-up.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

783 784 785 786 787 788 789
  flutter_gallery_sksl_warmup__transition_perf_e2e:
    description: >
      Measures the runtime performance of Flutter gallery transitions on Android
      with SkSL shader warm-up with e2e.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

790 791 792 793 794 795 796
  flutter_gallery__transition_perf_with_semantics:
    description: >
      Measures the delta in performance of screen transitions without and
      with semantics enabled.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

797 798 799 800 801 802 803 804 805 806 807
  flutter_gallery__memory_nav:
    description: >
      Measures memory usage after repeated navigation in Gallery.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__back_button_memory:
    description: >
      Measures memory usage after Android app suspend and resume.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
808

809 810 811 812 813 814
  flutter_gallery__image_cache_memory:
    description: >
      Measures memory usage for a list of large red squares in smaller containers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

815 816 817 818 819
  new_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in the new Flutter Gallery on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]
820

821 822 823
  new_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in the new Flutter Gallery on iOS.
824
    stage: devicelab_ios
825 826
    required_agent_capabilities: ["mac/ios"]

827 828 829 830 831 832
  new_gallery__crane_perf:
    description: >
      Measures the performance of the Crane page in the new Flutter Gallery on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

833 834 835 836 837 838
  fast_scroll_large_images__memory:
    description: >
      Measures memory usage for scrolling through a list of large images.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

839 840 841 842 843 844 845
  fast_scroll_heavy_gridview__memory:
    description: >
      Measures memory usage for scrolling through a grid view of heavy memory
      usage widgets.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

846 847 848 849 850 851 852 853 854 855 856 857
  large_image_changer_perf_android:
    description: >
      Measures memory usage when rotating through a series of large images.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  large_image_changer_perf_ios:
    description: >
      Measures memory, cpu, and gpu usage when rotating through a series of large images.
    stage: devicelab
    required_agent_capabilities: ["mac/ios"]

Dan Field's avatar
Dan Field committed
858 859 860 861 862 863 864 865
  animated_placeholder_perf:
    description: >
      Measures frame build and rasterizer times, as well as frame build counts
      for a grid of images that uses FadeInImage with an animated gif as the
      placeholder.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

866 867 868 869 870 871 872 873
  animated_placeholder_perf__e2e_summary:
    description: >
      Measures frame build and rasterizer times, as well as frame build counts
      for a grid of images that uses FadeInImage with an animated gif as the
      placeholder.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

874 875 876 877
  analyzer_benchmark:
    description: >
      Measures the speed of Dart analyzer.
    stage: devicelab
878
    required_agent_capabilities: ["linux/android"]
879

880 881 882 883 884 885 886 887 888 889 890 891
  web_benchmarks_html:
    description: >
      Runs Web benchmarks on Chrome on a Linux machine using the HTML rendering backend.
    stage: devicelab
    required_agent_capabilities: ["linux-vm"]

  web_benchmarks_canvaskit:
    description: >
      Runs Web benchmarks on Chrome on a Linux machine using the CanvasKit rendering backend.
    stage: devicelab
    required_agent_capabilities: ["linux-vm"]

892 893 894 895 896 897
  # android_splash_screen_integration_test:
  #   description: >
  #     Runs end-to-end test of Flutter's Android splash behavior.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android"]
  #   flaky: true
898

899 900 901 902 903 904 905 906 907 908 909 910
  flutter_gallery_v2_chrome_run_test:
    description: >
      Checks that the New Flutter Gallery runs successfully on Chrome.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery_v2_web_compile_test:
    description: >
      Measures the time to compile the New Flutter Gallery to JavaScript
      and the size of the compiled code.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]