manifest.yaml 24.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon

# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
Yegor's avatar
Yegor committed
13
# Required properties:
14
#
Yegor's avatar
Yegor committed
15 16 17 18 19
# * description: free form string
#       describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
#       tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
#       icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
20
#       run device tests on red Cirrus builds.
Yegor's avatar
Yegor committed
21 22
# * required_agent_capabilities: a list of strings
#       list of capabilities a devicelab agent must have to be able to run this task.
23
#
Yegor's avatar
Yegor committed
24 25 26 27
# Optional properties:
#
# * flaky: boolean true or false
#       whether the task is considered flaky; the result of running a flaky task does not affect
28 29 30
#       the overall build status. Tests should be marked flaky when newly added, until they have
#       been proved for a few cycles. Tests that are actually flaky but not being actively worked
#       on should be hidden because they just cause confusion otherwise.
Yegor's avatar
Yegor committed
31 32
# * timeout_in_minutes: integer
#       a custom task timeout, specified in minutes.
33 34

tasks:
35
  # Tests of compiling in a variety of modes
36

37 38 39 40 41 42
  complex_layout_android__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout sample app for Android from Linux.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
43

44
  complex_layout_ios__compile:
45
    description: >
46 47 48
      Collects various performance metrics of compiling the Complex
      Layout sample app for iOS from Mac.
    stage: devicelab_ios
49
    required_agent_capabilities: ["mac/ios"]
50 51 52 53 54 55

  complex_layout_win__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout for Android from Windows.
    stage: devicelab_win
56
    required_agent_capabilities: ["windows/android"]
57 58 59 60 61

  basic_material_app_android__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Linux.
62
    stage: devicelab
63 64 65 66 67 68 69
    required_agent_capabilities: ["linux/android"]

  basic_material_app_ios__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for iOS from Mac.
    stage: devicelab_ios
70
    required_agent_capabilities: ["mac/ios"]
71 72 73 74 75 76

  basic_material_app_win__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Windows.
    stage: devicelab_win
77
    required_agent_capabilities: ["windows/android"]
78

79 80 81 82 83 84 85
  # TODO(jonahwilliams): re-enable with https://github.com/flutter/flutter/issues/39597
  # codegen_integration_win:
  #   description: >
  #     Runs codegeneration and verifies that it can execute
  #     correctly.
  #   stage: devicelab_win
  #   required_agent_capabilities: ["windows/android"]
86

87 88 89 90 91 92 93
  codegen_integration_mac:
    description: >
      Runs codegeneration and verifies that it can execute
      correctly.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

94 95 96 97 98 99
  codegen_integration_linux:
    description: >
      Runs codegeneration and verifies that it can execute
      correctly.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
100

101 102 103 104 105 106
  uncaught_image_error_linux:
    description: >
      Ensures that an error thrown into the zone can be caught by the ImageStream
      completer
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
107

108
  flutter_gallery_android__compile:
109
    description: >
110 111
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Linux.
112
    stage: devicelab
113 114 115 116 117 118 119
    required_agent_capabilities: ["linux/android"]

  flutter_gallery_ios__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for iOS from Mac.
    stage: devicelab_ios
120
    required_agent_capabilities: ["mac/ios"]
121 122 123 124 125 126

  flutter_gallery_win__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Windows.
    stage: devicelab_win
127
    required_agent_capabilities: ["windows/android"]
128

129 130 131 132 133 134
  windows_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]
    flaky: true
135

136 137 138 139 140 141 142
  # Android on-device tests

  complex_layout_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      Android.
    stage: devicelab
143
    required_agent_capabilities: ["mac/android"]
144

145 146 147 148 149 150
  tiles_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

151 152 153 154 155 156 157
  home_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of scrolling the material page in the
      flutter_gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

158 159 160 161 162 163
  drive_perf_debug_warning:
    description: >
      Check that driver will print warnings when traceAction is called in debug mode.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

164 165 166 167 168 169
  cull_opacity_perf__timeline_summary:
    description: >
      Measures the runtime performance of culling opacity widgets on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

170 171 172 173 174 175
  picture_cache_perf__timeline_summary:
    description: >
      Measures the runtime performance of raster caching many pictures on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

176 177 178 179 180 181
  cubic_bezier_perf__timeline_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

182 183 184 185 186 187
  backdrop_filter_perf__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

188 189 190 191
  flavors_test:
    description: >
      Checks that flavored builds work on Android.
    stage: devicelab
192
    required_agent_capabilities: ["mac/android"]
193

194 195 196 197
  channels_integration_test:
    description: >
      Checks that platform channels work on Android.
    stage: devicelab
198
    required_agent_capabilities: ["mac/android"]
199

200 201 202 203
  external_ui_integration_test:
    description: >
      Checks that external UIs work on Android.
    stage: devicelab
204
    required_agent_capabilities: ["mac/android"]
205

206 207 208 209 210 211
  embedded_android_views_integration_test:
    description: >
      Tests embedded Android views.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

212 213 214 215 216 217
  android_semantics_integration_test:
    description: >
      Tests that the Android accessibility bridge produces correct semantics.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

218 219 220 221 222
  run_release_test:
    description: >
      Checks that `flutter run --release` does not crash.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]
223
    flaky: true # https://github.com/flutter/flutter/issues/45416.
224

225 226 227 228
  platform_interaction_test:
    description: >
      Checks platform interaction on Android.
    stage: devicelab
229
    required_agent_capabilities: ["mac/android"]
230

231
  platform_channel_sample_test:
232
    description: >
233
     Runs a driver test on the Platform Channel sample app on Android.
234
    stage: devicelab
235
    required_agent_capabilities: ["mac/android"]
236

237 238 239 240 241 242
  platform_view__start_up:
    description: >
      Verifies that Platform View can be used from an Android project.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

243 244 245 246
  complex_layout__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on Android.
    stage: devicelab
247
    required_agent_capabilities: ["mac/android"]
248

249 250 251 252
  hot_mode_dev_cycle__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature.
    stage: devicelab
253
    required_agent_capabilities: ["mac/android"]
254

255 256 257 258
  complex_layout_scroll_perf__memory:
    description: >
      Measures memory usage of the scroll performance test.
    stage: devicelab
259
    required_agent_capabilities: ["mac/android"]
260

261 262 263 264 265 266
  hello_world_android__compile:
    description: >
      Measures the APK size of Hello World.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

267 268 269 270
  hello_world__memory:
    description: >
      Measures starting memory usage of the simplest Flutter app.
    stage: devicelab
271
    required_agent_capabilities: ["mac/android"]
272

273 274 275 276 277 278
  # hello_world__start_up:
  #   description: >
  #     Verifies that Hello World can start on an array of devices.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android_esoteric"]

279 280 281 282
  microbenchmarks:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks.
    stage: devicelab
283
    required_agent_capabilities: ["mac/android"]
284
    timeout_in_minutes: 30
285

286 287 288 289
  flutter_view__start_up:
    description: >
      Verifies that Flutter View can be used from an Android project.
    stage: devicelab
290
    required_agent_capabilities: ["mac/android"]
291

292 293 294 295
  integration_ui:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
296
    required_agent_capabilities: ["mac/android"]
297

298 299 300 301
  commands_test:
    description: >
      Runs tests of flutter run commands.
    stage: devicelab
302
    required_agent_capabilities: ["mac/android"]
303

304 305 306 307 308 309
  run_machine_concurrent_hot_reload:
    description: >
      Runs tests of concurrent hot reload commands via flutter run --machine.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

310 311 312 313
  service_extensions_test:
    description: >
      Validates our service protocol extensions.
    stage: devicelab
314
    required_agent_capabilities: ["mac/android"]
315

316 317 318 319
  android_sample_catalog_generator:
    description: >
      Builds sample catalog markdown pages and Android screenshots
    stage: devicelab
320
    required_agent_capabilities: ["mac/android"]
321

322 323 324 325 326
  complex_layout_semantics_perf:
    description: >
      Measures duration of building the initial semantics tree.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
327

328
  routing_test:
329 330 331 332
    description: >
      Verifies that `flutter drive --route` still works. No performance numbers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
333

334 335 336 337 338 339 340 341
  # flutter_gallery_instrumentation_test:
  #   description: >
  #     Same as flutter_gallery__transition_perf but uses Android instrumentation
  #     framework, and therefore does not require a host computer to run. This
  #     test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android"]
  #   flaky: true
342

343
  flutter_attach_test_android:
344 345 346 347 348
    description: >
      Tests the `flutter attach` command.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

349 350 351 352 353 354
  named_isolates_test:
    description: >
      Tests naming and attaching to specific isolates.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

355 356 357 358 359
  linux_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
360

361 362 363 364 365
  web_size__compile_test:
    description: >
      Measures the size of a dart2js bundle.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
366

367 368
  image_list_reported_duration:
    description: >
369 370 371 372 373 374 375
      Measures image loading performance on release (aot) build.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  image_list_jit_reported_duration:
    description: >
      Measures image loading performance on debug (jit) build.
376 377
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
378

379 380 381 382 383
  build_benchmark:
    description: >
      Measures APK build performance across config changes.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
384

385 386 387 388 389 390
  time_to_development_benchmark__android:
    description: >
      Measures time from flutter run until resident runner setup is complete.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

391 392
  # iOS on-device tests

393 394 395 396 397 398 399 400 401 402
  tiles_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  tiles_scroll_perf_iphonexs__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone XS.
    stage: devicelab_ios
403 404
    # TODO(keyonghan): change with https://github.com/flutter/flutter/issues/50383
    required_agent_capabilities: ["mac/ios"]
405 406 407 408 409 410 411 412 413 414 415 416 417 418

  flutter_gallery_ios32__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]

  flutter_gallery_ios32__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      32-bit iOS (iPhone 4S).
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]

419 420
  flavors_test_ios:
    description: >
421
      Checks that flavored builds work on iPhone 6.
422
    stage: devicelab_ios
423
    required_agent_capabilities: ["mac/ios"]
424

425 426
  external_ui_integration_test_ios:
    description: >
427
      Checks that external UIs work on iPhone 6.
428
    stage: devicelab_ios
429
    required_agent_capabilities: ["mac/ios"]
430

431 432
  channels_integration_test_ios:
    description: >
433
      Checks that platform channels work on iPhone 6.
434
    stage: devicelab_ios
435
    required_agent_capabilities: ["mac/ios"]
436

437 438
  platform_interaction_test_ios:
    description: >
439
      Checks platform interaction on iPhone 6.
440
    stage: devicelab_ios
441
    required_agent_capabilities: ["mac/ios"]
442

443
  platform_channel_sample_test_ios:
444
    description: >
445
      Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
446
    stage: devicelab_ios
447
    required_agent_capabilities: ["mac/ios"]
448

449 450 451 452 453 454
  platform_channel_sample_test_swift:
    description: >
      Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

455 456 457 458 459 460
  platform_view_ios__start_up:
    description: >
      Verifies that Platform View can be used from an iOS project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

461 462 463 464 465 466
  backdrop_filter_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

467 468 469 470 471
  # post_backdrop_filter_perf_ios__timeline_summary:
  #   description: >
  #     Measures the runtime performance of animations after a backdrop filter is removed on iOS.
  #   stage: devicelab_ios
  #   required_agent_capabilities: ["mac/iphonexs"]
472

473 474 475 476 477
  complex_layout_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      iOS.
    stage: devicelab_ios
478
    required_agent_capabilities: ["mac/ios"]
479

480 481
  flutter_gallery_ios__start_up:
    description: >
482
      Measures the startup time of the Flutter Gallery app on iPhone 6.
483
    stage: devicelab_ios
484
    required_agent_capabilities: ["mac/ios"]
485

486 487
  complex_layout_ios__start_up:
    description: >
488
      Measures the startup time of the Complex Layout sample app on iPhone 6.
489
    stage: devicelab_ios
490
    required_agent_capabilities: ["mac/ios"]
491

492 493 494 495 496
  flutter_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      iOS.
    stage: devicelab_ios
497
    required_agent_capabilities: ["mac/ios"]
Yegor's avatar
Yegor committed
498

499 500 501
  hello_world_ios__compile:
    description: >
      Measures the IPA size of Hello World.
502
    stage: devicelab_ios
503 504
    required_agent_capabilities: ["mac/ios"]

505 506
  microbenchmarks_ios:
    description: >
507
      Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
508
    stage: devicelab_ios
509 510 511
    #TODO(godofredoc): Remove ios/12 capability once
    # https://github.com/flutter/flutter/issues/49635 is fixed.
    required_agent_capabilities: ["mac/ios", "ios/12"]
512
    timeout_in_minutes: 30
513

514 515 516 517
  flutter_view_ios__start_up:
    description: >
      Verifies that Flutter View can be used from an iOS project.
    stage: devicelab_ios
518
    required_agent_capabilities: ["mac/ios"]
519

520 521
  integration_ui_ios:
    description: >
522
      Runs end-to-end Flutter tests on iPhone 6.
523
    stage: devicelab_ios
524
    required_agent_capabilities: ["mac/ios"]
525
    timeout_in_minutes: 20
526

527 528 529 530 531 532 533 534 535 536
  flutter_driver_screenshot_test_ios:
    description: >
      Screenshot tests running on a specifc iPhone 6.
      The test makes sure that there is no regression while renderring an image with gl on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios", "ios/gl-render-image"]
    #TODO(cyanglaz): The flaky flag is added because it is the first screenshot test we added.
    # Remove the flaky flag when we are sure the test is stable.
    flaky: true

537 538 539 540 541 542 543
  # TODO(fujino): does not pass on iOS13 https://github.com/flutter/flutter/issues/41133
  # system_debug_ios:
  #   description: >
  #     Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps.
  #   stage: devicelab_ios
  #   required_agent_capabilities: ["mac/ios"]
  #   timeout_in_minutes: 10
544

545 546 547 548
  ios_sample_catalog_generator:
    description: >
      Builds sample catalog markdown pages and iOS screenshots
    stage: devicelab_ios
549
    required_agent_capabilities: ["mac/ios"]
550

551 552 553 554 555 556
  macos_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
    flaky: true # marked as flaky while infra is under heavy development
557

558 559 560 561 562 563
  build_benchmark_ios:
    description: >
      Measures iOS build performance across config changes.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

564 565 566 567 568
  # simple_animation_perf_iphonexs:
  #   description: >
  #     Measure CPU/GPU usage percentages of a simple animation.
  #   stage: devicelab_ios
  #   required_agent_capabilities: ["mac/iphonexs"]
569

570
  smoke_catalina_start_up_ios:
571 572 573 574 575
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]

576
  smoke_catalina_hot_mode_dev_cycle_ios__benchmark:
577 578 579 580 581
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]

582 583 584 585 586 587 588 589 590 591 592 593
  smoke_catalina_start_up:
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab
    required_agent_capabilities: ["mac-catalina/android"]

  smoke_catalina_hot_mode_dev_cycle__benchmark:
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab
    required_agent_capabilities: ["mac-catalina/android"]

594 595 596 597 598 599 600
  # macOS target platform tests
  hot_mode_dev_cycle_macos_target__benchmark:
    description: >
      Checks the functionality and performance of hot reload on a macOS target platform
    stage: devicelab
    required_agent_capabilities: ["mac/ios"]

601 602
  # Tests running on Windows host

603 604 605 606
  flavors_test_win:
    description: >
      Checks that flavored builds work on Windows.
    stage: devicelab_win
607
    required_agent_capabilities: ["windows/android"]
608

609 610 611 612
  channels_integration_test_win:
    description: >
      Checks that platform channels work when app is launched from Windows.
    stage: devicelab_win
613
    required_agent_capabilities: ["windows/android"]
614

615 616 617 618 619 620
  plugin_test_win:
    description: >
      Checks that the project template works and supports plugins on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]

621 622 623 624
  hot_mode_dev_cycle_win__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on Windows.
    stage: devicelab_win
625
    required_agent_capabilities: ["windows/android"]
626

627 628 629 630 631
  # run_without_leak_win:
  #   description: >
  #     Checks that `flutter run` does not leak dart.exe on Windows.
  #   stage: devicelab_win
  #   required_agent_capabilities: ["windows/android"]
632

633 634 635 636 637 638 639 640
  # Tests running on Linux hosts

  hot_mode_dev_cycle_linux__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on a Linux host.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

641 642 643 644 645 646
  flutter_test_performance:
    description: >
      Measures performance of running flutter test.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

647 648 649 650
  dartdocs:
    description: >
      Tracks how many members are still lacking documentation.
    stage: devicelab
651
    required_agent_capabilities: ["linux-vm"]
652

653 654 655 656
  technical_debt__cost:
    description: >
      Estimates our technical debt (TODOs, analyzer ignores, etc).
    stage: devicelab
657
    required_agent_capabilities: ["linux-vm"]
658 659 660 661 662 663 664 665 666 667 668 669 670 671

  flutter_gallery__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

672 673 674 675 676 677 678
  flutter_gallery__transition_perf_with_semantics:
    description: >
      Measures the delta in performance of screen transitions without and
      with semantics enabled.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

679 680 681 682 683 684 685 686 687 688 689
  flutter_gallery__memory_nav:
    description: >
      Measures memory usage after repeated navigation in Gallery.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__back_button_memory:
    description: >
      Measures memory usage after Android app suspend and resume.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
690

691 692 693 694 695 696
  flutter_gallery__image_cache_memory:
    description: >
      Measures memory usage for a list of large red squares in smaller containers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

697 698 699 700 701 702
  fast_scroll_large_images__memory:
    description: >
      Measures memory usage for scrolling through a list of large images.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

703 704 705 706
  analyzer_benchmark:
    description: >
      Measures the speed of Dart analyzer.
    stage: devicelab
707
    required_agent_capabilities: ["linux/android"]
708

709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
  web_benchmarks_html:
    description: >
      Runs Web benchmarks on Chrome on a Linux machine using the HTML rendering backend.
    stage: devicelab
    required_agent_capabilities: ["linux-vm"]
    # TODO(yjbanov): This is a new test. Marking temporarily as flaky while debugging on devicelab.
    flaky: true

  web_benchmarks_canvaskit:
    description: >
      Runs Web benchmarks on Chrome on a Linux machine using the CanvasKit rendering backend.
    stage: devicelab
    required_agent_capabilities: ["linux-vm"]
    # TODO(yjbanov): This is a new test. Marking temporarily as flaky while debugging on devicelab.
    flaky: true

725 726 727 728 729 730
  # run_without_leak_linux:
  #   description: >
  #     Checks that `flutter run` does not leak dart on Linux.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android"]
  #   flaky: true
731

732 733 734 735 736 737
  # run_without_leak_mac:
  #   description: >
  #     Checks that `flutter run` does not leak dart on macOS.
  #   stage: devicelab
  #   required_agent_capabilities: ["mac/android"]
  #   flaky: true
738

739 740 741 742 743 744
  # android_splash_screen_integration_test:
  #   description: >
  #     Runs end-to-end test of Flutter's Android splash behavior.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android"]
  #   flaky: true