manifest.yaml 20.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon

# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
Yegor's avatar
Yegor committed
13
# Required properties:
14
#
Yegor's avatar
Yegor committed
15 16 17 18 19
# * description: free form string
#       describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
#       tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
#       icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
20
#       run device tests on red Cirrus builds.
Yegor's avatar
Yegor committed
21 22
# * required_agent_capabilities: a list of strings
#       list of capabilities a devicelab agent must have to be able to run this task.
23
#
Yegor's avatar
Yegor committed
24 25 26 27
# Optional properties:
#
# * flaky: boolean true or false
#       whether the task is considered flaky; the result of running a flaky task does not affect
28 29 30
#       the overall build status. Tests should be marked flaky when newly added, until they have
#       been proved for a few cycles. Tests that are actually flaky but not being actively worked
#       on should be hidden because they just cause confusion otherwise.
Yegor's avatar
Yegor committed
31 32
# * timeout_in_minutes: integer
#       a custom task timeout, specified in minutes.
33 34

tasks:
35
  # Tests of compiling in a variety of modes
36

37 38 39 40 41 42
  complex_layout_android__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout sample app for Android from Linux.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
43

44
  complex_layout_ios__compile:
45
    description: >
46 47 48
      Collects various performance metrics of compiling the Complex
      Layout sample app for iOS from Mac.
    stage: devicelab_ios
49
    required_agent_capabilities: ["mac/ios"]
50 51 52 53 54 55

  complex_layout_win__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout for Android from Windows.
    stage: devicelab_win
56
    required_agent_capabilities: ["windows/android"]
57 58 59 60 61

  basic_material_app_android__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Linux.
62
    stage: devicelab
63 64 65 66 67 68 69
    required_agent_capabilities: ["linux/android"]

  basic_material_app_ios__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for iOS from Mac.
    stage: devicelab_ios
70
    required_agent_capabilities: ["mac/ios"]
71 72 73 74 75 76

  basic_material_app_win__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Windows.
    stage: devicelab_win
77
    required_agent_capabilities: ["windows/android"]
78

79
  codegen_integration_win:
80 81 82 83 84 85
    description: >
      Runs codegeneration and verifies that it can execute
      correctly.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]

86 87 88 89 90 91 92 93 94 95 96 97 98 99
  codegen_integration_mac:
    description: >
      Runs codegeneration and verifies that it can execute
      correctly.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  codegen_integration_linux:
    description: >
      Runs codegeneration and verifies that it can execute
      correctly.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

100 101 102 103 104 105 106
  uncaught_image_error_linux:
    description: >
      Ensures that an error thrown into the zone can be caught by the ImageStream
      completer
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

107
  flutter_gallery_android__compile:
108
    description: >
109 110
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Linux.
111
    stage: devicelab
112 113 114 115 116 117 118
    required_agent_capabilities: ["linux/android"]

  flutter_gallery_ios__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for iOS from Mac.
    stage: devicelab_ios
119
    required_agent_capabilities: ["mac/ios"]
120 121 122 123 124 125

  flutter_gallery_win__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Windows.
    stage: devicelab_win
126
    required_agent_capabilities: ["windows/android"]
127

128 129 130 131 132
  windows_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]
133
    flaky: true
134

135 136 137 138 139 140 141
  # Android on-device tests

  complex_layout_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      Android.
    stage: devicelab
142
    required_agent_capabilities: ["mac/android"]
143

144 145 146 147 148 149
  tiles_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

150 151 152
  tiles_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
153
    stage: devicelab_ios
154 155 156 157 158
    required_agent_capabilities: ["mac/ios"]

  tiles_scroll_perf_iphonexs__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone XS.
159
    stage: devicelab_ios
160 161
    required_agent_capabilities: ["mac/iphonexs"]

162 163 164 165 166 167 168
  home_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of scrolling the material page in the
      flutter_gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

169 170 171 172 173 174
  drive_perf_debug_warning:
    description: >
      Check that driver will print warnings when traceAction is called in debug mode.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

175 176 177 178 179 180
  cull_opacity_perf__timeline_summary:
    description: >
      Measures the runtime performance of culling opacity widgets on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

181 182 183 184 185 186
  cubic_bezier_perf__timeline_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

187 188 189 190 191 192
  backdrop_filter_perf__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

193 194 195 196
  flavors_test:
    description: >
      Checks that flavored builds work on Android.
    stage: devicelab
197
    required_agent_capabilities: ["mac/android"]
198

199 200 201 202
  channels_integration_test:
    description: >
      Checks that platform channels work on Android.
    stage: devicelab
203
    required_agent_capabilities: ["mac/android"]
204

205 206 207 208
  external_ui_integration_test:
    description: >
      Checks that external UIs work on Android.
    stage: devicelab
209
    required_agent_capabilities: ["mac/android"]
210

211 212 213 214 215 216
  embedded_android_views_integration_test:
    description: >
      Tests embedded Android views.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

217 218 219 220 221 222
  android_semantics_integration_test:
    description: >
      Tests that the Android accessibility bridge produces correct semantics.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

223 224 225 226 227 228
  run_release_test:
    description: >
      Checks that `flutter run --release` does not crash.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

229 230 231 232
  platform_interaction_test:
    description: >
      Checks platform interaction on Android.
    stage: devicelab
233
    required_agent_capabilities: ["mac/android"]
234

235
  platform_channel_sample_test:
236
    description: >
237
     Runs a driver test on the Platform Channel sample app on Android.
238
    stage: devicelab
239
    required_agent_capabilities: ["mac/android"]
240

241 242 243 244 245 246
  platform_view__start_up:
    description: >
      Verifies that Platform View can be used from an Android project.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

247 248 249 250
  complex_layout__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on Android.
    stage: devicelab
251
    required_agent_capabilities: ["mac/android"]
252

253 254 255 256
  hot_mode_dev_cycle__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature.
    stage: devicelab
257
    required_agent_capabilities: ["mac/android"]
258

259 260 261 262
  complex_layout_scroll_perf__memory:
    description: >
      Measures memory usage of the scroll performance test.
    stage: devicelab
263
    required_agent_capabilities: ["mac/android"]
264

265 266 267 268 269 270
  hello_world_android__compile:
    description: >
      Measures the APK size of Hello World.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

271 272 273 274
  hello_world__memory:
    description: >
      Measures starting memory usage of the simplest Flutter app.
    stage: devicelab
275
    required_agent_capabilities: ["mac/android"]
276

277 278 279 280 281 282
  # hello_world__start_up:
  #   description: >
  #     Verifies that Hello World can start on an array of devices.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android_esoteric"]

283 284 285 286
  microbenchmarks:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks.
    stage: devicelab
287
    required_agent_capabilities: ["mac/android"]
288
    timeout_in_minutes: 30
289

290 291 292 293
  flutter_view__start_up:
    description: >
      Verifies that Flutter View can be used from an Android project.
    stage: devicelab
294
    required_agent_capabilities: ["mac/android"]
295

296 297 298 299
  integration_ui:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
300
    required_agent_capabilities: ["mac/android"]
301

302 303 304 305
  commands_test:
    description: >
      Runs tests of flutter run commands.
    stage: devicelab
306
    required_agent_capabilities: ["mac/android"]
307

308 309 310 311 312 313
  run_machine_concurrent_hot_reload:
    description: >
      Runs tests of concurrent hot reload commands via flutter run --machine.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

314 315 316 317
  service_extensions_test:
    description: >
      Validates our service protocol extensions.
    stage: devicelab
318
    required_agent_capabilities: ["mac/android"]
319

320 321 322 323
  android_sample_catalog_generator:
    description: >
      Builds sample catalog markdown pages and Android screenshots
    stage: devicelab
324
    required_agent_capabilities: ["mac/android"]
325

326 327 328 329 330
  complex_layout_semantics_perf:
    description: >
      Measures duration of building the initial semantics tree.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
331

332
  routing_test:
333 334 335 336
    description: >
      Verifies that `flutter drive --route` still works. No performance numbers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
337 338 339 340 341 342 343 344

  flutter_gallery_instrumentation_test:
    description: >
      Same as flutter_gallery__transition_perf but uses Android instrumentation
      framework, and therefore does not require a host computer to run. This
      test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
345
    flaky: true
346

347 348 349 350 351 352
  flutter_attach_test:
    description: >
      Tests the `flutter attach` command.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

353 354 355 356 357 358
  named_isolates_test:
    description: >
      Tests naming and attaching to specific isolates.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

359 360 361 362 363 364 365
  linux_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
    flaky: true

366 367 368 369 370 371
  web_size__compile_test:
    description: >
      Measures the size of a dart2js bundle.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

372 373
  image_list_reported_duration:
    description: >
374 375 376 377 378 379 380
      Measures image loading performance on release (aot) build.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  image_list_jit_reported_duration:
    description: >
      Measures image loading performance on debug (jit) build.
381 382
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
383

384 385 386 387 388 389
  build_benchmark:
    description: >
      Measures APK build performance across config changes.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

390 391
  # iOS on-device tests

392 393
  flavors_test_ios:
    description: >
394
      Checks that flavored builds work on iPhone 6.
395
    stage: devicelab_ios
396
    required_agent_capabilities: ["mac/ios"]
397

398 399
  external_ui_integration_test_ios:
    description: >
400
      Checks that external UIs work on iPhone 6.
401
    stage: devicelab_ios
402
    required_agent_capabilities: ["mac/ios"]
403

404 405
  channels_integration_test_ios:
    description: >
406
      Checks that platform channels work on iPhone 6.
407
    stage: devicelab_ios
408
    required_agent_capabilities: ["mac/ios"]
409

410 411
  platform_interaction_test_ios:
    description: >
412
      Checks platform interaction on iPhone 6.
413
    stage: devicelab_ios
414
    required_agent_capabilities: ["mac/ios"]
415

416
  platform_channel_sample_test_ios:
417
    description: >
418
      Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
419
    stage: devicelab_ios
420
    required_agent_capabilities: ["mac/ios"]
421

422 423 424 425 426 427
  platform_channel_sample_test_swift:
    description: >
      Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

428 429 430 431 432 433
  platform_view_ios__start_up:
    description: >
      Verifies that Platform View can be used from an iOS project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

434 435 436 437 438 439 440
  backdrop_filter_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
    flaky: true

441 442 443 444 445
  complex_layout_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      iOS.
    stage: devicelab_ios
446
    required_agent_capabilities: ["mac/ios"]
447

448 449
  flutter_gallery_ios__start_up:
    description: >
450
      Measures the startup time of the Flutter Gallery app on iPhone 6.
451
    stage: devicelab_ios
452
    required_agent_capabilities: ["mac/ios"]
453

454 455
  complex_layout_ios__start_up:
    description: >
456
      Measures the startup time of the Complex Layout sample app on iPhone 6.
457
    stage: devicelab_ios
458
    required_agent_capabilities: ["mac/ios"]
459

460 461 462 463 464
  flutter_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      iOS.
    stage: devicelab_ios
465
    required_agent_capabilities: ["mac/ios"]
Yegor's avatar
Yegor committed
466

467 468 469
  hello_world_ios__compile:
    description: >
      Measures the IPA size of Hello World.
470
    stage: devicelab_ios
471 472
    required_agent_capabilities: ["mac/ios"]

473 474
  microbenchmarks_ios:
    description: >
475
      Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
476
    stage: devicelab_ios
477
    required_agent_capabilities: ["mac/ios"]
478
    timeout_in_minutes: 30
479

480 481 482 483
  flutter_view_ios__start_up:
    description: >
      Verifies that Flutter View can be used from an iOS project.
    stage: devicelab_ios
484
    flaky: true
485
    required_agent_capabilities: ["mac/ios"]
486

487 488
  integration_ui_ios:
    description: >
489
      Runs end-to-end Flutter tests on iPhone 6.
490
    stage: devicelab_ios
491
    required_agent_capabilities: ["mac/ios"]
492
    timeout_in_minutes: 20
493

494 495 496 497 498 499 500
  system_debug_ios:
    description: >
      Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
    timeout_in_minutes: 10

501 502 503 504
  ios_sample_catalog_generator:
    description: >
      Builds sample catalog markdown pages and iOS screenshots
    stage: devicelab_ios
505
    required_agent_capabilities: ["mac/ios"]
506

507 508 509 510 511
  macos_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
512
    flaky: true # marekd as flaky while infra is under heavy development
513

514 515 516 517 518 519
  build_benchmark_ios:
    description: >
      Measures iOS build performance across config changes.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

520 521 522 523 524 525 526 527 528 529 530 531 532
  smoke_catalina_start_up:
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]

  smoke_catalina_hot_mode_dev_cycle__benchmark:
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab_ios
    flaky: true
    required_agent_capabilities: ["mac-catalina/ios"]

533 534
  # Tests running on Windows host

535 536 537 538
  flavors_test_win:
    description: >
      Checks that flavored builds work on Windows.
    stage: devicelab_win
539
    required_agent_capabilities: ["windows/android"]
540

541 542 543 544
  channels_integration_test_win:
    description: >
      Checks that platform channels work when app is launched from Windows.
    stage: devicelab_win
545
    required_agent_capabilities: ["windows/android"]
546

547 548 549 550 551 552
  plugin_test_win:
    description: >
      Checks that the project template works and supports plugins on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]

553 554 555 556
  hot_mode_dev_cycle_win__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on Windows.
    stage: devicelab_win
557
    required_agent_capabilities: ["windows/android"]
558

559 560 561 562 563 564
  run_without_leak_win:
    description: >
      Checks that `flutter run` does not leak dart.exe on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]

565 566 567 568 569 570 571 572
  # Tests running on Linux hosts

  hot_mode_dev_cycle_linux__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on a Linux host.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

573 574 575 576 577 578
  flutter_test_performance:
    description: >
      Measures performance of running flutter test.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

579 580 581 582 583
  technical_debt__cost:
    description: >
      Estimates our technical debt (TODOs, analyzer ignores, etc).
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
584 585 586 587 588 589 590 591 592 593 594 595

  flutter_gallery__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android.
    stage: devicelab
596
    flaky: true
597 598
    required_agent_capabilities: ["linux/android"]

599 600 601 602 603
  flutter_gallery__transition_perf_with_semantics:
    description: >
      Measures the delta in performance of screen transitions without and
      with semantics enabled.
    stage: devicelab
604
    flaky: true
605 606
    required_agent_capabilities: ["linux/android"]

607 608 609 610 611 612 613 614 615 616 617
  flutter_gallery__memory_nav:
    description: >
      Measures memory usage after repeated navigation in Gallery.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__back_button_memory:
    description: >
      Measures memory usage after Android app suspend and resume.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
618

619 620 621 622 623 624
  flutter_gallery__image_cache_memory:
    description: >
      Measures memory usage for a list of large red squares in smaller containers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

625 626 627 628
  analyzer_benchmark:
    description: >
      Measures the speed of Dart analyzer.
    stage: devicelab
629
    required_agent_capabilities: ["linux/android"]
630

631 632 633 634 635 636 637
  run_without_leak_linux:
    description: >
      Checks that `flutter run` does not leak dart on Linux.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
    flaky: true

638 639
  flutter_gallery_ios32__start_up:
    description: >
640
      Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
641 642 643
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]

644 645 646
  flutter_gallery_ios32__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
647
      32-bit iOS (iPhone 4S).
648 649
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]
650 651 652 653 654 655 656

  run_without_leak_mac:
    description: >
      Checks that `flutter run` does not leak dart on macOS.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]
    flaky: true
657 658 659 660 661 662

  android_splash_screen_integration_test:
    description: >
      Runs end-to-end test of Flutter's Android splash behavior.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
663
    flaky: true