manifest.yaml 19.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon

# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
Yegor's avatar
Yegor committed
13
# Required properties:
14
#
Yegor's avatar
Yegor committed
15 16 17 18 19
# * description: free form string
#       describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
#       tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
#       icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
20
#       run device tests on red Cirrus builds.
Yegor's avatar
Yegor committed
21 22
# * required_agent_capabilities: a list of strings
#       list of capabilities a devicelab agent must have to be able to run this task.
23
#
Yegor's avatar
Yegor committed
24 25 26 27
# Optional properties:
#
# * flaky: boolean true or false
#       whether the task is considered flaky; the result of running a flaky task does not affect
28 29 30
#       the overall build status. Tests should be marked flaky when newly added, until they have
#       been proved for a few cycles. Tests that are actually flaky but not being actively worked
#       on should be hidden because they just cause confusion otherwise.
31 32 33 34
# * on_luci: boolean true or false
#       whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the
#       devicelab to LUCI migration. And when the migration is complete, on_luci flag will be
#       removed.
35 36

tasks:
37
  # Tests of compiling in a variety of modes
38

39
  complex_layout_ios__compile:
40
    description: >
41 42 43
      Collects various performance metrics of compiling the Complex
      Layout sample app for iOS from Mac.
    stage: devicelab_ios
44
    required_agent_capabilities: ["mac/ios"]
45 46 47 48 49 50

  complex_layout_win__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout for Android from Windows.
    stage: devicelab_win
51
    required_agent_capabilities: ["windows/android"]
52 53 54 55 56 57

  basic_material_app_ios__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for iOS from Mac.
    stage: devicelab_ios
58
    required_agent_capabilities: ["mac/ios"]
59 60 61 62 63 64

  basic_material_app_win__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Windows.
    stage: devicelab_win
65
    required_agent_capabilities: ["windows/android"]
66

67 68 69 70 71
  flutter_gallery_ios__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for iOS from Mac.
    stage: devicelab_ios
72
    required_agent_capabilities: ["mac/ios"]
73 74 75 76 77 78

  flutter_gallery_win__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Windows.
    stage: devicelab_win
79
    required_agent_capabilities: ["windows/android"]
80

81 82 83 84 85
  windows_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]
86

87 88 89 90 91 92 93
  # Android on-device tests

  complex_layout_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      Android.
    stage: devicelab
94
    required_agent_capabilities: ["mac/android"]
95

96 97 98 99 100 101
  tiles_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

102 103 104 105 106 107 108
  home_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of scrolling the material page in the
      flutter_gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

109 110 111 112 113 114
  drive_perf_debug_warning:
    description: >
      Check that driver will print warnings when traceAction is called in debug mode.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

115 116 117 118 119 120
  cull_opacity_perf__timeline_summary:
    description: >
      Measures the runtime performance of culling opacity widgets on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

121 122 123 124 125 126
  picture_cache_perf__timeline_summary:
    description: >
      Measures the runtime performance of raster caching many pictures on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

127 128 129 130 131 132
  cubic_bezier_perf__timeline_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

133 134 135 136 137 138 139
  cubic_bezier_perf_sksl_warmup__timeline_summary:
    description: >
      Measures the runtime performance of cubic bezier animations on Android
      with SkSL shader warm-up.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

140 141 142 143 144 145
  flutter_gallery_sksl_warmup__transition_perf_e2e_ios32:
    description: >
      Measures the runtime performance of Flutter gallery transitions on iPhone4s
      with SkSL shader warm-up with e2e.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]
146
    flaky: true
147

148 149 150 151 152 153
  backdrop_filter_perf__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

154 155 156 157 158 159
  textfield_perf__timeline_summary:
    description: >
      Measures the runtime performance of textfield on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

160 161 162 163 164 165
  color_filter_and_fade_perf__timeline_summary:
    description: >
      Measures the runtime performance of color filter with fade on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

166 167 168 169 170 171
  fading_child_animation_perf__timeline_summary:
    description: >
      Measures the runtime performance of opacity filter with fade on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

172 173 174 175 176 177
  imagefiltered_transform_animation_perf__timeline_summary:
    description: >
      Measures the runtime performance of imagefiltered widget with transform on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

178 179 180 181
  flavors_test:
    description: >
      Checks that flavored builds work on Android.
    stage: devicelab
182
    required_agent_capabilities: ["mac/android"]
183

184 185 186 187
  channels_integration_test:
    description: >
      Checks that platform channels work on Android.
    stage: devicelab
188
    required_agent_capabilities: ["mac/android"]
189

190 191 192 193
  external_ui_integration_test:
    description: >
      Checks that external UIs work on Android.
    stage: devicelab
194
    required_agent_capabilities: ["mac/android"]
195

196 197 198 199 200 201
  embedded_android_views_integration_test:
    description: >
      Tests embedded Android views.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

202 203 204 205 206 207
  hybrid_android_views_integration_test:
    description: >
      Tests hybrid Android views.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

208 209 210 211 212 213
  android_semantics_integration_test:
    description: >
      Tests that the Android accessibility bridge produces correct semantics.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

214 215 216 217 218 219
  android_plugin_example_app_build_test:
    description: >
      Tests that the plugin example app can be built using the Flutter Gradle plugin.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

220 221 222 223 224 225
  run_release_test:
    description: >
      Checks that `flutter run --release` does not crash.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

226 227 228 229
  platform_interaction_test:
    description: >
      Checks platform interaction on Android.
    stage: devicelab
230
    required_agent_capabilities: ["mac/android"]
231

232
  platform_channel_sample_test:
233
    description: >
234
     Runs a driver test on the Platform Channel sample app on Android.
235
    stage: devicelab
236
    required_agent_capabilities: ["mac/android"]
237

238 239 240 241 242 243
  platform_view__start_up:
    description: >
      Verifies that Platform View can be used from an Android project.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

244 245 246 247
  complex_layout__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on Android.
    stage: devicelab
248
    required_agent_capabilities: ["mac/android"]
249

250 251 252 253
  hot_mode_dev_cycle__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature.
    stage: devicelab
254
    required_agent_capabilities: ["mac/android"]
255

256 257 258 259
  complex_layout_scroll_perf__memory:
    description: >
      Measures memory usage of the scroll performance test.
    stage: devicelab
260
    required_agent_capabilities: ["mac/android"]
261

262 263 264 265 266 267
  hello_world_android__compile:
    description: >
      Measures the APK size of Hello World.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

268 269 270 271
  hello_world__memory:
    description: >
      Measures starting memory usage of the simplest Flutter app.
    stage: devicelab
272
    required_agent_capabilities: ["mac/android"]
273

274 275 276 277
  microbenchmarks:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks.
    stage: devicelab
278
    required_agent_capabilities: ["mac/android"]
279

280 281 282 283
  flutter_view__start_up:
    description: >
      Verifies that Flutter View can be used from an Android project.
    stage: devicelab
284
    required_agent_capabilities: ["mac/android"]
285

286
  integration_ui_driver:
287 288 289
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
290
    required_agent_capabilities: ["mac/android"]
291

292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
  integration_ui_keyboard_resize:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
    required_agent_capabilities: [ "mac/android" ]

  integration_ui_screenshot:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
    required_agent_capabilities: [ "mac/android" ]

  integration_ui_textfield:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
    required_agent_capabilities: [ "mac/android" ]

310 311 312 313
  service_extensions_test:
    description: >
      Validates our service protocol extensions.
    stage: devicelab
314
    required_agent_capabilities: ["mac/android"]
315

316 317
  # iOS on-device tests

318 319 320
  ios_defines_test:
    description: >
      Builds a Framework with a --dart-define and verifies it can be used as a constant
321
    stage: devicelab_ios
322 323
    required_agent_capabilities: ["mac/ios"]

324
  ios_content_validation_test:
325
    description: >
326
      Builds an obfuscated app and verifies contents and structure
327
    stage: devicelab_ios
328
    required_agent_capabilities: ["mac/ios"]
329
    on_luci: true
330

331 332 333 334 335 336
  tiles_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

337 338 339 340 341 342
  platform_views_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

343 344 345 346 347 348
  flutter_gallery_ios32__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]

349 350
  flavors_test_ios:
    description: >
351
      Checks that flavored builds work on iPhone 6.
352
    stage: devicelab_ios
353
    required_agent_capabilities: ["mac/ios"]
354

355 356
  external_ui_integration_test_ios:
    description: >
357
      Checks that external UIs work on iPhone 6.
358
    stage: devicelab_ios
359
    required_agent_capabilities: ["mac/ios"]
360

361 362
  channels_integration_test_ios:
    description: >
363
      Checks that platform channels work on iPhone 6.
364
    stage: devicelab_ios
365
    required_agent_capabilities: ["mac/ios"]
366

367 368
  platform_interaction_test_ios:
    description: >
369
      Checks platform interaction on iPhone 6.
370
    stage: devicelab_ios
371
    required_agent_capabilities: ["mac/ios"]
372

373
  platform_channel_sample_test_ios:
374
    description: >
375
      Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
376
    stage: devicelab_ios
377
    required_agent_capabilities: ["mac/ios"]
378

379 380 381 382 383 384
  platform_channel_sample_test_swift:
    description: >
      Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

385 386 387 388 389 390
  platform_view_ios__start_up:
    description: >
      Verifies that Platform View can be used from an iOS project.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

391 392 393 394 395 396
  backdrop_filter_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of backdrop filter blurs on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

397 398 399 400 401
  post_backdrop_filter_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of animations after a backdrop filter is removed on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
402

403 404 405 406 407
  complex_layout_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      iOS.
    stage: devicelab_ios
408
    required_agent_capabilities: ["mac/ios"]
409

410 411
  flutter_gallery_ios__start_up:
    description: >
412
      Measures the startup time of the Flutter Gallery app on iPhone 6.
413
    stage: devicelab_ios
414
    required_agent_capabilities: ["mac/ios"]
415

416 417
  complex_layout_ios__start_up:
    description: >
418
      Measures the startup time of the Complex Layout sample app on iPhone 6.
419
    stage: devicelab_ios
420
    required_agent_capabilities: ["mac/ios"]
421

422 423 424 425 426
  flutter_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      iOS.
    stage: devicelab_ios
427
    required_agent_capabilities: ["mac/ios"]
Yegor's avatar
Yegor committed
428

429 430 431
  hello_world_ios__compile:
    description: >
      Measures the IPA size of Hello World.
432
    stage: devicelab_ios
433 434
    required_agent_capabilities: ["mac/ios"]

435 436
  microbenchmarks_ios:
    description: >
437
      Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
438
    stage: devicelab_ios
439 440 441
    #TODO(godofredoc): Remove ios/12 capability once
    # https://github.com/flutter/flutter/issues/49635 is fixed.
    required_agent_capabilities: ["mac/ios", "ios/12"]
442

443 444 445 446
  flutter_view_ios__start_up:
    description: >
      Verifies that Flutter View can be used from an iOS project.
    stage: devicelab_ios
447
    required_agent_capabilities: ["mac/ios"]
448

449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
  integration_ui_ios_driver:
    description: >
      Runs end-to-end Flutter tests on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  integration_ui_ios_keyboard_resize:
    description: >
      Runs end-to-end Flutter tests on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  integration_ui_ios_screenshot:
    description: >
      Runs end-to-end Flutter tests on iPhone 6.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

  integration_ui_ios_textfield:
468
    description: >
469
      Runs end-to-end Flutter tests on iPhone 6.
470
    stage: devicelab_ios
471
    required_agent_capabilities: ["mac/ios"]
472

473 474 475 476 477 478
  ios_platform_view_tests:
    description: >
      Runs end-to-end tests with platform views in the scene.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

479 480 481 482 483
  macos_chrome_dev_mode:
    description: >
      Run flutter web on the devicelab and hot restart.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
484

485 486 487 488 489
  simple_animation_perf_ios:
    description: >
      Measure CPU/GPU usage percentages of a simple animation.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]
490

491
  smoke_catalina_start_up_ios:
492 493 494 495 496
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]

497
  smoke_catalina_hot_mode_dev_cycle_ios__benchmark:
498 499 500 501
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab_ios
    required_agent_capabilities: ["mac-catalina/ios"]
502
    flaky: true # https://github.com/flutter/flutter/issues/71178
503

504 505 506 507 508 509 510 511 512 513 514 515
  smoke_catalina_start_up:
    description: >
      A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
    stage: devicelab
    required_agent_capabilities: ["mac-catalina/android"]

  smoke_catalina_hot_mode_dev_cycle__benchmark:
    description: >
      A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
    stage: devicelab
    required_agent_capabilities: ["mac-catalina/android"]

516 517 518 519 520
  # macOS target platform tests
  hot_mode_dev_cycle_macos_target__benchmark:
    description: >
      Checks the functionality and performance of hot reload on a macOS target platform
    stage: devicelab
521
    required_agent_capabilities: ["mac/ios"]
522

523
  ios_app_with_extensions_test:
524
    description: >
525
      Checks that an iOS app with extensions can be built for physical and simulated devices.
526
    stage: devicelab_ios
527 528
    required_agent_capabilities: ["mac/ios"]

529 530
  # Tests running on Windows host

531 532 533 534
  flavors_test_win:
    description: >
      Checks that flavored builds work on Windows.
    stage: devicelab_win
535
    required_agent_capabilities: ["windows/android"]
536

537 538 539 540
  channels_integration_test_win:
    description: >
      Checks that platform channels work when app is launched from Windows.
    stage: devicelab_win
541
    required_agent_capabilities: ["windows/android"]
542

543 544 545 546
  hot_mode_dev_cycle_win__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on Windows.
    stage: devicelab_win
547
    required_agent_capabilities: ["windows/android"]
548 549 550

  # Tests running on Linux hosts

551 552 553 554 555 556
  flutter_gallery__transition_perf_e2e_ios32:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android with e2e.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios32"]
557
    flaky: true
558 559 560 561 562

  flutter_gallery__transition_perf_e2e_ios:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android with e2e.
563
    stage: devicelab_ios
564 565
    required_agent_capabilities: ["mac/ios"]

566 567 568 569 570
  new_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in the new Flutter Gallery on Android.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]
571

572 573 574
  new_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in the new Flutter Gallery on iOS.
575
    stage: devicelab_ios
576 577
    required_agent_capabilities: ["mac/ios"]

578 579 580 581 582 583
  fast_scroll_large_images__memory:
    description: >
      Measures memory usage for scrolling through a list of large images.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

584 585 586 587 588 589
  large_image_changer_perf_ios:
    description: >
      Measures memory, cpu, and gpu usage when rotating through a series of large images.
    stage: devicelab
    required_agent_capabilities: ["mac/ios"]

590 591 592 593 594 595
  # android_splash_screen_integration_test:
  #   description: >
  #     Runs end-to-end test of Flutter's Android splash behavior.
  #   stage: devicelab
  #   required_agent_capabilities: ["linux/android"]
  #   flaky: true