manifest.yaml 13.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon

# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
Yegor's avatar
Yegor committed
13
# Required properties:
14
#
Yegor's avatar
Yegor committed
15 16 17 18 19 20 21 22
# * description: free form string
#       describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
#       tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
#       icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
#       run device tests on red Travis.
# * required_agent_capabilities: a list of strings
#       list of capabilities a devicelab agent must have to be able to run this task.
23
#
Yegor's avatar
Yegor committed
24 25 26 27
# Optional properties:
#
# * flaky: boolean true or false
#       whether the task is considered flaky; the result of running a flaky task does not affect
28 29 30
#       the overall build status. Tests should be marked flaky when newly added, until they have
#       been proved for a few cycles. Tests that are actually flaky but not being actively worked
#       on should be hidden because they just cause confusion otherwise.
Yegor's avatar
Yegor committed
31 32
# * timeout_in_minutes: integer
#       a custom task timeout, specified in minutes.
33 34

tasks:
35
  # Tests of compiling in a variety of modes
36

37 38 39 40 41 42
  complex_layout_android__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout sample app for Android from Linux.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
43

44
  complex_layout_ios__compile:
45
    description: >
46 47 48
      Collects various performance metrics of compiling the Complex
      Layout sample app for iOS from Mac.
    stage: devicelab_ios
49
    required_agent_capabilities: ["mac/ios"]
50 51 52 53 54 55

  complex_layout_win__compile:
    description: >
      Collects various performance metrics of compiling the Complex
      Layout for Android from Windows.
    stage: devicelab_win
56
    required_agent_capabilities: ["windows/android"]
57 58 59 60 61

  basic_material_app_android__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Linux.
62
    stage: devicelab
63 64 65 66 67 68 69
    required_agent_capabilities: ["linux/android"]

  basic_material_app_ios__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for iOS from Mac.
    stage: devicelab_ios
70
    required_agent_capabilities: ["mac/ios"]
71 72 73 74 75 76

  basic_material_app_win__compile:
    description: >
      Collects various performance metrics of compiling the default
      app for Android from Windows.
    stage: devicelab_win
77
    required_agent_capabilities: ["windows/android"]
78

79
  flutter_gallery_android__compile:
80
    description: >
81 82
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Linux.
83
    stage: devicelab
84 85 86 87 88 89 90
    required_agent_capabilities: ["linux/android"]

  flutter_gallery_ios__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for iOS from Mac.
    stage: devicelab_ios
91
    required_agent_capabilities: ["mac/ios"]
92 93 94 95 96 97

  flutter_gallery_win__compile:
    description: >
      Collects various performance metrics of compiling the Flutter
      Gallery for Android from Windows.
    stage: devicelab_win
98
    required_agent_capabilities: ["windows/android"]
99 100 101 102 103 104 105 106

  # Android on-device tests

  complex_layout_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      Android.
    stage: devicelab
107
    required_agent_capabilities: ["mac/android"]
108

109 110 111 112
  flavors_test:
    description: >
      Checks that flavored builds work on Android.
    stage: devicelab
113
    required_agent_capabilities: ["mac/android"]
114

115 116 117 118
  channels_integration_test:
    description: >
      Checks that platform channels work on Android.
    stage: devicelab
119
    required_agent_capabilities: ["mac/android"]
120

121 122 123 124
  external_ui_integration_test:
    description: >
      Checks that external UIs work on Android.
    stage: devicelab
125
    required_agent_capabilities: ["mac/android"]
126

127 128 129 130
  platform_interaction_test:
    description: >
      Checks platform interaction on Android.
    stage: devicelab
131
    required_agent_capabilities: ["mac/android"]
132

133
  platform_channel_sample_test:
134
    description: >
135
     Runs a driver test on the Platform Channel sample app on Android.
136
    stage: devicelab
137
    required_agent_capabilities: ["mac/android"]
138

139 140 141 142
  complex_layout__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on Android.
    stage: devicelab
143
    required_agent_capabilities: ["mac/android"]
144

145 146 147 148
  hot_mode_dev_cycle__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature.
    stage: devicelab
149
    required_agent_capabilities: ["mac/android"]
150

151
  hot_mode_dev_cycle__benchmark_dart1:
152 153
    description: >
      Measures the performance of Dart VM hot patching feature under
154
      --no-preview-dart-2 option, that disables Dart 2.0 frontend.
155
    stage: devicelab
156
    required_agent_capabilities: ["mac/android"]
157

158
  hot_mode_dev_cycle_ios__benchmark_dart1:
159 160
    description: >
      Measures the performance of Dart VM hot patching feature under
161
      --no-preview-dart-2 option, that disables Dart 2.0 frontend.
162
    stage: devicelab_ios
163 164 165
    required_agent_capabilities: ["mac/ios"]
    flaky: true

166 167 168 169
  complex_layout_scroll_perf__memory:
    description: >
      Measures memory usage of the scroll performance test.
    stage: devicelab
170
    required_agent_capabilities: ["mac/android"]
171

172 173 174 175
  hello_world__memory:
    description: >
      Measures starting memory usage of the simplest Flutter app.
    stage: devicelab
176
    required_agent_capabilities: ["mac/android"]
177

178 179 180 181
  microbenchmarks:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks.
    stage: devicelab
182
    required_agent_capabilities: ["mac/android"]
183
    timeout_in_minutes: 30
184

185 186 187 188
  flutter_view__start_up:
    description: >
      Verifies that Flutter View can be used from an Android project.
    stage: devicelab
189
    required_agent_capabilities: ["mac/android"]
190

191 192 193 194
  integration_ui:
    description: >
      Runs end-to-end Flutter tests on Android.
    stage: devicelab
195
    required_agent_capabilities: ["mac/android"]
196

197 198 199 200
  commands_test:
    description: >
      Runs tests of flutter run commands.
    stage: devicelab
201
    required_agent_capabilities: ["mac/android"]
202

203 204 205 206 207 208
  run_machine_concurrent_hot_reload:
    description: >
      Runs tests of concurrent hot reload commands via flutter run --machine.
    stage: devicelab
    required_agent_capabilities: ["mac/android"]

209 210 211 212
  service_extensions_test:
    description: >
      Validates our service protocol extensions.
    stage: devicelab
213
    required_agent_capabilities: ["mac/android"]
214

215 216 217 218
  android_sample_catalog_generator:
    description: >
      Builds sample catalog markdown pages and Android screenshots
    stage: devicelab
219
    required_agent_capabilities: ["mac/android"]
220

221 222 223 224 225
  complex_layout_semantics_perf:
    description: >
      Measures duration of building the initial semantics tree.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
226

227
  routing_test:
228 229 230 231
    description: >
      Verifies that `flutter drive --route` still works. No performance numbers.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
232

233 234 235 236 237 238
  gradle_plugin_test:
    description: >
      Verifies that the Flutter Gradle plugin supports standard and custom Android build types.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

239 240 241 242 243 244
  plugin_test:
    description: >
      Checks that the project template works and supports plugins.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

245 246 247 248 249 250 251
  flutter_gallery_instrumentation_test:
    description: >
      Same as flutter_gallery__transition_perf but uses Android instrumentation
      framework, and therefore does not require a host computer to run. This
      test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
252

253 254
  # iOS on-device tests

255 256 257 258
  flavors_test_ios:
    description: >
      Checks that flavored builds work on iOS.
    stage: devicelab_ios
259
    required_agent_capabilities: ["mac/ios"]
260

261 262 263 264 265 266
  plugin_test_ios:
    description: >
      Checks that the project template works and supports plugins on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["mac/ios"]

267 268 269
  external_ui_integration_test_ios:
    description: >
      Checks that external UIs work on iOS.
270
    stage: devicelab_ios
271
    required_agent_capabilities: ["mac/ios"]
272

273 274 275 276
  channels_integration_test_ios:
    description: >
      Checks that platform channels work on iOS.
    stage: devicelab_ios
277
    required_agent_capabilities: ["mac/ios"]
278

279 280 281 282
  platform_interaction_test_ios:
    description: >
      Checks platform interaction on iOS.
    stage: devicelab_ios
283
    required_agent_capabilities: ["mac/ios"]
284

285
  platform_channel_sample_test_ios:
286
    description: >
287
      Runs a driver test on the Platform Channel sample app on iOS.
288
    stage: devicelab_ios
289
    required_agent_capabilities: ["mac/ios"]
290

291 292 293 294 295
  complex_layout_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      iOS.
    stage: devicelab_ios
296
    required_agent_capabilities: ["mac/ios"]
297

298 299 300 301
  flutter_gallery_ios__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on iOS.
    stage: devicelab_ios
302
    required_agent_capabilities: ["mac/ios"]
303

304 305 306 307
  complex_layout_ios__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on iOS.
    stage: devicelab_ios
308
    required_agent_capabilities: ["mac/ios"]
309

310 311 312 313 314
  flutter_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      iOS.
    stage: devicelab_ios
315
    required_agent_capabilities: ["mac/ios"]
Yegor's avatar
Yegor committed
316

317 318 319 320
  microbenchmarks_ios:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks on iOS.
    stage: devicelab_ios
321
    required_agent_capabilities: ["mac/ios"]
322
    timeout_in_minutes: 30
323

324 325 326 327
  flutter_view_ios__start_up:
    description: >
      Verifies that Flutter View can be used from an iOS project.
    stage: devicelab_ios
328
    required_agent_capabilities: ["mac/ios"]
329

330 331 332 333
  integration_ui_ios:
    description: >
      Runs end-to-end Flutter tests on iOS.
    stage: devicelab_ios
334
    required_agent_capabilities: ["mac/ios"]
335

336 337 338 339
  ios_sample_catalog_generator:
    description: >
      Builds sample catalog markdown pages and iOS screenshots
    stage: devicelab_ios
340
    required_agent_capabilities: ["mac/ios"]
341

342 343
  # Tests running on Windows host

344 345 346 347
  flavors_test_win:
    description: >
      Checks that flavored builds work on Windows.
    stage: devicelab_win
348
    required_agent_capabilities: ["windows/android"]
349

350 351 352 353
  channels_integration_test_win:
    description: >
      Checks that platform channels work when app is launched from Windows.
    stage: devicelab_win
354
    required_agent_capabilities: ["windows/android"]
355

356 357 358 359 360 361
  plugin_test_win:
    description: >
      Checks that the project template works and supports plugins on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows/android"]

362 363 364 365
  hot_mode_dev_cycle_win__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on Windows.
    stage: devicelab_win
366
    required_agent_capabilities: ["windows/android"]
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386

  # Tests running on Linux hosts

  hot_mode_dev_cycle_linux__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on a Linux host.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  dartdocs:
    description: >
      Tracks how many members are still lacking documentation.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  technical_debt__cost:
    description: >
      Estimates our technical debt (TODOs, analyzer ignores, etc).
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
387 388 389 390 391 392 393 394 395 396 397 398 399 400

  flutter_gallery__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

401 402 403 404 405 406 407
  flutter_gallery__transition_perf_with_semantics:
    description: >
      Measures the delta in performance of screen transitions without and
      with semantics enabled.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

408 409 410 411 412 413 414 415 416 417 418
  flutter_gallery__memory_nav:
    description: >
      Measures memory usage after repeated navigation in Gallery.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]

  flutter_gallery__back_button_memory:
    description: >
      Measures memory usage after Android app suspend and resume.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]
419 420 421 422 423 424

  analyzer_benchmark:
    description: >
      Measures the speed of Dart analyzer.
    stage: devicelab
    required_agent_capabilities: ["linux/android"]