manifest.yaml 7.61 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon

# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
Yegor's avatar
Yegor committed
13
# Required properties:
14
#
Yegor's avatar
Yegor committed
15 16 17 18 19 20 21 22
# * description: free form string
#       describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
#       tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
#       icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
#       run device tests on red Travis.
# * required_agent_capabilities: a list of strings
#       list of capabilities a devicelab agent must have to be able to run this task.
23
#
Yegor's avatar
Yegor committed
24 25 26 27 28 29 30
# Optional properties:
#
# * flaky: boolean true or false
#       whether the task is considered flaky; the result of running a flaky task does not affect
#       the overall build status.
# * timeout_in_minutes: integer
#       a custom task timeout, specified in minutes.
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53

tasks:
  # Deviceless tests

  # TODO: make these not require "has-android-device"; it is only there to
  #       ensure we have the Android SDK.

  flutter_gallery__build:
    description: >
      Collects various performance metrics from AOT builds of the Flutter
      Gallery.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  complex_layout__build:
    description: >
      Collects various performance metrics from AOT builds of the Complex
      Layout sample app.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  basic_material_app__size:
    description: >
Yegor's avatar
Yegor committed
54
      Measures the APK size of a basic material app.
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  analyzer_cli__analysis_time:
    description: >
      Measures the speed of analyzing Flutter itself in batch mode.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  analyzer_server__analysis_time:
    description: >
      Measures the speed of analyzing Flutter itself in server mode.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

70 71 72 73 74
  technical_debt__cost:
    description: >
      Estimates our technical debt (TODOs, analyzer ignores, etc).
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]
75 76 77 78 79 80 81 82 83

  # Android on-device tests

  complex_layout_scroll_perf__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      Android.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]
84
    flaky: true
85

86 87 88 89 90 91 92
  channels_integration_test:
    description: >
      Checks that platform channels work on Android.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  platform_channel_sample_test:
93
    description: >
94
     Runs a driver test on the Platform Channel sample app on Android.
95 96 97
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
  flutter_gallery__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on Android.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  complex_layout__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on Android.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

  flutter_gallery__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      Android.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]
116
    flaky: true
117

118 119 120 121 122 123
  hot_mode_dev_cycle__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

124 125 126 127 128 129
  complex_layout_scroll_perf__memory:
    description: >
      Measures memory usage of the scroll performance test.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

130 131 132 133 134 135
  hello_world__memory:
    description: >
      Measures starting memory usage of the simplest Flutter app.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

136 137 138 139 140 141
  flutter_gallery__memory_nav:
    description: >
      Measures memory usage after repeated navigation in Gallery.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

142 143 144 145 146 147
  flutter_gallery__back_button_memory:
    description: >
      Measures memory usage after Android app suspend and resume.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

148 149 150 151 152
  microbenchmarks:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]
153
    timeout_in_minutes: 30
154

155 156 157 158 159 160
  flutter_view__start_up:
    description: >
      Verifies that Flutter View can be used from an Android project.
    stage: devicelab
    required_agent_capabilities: ["has-android-device"]

161 162
  # iOS on-device tests

163 164 165 166 167
  channels_integration_test_ios:
    description: >
      Checks that platform channels work on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]
168
    flaky: true
169 170

  platform_channel_sample_test_ios:
171
    description: >
172
      Runs a driver test on the Platform Channel sample app on iOS.
173 174 175
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]

176 177 178 179 180 181 182 183 184 185
  complex_layout_scroll_perf_ios__timeline_summary:
    description: >
      Measures the runtime performance of the Complex Layout sample app on
      iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]

  flutter_gallery_ios__start_up:
    description: >
      Measures the startup time of the Flutter Gallery app on iOS.
Yegor's avatar
Yegor committed
186 187
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]
188 189 190 191 192 193 194

  complex_layout_ios__start_up:
    description: >
      Measures the startup time of the Complex Layout sample app on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]

195 196 197 198 199 200
  flutter_gallery_ios__transition_perf:
    description: >
      Measures the performance of screen transitions in Flutter Gallery on
      iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]
Yegor's avatar
Yegor committed
201 202 203 204 205

  basic_material_app_ios__size:
    description: >
      Measures the IPA size of a basic material app.
    stage: devicelab_ios
206 207 208 209 210 211 212
    required_agent_capabilities: ["has-ios-device"]

  microbenchmarks_ios:
    description: >
      Runs benchmarks from dev/benchmarks/microbenchmarks on iOS.
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]
213
    timeout_in_minutes: 30
214
    flaky: true
215

216 217 218 219 220 221
  flutter_view_ios__start_up:
    description: >
      Verifies that Flutter View can be used from an iOS project.
    stage: devicelab_ios
    required_agent_capabilities: ["has-ios-device"]

222 223
  # Tests running on Windows host

224 225 226 227 228 229 230
  channels_integration_test_win:
    description: >
      Checks that platform channels work when app is launched from Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows"]
    flaky: true

231 232 233 234 235 236
  flutter_gallery_win__build:
    description: >
      Collects various performance metrics from AOT builds of the Flutter
      Gallery on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows"]
237
    flaky: true
238

239 240 241 242 243
  hot_mode_dev_cycle_win__benchmark:
    description: >
      Measures the performance of Dart VM hot patching feature on Windows.
    stage: devicelab_win
    required_agent_capabilities: ["windows"]
244
    flaky: true