move CI tests into the main repo (#5758)

dev/bots/test.sh

@@ -35,6 +35,7 @@ fi
 (cd packages/flutter_test; flutter test)
 (cd packages/flutter_tools; dart -c test/all.dart)
 
+(cd dev/devicelab; dart -c test/all.dart)
 (cd dev/manual_tests; flutter test)
 (cd examples/hello_world; flutter test)
 (cd examples/layers; flutter test)
@@ -55,4 +56,4 @@ if [ -n "$COVERAGE_FLAG" ]; then
 fi
 
 # generate the API docs, upload them
-dev/bots/docs.sh
+dev/bots/docs.sh
\ No newline at end of file

dev/devicelab/.gitignore

+.DS_Store
+.buildlog
+.idea
+.packages
+.pub/
+build/
+packages
+pubspec.lock
+.atom/

dev/devicelab/README.md

+# Flutter devicelab
+
+This package contains the test framework and tests that run on physical devices.
+More generally the tests are referred to as "tasks" in the API, but since we
+primarily use it for testing, this document refers to them as "tests".
+
+# Writing tests
+
+A test is a simple Dart program that lives under `bin/tests` and uses
+`package:flutter_devicelab/framework/framework.dart` to define and run a _task_.
+
+Example:
+
+```dart
+import 'dart:async';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(() async {
+    ... do something interesting ...
+
+    // Aggregate results into a JSONable Map structure.
+    Map<String, dynamic> testResults = ...;
+
+    // Report success.
+    return new TaskResult.success(testResults);
+
+    // Or you can also report a failure.
+    return new TaskResult.failure('Something went wrong!');
+  });
+}
+```
+
+Only one `task` is permitted per program. However, that task can run any number
+of tests internally. A task has a name. It succeeds and fails independently of
+other tasks, and is reported to the dashboard independently of other tasks.
+
+A task runs in its own standalone Dart VM and reports results via Dart VM
+service protocol. This ensures that tasks do not interfere with each other and
+lets the CI system time out and clean up tasks that get stuck.
+
+# Adding tests to the CI environment
+
+The `manifest.yaml` file describes a subset of tests we run in the CI. To add
+your test edit `manifest.yaml` and add the following in the "tasks" dictionary:
+
+```
+  {NAME_OF_TEST}:
+    description: {DESCRIPTION}
+    stage: {STAGE}
+    required_agent_capabilities: {CAPABILITIES}
+```
+
+Where:
+
+ - `{NAME_OF_TEST}` is the name of your test that also matches the name of the
+ file in `bin/tests` without the `.dart` extension.
+ - `{DESCRIPTION}` is the plain English description of your test that helps
+ others understand what this test is testing.
+ - `{STAGE}` is `devicelab` if you want to run on Android, or `devicelab_ios` if
+ you want to run on iOS.
+ - `{CAPABILITIES}` is an array that lists the capabilities required of
+ the test agent (the computer that runs the test) to run your test. Available
+ capabilities are: `has-android-device`, `has-ios-device`.
+
+# Running tests locally
+
+Do make sure your tests pass locally before deploying to the CI environment.
+Below is a handful of commands that run tests in a fashion very close to how the
+CI environment runs them. These commands are also useful when you need to
+reproduce a CI test failure locally.
+
+To run a test use option `-t` (`--task`):
+
+```sh
+dart bin/run.dart -t {NAME_OF_TEST}
+```
+
+To run multiple tests repeat option `-t` (`--task`) multiple times:
+
+```sh
+dart bin/run.dart -t test1 -t test2 -t test3
+```
+
+To run all tests defined in `manifest.yaml` use option `-a` (`--all`):
+
+```sh
+dart bin/run.dart -a
+```
+
+To run tests from a specific stage use option `-s` (`--stage`):
+
+```sh
+dart bin/run.dart -s {NAME_OF_STAGE}
+```

dev/devicelab/bin/run.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:args/args.dart';
+
+import 'package:flutter_devicelab/framework/manifest.dart';
+import 'package:flutter_devicelab/framework/runner.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+/// Runs tasks.
+///
+/// The tasks are chosen depending on the command-line options
+/// (see [_argParser]).
+Future<Null> main(List<String> rawArgs) async {
+  ArgResults args;
+  try {
+    args = _argParser.parse(rawArgs);
+  } on FormatException catch(error) {
+    stderr.writeln('${error.message}\n');
+    stderr.writeln('Usage:\n');
+    stderr.writeln(_argParser.usage);
+    exitCode = 1;
+    return null;
+  }
+
+  List<String> taskNames = <String>[];
+  if (args.wasParsed('task')) {
+    taskNames.addAll(args['task']);
+  } else if (args.wasParsed('stage')) {
+    String stageName = args['stage'];
+    List<ManifestTask> tasks = loadTaskManifest().tasks;
+    for (ManifestTask task in tasks) {
+      if (task.stage == stageName)
+        taskNames.add(task.name);
+    }
+  } else if (args.wasParsed('all')) {
+    List<ManifestTask> tasks = loadTaskManifest().tasks;
+    for (ManifestTask task in tasks) {
+      taskNames.add(task.name);
+    }
+  }
+
+  if (taskNames.isEmpty) {
+    stderr.writeln('Failed to find tasks to run based on supplied options.');
+    exitCode = 1;
+    return null;
+  }
+
+  for (String taskName in taskNames) {
+    section('Running task "$taskName"');
+    Map<String, dynamic> result = await runTask(taskName);
+
+    if (!result['success'])
+      exitCode = 1;
+
+    print('Task result:');
+    print(new JsonEncoder.withIndent('  ').convert(result));
+    section('Finished task "$taskName"');
+  }
+}
+
+/// Command-line options for the `run.dart` command.
+final ArgParser _argParser = new ArgParser()
+  ..addOption(
+    'task',
+    abbr: 't',
+    allowMultiple: true,
+    splitCommas: true,
+    help: 'Name of the task to run. This option may be repeated to '
+    'specify multiple tasks. A task selected by name does not have to be '
+    'defined in manifest.yaml. It only needs a Dart executable in bin/tasks.',
+  )
+  ..addOption(
+    'stage',
+    abbr: 's',
+    help: 'Name of the stage. Runs all tasks for that stage. '
+        'The tasks and their stages are read from manifest.yaml.',
+  )
+  ..addOption(
+    'all',
+    abbr: 'a',
+    help: 'Runs all tasks defined in manifest.yaml.',
+  )
+  ..addOption(
+    'test',
+    hide: true,
+    allowMultiple: true,
+    splitCommas: true,
+    callback: (List<String> value) {
+      if (value.isNotEmpty) {
+        throw new FormatException(
+          'Invalid option --test. Did you mean --task (-t)?',
+        );
+      }
+    },
+  );

dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/analysis.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+Future<Null> main() async {
+  String revision = await getCurrentFlutterRepoCommit();
+  DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
+  String dartSdkVersion = await getDartVersion();
+  await task(createAnalyzerCliTest(
+    sdk: dartSdkVersion,
+    commit: revision,
+    timestamp: revisionTimestamp,
+  ));
+}

dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/analysis.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+Future<Null> main() async {
+  String revision = await getCurrentFlutterRepoCommit();
+  DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
+  String dartSdkVersion = await getDartVersion();
+  await task(createAnalyzerServerTest(
+    sdk: dartSdkVersion,
+    commit: revision,
+    timestamp: revisionTimestamp,
+  ));
+}

dev/devicelab/bin/tasks/basic_material_app__size.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/size_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createBasicMaterialAppSizeTest());
+}

dev/devicelab/bin/tasks/complex_layout__build.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createComplexLayoutBuildTest());
+}

dev/devicelab/bin/tasks/complex_layout__start_up.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createComplexLayoutStartupTest(ios: false));
+}

dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createComplexLayoutStartupTest(ios: true));
+}

dev/devicelab/bin/tasks/complex_layout_scroll_perf__timeline_summary.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createComplexLayoutScrollPerfTest(ios: false));
+}

dev/devicelab/bin/tasks/complex_layout_scroll_perf_ios__timeline_summary.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createComplexLayoutScrollPerfTest(ios: true));
+}

dev/devicelab/bin/tasks/flutter_gallery__build.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createFlutterGalleryBuildTest());
+}

dev/devicelab/bin/tasks/flutter_gallery__start_up.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createFlutterGalleryStartupTest(ios: false));
+}

dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/gallery.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createGalleryTransitionTest(ios: false));
+}

dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createFlutterGalleryStartupTest(ios: true));
+}

dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/gallery.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<Null> main() async {
+  await task(createGalleryTransitionTest(ios: true));
+}

dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/refresh.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+Future<Null> main() async {
+  String revision = await getCurrentFlutterRepoCommit();
+  DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
+  await task(createRefreshTest(
+    commit: revision,
+    timestamp: revisionTimestamp,
+  ));
+}

dev/devicelab/bin/tasks/smoke_test_failure.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+
+/// Smoke test of a task that fails by returning an unsuccessful response.
+Future<Null> main() async {
+  await task(() async {
+    return new TaskResult.failure('Failed');
+  });
+}

dev/devicelab/bin/tasks/smoke_test_setup_failure.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+/// Creates a situation when the test framework was not properly initialized.
+///
+/// By not calling `task()` the VM service extension is not registered and
+/// therefore will not accept requests to run tasks. When the runner attempts to
+/// connect and run the test it will receive a "method not found" error from the
+/// VM service, will likely retry and finally time out.
+Future<Null> main() async {}

dev/devicelab/bin/tasks/smoke_test_success.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+
+/// Smoke test of a successful task.
+Future<Null> main() async {
+  await task(() async {
+    return new TaskResult.success(<String, dynamic>{});
+  });
+}

dev/devicelab/bin/tasks/smoke_test_throws.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+
+/// Smoke test of a task that fails with an exception.
+Future<Null> main() async {
+  await task(() async {
+    throw 'failed';
+  });
+}

dev/devicelab/lib/framework/adb.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+import 'dart:math' as math;
+
+import 'package:path/path.dart' as path;
+
+import 'utils.dart';
+
+typedef Future<Adb> AdbGetter();
+
+/// Get an instance of [Adb].
+///
+/// See [realAdbGetter] for signature. This can be overwritten for testing.
+AdbGetter adb = realAdbGetter;
+
+Adb _currentDevice;
+
+/// Picks a random Android device out of connected devices and sets it as
+/// [_currentDevice].
+Future<Null> pickNextDevice() async {
+  List<Adb> allDevices =
+      (await Adb.deviceIds).map((String id) => new Adb(deviceId: id)).toList();
+
+  if (allDevices.length == 0) throw 'No Android devices detected';
+
+  // TODO(yjbanov): filter out and warn about those with low battery level
+  _currentDevice = allDevices[new math.Random().nextInt(allDevices.length)];
+}
+
+Future<Adb> realAdbGetter() async {
+  if (_currentDevice == null) await pickNextDevice();
+  return _currentDevice;
+}
+
+/// Gets the ID of an unlocked device, unlocking it if necessary.
+// TODO(yjbanov): abstract away iOS from Android.
+Future<String> getUnlockedDeviceId({ bool ios: false }) async {
+  if (ios) {
+    // We currently do not have a way to lock/unlock iOS devices, or even to
+    // pick one out of many. So we pick the first random iPhone and assume it's
+    // already unlocked. For now we'll just keep them at minimum screen
+    // brightness so they don't drain battery too fast.
+    List<String> iosDeviceIds =
+        grep('UniqueDeviceID', from: await eval('ideviceinfo', <String>[]))
+            .map((String line) => line.split(' ').last)
+            .toList();
+
+    if (iosDeviceIds.isEmpty) throw 'No connected iOS devices found.';
+
+    return iosDeviceIds.first;
+  }
+
+  Adb device = await adb();
+  await device.unlock();
+  return device.deviceId;
+}
+
+/// Android Debug Bridge (`adb`) client that exposes a subset of functions
+/// relevant to on-device testing.
+class Adb {
+  Adb({ this.deviceId });
+
+  final String deviceId;
+
+  // Parses information about a device. Example:
+  //
+  // 015d172c98400a03       device usb:340787200X product:nakasi model:Nexus_7 device:grouper
+  static final RegExp _kDeviceRegex = new RegExp(r'^(\S+)\s+(\S+)(.*)');
+
+  /// Reports connection health for every device.
+  static Future<Map<String, HealthCheckResult>> checkDevices() async {
+    Map<String, HealthCheckResult> results = <String, HealthCheckResult>{};
+    for (String deviceId in await deviceIds) {
+      try {
+        Adb device = new Adb(deviceId: deviceId);
+        // Just a smoke test that we can read wakefulness state
+        // TODO(yjbanov): also check battery level
+        await device._getWakefulness();
+        results['android-device-$deviceId'] = new HealthCheckResult.success();
+      } catch (e, s) {
+        results['android-device-$deviceId'] = new HealthCheckResult.error(e, s);
+      }
+    }
+    return results;
+  }
+
+  /// Kills the `adb` server causing it to start a new instance upon next
+  /// command.
+  ///
+  /// Restarting `adb` helps with keeping device connections alive. When `adb`
+  /// runs non-stop for too long it loses connections to devices.
+  static Future<Null> restart() async {
+    await exec(adbPath, <String>['kill-server'], canFail: false);
+  }
+
+  /// List of device IDs visible to `adb`.
+  static Future<List<String>> get deviceIds async {
+    List<String> output =
+        (await eval(adbPath, <String>['devices', '-l'], canFail: false))
+            .trim()
+            .split('\n');
+    List<String> results = <String>[];
+    for (String line in output) {
+      // Skip lines like: * daemon started successfully *
+      if (line.startsWith('* daemon ')) continue;
+
+      if (line.startsWith('List of devices')) continue;
+
+      if (_kDeviceRegex.hasMatch(line)) {
+        Match match = _kDeviceRegex.firstMatch(line);
+
+        String deviceID = match[1];
+        String deviceState = match[2];
+
+        if (!const <String>['unauthorized', 'offline'].contains(deviceState)) {
+          results.add(deviceID);
+        }
+      } else {
+        throw 'Failed to parse device from adb output: $line';
+      }
+    }
+
+    return results;
+  }
+
+  /// Whether the device is awake.
+  Future<bool> isAwake() async {
+    return await _getWakefulness() == 'Awake';
+  }
+
+  /// Whether the device is asleep.
+  Future<bool> isAsleep() async {
+    return await _getWakefulness() == 'Asleep';
+  }
+
+  /// Wake up the device if it is not awake using [togglePower].
+  Future<Null> wakeUp() async {
+    if (!(await isAwake())) await togglePower();
+  }
+
+  /// Send the device to sleep mode if it is not asleep using [togglePower].
+  Future<Null> sendToSleep() async {
+    if (!(await isAsleep())) await togglePower();
+  }
+
+  /// Sends `KEYCODE_POWER` (26), which causes the device to toggle its mode
+  /// between awake and asleep.
+  Future<Null> togglePower() async {
+    await shellExec('input', const <String>['keyevent', '26']);
+  }
+
+  /// Unlocks the device by sending `KEYCODE_MENU` (82).
+  ///
+  /// This only works when the device doesn't have a secure unlock pattern.
+  Future<Null> unlock() async {
+    await wakeUp();
+    await shellExec('input', const <String>['keyevent', '82']);
+  }
+
+  /// Retrieves device's wakefulness state.
+  ///
+  /// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java
+  Future<String> _getWakefulness() async {
+    String powerInfo = await shellEval('dumpsys', <String>['power']);
+    String wakefulness =
+        grep('mWakefulness=', from: powerInfo).single.split('=')[1].trim();
+    return wakefulness;
+  }
+
+  /// Executes [command] on `adb shell` and returns its exit code.
+  Future<Null> shellExec(String command, List<String> arguments,
+      { Map<String, String> env }) async {
+    await exec(adbPath, <String>['shell', command]..addAll(arguments),
+        env: env, canFail: false);
+  }
+
+  /// Executes [command] on `adb shell` and returns its standard output as a [String].
+  Future<String> shellEval(String command, List<String> arguments,
+      { Map<String, String> env }) {
+    return eval(adbPath, <String>['shell', command]..addAll(arguments),
+        env: env, canFail: false);
+  }
+}
+
+/// Path to the `adb` executable.
+String get adbPath {
+  String androidHome = Platform.environment['ANDROID_HOME'];
+
+  if (androidHome == null)
+    throw 'ANDROID_HOME environment variable missing. This variable must '
+        'point to the Android SDK directory containing platform-tools.';
+
+  File adbPath = file(path.join(androidHome, 'platform-tools/adb'));
+
+  if (!adbPath.existsSync()) throw 'adb not found at: $adbPath';
+
+  return adbPath.absolute.path;
+}

dev/devicelab/lib/framework/benchmarks.dart

+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'framework.dart';
+
+/// A benchmark harness used to run a benchmark multiple times and report the
+/// best result.
+abstract class Benchmark {
+  Benchmark(this.name);
+
+  final String name;
+
+  TaskResult bestResult;
+
+  Future<Null> init() => new Future<Null>.value();
+
+  Future<num> run();
+  TaskResult get lastResult;
+
+  @override
+  String toString() => name;
+}
+
+/// Runs a [benchmark] [iterations] times and reports the best result.
+///
+/// Use [warmUpBenchmark] to discard cold performance results.
+Future<num> runBenchmark(Benchmark benchmark, {
+  int iterations: 1,
+  bool warmUpBenchmark: false
+}) async {
+  await benchmark.init();
+
+  List<num> allRuns = <num>[];
+
+  num minValue;
+
+  if (warmUpBenchmark)
+    await benchmark.run();
+
+  while (iterations > 0) {
+    iterations--;
+
+    print('');
+
+    try {
+      num result = await benchmark.run();
+      allRuns.add(result);
+
+      if (minValue == null || result < minValue) {
+        benchmark.bestResult = benchmark.lastResult;
+        minValue = result;
+      }
+    } catch (error) {
+      print('benchmark failed with error: $error');
+    }
+  }
+
+  return minValue;
+}

dev/devicelab/lib/framework/framework.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:developer';
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:logging/logging.dart';
+
+import 'utils.dart';
+
+/// Maximum amount of time a single task is allowed to take to run.
+///
+/// If exceeded the task is considered to have failed.
+const Duration taskTimeout = const Duration(minutes: 10);
+
+/// Represents a unit of work performed in the CI environment that can
+/// succeed, fail and be retried independently of others.
+typedef Future<TaskResult> TaskFunction();
+
+bool _isTaskRegistered = false;
+
+/// Registers a [task] to run, returns the result when it is complete.
+///
+/// Note, the task does not run immediately but waits for the request via the
+/// VM service protocol to run it.
+///
+/// It is ok for a [task] to perform many things. However, only one task can be
+/// registered per Dart VM.
+Future<TaskResult> task(TaskFunction task) {
+  if (_isTaskRegistered)
+    throw new StateError('A task is already registered');
+
+  _isTaskRegistered = true;
+
+  // TODO: allow overriding logging.
+  Logger.root.level = Level.ALL;
+  Logger.root.onRecord.listen((LogRecord rec) {
+    print('${rec.level.name}: ${rec.time}: ${rec.message}');
+  });
+
+  _TaskRunner runner = new _TaskRunner(task);
+  runner.keepVmAliveUntilTaskRunRequested();
+  return runner.whenDone;
+}
+
+class _TaskRunner {
+  static final Logger logger = new Logger('TaskRunner');
+
+  final TaskFunction task;
+
+  // TODO: workaround for https://github.com/dart-lang/sdk/issues/23797
+  RawReceivePort _keepAlivePort;
+  Timer _startTaskTimeout;
+  bool _taskStarted = false;
+
+  final Completer<TaskResult> _completer = new Completer<TaskResult>();
+
+  _TaskRunner(this.task) {
+    registerExtension('ext.cocoonRunTask',
+        (String method, Map<String, String> parameters) async {
+      TaskResult result = await run();
+      return new ServiceExtensionResponse.result(JSON.encode(result.toJson()));
+    });
+    registerExtension('ext.cocoonRunnerReady',
+        (String method, Map<String, String> parameters) async {
+      return new ServiceExtensionResponse.result('"ready"');
+    });
+  }
+
+  /// Signals that this task runner finished running the task.
+  Future<TaskResult> get whenDone => _completer.future;
+
+  Future<TaskResult> run() async {
+    try {
+      _taskStarted = true;
+      TaskResult result = await _performTask().timeout(taskTimeout);
+      _completer.complete(result);
+      return result;
+    } on TimeoutException catch (_) {
+      return new TaskResult.failure('Task timed out after $taskTimeout');
+    } finally {
+      forceQuitRunningProcesses();
+      _closeKeepAlivePort();
+    }
+  }
+
+  /// Causes the Dart VM to stay alive until a request to run the task is
+  /// received via the VM service protocol.
+  void keepVmAliveUntilTaskRunRequested() {
+    if (_taskStarted)
+      throw new StateError('Task already started.');
+
+    // Merely creating this port object will cause the VM to stay alive and keep
+    // the VM service server running until the port is disposed of.
+    _keepAlivePort = new RawReceivePort();
+
+    // Timeout if nothing bothers to connect and ask us to run the task.
+    const Duration taskStartTimeout = const Duration(seconds: 10);
+    _startTaskTimeout = new Timer(taskStartTimeout, () {
+      if (!_taskStarted) {
+        logger.severe('Task did not start in $taskStartTimeout.');
+        _closeKeepAlivePort();
+        exitCode = 1;
+      }
+    });
+  }
+
+  /// Disables the keep-alive port, allowing the VM to exit.
+  void _closeKeepAlivePort() {
+    _startTaskTimeout?.cancel();
+    _keepAlivePort?.close();
+  }
+
+  Future<TaskResult> _performTask() async {
+    try {
+      return await task();
+    } catch (taskError, taskErrorStack) {
+      String message = 'Task failed: $taskError';
+      if (taskErrorStack != null) {
+        message += '\n\n$taskErrorStack';
+      }
+      return new TaskResult.failure(message);
+    }
+  }
+}
+
+/// A result of running a single task.
+class TaskResult {
+  /// Constructs a successful result.
+  TaskResult.success(this.data, {this.benchmarkScoreKeys: const <String>[]})
+      : this.succeeded = true,
+        this.message = 'success' {
+    const JsonEncoder prettyJson = const JsonEncoder.withIndent('  ');
+    if (benchmarkScoreKeys != null) {
+      for (String key in benchmarkScoreKeys) {
+        if (!data.containsKey(key)) {
+          throw 'Invalid Golem score key "$key". It does not exist in task '
+              'result data ${prettyJson.convert(data)}';
+        } else if (data[key] is! num) {
+          throw 'Invalid Golem score for key "$key". It is expected to be a num '
+              'but was ${data[key].runtimeType}: ${prettyJson.convert(data[key])}';
+        }
+      }
+    }
+  }
+
+  /// Constructs a successful result using JSON data stored in a file.
+  factory TaskResult.successFromFile(File file,
+      {List<String> benchmarkScoreKeys}) {
+    return new TaskResult.success(JSON.decode(file.readAsStringSync()),
+        benchmarkScoreKeys: benchmarkScoreKeys);
+  }
+
+  /// Constructs an unsuccessful result.
+  TaskResult.failure(this.message)
+      : this.succeeded = false,
+        this.data = null,
+        this.benchmarkScoreKeys = const <String>[];
+
+  /// Whether the task succeeded.
+  final bool succeeded;
+
+  /// Task-specific JSON data
+  final Map<String, dynamic> data;
+
+  /// Keys in [data] that store scores that will be submitted to Golem.
+  ///
+  /// Each key is also part of a benchmark's name tracked by Golem.
+  /// A benchmark name is computed by combining [Task.name] with a key
+  /// separated by a dot. For example, if a task's name is
+  /// `"complex_layout__start_up"` and score key is
+  /// `"engineEnterTimestampMicros"`, the score will be submitted to Golem under
+  /// `"complex_layout__start_up.engineEnterTimestampMicros"`.
+  ///
+  /// This convention reduces the amount of configuration that needs to be done
+  /// to submit benchmark scores to Golem.
+  final List<String> benchmarkScoreKeys;
+
+  /// Whether the task failed.
+  bool get failed => !succeeded;
+
+  /// Explains the result in a human-readable format.
+  final String message;
+
+  /// Serializes this task result to JSON format.
+  ///
+  /// The JSON format is as follows:
+  ///
+  ///     {
+  ///       "success": true|false,
+  ///       "data": arbitrary JSON data valid only for successful results,
+  ///       "benchmarkScoreKeys": [
+  ///         contains keys into "data" that represent benchmarks scores, which
+  ///         can be uploaded, for example. to golem, valid only for successful
+  ///         results
+  ///       ],
+  ///       "reason": failure reason string valid only for unsuccessful results
+  ///     }
+  Map<String, dynamic> toJson() {
+    Map<String, dynamic> json = <String, dynamic>{
+      'success': succeeded,
+    };
+
+    if (succeeded) {
+      json['data'] = data;
+      json['benchmarkScoreKeys'] = benchmarkScoreKeys;
+    } else {
+      json['reason'] = message;
+    }
+
+    return json;
+  }
+}

dev/devicelab/lib/framework/manifest.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:meta/meta.dart';
+import 'package:yaml/yaml.dart';
+
+import 'utils.dart';
+
+/// Loads manifest data from `manifest.yaml` file or from [yaml], if present.
+Manifest loadTaskManifest([ String yaml ]) {
+  dynamic manifestYaml = yaml == null
+    ? loadYaml(file('manifest.yaml').readAsStringSync())
+    : loadYamlNode(yaml);
+
+  _checkType(manifestYaml is Map, manifestYaml, 'Manifest', 'dictionary');
+  return _validateAndParseManifest(manifestYaml);
+}
+
+/// Contains CI task information.
+class Manifest {
+  Manifest._(this.tasks);
+
+  /// CI tasks.
+  final List<ManifestTask> tasks;
+}
+
+/// A CI task.
+class ManifestTask {
+  ManifestTask._({
+    @required this.name,
+    @required this.description,
+    @required this.stage,
+    @required this.requiredAgentCapabilities,
+  }) {
+    String taskName = 'task "$name"';
+    _checkIsNotBlank(name, 'Task name', taskName);
+    _checkIsNotBlank(description, 'Task description', taskName);
+    _checkIsNotBlank(stage, 'Task stage', taskName);
+    _checkIsNotBlank(requiredAgentCapabilities, 'requiredAgentCapabilities', taskName);
+  }
+
+  /// Task name as it appears on the dashboard.
+  final String name;
+
+  /// A human-readable description of the task.
+  final String description;
+
+  /// The stage this task should run in.
+  final String stage;
+
+  /// Capabilities required of the build agent to be able to perform this task.
+  final List<String> requiredAgentCapabilities;
+}
+
+/// Thrown when the manifest YAML is not valid.
+class ManifestError extends Error {
+  ManifestError(this.message);
+
+  final String message;
+
+  @override
+  String toString() => '$ManifestError: $message';
+}
+
+// There's no good YAML validator, at least not for Dart, so we validate
+// manually. It's not too much code and produces good error messages.
+Manifest _validateAndParseManifest(Map<String, dynamic> manifestYaml) {
+  _checkKeys(manifestYaml, 'manifest', const <String>['tasks']);
+  return new Manifest._(_validateAndParseTasks(manifestYaml['tasks']));
+}
+
+List<ManifestTask> _validateAndParseTasks(dynamic tasksYaml) {
+  _checkType(tasksYaml is Map, tasksYaml, 'Value of "tasks"', 'dictionary');
+  return tasksYaml.keys.map((dynamic taskName) => _validateAndParseTask(taskName, tasksYaml[taskName])).toList();
+}
+
+ManifestTask _validateAndParseTask(dynamic taskName, dynamic taskYaml) {
+  _checkType(taskName is String, taskName, 'Task name', 'string');
+  _checkType(taskYaml is Map, taskYaml, 'Value of task "$taskName"', 'dictionary');
+  _checkKeys(taskYaml, 'Value of task "$taskName"', const <String>[
+    'description',
+    'stage',
+    'required_agent_capabilities',
+  ]);
+
+  List<String> capabilities = _validateAndParseCapabilities(taskName, taskYaml['required_agent_capabilities']);
+  return new ManifestTask._(
+    name: taskName,
+    description: taskYaml['description'],
+    stage: taskYaml['stage'],
+    requiredAgentCapabilities: capabilities,
+  );
+}
+
+List<String> _validateAndParseCapabilities(String taskName, dynamic capabilitiesYaml) {
+  _checkType(capabilitiesYaml is List, capabilitiesYaml, 'required_agent_capabilities', 'list');
+  for (int i = 0; i < capabilitiesYaml.length; i++) {
+    dynamic capability = capabilitiesYaml[i];
+    _checkType(capability is String, capability, 'required_agent_capabilities[$i]', 'string');
+  }
+  return capabilitiesYaml;
+}
+
+void _checkType(bool isValid, dynamic value, String variableName, String typeName) {
+  if (!isValid) {
+    throw new ManifestError(
+      '$variableName must be a $typeName but was ${value.runtimeType}: $value',
+    );
+  }
+}
+
+void _checkIsNotBlank(dynamic value, String variableName, String ownerName) {
+  if (value == null || value.isEmpty) {
+    throw new ManifestError('$variableName must not be empty in $ownerName.');
+  }
+}
+
+void _checkKeys(Map<String, dynamic> map, String variableName, List<String> allowedKeys) {
+  for (String key in map.keys) {
+    if (!allowedKeys.contains(key)) {
+      throw new ManifestError(
+        'Unrecognized property "$key" in $variableName. '
+        'Allowed properties: ${allowedKeys.join(', ')}');
+    }
+  }
+}

dev/devicelab/lib/framework/runner.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:vm_service_client/vm_service_client.dart';
+
+import 'package:flutter_devicelab/framework/utils.dart';
+
+/// Slightly longer than task timeout that gives the task runner a chance to
+/// clean-up before forcefully quitting it.
+const Duration taskTimeoutWithGracePeriod = const Duration(minutes: 11);
+
+/// Runs a task in a separate Dart VM and collects the result using the VM
+/// service protocol.
+///
+/// [taskName] is the name of the task. The corresponding task executable is
+/// expected to be found under `bin/tasks`.
+Future<Map<String, dynamic>> runTask(String taskName) async {
+  String taskExecutable = 'bin/tasks/$taskName.dart';
+
+  if (!file(taskExecutable).existsSync())
+    throw 'Executable Dart file not found: $taskExecutable';
+
+  int vmServicePort = await _findAvailablePort();
+  Process runner = await startProcess(dartBin, <String>[
+    '--enable-vm-service=$vmServicePort',
+    '--no-pause-isolates-on-exit',
+    taskExecutable,
+  ]);
+
+  bool runnerFinished = false;
+
+  runner.exitCode.then((_) {
+    runnerFinished = true;
+  });
+
+  StreamSubscription<String> stdoutSub = runner.stdout
+      .transform(new Utf8Decoder())
+      .transform(new LineSplitter())
+      .listen((String line) {
+    stdout.writeln('[$taskName] [STDOUT] $line');
+  });
+
+  StreamSubscription<String> stderrSub = runner.stderr
+      .transform(new Utf8Decoder())
+      .transform(new LineSplitter())
+      .listen((String line) {
+    stderr.writeln('[$taskName] [STDERR] $line');
+  });
+
+  String waitingFor = 'connection';
+  try {
+    VMIsolate isolate = await _connectToRunnerIsolate(vmServicePort);
+    waitingFor = 'task completion';
+    Map<String, dynamic> taskResult =
+        await isolate.invokeExtension('ext.cocoonRunTask').timeout(taskTimeoutWithGracePeriod);
+    waitingFor = 'task process to exit';
+    await runner.exitCode.timeout(const Duration(seconds: 1));
+    return taskResult;
+  } on TimeoutException catch (timeout) {
+    runner.kill(ProcessSignal.SIGINT);
+    return <String, dynamic>{
+      'success': false,
+      'reason': 'Timeout waiting for $waitingFor: ${timeout.message}',
+    };
+  } finally {
+    if (!runnerFinished)
+      runner.kill(ProcessSignal.SIGKILL);
+    await stdoutSub.cancel();
+    await stderrSub.cancel();
+  }
+}
+
+Future<VMIsolate> _connectToRunnerIsolate(int vmServicePort) async {
+  String url = 'ws://localhost:$vmServicePort/ws';
+  DateTime started = new DateTime.now();
+
+  // TODO(yjbanov): due to lack of imagination at the moment the handshake with
+  //                the task process is very rudimentary and requires this small
+  //                delay to let the task process open up the VM service port.
+  //                Otherwise we almost always hit the non-ready case first and
+  //                wait a whole 1 second, which is annoying.
+  await new Future<Null>.delayed(const Duration(milliseconds: 100));
+
+  while (true) {
+    try {
+      // Make sure VM server is up by successfully opening and closing a socket.
+      await (await WebSocket.connect(url)).close();
+
+      // Look up the isolate.
+      VMServiceClient client = new VMServiceClient.connect(url);
+      VM vm = await client.getVM();
+      VMIsolate isolate = vm.isolates.single;
+      String response = await isolate.invokeExtension('ext.cocoonRunnerReady');
+      if (response != 'ready') throw 'not ready yet';
+      return isolate;
+    } catch (error) {
+      const Duration connectionTimeout = const Duration(seconds: 2);
+      if (new DateTime.now().difference(started) > connectionTimeout) {
+        throw new TimeoutException(
+          'Failed to connect to the task runner process',
+          connectionTimeout,
+        );
+      }
+      print('VM service not ready yet: $error');
+      const Duration pauseBetweenRetries = const Duration(milliseconds: 200);
+      print('Will retry in $pauseBetweenRetries.');
+      await new Future<Null>.delayed(pauseBetweenRetries);
+    }
+  }
+}
+
+Future<int> _findAvailablePort() async {
+  int port = 20000;
+  while (true) {
+    try {
+      ServerSocket socket =
+          await ServerSocket.bind(InternetAddress.LOOPBACK_IP_V4, port);
+      await socket.close();
+      return port;
+    } catch (_) {
+      port++;
+    }
+  }
+}

dev/devicelab/lib/tasks/analysis.dart

+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:meta/meta.dart';
+import 'package:path/path.dart' as path;
+
+import '../framework/benchmarks.dart';
+import '../framework/framework.dart';
+import '../framework/utils.dart';
+
+TaskFunction createAnalyzerCliTest({
+  @required String sdk,
+  @required String commit,
+  @required DateTime timestamp,
+}) {
+  return new AnalyzerCliTask(sdk, commit, timestamp);
+}
+
+TaskFunction createAnalyzerServerTest({
+  @required String sdk,
+  @required String commit,
+  @required DateTime timestamp,
+}) {
+  return new AnalyzerServerTask(sdk, commit, timestamp);
+}
+
+abstract class AnalyzerTask {
+  Benchmark benchmark;
+
+  Future<TaskResult> call() async {
+    section(benchmark.name);
+    await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true);
+    return benchmark.bestResult;
+  }
+}
+
+class AnalyzerCliTask extends AnalyzerTask {
+  AnalyzerCliTask(String sdk, String commit, DateTime timestamp) {
+    this.benchmark = new FlutterAnalyzeBenchmark(sdk, commit, timestamp);
+  }
+}
+
+class AnalyzerServerTask extends AnalyzerTask {
+  AnalyzerServerTask(String sdk, String commit, DateTime timestamp) {
+    this.benchmark = new FlutterAnalyzeAppBenchmark(sdk, commit, timestamp);
+  }
+}
+
+class FlutterAnalyzeBenchmark extends Benchmark {
+  FlutterAnalyzeBenchmark(this.sdk, this.commit, this.timestamp)
+      : super('flutter analyze --flutter-repo');
+
+  final String sdk;
+  final String commit;
+  final DateTime timestamp;
+
+  File get benchmarkFile =>
+      file(path.join(flutterDirectory.path, 'analysis_benchmark.json'));
+
+  @override
+  TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
+
+  @override
+  Future<num> run() async {
+    rm(benchmarkFile);
+    await inDirectory(flutterDirectory, () async {
+      await flutter('analyze', options: <String>[
+        '--flutter-repo',
+        '--benchmark',
+      ]);
+    });
+    return addBuildInfo(benchmarkFile,
+        timestamp: timestamp, expected: 25.0, sdk: sdk, commit: commit);
+  }
+}
+
+class FlutterAnalyzeAppBenchmark extends Benchmark {
+  FlutterAnalyzeAppBenchmark(this.sdk, this.commit, this.timestamp)
+      : super('analysis server mega_gallery');
+
+  final String sdk;
+  final String commit;
+  final DateTime timestamp;
+
+  @override
+  TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
+
+  Directory get megaDir => dir(
+      path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery'));
+  File get benchmarkFile =>
+      file(path.join(megaDir.path, 'analysis_benchmark.json'));
+
+  @override
+  Future<Null> init() {
+    return inDirectory(flutterDirectory, () async {
+      await dart(<String>['dev/tools/mega_gallery.dart']);
+    });
+  }
+
+  @override
+  Future<num> run() async {
+    rm(benchmarkFile);
+    await inDirectory(megaDir, () async {
+      await flutter('analyze', options: <String>[
+        '--watch',
+        '--benchmark',
+      ]);
+    });
+    return addBuildInfo(benchmarkFile,
+        timestamp: timestamp, expected: 10.0, sdk: sdk, commit: commit);
+  }
+}

dev/devicelab/lib/tasks/gallery.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:meta/meta.dart';
+
+import '../framework/adb.dart';
+import '../framework/framework.dart';
+import '../framework/utils.dart';
+
+TaskFunction createGalleryTransitionTest({ @required bool ios: false }) {
+  return new GalleryTransitionTest(ios: ios);
+}
+
+class GalleryTransitionTest {
+  GalleryTransitionTest({ this.ios });
+
+  final bool ios;
+
+  Future<TaskResult> call() async {
+    String deviceId = await getUnlockedDeviceId(ios: ios);
+    Directory galleryDirectory =
+        dir('${flutterDirectory.path}/examples/flutter_gallery');
+    await inDirectory(galleryDirectory, () async {
+      await pub('get');
+
+      if (ios) {
+        // This causes an Xcode project to be created.
+        await flutter('build', options: <String>['ios', '--profile']);
+      }
+
+      await flutter('drive', options: <String>[
+        '--profile',
+        '--trace-startup',
+        '-t',
+        'test_driver/transitions_perf.dart',
+        '-d',
+        deviceId,
+      ]);
+    });
+
+    // Route paths contains slashes, which Firebase doesn't accept in keys, so we
+    // remove them.
+    Map<String, dynamic> original = JSON.decode(file(
+            '${galleryDirectory.path}/build/transition_durations.timeline.json')
+        .readAsStringSync());
+    Map<String, dynamic> clean = new Map<String, dynamic>.fromIterable(
+        original.keys,
+        key: (String key) => key.replaceAll('/', ''),
+        value: (String key) => original[key]);
+
+    return new TaskResult.success(clean);
+  }
+}

dev/devicelab/lib/tasks/perf_tests.dart

+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert' show JSON;
+
+import 'package:meta/meta.dart';
+
+import '../framework/adb.dart';
+import '../framework/framework.dart';
+import '../framework/utils.dart';
+
+TaskFunction createComplexLayoutScrollPerfTest({ @required bool ios: false }) {
+  return new PerfTest(
+    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
+    'test_driver/scroll_perf.dart',
+    'complex_layout_scroll_perf',
+    ios: ios
+  );
+}
+
+TaskFunction createFlutterGalleryStartupTest({ bool ios: false }) {
+  return new StartupTest(
+    '${flutterDirectory.path}/examples/flutter_gallery',
+    ios: ios
+  );
+}
+
+TaskFunction createComplexLayoutStartupTest({ bool ios: false }) {
+  return new StartupTest(
+    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
+    ios: ios
+  );
+}
+
+TaskFunction createFlutterGalleryBuildTest() {
+  return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery');
+}
+
+TaskFunction createComplexLayoutBuildTest() {
+  return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout');
+}
+
+/// Measure application startup performance.
+class StartupTest {
+  static const Duration _startupTimeout = const Duration(minutes: 2);
+
+  StartupTest(this.testDirectory, { this.ios });
+
+  final String testDirectory;
+  final bool ios;
+
+  Future<TaskResult> call() async {
+    return await inDirectory(testDirectory, () async {
+      String deviceId = await getUnlockedDeviceId(ios: ios);
+      await pub('get');
+
+      if (ios) {
+        // This causes an Xcode project to be created.
+        await flutter('build', options: <String>['ios', '--profile']);
+      }
+
+      await flutter('run', options: <String>[
+        '--profile',
+        '--trace-startup',
+        '-d',
+        deviceId,
+      ]).timeout(_startupTimeout);
+      Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
+      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
+        'engineEnterTimestampMicros',
+        'timeToFirstFrameMicros',
+      ]);
+    });
+  }
+}
+
+/// Measures application runtime performance, specifically per-frame
+/// performance.
+class PerfTest {
+
+  PerfTest(this.testDirectory, this.testTarget, this.timelineFileName, { this.ios });
+
+  final String testDirectory;
+  final String testTarget;
+  final String timelineFileName;
+  final bool ios;
+
+  Future<TaskResult> call() {
+    return inDirectory(testDirectory, () async {
+      String deviceId = await getUnlockedDeviceId(ios: ios);
+      await pub('get');
+
+      if (ios) {
+        // This causes an Xcode project to be created.
+        await flutter('build', options: <String>['ios', '--profile']);
+      }
+
+      await flutter('drive', options: <String>[
+        '-v',
+        '--profile',
+        '--trace-startup', // Enables "endless" timeline event buffering.
+        '-t',
+        testTarget,
+        '-d',
+        deviceId,
+      ]);
+      Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
+      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
+        'average_frame_build_time_millis',
+        'worst_frame_build_time_millis',
+        'missed_frame_build_budget_count',
+      ]);
+    });
+  }
+}
+
+class BuildTest {
+
+  BuildTest(this.testDirectory);
+
+  final String testDirectory;
+
+  Future<TaskResult> call() async {
+    return await inDirectory(testDirectory, () async {
+      Adb device = await adb();
+      await device.unlock();
+      await pub('get');
+
+      Stopwatch watch = new Stopwatch()..start();
+      await flutter('build', options: <String>[
+        'aot',
+        '--profile',
+        '--no-pub',
+        '--target-platform', 'android-arm'  // Generate blobs instead of assembly.
+      ]);
+      watch.stop();
+
+      int vmisolateSize = file("$testDirectory/build/aot/snapshot_aot_vmisolate").lengthSync();
+      int isolateSize = file("$testDirectory/build/aot/snapshot_aot_isolate").lengthSync();
+      int instructionsSize = file("$testDirectory/build/aot/snapshot_aot_instr").lengthSync();
+      int rodataSize = file("$testDirectory/build/aot/snapshot_aot_rodata").lengthSync();
+      int totalSize = vmisolateSize + isolateSize + instructionsSize + rodataSize;
+
+      Map<String, dynamic> data = <String, dynamic>{
+        'aot_snapshot_build_millis': watch.elapsedMilliseconds,
+        'aot_snapshot_size_vmisolate': vmisolateSize,
+        'aot_snapshot_size_isolate': isolateSize,
+        'aot_snapshot_size_instructions': instructionsSize,
+        'aot_snapshot_size_rodata': rodataSize,
+        'aot_snapshot_size_total': totalSize,
+      };
+      return new TaskResult.success(data, benchmarkScoreKeys: <String>[
+        'aot_snapshot_build_millis',
+        'aot_snapshot_size_vmisolate',
+        'aot_snapshot_size_isolate',
+        'aot_snapshot_size_instructions',
+        'aot_snapshot_size_rodata',
+        'aot_snapshot_size_total',
+      ]);
+    });
+  }
+}

dev/devicelab/lib/tasks/refresh.dart

+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import '../framework/adb.dart';
+import '../framework/benchmarks.dart';
+import '../framework/framework.dart';
+import '../framework/utils.dart';
+
+TaskFunction createRefreshTest({ String commit, DateTime timestamp }) =>
+    new EditRefreshTask(commit, timestamp);
+
+class EditRefreshTask {
+  EditRefreshTask(this.commit, this.timestamp) {
+    assert(commit != null);
+    assert(timestamp != null);
+  }
+
+  final String commit;
+  final DateTime timestamp;
+
+  Future<TaskResult> call() async {
+    Adb device = await adb();
+    await device.unlock();
+    Benchmark benchmark = new EditRefreshBenchmark(commit, timestamp);
+    section(benchmark.name);
+    await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true);
+    return benchmark.bestResult;
+  }
+}
+
+class EditRefreshBenchmark extends Benchmark {
+  EditRefreshBenchmark(this.commit, this.timestamp) : super('edit refresh');
+
+  final String commit;
+  final DateTime timestamp;
+
+  Directory get megaDir => dir(
+      path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery'));
+  File get benchmarkFile =>
+      file(path.join(megaDir.path, 'refresh_benchmark.json'));
+
+  @override
+  TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
+
+  @override
+  Future<Null> init() {
+    return inDirectory(flutterDirectory, () async {
+      await dart(<String>['dev/tools/mega_gallery.dart']);
+    });
+  }
+
+  @override
+  Future<num> run() async {
+    Adb device = await adb();
+    rm(benchmarkFile);
+    int exitCode = await inDirectory(megaDir, () async {
+      return await flutter('run',
+          options: <String>['-d', device.deviceId, '--benchmark'],
+          canFail: true);
+    });
+    if (exitCode != 0) return new Future<num>.error(exitCode);
+    return addBuildInfo(
+      benchmarkFile,
+      timestamp: timestamp,
+      expected: 200,
+      commit: commit,
+    );
+  }
+}

dev/devicelab/lib/tasks/size_tests.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import '../framework/framework.dart';
+import '../framework/utils.dart';
+
+TaskFunction createBasicMaterialAppSizeTest() {
+  return () async {
+    const String sampleAppName = 'sample_flutter_app';
+    Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
+
+    if (await sampleDir.exists())
+      rmTree(sampleDir);
+
+    int apkSizeInBytes;
+
+    await inDirectory(Directory.systemTemp, () async {
+      await flutter('create', options: <String>[sampleAppName]);
+
+      if (!(await sampleDir.exists()))
+        throw 'Failed to create sample Flutter app in ${sampleDir.path}';
+
+      await inDirectory(sampleDir, () async {
+        await pub('get');
+        await flutter('build', options: <String>['clean']);
+        await flutter('build', options: <String>['apk', '--release']);
+        apkSizeInBytes = await file('${sampleDir.path}/build/app.apk').length();
+      });
+    });
+
+    return new TaskResult.success(
+        <String, dynamic>{'release_size_in_bytes': apkSizeInBytes},
+        benchmarkScoreKeys: <String>['release_size_in_bytes']);
+  };
+}

dev/devicelab/manifest.yaml

+# Describes the tasks we run in the continuous integration (CI) environment.
+#
+# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
+# every master commit.
+#
+# [1] github.com/flutter/cocoon
+
+# CI tasks.
+#
+# Each key in this dictionary is the unique name of a task, which also
+# corresponds to a file in the "bin/" directory that the task runner will run.
+#
+# Due to historic reasons that may go away at some point, the suffix of the task
+# name is significant. It is used by the dashboard to pick the right HTML
+# template to display the results in a card. If you use a known name suffix also
+# make sure that your task outputs data in the expected format for that card.
+#
+# Known suffixes:
+#
+# __analysis_time:
+#     Analyzer performance benchmarks.
+# __refresh_time:
+#     Edit refresh cycle benchmarks.
+# __start_up:
+#     Application startup speed benchmarks.
+# __timeline_summary:
+#     Per-frame timings and missed/average/total counts.
+# __transition_perf:
+#     Flutter Gallery app transitions benchmark.
+# __size:
+#     Application size benchmarks.
+
+tasks:
+  # Deviceless tests
+
+  # TODO: make these not require "has-android-device"; it is only there to
+  #       ensure we have the Android SDK.
+
+  flutter_gallery__build:
+    description: >
+      Collects various performance metrics from AOT builds of the Flutter
+      Gallery.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  complex_layout__build:
+    description: >
+      Collects various performance metrics from AOT builds of the Complex
+      Layout sample app.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  basic_material_app__size:
+    description: >
+      Measures the APK/IPA sizes of a basic material app.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  analyzer_cli__analysis_time:
+    description: >
+      Measures the speed of analyzing Flutter itself in batch mode.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  analyzer_server__analysis_time:
+    description: >
+      Measures the speed of analyzing Flutter itself in server mode.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+
+  # Android on-device tests
+
+  complex_layout_scroll_perf__timeline_summary:
+    description: >
+      Measures the runtime performance of the Complex Layout sample app on
+      Android.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  flutter_gallery__start_up:
+    description: >
+      Measures the startup time of the Flutter Gallery app on Android.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  complex_layout__start_up:
+    description: >
+      Measures the startup time of the Complex Layout sample app on Android.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  flutter_gallery__transition_perf:
+    description: >
+      Measures the performance of screen transitions in Flutter Gallery on
+      Android.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+  mega_gallery__refresh_time:
+    description: >
+      Measures AOT snapshot rebuild performance on a generated large app.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
+
+  # iOS on-device tests
+
+  complex_layout_scroll_perf_ios__timeline_summary:
+    description: >
+      Measures the runtime performance of the Complex Layout sample app on
+      iOS.
+    stage: devicelab_ios
+    required_agent_capabilities: ["has-ios-device"]
+
+  flutter_gallery_ios__start_up:
+    stage: devicelab_ios
+    required_agent_capabilities: ["has-ios-device"]
+    description: >
+      Measures the startup time of the Flutter Gallery app on iOS.
+
+  complex_layout_ios__start_up:
+    description: >
+      Measures the startup time of the Complex Layout sample app on iOS.
+    stage: devicelab_ios
+    required_agent_capabilities: ["has-ios-device"]
+
+  flutter_gallery_ios__transition_perf:
+    stage: devicelab_ios
+    required_agent_capabilities: ["has-ios-device"]
+    description: >
+      Measures the performance of screen transitions in Flutter Gallery on
+      iOS.

dev/devicelab/pubspec.yaml

+name: flutter_devicelab
+version: 0.0.1
+author: Flutter Authors <flutter-dev@googlegroups.com>
+description: Flutter continuous integration performance and correctness tests.
+homepage: https://github.com/flutter/flutter
+
+environment:
+  sdk: '>=1.12.0 <2.0.0'
+
+dependencies:
+  args: ^0.13.4
+  meta: ^1.0.3
+  path: ^1.3.0
+  stack_trace: ^1.4.0
+  vm_service_client: '^0.2.0'
+
+  # See packages/flutter_test/pubspec.yaml for why we're pinning this version
+  analyzer: 0.28.2-alpha.0
+
+dev_dependencies:
+  # See packages/flutter_test/pubspec.yaml for why we're pinning this version
+  test: 0.12.15+4

dev/devicelab/test/adb_test.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:test/test.dart';
+import 'package:collection/collection.dart';
+
+import 'package:flutter_devicelab/framework/adb.dart';
+
+void main() {
+  group('adb', () {
+    Adb device;
+
+    setUp(() {
+      FakeAdb.resetLog();
+      adb = null;
+      device = new FakeAdb();
+    });
+
+    tearDown(() {
+      adb = realAdbGetter;
+    });
+
+    group('isAwake/isAsleep', () {
+      test('reads Awake', () async {
+        FakeAdb.pretendAwake();
+        expect(await device.isAwake(), isTrue);
+        expect(await device.isAsleep(), isFalse);
+      });
+
+      test('reads Asleep', () async {
+        FakeAdb.pretendAsleep();
+        expect(await device.isAwake(), isFalse);
+        expect(await device.isAsleep(), isTrue);
+      });
+    });
+
+    group('togglePower', () {
+      test('sends power event', () async {
+        await device.togglePower();
+        expectLog(<CommandArgs>[
+          cmd(command: 'input', arguments: <String>['keyevent', '26']),
+        ]);
+      });
+    });
+
+    group('wakeUp', () {
+      test('when awake', () async {
+        FakeAdb.pretendAwake();
+        await device.wakeUp();
+        expectLog(<CommandArgs>[
+          cmd(command: 'dumpsys', arguments: <String>['power']),
+        ]);
+      });
+
+      test('when asleep', () async {
+        FakeAdb.pretendAsleep();
+        await device.wakeUp();
+        expectLog(<CommandArgs>[
+          cmd(command: 'dumpsys', arguments: <String>['power']),
+          cmd(command: 'input', arguments: <String>['keyevent', '26']),
+        ]);
+      });
+    });
+
+    group('sendToSleep', () {
+      test('when asleep', () async {
+        FakeAdb.pretendAsleep();
+        await device.sendToSleep();
+        expectLog(<CommandArgs>[
+          cmd(command: 'dumpsys', arguments: <String>['power']),
+        ]);
+      });
+
+      test('when awake', () async {
+        FakeAdb.pretendAwake();
+        await device.sendToSleep();
+        expectLog(<CommandArgs>[
+          cmd(command: 'dumpsys', arguments: <String>['power']),
+          cmd(command: 'input', arguments: <String>['keyevent', '26']),
+        ]);
+      });
+    });
+
+    group('unlock', () {
+      test('sends unlock event', () async {
+        FakeAdb.pretendAwake();
+        await device.unlock();
+        expectLog(<CommandArgs>[
+          cmd(command: 'dumpsys', arguments: <String>['power']),
+          cmd(command: 'input', arguments: <String>['keyevent', '82']),
+        ]);
+      });
+    });
+  });
+}
+
+void expectLog(List<CommandArgs> log) {
+  expect(FakeAdb.commandLog, log);
+}
+
+CommandArgs cmd({ String command, List<String> arguments, Map<String, String> env }) => new CommandArgs(
+  command: command,
+  arguments: arguments,
+  env: env
+);
+
+typedef dynamic ExitErrorFactory();
+
+class CommandArgs {
+  CommandArgs({ this.command, this.arguments, this.env });
+
+  final String command;
+  final List<String> arguments;
+  final Map<String, String> env;
+
+  @override
+  String toString() => 'CommandArgs(command: $command, arguments: $arguments, env: $env)';
+
+  @override
+  bool operator==(Object other) {
+    if (other.runtimeType != CommandArgs)
+      return false;
+
+    CommandArgs otherCmd = other;
+    return otherCmd.command == this.command &&
+      const ListEquality<String>().equals(otherCmd.arguments, this.arguments) &&
+      const MapEquality<String, String>().equals(otherCmd.env, this.env);
+  }
+
+  @override
+  int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnv;
+
+  int get _hashArguments => arguments != null
+    ? const ListEquality<String>().hash(arguments)
+    : null.hashCode;
+
+  int get _hashEnv => env != null
+    ? const MapEquality<String, String>().hash(env)
+    : null.hashCode;
+}
+
+class FakeAdb extends Adb {
+  FakeAdb({ String deviceId: null }) : super(deviceId: deviceId);
+
+  static String output = '';
+  static ExitErrorFactory exitErrorFactory = () => null;
+
+  static List<CommandArgs> commandLog = <CommandArgs>[];
+
+  static void resetLog() {
+    commandLog.clear();
+  }
+
+  static void pretendAwake() {
+    output = '''
+      mWakefulness=Awake
+    ''';
+  }
+
+  static void pretendAsleep() {
+    output = '''
+      mWakefulness=Asleep
+    ''';
+  }
+
+  @override
+  Future<String> shellEval(String command, List<String> arguments, {Map<String, String> env}) async {
+    commandLog.add(new CommandArgs(
+      command: command,
+      arguments: arguments,
+      env: env
+    ));
+    return output;
+  }
+
+  @override
+  Future<Null> shellExec(String command, List<String> arguments, {Map<String, String> env}) async {
+    commandLog.add(new CommandArgs(
+      command: command,
+      arguments: arguments,
+      env: env
+    ));
+    dynamic exitError = exitErrorFactory();
+    if (exitError != null)
+      throw exitError;
+  }
+}

dev/devicelab/test/all.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'adb_test.dart' as adb_test;
+import 'manifest_test.dart' as manifest_test;
+import 'run_test.dart' as run_test;
+
+void main() {
+  adb_test.main();
+  manifest_test.main();
+  run_test.main();
+}

dev/devicelab/test/manifest_test.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:test/test.dart';
+
+import 'package:flutter_devicelab/framework/manifest.dart';
+
+void main() {
+  group('production manifest', () {
+    test('must be valid', () {
+      Manifest manifest = loadTaskManifest();
+      expect(manifest.tasks, isNotEmpty);
+
+      ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'flutter_gallery__start_up');
+      expect(task.description, 'Measures the startup time of the Flutter Gallery app on Android.\n');
+      expect(task.stage, 'devicelab');
+      expect(task.requiredAgentCapabilities, <String>['has-android-device']);
+    });
+  });
+
+  group('manifest parser', () {
+    void testManifestError(
+      String testDescription,
+      String errorMessage,
+      String yaml,
+    ) {
+      test(testDescription, () {
+        try {
+          loadTaskManifest(yaml);
+        } on ManifestError catch(error) {
+          expect(error.message, errorMessage);
+        }
+      });
+    }
+
+    testManifestError(
+      'invalid top-level type',
+      'Manifest must be a dictionary but was YamlScalar: null',
+      '',
+    );
+
+    testManifestError(
+      'invalid top-level key',
+      'Unrecognized property "bad" in manifest. Allowed properties: tasks',
+      '''
+      bad:
+        key: yes
+      ''',
+    );
+
+    testManifestError(
+      'invalid tasks list type',
+      'Value of "tasks" must be a dictionary but was YamlList: [a, b]',
+      '''
+      tasks:
+        - a
+        - b
+      '''
+    );
+
+    testManifestError(
+      'invalid task name type',
+      'Task name must be a string but was int: 1',
+      '''
+      tasks:
+        1: 2
+      '''
+    );
+
+    testManifestError(
+      'invalid task type',
+      'Value of task "foo" must be a dictionary but was int: 2',
+      '''
+      tasks:
+        foo: 2
+      '''
+    );
+
+    testManifestError(
+      'invalid task property',
+      'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities',
+      '''
+      tasks:
+        foo:
+          bar: 2
+      '''
+    );
+
+    testManifestError(
+      'invalid required_agent_capabilities type',
+      'required_agent_capabilities must be a list but was int: 1',
+      '''
+      tasks:
+        foo:
+          required_agent_capabilities: 1
+      '''
+    );
+
+    testManifestError(
+      'invalid required_agent_capabilities element type',
+      'required_agent_capabilities[0] must be a string but was int: 1',
+      '''
+      tasks:
+        foo:
+          required_agent_capabilities: [1]
+      '''
+    );
+
+    testManifestError(
+      'missing description',
+      'Task description must not be empty in task "foo".',
+      '''
+      tasks:
+        foo:
+          required_agent_capabilities: ["a"]
+      '''
+    );
+
+    testManifestError(
+      'missing stage',
+      'Task stage must not be empty in task "foo".',
+      '''
+      tasks:
+        foo:
+          description: b
+          required_agent_capabilities: ["a"]
+      '''
+    );
+
+    testManifestError(
+      'missing stage',
+      'requiredAgentCapabilities must not be empty in task "foo".',
+      '''
+      tasks:
+        foo:
+          description: b
+          stage: c
+          required_agent_capabilities: []
+      '''
+    );
+  });
+}

dev/devicelab/test/run_test.dart

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+void main() {
+  group('run.dart script', () {
+    Future<int> runScript(List<String> testNames) async {
+      List<String> options = <String>['bin/run.dart'];
+      for (String testName in testNames) {
+        options..addAll(<String>['-t', testName]);
+      }
+      Process scriptProcess = await Process.start(
+        '../../bin/cache/dart-sdk/bin/dart',
+        options,
+      );
+      return scriptProcess.exitCode;
+    }
+
+    test('Exits with code 0 when succeeds', () async {
+      expect(await runScript(<String>['smoke_test_success']), 0);
+    });
+
+    test('Exits with code 1 when task throws', () async {
+      expect(await runScript(<String>['smoke_test_throws']), 1);
+    });
+
+    test('Exits with code 1 when fails', () async {
+      expect(await runScript(<String>['smoke_test_failure']), 1);
+    });
+
+    test('Exits with code 1 when fails to connect', () async {
+      expect(await runScript(<String>['smoke_test_setup_failure']), 1);
+    });
+
+    test('Exits with code 1 when results are mixed', () async {
+      expect(
+        await runScript(<String>[
+          'smoke_test_failure',
+          'smoke_test_success',
+        ]),
+        1,
+      );
+    });
+  });
+}

dev/devicelab/test/utils_test.dart

+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:test/test.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+void main() {
+  group('grep', () {
+    test('greps lines', () {
+      expect(grep('b', from: 'ab\ncd\nba'), <String>['ab', 'ba']);
+    });
+
+    test('understands RegExp', () {
+      expect(grep(new RegExp('^b'), from: 'ab\nba'), <String>['ba']);
+    });
+  });
+}

packages/flutter/pubspec.yaml

@@ -7,7 +7,7 @@ homepage: http://flutter.io
 dependencies:
   collection: '>=1.9.1 <2.0.0'
   intl: '>=0.14.0 <0.15.0'
-  meta: ^1.0.2
+  meta: ^1.0.3
   vector_math: '>=2.0.3 <3.0.0'
 
   sky_engine:

packages/flutter_tools/pubspec.yaml

@@ -22,7 +22,7 @@ dependencies:
   json_schema: 1.0.3
 
   linter: ^0.1.21
-  meta: ^1.0.0
+  meta: ^1.0.3
   mustache4dart: ^1.0.0
   package_config: '>=0.1.5 <2.0.0'
   path: ^1.3.0