Merge pull request #1273 from eseidelGoogle/infra

Add support for running flutter tests on Chromium Infra

infra/.gitignore

+.recipe_deps

infra/README.md

+Directory to support running Flutter builds/tests on Chromium Infra.
+
+Following documentation at:
+https://github.com/luci/recipes-py/blob/master/doc/cross_repo.md
+
+recipes.cfg is a protobuf dump (no comment support) explaining
+where to store build and recipe_engine dependencies needed for running
+a recipe.
+
+recipes.py is for bootstrapping and is taken from
+https://chromium.googlesource.com/chromium/tools/build.git/+/master/scripts/slave/recipes.py
+at 18df86c, modified to have correct hard-coded paths for flutter.

infra/config/recipes.cfg

+api_version: 1
+project_id: "flutter"
+recipes_path: "infra"
+deps {
+  project_id: "build"
+  url: "https://chromium.googlesource.com/chromium/tools/build"
+  branch: "master"
+  revision: "7e696f8d7969be66a13a2ad54970a43f2eae930e"
+}
+deps {
+  project_id: "recipe_engine"
+  url: "https://chromium.googlesource.com/external/github.com/luci/recipes-py.git"
+  branch: "master"
+  revision: "636414454465da499a9ea443364eec7b830ed34c"
+}

infra/recipes.py

+#!/usr/bin/env python
+
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Bootstrap script to clone and forward to the recipe engine tool."""
+
+import ast
+import logging
+import os
+import random
+import re
+import subprocess
+import sys
+import time
+import traceback
+
+BOOTSTRAP_VERSION = 1
+# The root of the repository relative to the directory of this file.
+REPO_ROOT = os.path.join(os.pardir)
+# The path of the recipes.cfg file relative to the root of the repository.
+RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg')
+
+
+def parse_protobuf(fh):
+  """Parse the protobuf text format just well enough to understand recipes.cfg.
+
+  We don't use the protobuf library because we want to be as self-contained
+  as possible in this bootstrap, so it can be simply vendored into a client
+  repo.
+
+  We assume all fields are repeated since we don't have a proto spec to work
+  with.
+
+  Args:
+    fh: a filehandle containing the text format protobuf.
+  Returns:
+    A recursive dictionary of lists.
+  """
+  def parse_atom(text):
+    if text == 'true': return True
+    if text == 'false': return False
+    return ast.literal_eval(text)
+
+  ret = {}
+  for line in fh:
+    line = line.strip()
+    m = re.match(r'(\w+)\s*:\s*(.*)', line)
+    if m:
+      ret.setdefault(m.group(1), []).append(parse_atom(m.group(2)))
+      continue
+
+    m = re.match(r'(\w+)\s*{', line)
+    if m:
+      subparse = parse_protobuf(fh)
+      ret.setdefault(m.group(1), []).append(subparse)
+      continue
+
+    if line == '}': return ret
+    if line == '': continue
+
+    raise Exception('Could not understand line: <%s>' % line)
+
+  return ret
+
+
+def get_unique(things):
+  if len(things) == 1:
+    return things[0]
+  elif len(things) == 0:
+    raise ValueError("Expected to get one thing, but dinna get none.")
+  else:
+    logging.warn('Expected to get one thing, but got a bunch: %s\n%s' %
+                 (things, traceback.format_stack()))
+    return things[0]
+
+
+def main():
+  if sys.platform.startswith(('win', 'cygwin')):
+    git = 'git.bat'
+  else:
+    git = 'git'
+
+  # Find the repository and config file to operate on.
+  repo_root = os.path.abspath(
+      os.path.join(os.path.dirname(__file__), REPO_ROOT))
+  recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG)
+
+  with open(recipes_cfg_path, 'rU') as fh:
+    protobuf = parse_protobuf(fh)
+
+  engine_buf = get_unique([
+      b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ])
+  engine_url = get_unique(engine_buf['url'])
+  engine_revision = get_unique(engine_buf['revision'])
+  engine_subpath = (get_unique(engine_buf.get('path_override', ['']))
+                    .replace('/', os.path.sep))
+
+  recipes_path = os.path.join(repo_root,
+      get_unique(protobuf['recipes_path']).replace('/', os.path.sep))
+  deps_path = os.path.join(recipes_path, '.recipe_deps')
+  engine_path = os.path.join(deps_path, 'recipe_engine')
+
+  # Ensure that we have the recipe engine cloned.
+  def ensure_engine():
+    if not os.path.exists(deps_path):
+      os.makedirs(deps_path)
+    if not os.path.exists(engine_path):
+      subprocess.check_call([git, 'clone', engine_url, engine_path])
+
+    needs_fetch = subprocess.call(
+        [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision],
+        cwd=engine_path, stdout=open(os.devnull, 'w'))
+    if needs_fetch:
+      subprocess.check_call([git, 'fetch'], cwd=engine_path)
+    subprocess.check_call(
+        [git, 'checkout', '--quiet', engine_revision], cwd=engine_path)
+
+  try:
+    ensure_engine()
+  except subprocess.CalledProcessError as e:
+    if e.returncode == 128:  # Thrown when git gets a lock error.
+      time.sleep(random.uniform(2,5))
+      ensure_engine()
+    else:
+      raise
+
+  args = ['--package', recipes_cfg_path,
+          '--bootstrap-script', __file__] + sys.argv[1:]
+  return subprocess.call([
+      sys.executable, '-u',
+      os.path.join(engine_path, engine_subpath, 'recipes.py')] + args)
+
+if __name__ == '__main__':
+  sys.exit(main())

infra/recipes/flutter.expected/basic.json

+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE[build]/git_setup.py",
+      "--path",
+      "[SLAVE_BUILD]/flutter",
+      "--url",
+      "https://github.com/flutter/flutter.git"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "git setup"
+  },
+  {
+    "cmd": [
+      "git",
+      "retry",
+      "fetch",
+      "origin",
+      "master",
+      "--recurse-submodules"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter",
+    "name": "git fetch"
+  },
+  {
+    "cmd": [
+      "git",
+      "checkout",
+      "-f",
+      "FETCH_HEAD"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter",
+    "name": "git checkout"
+  },
+  {
+    "cmd": [
+      "git",
+      "clean",
+      "-f",
+      "-d",
+      "-x"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter",
+    "name": "git clean"
+  },
+  {
+    "cmd": [
+      "git",
+      "submodule",
+      "sync"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter",
+    "name": "submodule sync"
+  },
+  {
+    "cmd": [
+      "git",
+      "submodule",
+      "update",
+      "--init",
+      "--recursive"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter",
+    "name": "submodule update"
+  },
+  {
+    "cmd": [
+      "dart",
+      "[SLAVE_BUILD]/flutter/dev/update_packages.dart"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "update packages"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/flutter/bin/flutter",
+      "cache",
+      "populate"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/packages/flutter",
+    "name": "populate flutter cache"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/flutter/bin/flutter",
+      "analyze",
+      "--flutter-repo",
+      "--no-current-directory",
+      "--no-current-package",
+      "--congratulate"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "flutter analyze"
+  },
+  {
+    "cmd": [
+      "pub",
+      "run",
+      "test",
+      "-j1"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/packages/cassowary",
+    "name": "test packages/cassowary"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/flutter/bin/flutter",
+      "test"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/packages/flutter",
+    "name": "test packages/flutter"
+  },
+  {
+    "cmd": [
+      "pub",
+      "run",
+      "test",
+      "-j1"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/packages/flutter_tools",
+    "name": "test packages/flutter_tools"
+  },
+  {
+    "cmd": [
+      "pub",
+      "run",
+      "test",
+      "-j1"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/packages/flx",
+    "name": "test packages/flx"
+  },
+  {
+    "cmd": [
+      "pub",
+      "run",
+      "test",
+      "-j1"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/packages/newton",
+    "name": "test packages/newton"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/flutter/bin/flutter",
+      "test"
+    ],
+    "cwd": "[SLAVE_BUILD]/flutter/examples/stocks",
+    "name": "test examples/stocks"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file

infra/recipes/flutter.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+  'recipe_engine/path',
+  'recipe_engine/step',
+  'build/git',
+]
+
+def RunSteps(api):
+  api.git.checkout('https://github.com/flutter/flutter.git', recursive=True)
+
+  checkout = api.path['checkout']
+  update_packages = checkout.join('dev', 'update_packages.dart')
+
+  api.step('update packages', ['dart', update_packages])
+
+  flutter_cli = checkout.join('bin', 'flutter')
+  flutter_package = checkout.join('packages', 'flutter')
+  populate_cmd = [flutter_cli, 'cache', 'populate']
+  api.step('populate flutter cache', populate_cmd, cwd=flutter_package)
+
+  analyze_cmd = [
+    flutter_cli,
+    'analyze',
+    '--flutter-repo',
+    '--no-current-directory',
+    '--no-current-package',
+    '--congratulate'
+  ]
+  api.step('flutter analyze', analyze_cmd)
+
+
+  def _pub_test(path):
+    api.step('test %s' % path, ['pub', 'run', 'test', '-j1'],
+      cwd=checkout.join(path))
+
+  def _flutter_test(path):
+    api.step('test %s' % path, [flutter_cli, 'test'],
+      cwd=checkout.join(path))
+
+  _pub_test('packages/cassowary')
+  _flutter_test('packages/flutter')
+  _pub_test('packages/flutter_tools')
+  _pub_test('packages/flx')
+  _pub_test('packages/newton')
+
+  _flutter_test('examples/stocks')
+
+def GenTests(api):
+  yield api.test('basic')