Move docs and gallery deployment to Cirrus, add Docker image for Linux (#20097)

This adds a Docker image for the linux builds, replacing a lot of the setup code with a Docker build.

Added a docker image build step that has the right gcloud credentials in it.

Also, this finally moves the gallery deployment and docs publishing steps to Cirrus. They were dependent upon some environment setup that was a lot easier to do in Docker than in a setup bash script.

.cirrus.yml

 container:
-  image: cirrusci/flutter:base
+  image: gcr.io/flutter-cirrus/build-flutter-image:latest
 
 task:
   env:
+    # Name the SDK directory to include a space so that we constantly
+    # test path names with spaces in them.
     CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
     PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH"
-
-  git_fetch_script: git fetch origin
+    ANDROID_HOME: "/opt/android_sdk"
+  git_fetch_script:
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
   pub_cache:
     folder: $HOME/.pub-cache
     fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
@@ -16,22 +20,26 @@ task:
   artifacts_cache:
     folder: bin/cache/artifacts
     fingerprint_script: echo $OS; cat bin/internal/engine.version
-  setup_script: |
-    echo "SDK directory is: $PWD"
-    ./bin/flutter --version
-
-    # disable analytics on the bots and download Flutter dependencies
-    ./bin/flutter config --no-analytics
-
-    # run pub get in all the repo packages
-    ./bin/flutter update-packages
-
-    git fetch origin master
+  setup_script: ./dev/bots/cirrus_setup.sh
   matrix:
     - name: docs
       env:
         SHARD: docs
+        # For uploading docs to Firebase
+        FIREBASE_TOKEN: ENCRYPTED[a40fcb68452b92254b7f532a460529b5e4ca51d6d338a8fae8db9db832ad3c2a7efb962e257de79686a9345f4a18661a]
       docs_script: ./dev/bots/docs.sh
+    - name: deploy_gallery
+      only_if: $CIRRUS_BRANCH == 'dev'
+      depends_on:
+        - docs
+        - analyze
+        - tests-linux
+        - tool_tests-linux
+      env:
+        SHARD: deploy_gallery
+        GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346]
+        ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0b3e681b4507aec433ef29c79b715f15f8c75ecd25315ea286b0b2bcb8b28d578634eead5aa2c54086a25e8da1bb219a]
+      test_script: ./dev/bots/deploy_gallery.sh
     - name: analyze
       env:
         SHARD: analyze
@@ -61,7 +69,9 @@ task:
     cpu: 4
   env:
     CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk"
-  git_fetch_script: git fetch origin
+  git_fetch_script:
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
   pub_cache:
     folder: $APPDATA\Pub\Cache
     fingerprint_script:
@@ -87,13 +97,46 @@ task:
         SHARD: tool_tests
 
 task:
+  name: deploy_gallery-macos
+  only_if: $CIRRUS_BRANCH == 'dev'
+  pub_cache:
+    folder: ~/.pub-cache
   depends_on:
     - analyze
+    - tests-macos
+    - tool_tests-macos
+  env:
+    # Name the SDK directory to include a space so that we constantly
+    # test path names with spaces in them.
+    CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
+    SHARD: deploy_gallery
+    # Apple Certificates Match Passphrase
+    MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73]
+    # Apple Fastlane Password
+    FASTLANE_PASSWORD: ENCRYPTED[0bf9bb0cc2cb32a0ed18470cf2c9df0f587cce5f8b04adbd6cff15ca5bde7a74f721ee580227b132ab6b032f08e52ae0]
+    # Private repo for publishing certificates.
+    PUBLISHING_MATCH_CERTIFICATE_REPO: git@github.com:flutter/private_publishing_certificates.git
+  osx_instance:
+    image: high-sierra-xcode-9.4.1
+  git_fetch_script:
+  - git fetch origin
+  - git fetch origin master # To set FETCH_HEAD
+  setup_script:
+    - bin/flutter config --no-analytics
+    - bin/flutter update-packages
+  test_all_script:
+    - ./dev/bots/deploy_gallery.sh
+
+task:
   osx_instance:
     image: high-sierra-xcode-9.4.1
+  depends_on:
+    - analyze
   env:
     CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
-  git_fetch_script: git fetch origin
+  git_fetch_script:
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
   pub_cache:
     folder: $HOME/.pub-cache
     fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
@@ -116,3 +159,19 @@ task:
     - name: tool_tests-macos
       env:
         SHARD: tool_tests
+
+
+docker_builder:
+  # Only build a new docker image when we tag a release (for dev, beta, or release.)
+  only_if: $CIRRUS_TAG != ''
+  env:
+    GCLOUD_CREDENTIALS: ENCRYPTED[f7c098d4dd7f5ee1bfee0bb7e944cce72efbe10e97ad6440ae72de4de6a1c24d23f421a2619c668e94377fb64b0bb3e6]
+  depends_on:
+    - docs
+    - analyze
+    - tests-linux
+    - tool_tests-linux
+  build_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh"
+  login_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh"
+  push_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh"
+

dev/bots/cirrus_setup.sh

+#!/bin/bash
+set -e
+
+# This script is only meant to be run by the Cirrus CI system, not locally.
+# It must be run from the root of the Flutter repo.
+
+# Collects log output in a tmpfile, but only prints it if the command fails.
+function log_on_fail() {
+  local COMMAND="$@"
+  local TMPDIR="$(mktemp -d)"
+  local TMPFILE="$TMPDIR/command.log"
+  local EXIT=0
+  if ("$@" > "$TMPFILE" 2>&1); then
+    echo "'$COMMAND' succeeded."
+  else
+    EXIT=$?
+    cat "$TMPFILE" 1>&2
+    echo "FAIL: '$COMMAND' exited with code $EXIT" 1>&2
+  fi
+  rm -rf "$TMPDIR"
+  return "$EXIT"
+}
+
+function accept_android_licenses() {
+  yes "y" | flutter doctor --android-licenses
+}
+
+echo "Flutter SDK directory is: $PWD"
+
+# Run flutter to download dependencies and precompile things, and to disable
+# analytics on the bots.
+echo "Downloading build dependencies and pre-compiling Flutter snapshot"
+log_on_fail ./bin/flutter config --no-analytics
+
+# Run doctor, to print it to the log for debugging purposes.
+./bin/flutter doctor -v
+
+# Accept licenses.
+log_on_fail accept_android_licenses && echo "Android licenses accepted."
+
+# Run pub get in all the repo packages.
+echo "Updating packages for Flutter."
+log_on_fail ./bin/flutter update-packages

dev/bots/deploy_gallery.sh

+#!/bin/bash
+
+set -e
+
+function script_location() {
+  local script_location="${BASH_SOURCE[0]}"
+  # Resolve symlinks
+  while [[ -h "$script_location" ]]; do
+    DIR="$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
+    script_location="$(readlink "$script_location")"
+    [[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
+  done
+  echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
+}
+
+# So that users can run this script locally from any directory and it will work as
+# expected.
+SCRIPT_LOCATION="$(script_location)"
+FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"
+
+export PATH="$FLUTTER_ROOT/bin:$FLUTTER_ROOT/bin/cache/dart-sdk/bin:$PATH"
+
+set -x
+
+cd "$FLUTTER_ROOT"
+
+if [[ "$SHARD" = "deploy_gallery" ]]; then
+  version="$(<version)"
+  if [[ "$OS" == "linux" ]]; then
+    echo "Building Flutter Gallery $version for Android..."
+    export ANDROID_HOME="$PWD/android-sdk"
+    (
+      cd examples/flutter_gallery
+      flutter build apk --release -t lib/main_publish.dart
+    )
+    echo "Android Flutter Gallery built"
+    if [[ -z "$CIRRUS_PULL_REQUEST" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
+      echo "Deploying Flutter Gellery $version to Play Store..."
+      (
+        cd examples/flutter_gallery/android
+        bundle install
+        bundle exec fastlane deploy_play_store
+      )
+    else
+      echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits"
+    fi
+  elif [[ "$OS" == "macos" ]]; then
+    echo "Building Flutter Gallery $version for iOS..."
+    (
+      cd examples/flutter_gallery
+      flutter build ios --release --no-codesign -t lib/main_publish.dart
+    )
+    echo "iOS Flutter Gallery built"
+    if [[ -z "$CIRRUS_PULL_REQUEST" ]]; then
+      if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
+        echo "Archiving with distribution profile and deploying to TestFlight..."
+        (
+          cd examples/flutter_gallery/ios
+          bundle install
+          bundle exec fastlane build_and_deploy_testflight upload:true
+        )
+      else
+        echo "Archiving with distribution profile..."
+        (
+          cd examples/flutter_gallery/ios
+          bundle install
+          bundle exec fastlane build_and_deploy_testflight
+        )
+        echo "Archive is only deployed to TestFlight on tagged dev branch commits"
+      fi
+    else
+      echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits"
+    fi
+  fi
+else
+  echo "Doing nothing: not on the 'deploy_gallery' SHARD."
+fi

dev/bots/docs.sh

 #!/bin/bash
 set -e
 
-echo "Running docs.sh"
+function script_location() {
+  local script_location="${BASH_SOURCE[0]}"
+  # Resolve symlinks
+  while [[ -h "$script_location" ]]; do
+    DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)"
+    script_location="$(readlink "$script_location")"
+    [[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
+  done
+  echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
+}
 
-# If you want to run this script locally, make sure you run it from
-# the root of the flutter repository.
-export FLUTTER_ROOT="$PWD"
-export PATH="$PWD/bin:$PATH"
+# So that users can run this script from anywhere and it will work as expected.
+SCRIPT_LOCATION="$(script_location)"
+FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"
+
+if [[ ! -d "$FLUTTER_ROOT" || ! -f "$FLUTTER_ROOT/bin/flutter" ]]; then
+  echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)"
+  exit 1
+fi
+
+FLUTTER_BIN="$FLUTTER_ROOT/bin"
+DART_BIN="$FLUTTER_ROOT/bin/cache/dart-sdk/bin"
+FLUTTER="$FLUTTER_BIN/flutter"
+DART="$DART_BIN/dart"
+PUB="$DART_BIN/pub"
+export PATH="$FLUTTER_BIN:$DART_BIN:$PATH"
 
 # This is called from travis_upload.sh on Travis.
 
-# Make sure dart is installed
-bin/flutter --version
+# Make sure dart is installed by invoking flutter to download it.
+"$FLUTTER" --version
 
 # If the pub cache directory exists in the root, then use that.
 FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache"
-if [ -d "$FLUTTER_PUB_CACHE" ]; then
+if [[ -d "$FLUTTER_PUB_CACHE" ]]; then
   # This has to be exported, because pub interprets setting it
   # to the empty string in the same way as setting it to ".".
   export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
 fi
 
 # Install dartdoc.
-bin/cache/dart-sdk/bin/pub global activate dartdoc 0.20.2
+"$PUB" global activate dartdoc 0.20.2
 
 # This script generates a unified doc set, and creates
 # a custom index.html, placing everything into dev/docs/doc.
-(cd dev/tools; ../../bin/cache/dart-sdk/bin/pub get)
-bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart
-bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart
+(cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get)
+(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
+(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")
 
 # Ensure google webmaster tools can verify our site.
-cp dev/docs/google2ed1af765c529f57.html dev/docs/doc
-
-# Upload new API docs when on Travis
-if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-  echo "This is not a pull request; considering whether to upload docs... (branch=$TRAVIS_BRANCH)"
-  if [ "$TRAVIS_BRANCH" == "master" -o "$TRAVIS_BRANCH" == "beta" ]; then
-    cd dev/docs
+cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc"
 
-    if [ "$TRAVIS_BRANCH" == "master" ]; then
+# Upload new API docs when on Cirrus
+if [[ -n "$CIRRUS_CI" && -z "$CIRRUS_PR" ]]; then
+  echo "This is not a pull request; considering whether to upload docs... (branch=$CIRRUS_BRANCH)"
+  if [[ "$CIRRUS_BRANCH" == "master" || "$CIRRUS_BRANCH" == "beta" ]]; then
+    if [[ "$CIRRUS_BRANCH" == "master" ]]; then
       echo "Updating master docs: https://master-docs-flutter-io.firebaseapp.com/"
-      echo -e "User-agent: *\nDisallow: /" > doc/robots.txt
-      while : ; do
-        firebase deploy --project master-docs-flutter-io && break
+      echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
+      while true; do
+        (cd "$FLUTTER_ROOT/dev/docs" && firebase deploy --project master-docs-flutter-io) && break
         echo Error: Unable to deploy documentation to firebase. Retrying in five seconds...
         sleep 5
       done
     fi
 
-    if [ "$TRAVIS_BRANCH" == "beta" ]; then
+    if [[ "$CIRRUS_BRANCH" == "beta" ]]; then
       echo "Updating beta docs: https://docs.flutter.io/"
-      while : ; do
-        firebase deploy --project docs-flutter-io && break
+      while true; do
+        (cd "$FLUTTER_ROOT/dev/docs" && firebase deploy --project docs-flutter-io) && break
         echo Error: Unable to deploy documentation to firebase. Retrying in five seconds...
         sleep 5
       done

dev/ci/README.md

+This directory includes scripts and tools for continuous integration builds and tests.

dev/ci/docker_linux/Dockerfile

+# Flutter (https://flutter.io) Developement Environment for Linux
+# ===============================================================
+#
+# This environment passes all Linux Flutter Doctor checks and is sufficient
+# for building Android applications and running Flutter tests.
+#
+# To build iOS applications, a Mac development environment is necessary.
+#
+# This includes applications and sdks that are needed only by the CI system
+# for performing pushes to production, and so this image is quite a bit larger
+# than strictly needed for just building Flutter apps.
+
+FROM debian:stretch
+MAINTAINER Flutter Developers <flutter-dev@googlegroups.com>
+
+RUN apt-get update -y
+RUN apt-get upgrade -y
+
+# Install basics
+RUN apt-get install -y --no-install-recommends \
+  git \
+  wget \
+  curl \
+  unzip \
+  ca-certificates \
+  gnupg
+
+# Add nodejs repository to apt sources and install it.
+ENV NODEJS_INSTALL="/opt/nodejs_install"
+RUN mkdir -p "${NODEJS_INSTALL}"
+RUN wget -q https://deb.nodesource.com/setup_10.x -O "${NODEJS_INSTALL}/nodejs_install.sh"
+RUN bash "${NODEJS_INSTALL}/nodejs_install.sh"
+
+# Install the rest of the dependencies.
+RUN apt-get install -y --no-install-recommends \
+  locales \
+  ruby \
+  nodejs \
+  lib32stdc++6 \
+  libstdc++6 \
+  libglu1-mesa \
+  build-essential \
+  default-jdk-headless
+
+# Install the Android SDK Dependency.
+ENV ANDROID_SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip"
+ENV ANDROID_TOOLS_ROOT="/opt/android_sdk"
+RUN mkdir -p "${ANDROID_TOOLS_ROOT}"
+RUN mkdir -p ~/.android
+# Silence warning.
+RUN touch ~/.android/repositories.cfg
+ENV ANDROID_SDK_ARCHIVE="${ANDROID_TOOLS_ROOT}/archive"
+RUN wget --progress=dot:giga "${ANDROID_SDK_URL}" -O "${ANDROID_SDK_ARCHIVE}"
+RUN unzip -q -d "${ANDROID_TOOLS_ROOT}" "${ANDROID_SDK_ARCHIVE}"
+# Suppressing output of sdkmanager to keep log size down
+# (it prints install progress WAY too often).
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "tools" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "build-tools;28.0.0" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "platforms;android-28" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "platform-tools" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "extras;android;m2repository" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "extras;google;m2repository" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "patcher;v4" > /dev/null
+RUN rm "${ANDROID_SDK_ARCHIVE}"
+ENV PATH="${ANDROID_TOOLS_ROOT}/tools:${PATH}"
+ENV PATH="${ANDROID_TOOLS_ROOT}/tools/bin:${PATH}"
+# Silence warnings when accepting android licenses.
+RUN mkdir -p ~/.android
+RUN touch ~/.android/repositories.cfg
+
+# Setup gradle
+ENV GRADLE_ROOT="/opt/gradle"
+RUN mkdir -p "${GRADLE_ROOT}"
+ENV GRADLE_ARCHIVE="${GRADLE_ROOT}/gradle.zip"
+ENV GRADLE_URL="http://services.gradle.org/distributions/gradle-4.4-bin.zip"
+RUN wget --progress=dot:giga "$GRADLE_URL" -O "${GRADLE_ARCHIVE}"
+RUN unzip -q -d "${GRADLE_ROOT}" "${GRADLE_ARCHIVE}"
+ENV PATH="$GRADLE_ROOT/bin:$PATH"
+
+# Add npm to path.
+ENV PATH="/usr/bin:${PATH}"
+RUN dpkg-query -L nodejs
+# Install Firebase
+RUN /usr/bin/npm install -g firebase-tools
+
+# Set locale to en_US
+RUN locale-gen en_US "en_US.UTF-8" && dpkg-reconfigure locales
+ENV LANG en_US.UTF-8
+
+# Install coveralls and Firebase
+RUN gem install coveralls
+RUN gem install bundler
+

dev/ci/docker_linux/README.md

+This directory includes scripts to build the docker container image used for
+building flutter/flutter in our CI system (currently [Cirrus](cirrus-ci.org)).
+
+In order to run the scripts, you have to setup `docker` and `gcloud`. Please
+refer to the [internal flutter team doc](go/flutter-team) for how to setup in a
+Google internal environment.
+
+After setup,
+* edit `Dockerfile` to change how the container image is built.
+* run `./build_docker.sh` to build the container image.
+* run `./push_docker.sh` to push the image to google cloud registry. This will
+  affect our CI tests.
+

dev/ci/docker_linux/docker_build.sh

+#!/bin/bash
+
+TAG="${CIRRUS_TAG:-latest}"
+
+# pull to make sure we are not rebuilding for nothing
+docker pull "gcr.io/flutter-cirrus/build-flutter-image:$TAG"
+
+docker build --tag "gcr.io/flutter-cirrus/build-flutter-image:$TAG" .

dev/ci/docker_linux/docker_login.sh

+#!/bin/bash
+
+if [[ -n "$CIRRUS_CI" && -n "$GCLOUD_CREDENTIALS" ]]; then
+  echo "$GCLOUD_CREDENTIALS" | base64 --decode | docker login -u _json_key --password-stdin https://gcr.io
+else
+  gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
+fi
\ No newline at end of file

dev/ci/docker_linux/docker_push.sh

+#!/bin/bash
+
+TAG="${CIRRUS_TAG:-latest}"
+
+docker push "gcr.io/flutter-cirrus/build-flutter-image::$TAG"
+