Try again to switch docs and gallery deployment to Cirrus (#20276)

This switches docs and gallery build and deployment to use Cirrus CI instead of Travis, reapplying the changes from #19925, but updating the Firebase tokens for upload, and refining docs.sh some to limit retries to five times (instead of indefinitely).

.cirrus.yml

 container:
-  image: cirrusci/flutter:base
+  image: gcr.io/flutter-cirrus/build-flutter-image:latest
 
 task:
   env:
+    # Name the SDK directory to include a space so that we constantly
+    # test path names with spaces in them.
     CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
     PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH"
-
-  git_fetch_script: git fetch origin
+    ANDROID_HOME: "/opt/android_sdk"
+  git_fetch_script:
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
   pub_cache:
     folder: $HOME/.pub-cache
     fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
@@ -16,22 +20,28 @@ task:
   artifacts_cache:
     folder: bin/cache/artifacts
     fingerprint_script: echo $OS; cat bin/internal/engine.version
-  setup_script: |
-    echo "SDK directory is: $PWD"
-    ./bin/flutter --version
-
-    # disable analytics on the bots and download Flutter dependencies
-    ./bin/flutter config --no-analytics
-
-    # run pub get in all the repo packages
-    ./bin/flutter update-packages
-
-    git fetch origin master
+  setup_script: ./dev/bots/cirrus_setup.sh
   matrix:
     - name: docs
       env:
         SHARD: docs
+        # For uploading master docs to Firebase master branch staging site
+        FIREBASE_MASTER_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92]
+        # For uploading beta docs to Firebase public live site
+        FIREBASE_PUBLIC_TOKEN: ENCRYPTED[c422da192f06da7b4449ca8e7aa866dabeb8a0f8d7488497c2e7e447e6fd31d917e6c813db081dc4e2a7a63afdf41864]
       docs_script: ./dev/bots/docs.sh
+    - name: deploy_gallery
+      only_if: $CIRRUS_BRANCH == 'dev'
+      depends_on:
+        - docs
+        - analyze
+        - tests-linux
+        - tool_tests-linux
+      env:
+        SHARD: deploy_gallery
+        GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346]
+        ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0b3e681b4507aec433ef29c79b715f15f8c75ecd25315ea286b0b2bcb8b28d578634eead5aa2c54086a25e8da1bb219a]
+      test_script: ./dev/bots/deploy_gallery.sh
     - name: analyze
       env:
         SHARD: analyze
@@ -61,7 +71,9 @@ task:
     cpu: 4
   env:
     CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk"
-  git_fetch_script: git fetch origin
+  git_fetch_script:
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
   pub_cache:
     folder: $APPDATA\Pub\Cache
     fingerprint_script:
@@ -87,13 +99,46 @@ task:
         SHARD: tool_tests
 
 task:
+  name: deploy_gallery-macos
+  only_if: $CIRRUS_BRANCH == 'dev'
+  pub_cache:
+    folder: ~/.pub-cache
   depends_on:
     - analyze
+    - tests-macos
+    - tool_tests-macos
+  env:
+    # Name the SDK directory to include a space so that we constantly
+    # test path names with spaces in them.
+    CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
+    SHARD: deploy_gallery
+    # Apple Certificates Match Passphrase
+    MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73]
+    # Apple Fastlane Password
+    FASTLANE_PASSWORD: ENCRYPTED[0bf9bb0cc2cb32a0ed18470cf2c9df0f587cce5f8b04adbd6cff15ca5bde7a74f721ee580227b132ab6b032f08e52ae0]
+    # Private repo for publishing certificates.
+    PUBLISHING_MATCH_CERTIFICATE_REPO: git@github.com:flutter/private_publishing_certificates.git
+  osx_instance:
+    image: high-sierra-xcode-9.4.1
+  git_fetch_script:
+  - git fetch origin
+  - git fetch origin master # To set FETCH_HEAD
+  setup_script:
+    - bin/flutter config --no-analytics
+    - bin/flutter update-packages
+  test_all_script:
+    - ./dev/bots/deploy_gallery.sh
+
+task:
   osx_instance:
     image: high-sierra-xcode-9.4.1
+  depends_on:
+    - analyze
   env:
     CIRRUS_WORKING_DIR: "/tmp/flutter sdk"
-  git_fetch_script: git fetch origin
+  git_fetch_script:
+    - git fetch origin
+    - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
   pub_cache:
     folder: $HOME/.pub-cache
     fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR"
@@ -116,3 +161,19 @@ task:
     - name: tool_tests-macos
       env:
         SHARD: tool_tests
+
+
+docker_builder:
+  # Only build a new docker image when we tag a release (for dev, beta, or release.)
+  only_if: $CIRRUS_TAG != ''
+  env:
+    GCLOUD_CREDENTIALS: ENCRYPTED[f7c098d4dd7f5ee1bfee0bb7e944cce72efbe10e97ad6440ae72de4de6a1c24d23f421a2619c668e94377fb64b0bb3e6]
+  depends_on:
+    - docs
+    - analyze
+    - tests-linux
+    - tool_tests-linux
+  build_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh"
+  login_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh"
+  push_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh"
+

dev/bots/cirrus_setup.sh

+#!/bin/bash
+set -e
+
+# This script is only meant to be run by the Cirrus CI system, not locally.
+# It must be run from the root of the Flutter repo.
+
+# Collects log output in a tmpfile, but only prints it if the command fails.
+function log_on_fail() {
+  local COMMAND="$@"
+  local TMPDIR="$(mktemp -d)"
+  local TMPFILE="$TMPDIR/command.log"
+  local EXIT=0
+  if ("$@" > "$TMPFILE" 2>&1); then
+    echo "'$COMMAND' succeeded."
+  else
+    EXIT=$?
+    cat "$TMPFILE" 1>&2
+    echo "FAIL: '$COMMAND' exited with code $EXIT" 1>&2
+  fi
+  rm -rf "$TMPDIR"
+  return "$EXIT"
+}
+
+function accept_android_licenses() {
+  yes "y" | flutter doctor --android-licenses
+}
+
+echo "Flutter SDK directory is: $PWD"
+
+# Run flutter to download dependencies and precompile things, and to disable
+# analytics on the bots.
+echo "Downloading build dependencies and pre-compiling Flutter snapshot"
+log_on_fail ./bin/flutter config --no-analytics
+
+# Run doctor, to print it to the log for debugging purposes.
+./bin/flutter doctor -v
+
+# Accept licenses.
+log_on_fail accept_android_licenses && echo "Android licenses accepted."
+
+# Run pub get in all the repo packages.
+echo "Updating packages for Flutter."
+log_on_fail ./bin/flutter update-packages

dev/bots/deploy_gallery.sh

+#!/bin/bash
+
+set -e
+
+function script_location() {
+  local script_location="${BASH_SOURCE[0]}"
+  # Resolve symlinks
+  while [[ -h "$script_location" ]]; do
+    DIR="$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
+    script_location="$(readlink "$script_location")"
+    [[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
+  done
+  echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
+}
+
+# So that users can run this script locally from any directory and it will work as
+# expected.
+SCRIPT_LOCATION="$(script_location)"
+FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"
+
+export PATH="$FLUTTER_ROOT/bin:$FLUTTER_ROOT/bin/cache/dart-sdk/bin:$PATH"
+
+set -x
+
+cd "$FLUTTER_ROOT"
+
+if [[ "$SHARD" = "deploy_gallery" ]]; then
+  version="$(<version)"
+  if [[ "$OS" == "linux" ]]; then
+    echo "Building Flutter Gallery $version for Android..."
+    export ANDROID_HOME="$PWD/android-sdk"
+    (
+      cd examples/flutter_gallery
+      flutter build apk --release -t lib/main_publish.dart
+    )
+    echo "Android Flutter Gallery built"
+    if [[ -z "$CIRRUS_PULL_REQUEST" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
+      echo "Deploying Flutter Gellery $version to Play Store..."
+      (
+        cd examples/flutter_gallery/android
+        bundle install
+        bundle exec fastlane deploy_play_store
+      )
+    else
+      echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits"
+    fi
+  elif [[ "$OS" == "macos" ]]; then
+    echo "Building Flutter Gallery $version for iOS..."
+    (
+      cd examples/flutter_gallery
+      flutter build ios --release --no-codesign -t lib/main_publish.dart
+    )
+    echo "iOS Flutter Gallery built"
+    if [[ -z "$CIRRUS_PULL_REQUEST" ]]; then
+      if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
+        echo "Archiving with distribution profile and deploying to TestFlight..."
+        (
+          cd examples/flutter_gallery/ios
+          bundle install
+          bundle exec fastlane build_and_deploy_testflight upload:true
+        )
+      else
+        echo "Archiving with distribution profile..."
+        (
+          cd examples/flutter_gallery/ios
+          bundle install
+          bundle exec fastlane build_and_deploy_testflight
+        )
+        echo "Archive is only deployed to TestFlight on tagged dev branch commits"
+      fi
+    else
+      echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits"
+    fi
+  fi
+else
+  echo "Doing nothing: not on the 'deploy_gallery' SHARD."
+fi

dev/bots/docs.sh

 #!/bin/bash
 set -e
 
+function deploy {
+  local total_tries="$1"
+  local remaining_tries=$(($total_tries - 1))
+  shift
+  while [[ "$remaining_tries" > 0 ]]; do
+    (cd "$FLUTTER_ROOT/dev/docs" && firebase deploy --token "$FIREBASE_TOKEN" --project "$@") && break
+    remaining_tries=$(($remaining_tries - 1))
+    echo "Error: Unable to deploy documentation to Firebase. Retrying in five seconds... ($remaining_tries tries left)"
+    sleep 5
+  done
+
+  [[ "$remaining_tries" == 0 ]] && {
+    echo "Command still failed after $total_tries tries: '$@'"
+    return 1
+  }
+  return 0
+}
+
+function script_location() {
+  local script_location="${BASH_SOURCE[0]}"
+  # Resolve symlinks
+  while [[ -h "$script_location" ]]; do
+    DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)"
+    script_location="$(readlink "$script_location")"
+    [[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
+  done
+  echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
+}
+
+# So that users can run this script from anywhere and it will work as expected.
+SCRIPT_LOCATION="$(script_location)"
+# Sets the Flutter root to be "$(script_location)/../..": This script assumes
+# that it resides two directory levels down from the root, so if that changes,
+# then this line will need to as well.
+FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"
+
 echo "Running docs.sh"
 
-# If you want to run this script locally, make sure you run it from
-# the root of the flutter repository.
-export FLUTTER_ROOT="$PWD"
-export PATH="$PWD/bin:$PATH"
+if [[ ! -d "$FLUTTER_ROOT" || ! -f "$FLUTTER_ROOT/bin/flutter" ]]; then
+  echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)"
+  exit 1
+fi
 
-# This is called from travis_upload.sh on Travis.
+FLUTTER_BIN="$FLUTTER_ROOT/bin"
+DART_BIN="$FLUTTER_ROOT/bin/cache/dart-sdk/bin"
+FLUTTER="$FLUTTER_BIN/flutter"
+DART="$DART_BIN/dart"
+PUB="$DART_BIN/pub"
+export PATH="$FLUTTER_BIN:$DART_BIN:$PATH"
 
-# Make sure dart is installed
-bin/flutter --version
+# Make sure dart is installed by invoking flutter to download it.
+"$FLUTTER" --version
 
 # If the pub cache directory exists in the root, then use that.
 FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache"
-if [ -d "$FLUTTER_PUB_CACHE" ]; then
+if [[ -d "$FLUTTER_PUB_CACHE" ]]; then
   # This has to be exported, because pub interprets setting it
   # to the empty string in the same way as setting it to ".".
   export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
 fi
 
-# Install dartdoc.
-bin/cache/dart-sdk/bin/pub global activate dartdoc 0.20.2
+# Install and activate dartdoc.
+"$PUB" global activate dartdoc 0.20.2
 
 # This script generates a unified doc set, and creates
 # a custom index.html, placing everything into dev/docs/doc.
-(cd dev/tools; ../../bin/cache/dart-sdk/bin/pub get)
-bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart
-bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart
+(cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" packages get)
+(cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get)
+(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
+(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")
 
 # Ensure google webmaster tools can verify our site.
-cp dev/docs/google2ed1af765c529f57.html dev/docs/doc
+cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc"
 
 # Upload new API docs when on Travis
-if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-  echo "This is not a pull request; considering whether to upload docs... (branch=$TRAVIS_BRANCH)"
-  if [ "$TRAVIS_BRANCH" == "master" -o "$TRAVIS_BRANCH" == "beta" ]; then
-    cd dev/docs
-
-    if [ "$TRAVIS_BRANCH" == "master" ]; then
+if [[ -n "$CIRRUS_CI" && -z "$CIRRUS_PR" ]]; then
+  echo "This is not a pull request; considering whether to upload docs... (branch=$CIRRUS_BRANCH)"
+  if [[ "$CIRRUS_BRANCH" == "master" || "$CIRRUS_BRANCH" == "beta" ]]; then
+    if [[ "$CIRRUS_BRANCH" == "master" ]]; then
       echo "Updating master docs: https://master-docs-flutter-io.firebaseapp.com/"
-      echo -e "User-agent: *\nDisallow: /" > doc/robots.txt
-      while : ; do
-        firebase deploy --project master-docs-flutter-io && break
-        echo Error: Unable to deploy documentation to firebase. Retrying in five seconds...
-        sleep 5
-      done
+      # Disable search indexing on the master staging site so searches get only
+      # the beta site.
+      echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
+      export FIREBASE_TOKEN="$FIREBASE_MASTER_TOKEN"
+      deploy 5 master-docs-flutter-io
     fi
 
-    if [ "$TRAVIS_BRANCH" == "beta" ]; then
+    if [[ "$CIRRUS_BRANCH" == "beta" ]]; then
       echo "Updating beta docs: https://docs.flutter.io/"
-      while : ; do
-        firebase deploy --project docs-flutter-io && break
-        echo Error: Unable to deploy documentation to firebase. Retrying in five seconds...
-        sleep 5
-      done
+      export FIREBASE_TOKEN="$FIREBASE_PUBLIC_TOKEN"
+      deploy 5 docs-flutter-io
     fi
   fi
 fi
+

dev/ci/README.md

+This directory includes scripts and tools for continuous integration builds and tests.

dev/ci/docker_linux/Dockerfile

+# Flutter (https://flutter.io) Developement Environment for Linux
+# ===============================================================
+#
+# This environment passes all Linux Flutter Doctor checks and is sufficient
+# for building Android applications and running Flutter tests.
+#
+# To build iOS applications, a Mac development environment is necessary.
+#
+# This includes applications and sdks that are needed only by the CI system
+# for performing pushes to production, and so this image is quite a bit larger
+# than strictly needed for just building Flutter apps.
+
+FROM debian:stretch
+MAINTAINER Flutter Developers <flutter-dev@googlegroups.com>
+
+RUN apt-get update -y
+RUN apt-get upgrade -y
+
+# Install basics
+RUN apt-get install -y --no-install-recommends \
+  git \
+  wget \
+  curl \
+  unzip \
+  ca-certificates \
+  gnupg
+
+# Add nodejs repository to apt sources and install it.
+ENV NODEJS_INSTALL="/opt/nodejs_install"
+RUN mkdir -p "${NODEJS_INSTALL}"
+RUN wget -q https://deb.nodesource.com/setup_10.x -O "${NODEJS_INSTALL}/nodejs_install.sh"
+RUN bash "${NODEJS_INSTALL}/nodejs_install.sh"
+
+# Install the rest of the dependencies.
+RUN apt-get install -y --no-install-recommends \
+  locales \
+  ruby \
+  nodejs \
+  lib32stdc++6 \
+  libstdc++6 \
+  libglu1-mesa \
+  build-essential \
+  default-jdk-headless
+
+# Install the Android SDK Dependency.
+ENV ANDROID_SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip"
+ENV ANDROID_TOOLS_ROOT="/opt/android_sdk"
+RUN mkdir -p "${ANDROID_TOOLS_ROOT}"
+RUN mkdir -p ~/.android
+# Silence warning.
+RUN touch ~/.android/repositories.cfg
+ENV ANDROID_SDK_ARCHIVE="${ANDROID_TOOLS_ROOT}/archive"
+RUN wget --progress=dot:giga "${ANDROID_SDK_URL}" -O "${ANDROID_SDK_ARCHIVE}"
+RUN unzip -q -d "${ANDROID_TOOLS_ROOT}" "${ANDROID_SDK_ARCHIVE}"
+# Suppressing output of sdkmanager to keep log size down
+# (it prints install progress WAY too often).
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "tools" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "build-tools;28.0.0" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "platforms;android-28" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "platform-tools" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "extras;android;m2repository" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "extras;google;m2repository" > /dev/null
+RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "patcher;v4" > /dev/null
+RUN rm "${ANDROID_SDK_ARCHIVE}"
+ENV PATH="${ANDROID_TOOLS_ROOT}/tools:${PATH}"
+ENV PATH="${ANDROID_TOOLS_ROOT}/tools/bin:${PATH}"
+# Silence warnings when accepting android licenses.
+RUN mkdir -p ~/.android
+RUN touch ~/.android/repositories.cfg
+
+# Setup gradle
+ENV GRADLE_ROOT="/opt/gradle"
+RUN mkdir -p "${GRADLE_ROOT}"
+ENV GRADLE_ARCHIVE="${GRADLE_ROOT}/gradle.zip"
+ENV GRADLE_URL="http://services.gradle.org/distributions/gradle-4.4-bin.zip"
+RUN wget --progress=dot:giga "$GRADLE_URL" -O "${GRADLE_ARCHIVE}"
+RUN unzip -q -d "${GRADLE_ROOT}" "${GRADLE_ARCHIVE}"
+ENV PATH="$GRADLE_ROOT/bin:$PATH"
+
+# Add npm to path.
+ENV PATH="/usr/bin:${PATH}"
+RUN dpkg-query -L nodejs
+# Install Firebase
+RUN /usr/bin/npm install -g firebase-tools
+
+# Set locale to en_US
+RUN locale-gen en_US "en_US.UTF-8" && dpkg-reconfigure locales
+ENV LANG en_US.UTF-8
+
+# Install coveralls and Firebase
+RUN gem install coveralls
+RUN gem install bundler
+

dev/ci/docker_linux/README.md

+This directory includes scripts to build the docker container image used for
+building flutter/flutter in our CI system (currently [Cirrus](cirrus-ci.org)).
+
+In order to run the scripts, you have to setup `docker` and `gcloud`. Please
+refer to the [internal flutter team doc](go/flutter-team) for how to setup in a
+Google internal environment.
+
+After setup,
+* edit `Dockerfile` to change how the container image is built.
+* run `./build_docker.sh` to build the container image.
+* run `./push_docker.sh` to push the image to google cloud registry. This will
+  affect our CI tests.
+

dev/ci/docker_linux/docker_build.sh

+#!/bin/bash
+
+TAG="${CIRRUS_TAG:-latest}"
+
+# pull to make sure we are not rebuilding for nothing
+docker pull "gcr.io/flutter-cirrus/build-flutter-image:$TAG"
+
+docker build --tag "gcr.io/flutter-cirrus/build-flutter-image:$TAG" .

dev/ci/docker_linux/docker_login.sh

+#!/bin/bash
+
+if [[ -n "$CIRRUS_CI" && -n "$GCLOUD_CREDENTIALS" ]]; then
+  echo "$GCLOUD_CREDENTIALS" | base64 --decode | docker login -u _json_key --password-stdin https://gcr.io
+else
+  gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
+fi
\ No newline at end of file

dev/ci/docker_linux/docker_push.sh

+#!/bin/bash
+
+TAG="${CIRRUS_TAG:-latest}"
+
+docker push "gcr.io/flutter-cirrus/build-flutter-image::$TAG"
+