[flutter_tools] only lock if an upgrade/download will be performed...

[flutter_tools] only lock if an upgrade/download will be performed (linux/macos) and output building messages to stderr (#65422)

Currently an invocation of flutter/dart will always attempt to acquire a lock. This can pose problems for tools that attempt to run multiple dart/flutter instances.

Instead update the lock logic (on Linux/macOS) so that we only attempt to acquire it if an update/snapshot needs to be performed. To avoid repeatedly performing downloads/snapshots if multiple flutter/dart invocations are fired off concurrently when an update needs to be performed, do a second check of the download/snapshot condition after the lock is released.

Additionally, moves all of the building/debug output to stderr on both the bash and batch scripts. This allows machine mode consumption of the tool to ignore needing to parse/handle the rebuild messages.

bin/flutter.bat

@@ -27,11 +27,11 @@ IF NOT EXIST "%flutter_root%\.git" (
   ECHO        The flutter tool requires Git in order to operate properly;
   ECHO        to set up Flutter, run the following command:
   ECHO        git clone -b stable https://github.com/flutter/flutter.git
-  EXIT /B 1
+  EXIT 1
 )
 
 REM Include shared scripts in shared.bat
-SET shared_bin=%FLUTTER_ROOT%/bin/internal/shared.bat
+SET shared_bin=%FLUTTER_ROOT%\bin\internal\shared.bat
 CALL "%shared_bin%"
 
 SET flutter_tools_dir=%FLUTTER_ROOT%\packages\flutter_tools

bin/internal/shared.bat

@@ -34,9 +34,9 @@ WHERE /Q pwsh.exe && (
 ) || WHERE /Q PowerShell.exe && (
     SET powershell_executable=PowerShell.exe
 ) || (
-    ECHO Error: PowerShell executable not found.
-    ECHO        Either pwsh.exe or PowerShell.exe must be in your PATH.
-    EXIT /B 1
+    ECHO Error: PowerShell executable not found.                        1>&2
+    ECHO        Either pwsh.exe or PowerShell.exe must be in your PATH. 1>&2
+    EXIT 1
 )
 
 REM Ensure that bin/cache exists.
@@ -44,13 +44,13 @@ IF NOT EXIST "%cache_dir%" MKDIR "%cache_dir%"
 
 REM If the cache still doesn't exist, fail with an error that we probably don't have permissions.
 IF NOT EXIST "%cache_dir%" (
-  ECHO Error: Unable to create cache directory at
-  ECHO            %cache_dir%
-  ECHO.
-  ECHO        This may be because flutter doesn't have write permissions for
-  ECHO        this path. Try moving the flutter directory to a writable location,
-  ECHO        such as within your home directory.
-  EXIT /B 1
+  ECHO Error: Unable to create cache directory at                                 1>&2
+  ECHO            %cache_dir%                                                     1>&2
+  ECHO.                                                                           1>&2
+  ECHO        This may be because flutter doesn't have write permissions for      1>&2
+  ECHO        this path. Try moving the flutter directory to a writable location, 1>&2
+  ECHO        such as within your home directory.                                 1>&2
+  EXIT 1
 )
 
 :acquire_lock
@@ -88,19 +88,20 @@ GOTO :after_subroutine
   EXIT /B
 
   :do_sdk_update_and_snapshot
-    ECHO Checking Dart SDK version...
-    SET update_dart_bin=%FLUTTER_ROOT%/bin/internal/update_dart_sdk.ps1
+    ECHO Checking Dart SDK version... 1>&2
+    SET update_dart_bin=%FLUTTER_ROOT%\bin\internal\update_dart_sdk.ps1
     REM Escape apostrophes from the executable path
     SET "update_dart_bin=!update_dart_bin:'=''!"
     REM PowerShell command must have exit code set in order to prevent all non-zero exit codes from being translated
     REM into 1. The exit code 2 is used to detect the case where the major version is incorrect and there should be
     REM no subsequent retries.
+    ECHO Downloading Dart SDK from Flutter engine %dart_required_version%... 1>&2
     %powershell_executable% -ExecutionPolicy Bypass -Command "Unblock-File -Path '%update_dart_bin%'; & '%update_dart_bin%'; exit $LASTEXITCODE;"
     IF "%ERRORLEVEL%" EQU "2" (
       EXIT 1
     )
     IF "%ERRORLEVEL%" NEQ "0" (
-      ECHO Error: Unable to update Dart SDK. Retrying...
+      ECHO Error: Unable to update Dart SDK. Retrying... 1>&2
       timeout /t 5 /nobreak
       GOTO :do_sdk_update_and_snapshot
     )
@@ -108,7 +109,7 @@ GOTO :after_subroutine
   :do_snapshot
     IF EXIST "%FLUTTER_ROOT%\version" DEL "%FLUTTER_ROOT%\version"
     ECHO: > "%cache_dir%\.dartignore"
-    ECHO Building flutter tool...
+    ECHO Building flutter tool... 1>&2
     PUSHD "%flutter_tools_dir%"
 
     REM Makes changes to PUB_ENVIRONMENT only visible to commands within SETLOCAL/ENDLOCAL
@@ -131,17 +132,17 @@ GOTO :after_subroutine
       SET /A total_tries=10
       SET /A remaining_tries=%total_tries%-1
       :retry_pub_upgrade
-        ECHO Running pub upgrade...
+        ECHO Running pub upgrade... 1>&2
         CALL "%pub%" upgrade "%VERBOSITY%" --no-precompile
         IF "%ERRORLEVEL%" EQU "0" goto :upgrade_succeeded
-        ECHO Error (%ERRORLEVEL%): Unable to 'pub upgrade' flutter tool. Retrying in five seconds... (%remaining_tries% tries left)
+        ECHO Error (%ERRORLEVEL%): Unable to 'pub upgrade' flutter tool. Retrying in five seconds... (%remaining_tries% tries left) 1>&2
         timeout /t 5 /nobreak 2>NUL
         SET /A remaining_tries-=1
         IF "%remaining_tries%" EQU "0" GOTO upgrade_retries_exhausted
         GOTO :retry_pub_upgrade
       :upgrade_retries_exhausted
         SET exit_code=%ERRORLEVEL%
-        ECHO Error: 'pub upgrade' still failing after %total_tries% tries, giving up.
+        ECHO Error: 'pub upgrade' still failing after %total_tries% tries. 1>&2
         GOTO final_exit
       :upgrade_succeeded
     ENDLOCAL
@@ -154,7 +155,7 @@ GOTO :after_subroutine
       "%dart%" "%FLUTTER_TOOL_ARGS%" --snapshot="%snapshot_path%" --packages="%flutter_tools_dir%\.packages" "%script_path%"
     )
     IF "%ERRORLEVEL%" NEQ "0" (
-      ECHO Error: Unable to create dart snapshot for flutter tool.
+      ECHO Error: Unable to create dart snapshot for flutter tool. 1>&2
       SET exit_code=%ERRORLEVEL%
       GOTO :final_exit
     )

bin/internal/shared.sh

@@ -22,13 +22,13 @@ function retry_upgrade {
   local remaining_tries=$((total_tries - 1))
   while [[ "$remaining_tries" -gt 0 ]]; do
     (cd "$FLUTTER_TOOLS_DIR" && "$PUB" upgrade "$VERBOSITY" --no-precompile) && break
-    echo "Error: Unable to 'pub upgrade' flutter tool. Retrying in five seconds... ($remaining_tries tries left)"
+    >&2 echo "Error: Unable to 'pub upgrade' flutter tool. Retrying in five seconds... ($remaining_tries tries left)"
     remaining_tries=$((remaining_tries - 1))
     sleep 5
   done
 
   if [[ "$remaining_tries" == 0 ]]; then
-    echo "Command 'pub upgrade' still failed after $total_tries tries, giving up."
+    >&2 echo "Command 'pub upgrade' still failed after $total_tries tries, giving up."
     return 1
   fi
   return 0
@@ -94,14 +94,14 @@ function _wait_for_lock () {
       # Print with a return so that if the Dart code also prints this message
       # when it does its own lock, the message won't appear twice. Be sure that
       # the clearing printf below has the same number of space characters.
-      printf "Waiting for another flutter command to release the startup lock...\r";
+      printf "Waiting for another flutter command to release the startup lock...\r" >&2;
       waiting_message_displayed="true"
     fi
     sleep .1;
   done
   if [[ $waiting_message_displayed == "true" ]]; then
     # Clear the waiting message so it doesn't overlap any following text.
-    printf "                                                                  \r";
+    printf "                                                                  \r" >&2;
   fi
   unset waiting_message_displayed
   # If the lock file is acquired, make sure that it is removed on exit.
@@ -114,9 +114,6 @@ function _wait_for_lock () {
 function upgrade_flutter () (
   mkdir -p "$FLUTTER_ROOT/bin/cache"
 
-  # Waits for the update lock to be acquired.
-  _wait_for_lock
-
   local revision="$(cd "$FLUTTER_ROOT"; git rev-parse HEAD)"
 
   # Invalidate cache if:
@@ -125,12 +122,23 @@ function upgrade_flutter () (
   #  * Contents of STAMP_PATH is not our local git HEAD revision, or
   #  * pubspec.yaml last modified after pubspec.lock
   if [[ ! -f "$SNAPSHOT_PATH" || ! -s "$STAMP_PATH" || "$(cat "$STAMP_PATH")" != "$revision" || "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]]; then
+    # Waits for the update lock to be acquired. Placing this check inside the
+    # conditional allows the majority of flutter/dart installations to bypass
+    # the lock entirely, but as a result this required a second verification that
+    # the SDK is up to date.
+    _wait_for_lock
+
+    # A different shell process might have updated the tool/SDK.
+    if [[ -f "$SNAPSHOT_PATH" && -s "$STAMP_PATH" && "$(cat "$STAMP_PATH")" == "$revision" && "$FLUTTER_TOOLS_DIR/pubspec.yaml" -ot "$FLUTTER_TOOLS_DIR/pubspec.lock" ]]; then
+      exit $?
+    fi
+
     rm -f "$FLUTTER_ROOT/version"
     touch "$FLUTTER_ROOT/bin/cache/.dartignore"
     "$FLUTTER_ROOT/bin/internal/update_dart_sdk.sh"
     VERBOSITY="--verbosity=error"
 
-    echo Building flutter tool...
+    >&2 echo Building flutter tool...
     if [[ "$CI" == "true" || "$BOT" == "true" || "$CONTINUOUS_INTEGRATION" == "true" || "$CHROME_HEADLESS" == "1" ]]; then
       PUB_ENVIRONMENT="$PUB_ENVIRONMENT:flutter_bot"
       VERBOSITY="--verbosity=normal"
@@ -178,24 +186,24 @@ function shared::execute() {
 
   # Test if running as superuser – but don't warn if running within Docker
   if [[ "$EUID" == "0" && ! -f /.dockerenv ]]; then
-    echo "   Woah! You appear to be trying to run flutter as root."
-    echo "   We strongly recommend running the flutter tool without superuser privileges."
-    echo "  /"
-    echo "📎"
+    >&2 echo "   Woah! You appear to be trying to run flutter as root."
+    >&2 echo "   We strongly recommend running the flutter tool without superuser privileges."
+    >&2 echo "  /"
+    >&2 echo "📎"
   fi
 
   # Test if Git is available on the Host
   if ! hash git 2>/dev/null; then
-    echo "Error: Unable to find git in your PATH."
+    >&2 echo "Error: Unable to find git in your PATH."
     exit 1
   fi
   # Test if the flutter directory is a git clone (otherwise git rev-parse HEAD
   # would fail)
   if [[ ! -e "$FLUTTER_ROOT/.git" ]]; then
-    echo "Error: The Flutter directory is not a clone of the GitHub project."
-    echo "       The flutter tool requires Git in order to operate properly;"
-    echo "       to install Flutter, see the instructions at:"
-    echo "       https://flutter.dev/get-started"
+    >&2 echo "Error: The Flutter directory is not a clone of the GitHub project."
+    >&2 echo "       The flutter tool requires Git in order to operate properly;"
+    >&2 echo "       to install Flutter, see the instructions at:"
+    >&2 echo "       https://flutter.dev/get-started"
     exit 1
   fi
 
@@ -217,7 +225,7 @@ function shared::execute() {
       "$DART" "$@"
       ;;
     *)
-      echo "Error! Executable name $BIN_NAME not recognized!"
+      >&2 echo "Error! Executable name $BIN_NAME not recognized!"
       exit 1
       ;;
   esac

bin/internal/update_dart_sdk.ps1

@@ -38,7 +38,6 @@ if ((Test-Path $engineStamp) -and ($engineVersion -eq (Get-Content $engineStamp)
     return
 }
 
-Write-Host "Downloading Dart SDK from Flutter engine $engineVersion..."
 $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL
 if (-not $dartSdkBaseUrl) {
     $dartSdkBaseUrl = "https://storage.googleapis.com"
@@ -71,7 +70,6 @@ Catch {
     $ProgressPreference = $OriginalProgressPreference
 }
 
-Write-Host "Unzipping Dart SDK..."
 If (Get-Command 7z -errorAction SilentlyContinue) {
     # The built-in unzippers are painfully slow. Use 7-Zip, if available.
     & 7z x $dartSdkZip "-o$cachePath" -bd | Out-Null

bin/internal/update_dart_sdk.sh

@@ -23,25 +23,25 @@ ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/internal/engine.version"`
 
 if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; then
   command -v curl > /dev/null 2>&1 || {
-    echo
-    echo 'Missing "curl" tool. Unable to download Dart SDK.'
+    >&2 echo
+    >&2 echo 'Missing "curl" tool. Unable to download Dart SDK.'
     case "$(uname -s)" in
       Darwin)
-        echo 'Consider running "brew install curl".'
+        >&2 echo 'Consider running "brew install curl".'
         ;;
       Linux)
-        echo 'Consider running "sudo apt-get install curl".'
+        >&2 echo 'Consider running "sudo apt-get install curl".'
         ;;
       *)
-        echo "Please install curl."
+        >&2 echo "Please install curl."
         ;;
     esac
     echo
     exit 1
   }
   command -v unzip > /dev/null 2>&1 || {
-    echo
-    echo 'Missing "unzip" tool. Unable to extract Dart SDK.'
+    >&2 echo
+    >&2 echo 'Missing "unzip" tool. Unable to extract Dart SDK.'
     case "$(uname -s)" in
       Darwin)
         echo 'Consider running "brew install unzip".'
@@ -56,7 +56,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t
     echo
     exit 1
   }
-  echo "Downloading Dart SDK from Flutter engine $ENGINE_VERSION..."
+  >&2 echo "Downloading Dart SDK from Flutter engine $ENGINE_VERSION..."
 
   case "$(uname -s)" in
     Darwin)
@@ -99,20 +99,20 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t
   DART_SDK_ZIP="$FLUTTER_ROOT/bin/cache/$DART_ZIP_NAME"
 
   curl --retry 3 --continue-at - --location --output "$DART_SDK_ZIP" "$DART_SDK_URL" 2>&1 || {
-    echo
-    echo "Failed to retrieve the Dart SDK from: $DART_SDK_URL"
-    echo "If you're located in China, please see this page:"
-    echo "  https://flutter.dev/community/china"
-    echo
+    >&2 echo
+    >&2 echo "Failed to retrieve the Dart SDK from: $DART_SDK_URL"
+    >&2 echo "If you're located in China, please see this page:"
+    >&2 echo "  https://flutter.dev/community/china"
+    >&2 echo
     rm -f -- "$DART_SDK_ZIP"
     exit 1
   }
   unzip -o -q "$DART_SDK_ZIP" -d "$FLUTTER_ROOT/bin/cache" || {
-    echo
-    echo "It appears that the downloaded file is corrupt; please try again."
-    echo "If this problem persists, please report the problem at:"
-    echo "  https://github.com/flutter/flutter/issues/new?template=ACTIVATION.md"
-    echo
+    >&2 echo
+    >&2 echo "It appears that the downloaded file is corrupt; please try again."
+    >&2 echo "If this problem persists, please report the problem at:"
+    >&2 echo "  https://github.com/flutter/flutter/issues/new?template=ACTIVATION.md"
+    >&2 echo
     rm -f -- "$DART_SDK_ZIP"
     exit 1
   }

dev/bots/analyze-sample-code.dart

@@ -529,6 +529,10 @@ linter:
     );
     final List<String> stderr = result.stderr.toString().trim().split('\n');
     final List<String> stdout = result.stdout.toString().trim().split('\n');
+    // Remove output from building the flutter tool.
+    stderr.removeWhere((String line) {
+      return line.startsWith('Building flutter tool...');
+    });
     // Check out the stderr to see if the analyzer had it's own issues.
     if (stderr.isNotEmpty && (stderr.first.contains(' issues found. (ran in ') || stderr.first.contains(' issue found. (ran in '))) {
       // The "23 issues found" message goes onto stderr, which is concatenated first.

dev/bots/test/analyze-sample-code_test.dart

@@ -14,7 +14,7 @@ void main() {
     );
     final List<String> stdoutLines = process.stdout.toString().split('\n');
     final List<String> stderrLines = process.stderr.toString().split('\n')
-      ..removeWhere((String line) => line.startsWith('Analyzer output:'));
+      ..removeWhere((String line) => line.startsWith('Analyzer output:') || line.startsWith('Building flutter tool...'));
     expect(process.exitCode, isNot(equals(0)));
     expect(stderrLines, <String>[
       'known_broken_documentation.dart:30:9: new Opacity(',