// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'package:crypto/crypto.dart'; import 'package:file/memory.dart'; import 'package:meta/meta.dart'; import 'package:process/process.dart'; import 'base/common.dart'; import 'base/error_handling_io.dart'; import 'base/file_system.dart'; import 'base/io.dart' show HttpClient, HttpClientRequest, HttpClientResponse, HttpHeaders, HttpStatus, SocketException; import 'base/logger.dart'; import 'base/net.dart'; import 'base/os.dart' show OperatingSystemUtils; import 'base/platform.dart'; import 'base/terminal.dart'; import 'base/user_messages.dart'; import 'build_info.dart'; import 'convert.dart'; import 'features.dart'; const String kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo) const String kFlutterEngineEnvironmentVariableName = 'FLUTTER_ENGINE'; // should point to //engine/src/ (root of flutter/engine repo) const String kSnapshotFileName = 'flutter_tools.snapshot'; // in //flutter/bin/cache/ const String kFlutterToolsScriptFileName = 'flutter_tools.dart'; // in //flutter/packages/flutter_tools/bin/ const String kFlutterEnginePackageName = 'sky_engine'; /// A tag for a set of development artifacts that need to be cached. class DevelopmentArtifact { const DevelopmentArtifact._(this.name, {this.feature}); /// The name of the artifact. /// /// This should match the flag name in precache.dart. final String name; /// A feature to control the visibility of this artifact. final Feature? feature; /// Artifacts required for Android development. static const DevelopmentArtifact androidGenSnapshot = DevelopmentArtifact._('android_gen_snapshot', feature: flutterAndroidFeature); static const DevelopmentArtifact androidMaven = DevelopmentArtifact._('android_maven', feature: flutterAndroidFeature); // Artifacts used for internal builds. static const DevelopmentArtifact androidInternalBuild = DevelopmentArtifact._('android_internal_build', feature: flutterAndroidFeature); /// Artifacts required for iOS development. static const DevelopmentArtifact iOS = DevelopmentArtifact._('ios', feature: flutterIOSFeature); /// Artifacts required for web development. static const DevelopmentArtifact web = DevelopmentArtifact._('web', feature: flutterWebFeature); /// Artifacts required for desktop macOS. static const DevelopmentArtifact macOS = DevelopmentArtifact._('macos', feature: flutterMacOSDesktopFeature); /// Artifacts required for desktop Windows. static const DevelopmentArtifact windows = DevelopmentArtifact._('windows', feature: flutterWindowsDesktopFeature); /// Artifacts required for desktop Linux. static const DevelopmentArtifact linux = DevelopmentArtifact._('linux', feature: flutterLinuxDesktopFeature); /// Artifacts required for Fuchsia. static const DevelopmentArtifact fuchsia = DevelopmentArtifact._('fuchsia', feature: flutterFuchsiaFeature); /// Artifacts required for the Flutter Runner. static const DevelopmentArtifact flutterRunner = DevelopmentArtifact._('flutter_runner', feature: flutterFuchsiaFeature); /// Artifacts required for any development platform. /// /// This does not need to be explicitly returned from requiredArtifacts as /// it will always be downloaded. static const DevelopmentArtifact universal = DevelopmentArtifact._('universal'); /// The values of DevelopmentArtifacts. static final List<DevelopmentArtifact> values = <DevelopmentArtifact>[ androidGenSnapshot, androidMaven, androidInternalBuild, iOS, web, macOS, windows, linux, fuchsia, universal, flutterRunner, ]; @override String toString() => 'Artifact($name)'; } /// A wrapper around the `bin/cache/` directory. /// /// This does not provide any artifacts by default. See [FlutterCache] for the default /// artifact set. /// /// ## Artifact mirrors /// /// Some environments cannot reach the Google Cloud Storage buckets and CIPD due /// to regional or corporate policies. /// /// To enable Flutter users in these environments, the Flutter tool supports /// custom artifact mirrors that the administrators of such environments may /// provide. To use an artifact mirror, the user defines the /// `FLUTTER_STORAGE_BASE_URL` environment variable that points to the mirror. /// Flutter tool reads this variable and uses it instead of the default URLs. /// /// For more details on specific URLs used to download artifacts, see /// [storageBaseUrl] and [cipdBaseUrl]. class Cache { /// [rootOverride] is configurable for testing. /// [artifacts] is configurable for testing. Cache({ @protected Directory? rootOverride, @protected List<ArtifactSet>? artifacts, required Logger logger, required FileSystem fileSystem, required Platform platform, required OperatingSystemUtils osUtils, }) : _rootOverride = rootOverride, _logger = logger, _fileSystem = fileSystem, _platform = platform, _osUtils = osUtils, _net = Net(logger: logger, platform: platform), _fsUtils = FileSystemUtils(fileSystem: fileSystem, platform: platform), _artifacts = artifacts ?? <ArtifactSet>[]; /// Create a [Cache] for testing. /// /// Defaults to a memory file system, fake platform, /// buffer logger, and no accessible artifacts. /// By default, the root cache directory path is "cache". factory Cache.test({ Directory? rootOverride, List<ArtifactSet>? artifacts, Logger? logger, FileSystem? fileSystem, Platform? platform, required ProcessManager processManager, }) { fileSystem ??= rootOverride?.fileSystem ?? MemoryFileSystem.test(); platform ??= FakePlatform(environment: <String, String>{}); logger ??= BufferLogger.test(); return Cache( rootOverride: rootOverride ?? fileSystem.directory('cache'), artifacts: artifacts ?? <ArtifactSet>[], logger: logger, fileSystem: fileSystem, platform: platform, osUtils: OperatingSystemUtils( fileSystem: fileSystem, logger: logger, platform: platform, processManager: processManager, ), ); } final Logger _logger; final Platform _platform; final FileSystem _fileSystem; final OperatingSystemUtils _osUtils; final Directory? _rootOverride; final List<ArtifactSet> _artifacts; final Net _net; final FileSystemUtils _fsUtils; late final ArtifactUpdater _artifactUpdater = _createUpdater(); @visibleForTesting @protected void registerArtifact(ArtifactSet artifactSet) { _artifacts.add(artifactSet); } /// This has to be lazy because it requires FLUTTER_ROOT to be initialized. ArtifactUpdater _createUpdater() { return ArtifactUpdater( operatingSystemUtils: _osUtils, logger: _logger, fileSystem: _fileSystem, tempStorage: getDownloadDir(), platform: _platform, httpClient: HttpClient(), allowedBaseUrls: <String>[ storageBaseUrl, cipdBaseUrl, ], ); } static const List<String> _hostsBlockedInChina = <String> [ 'storage.googleapis.com', 'chrome-infra-packages.appspot.com', ]; // Initialized by FlutterCommandRunner on startup. // Explore making this field lazy to catch non-initialized access. static String? flutterRoot; /// Determine the absolute and normalized path for the root of the current /// Flutter checkout. /// /// This method has a series of fallbacks for determining the repo location. The /// first success will immediately return the root without further checks. /// /// The order of these tests is: /// 1. FLUTTER_ROOT environment variable contains the path. /// 2. Platform script is a data URI scheme, returning `../..` to support /// tests run from `packages/flutter_tools`. /// 3. Platform script is package URI scheme, returning the grandparent directory /// of the package config file location from `packages/flutter_tools/.packages`. /// 4. Platform script file path is the snapshot path generated by `bin/flutter`, /// returning the grandparent directory from `bin/cache`. /// 5. Platform script file name is the entrypoint in `packages/flutter_tools/bin/flutter_tools.dart`, /// returning the 4th parent directory. /// 6. The current directory /// /// If an exception is thrown during any of these checks, an error message is /// printed and `.` is returned by default (6). static String defaultFlutterRoot({ required Platform platform, required FileSystem fileSystem, required UserMessages userMessages, }) { String normalize(String path) { return fileSystem.path.normalize(fileSystem.path.absolute(path)); } if (platform.environment.containsKey(kFlutterRootEnvironmentVariableName)) { return normalize(platform.environment[kFlutterRootEnvironmentVariableName]!); } try { if (platform.script.scheme == 'data') { return normalize('../..'); // The tool is running as a test. } final String Function(String) dirname = fileSystem.path.dirname; if (platform.script.scheme == 'package') { final String packageConfigPath = Uri.parse(platform.packageConfig!).toFilePath( windows: platform.isWindows, ); return normalize(dirname(dirname(dirname(packageConfigPath)))); } if (platform.script.scheme == 'file') { final String script = platform.script.toFilePath( windows: platform.isWindows, ); if (fileSystem.path.basename(script) == kSnapshotFileName) { return normalize(dirname(dirname(fileSystem.path.dirname(script)))); } if (fileSystem.path.basename(script) == kFlutterToolsScriptFileName) { return normalize(dirname(dirname(dirname(dirname(script))))); } } } on Exception catch (error) { // There is currently no logger attached since this is computed at startup. // ignore: avoid_print print(userMessages.runnerNoRoot('$error')); } return normalize('.'); } // Whether to cache artifacts for all platforms. Defaults to only caching // artifacts for the current platform. bool includeAllPlatforms = false; // Names of artifacts which should be cached even if they would normally // be filtered out for the current platform. Set<String>? platformOverrideArtifacts; // Whether to cache the unsigned mac binaries. Defaults to caching the signed binaries. bool useUnsignedMacBinaries = false; static RandomAccessFile? _lock; static bool _lockEnabled = true; /// Turn off the [lock]/[releaseLock] mechanism. /// /// This is used by the tests since they run simultaneously and all in one /// process and so it would be a mess if they had to use the lock. @visibleForTesting static void disableLocking() { _lockEnabled = false; } /// Turn on the [lock]/[releaseLock] mechanism. /// /// This is used by the tests. @visibleForTesting static void enableLocking() { _lockEnabled = true; } /// Check if lock acquired, skipping FLUTTER_ALREADY_LOCKED reentrant checks. /// /// This is used by the tests. @visibleForTesting static bool isLocked() { return _lock != null; } /// Lock the cache directory. /// /// This happens while required artifacts are updated /// (see [FlutterCommandRunner.runCommand]). /// /// This uses normal POSIX flock semantics. Future<void> lock() async { if (!_lockEnabled) { return; } assert(_lock == null); final File lockFile = _fileSystem.file(_fileSystem.path.join(flutterRoot!, 'bin', 'cache', 'lockfile')); try { _lock = lockFile.openSync(mode: FileMode.write); } on FileSystemException catch (e) { _logger.printError('Failed to open or create the artifact cache lockfile: "$e"'); _logger.printError('Please ensure you have permissions to create or open ${lockFile.path}'); throwToolExit('Failed to open or create the lockfile'); } bool locked = false; bool printed = false; while (!locked) { try { _lock!.lockSync(); locked = true; } on FileSystemException { if (!printed) { _logger.printTrace('Waiting to be able to obtain lock of Flutter binary artifacts directory: ${_lock!.path}'); // This needs to go to stderr to avoid cluttering up stdout if a // parent process is collecting stdout (e.g. when calling "flutter // version --machine"). It's not really a "warning" though, so print it // in grey. Also, make sure that it isn't counted as a warning for // Logger.warningsAreFatal. final bool oldWarnings = _logger.hadWarningOutput; _logger.printWarning( 'Waiting for another flutter command to release the startup lock...', color: TerminalColor.grey, ); _logger.hadWarningOutput = oldWarnings; printed = true; } await Future<void>.delayed(const Duration(milliseconds: 50)); } } } /// Releases the lock. /// /// This happens automatically on startup (see [FlutterCommand.verifyThenRunCommand]) /// after the command's required artifacts are updated. void releaseLock() { if (!_lockEnabled || _lock == null) { return; } _lock!.closeSync(); _lock = null; } /// Checks if the current process owns the lock for the cache directory at /// this very moment; throws a [StateError] if it doesn't. void checkLockAcquired() { if (_lockEnabled && _lock == null && _platform.environment['FLUTTER_ALREADY_LOCKED'] != 'true') { throw StateError( 'The current process does not own the lock for the cache directory. This is a bug in Flutter CLI tools.', ); } } String get devToolsVersion { if (_devToolsVersion == null) { const String devToolsDirPath = 'dart-sdk/bin/resources/devtools'; final Directory devToolsDir = getCacheDir(devToolsDirPath, shouldCreate: false); if (!devToolsDir.existsSync()) { throw Exception('Could not find directory at ${devToolsDir.path}'); } final String versionFilePath = '${devToolsDir.path}/version.json'; final File versionFile = _fileSystem.file(versionFilePath); if (!versionFile.existsSync()) { throw Exception('Could not find file at $versionFilePath'); } final dynamic data = jsonDecode(versionFile.readAsStringSync()); if (data is! Map<String, Object?>) { throw Exception("Expected object of type 'Map<String, Object?>' but got one of type '${data.runtimeType}'"); } final Object? version = data['version']; if (version == null) { throw Exception('Could not parse DevTools version from $version'); } if (version is! String) { throw Exception("Could not parse DevTools version. Expected object of type 'String', but got one of type '${version.runtimeType}'"); } return _devToolsVersion = version; } return _devToolsVersion!; } String ? _devToolsVersion; /// The current version of Dart used to build Flutter and run the tool. String get dartSdkVersion { if (_dartSdkVersion == null) { // Make the version string more customer-friendly. // Changes '2.1.0-dev.8.0.flutter-4312ae32' to '2.1.0 (build 2.1.0-dev.8.0 4312ae32)' final String justVersion = _platform.version.split(' ')[0]; _dartSdkVersion = justVersion.replaceFirstMapped(RegExp(r'(\d+\.\d+\.\d+)(.+)'), (Match match) { final String noFlutter = match[2]!.replaceAll('.flutter-', ' '); return '${match[1]} (build ${match[1]}$noFlutter)'; }); } return _dartSdkVersion!; } String? _dartSdkVersion; /// The current version of Dart used to build Flutter and run the tool. String get dartSdkBuild { if (_dartSdkBuild == null) { // Make the version string more customer-friendly. // Changes '2.1.0-dev.8.0.flutter-4312ae32' to '2.1.0 (build 2.1.0-dev.8.0 4312ae32)' final String justVersion = _platform.version.split(' ')[0]; _dartSdkBuild = justVersion.replaceFirstMapped(RegExp(r'(\d+\.\d+\.\d+)(.+)'), (Match match) { final String noFlutter = match[2]!.replaceAll('.flutter-', ' '); return '${match[1]}$noFlutter'; }); } return _dartSdkBuild!; } String? _dartSdkBuild; /// The current version of the Flutter engine the flutter tool will download. String get engineRevision { _engineRevision ??= getVersionFor('engine'); if (_engineRevision == null) { throwToolExit('Could not determine engine revision.'); } return _engineRevision!; } String? _engineRevision; /// The base for URLs that store Flutter engine artifacts that are fetched /// during the installation of the Flutter SDK. /// /// By default the base URL is https://storage.googleapis.com. However, if /// `FLUTTER_STORAGE_BASE_URL` environment variable is provided, the /// environment variable value is returned instead. /// /// See also: /// /// * [cipdBaseUrl], which determines how CIPD artifacts are fetched. /// * [Cache] class-level dartdocs that explain how artifact mirrors work. String get storageBaseUrl { final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL']; if (overrideUrl == null) { return 'https://storage.googleapis.com'; } // verify that this is a valid URI. try { Uri.parse(overrideUrl); } on FormatException catch (err) { throwToolExit('"FLUTTER_STORAGE_BASE_URL" contains an invalid URI:\n$err'); } _maybeWarnAboutStorageOverride(overrideUrl); return overrideUrl; } /// The base for URLs that store Flutter engine artifacts in CIPD. /// /// For some platforms, such as Web and Fuchsia, CIPD artifacts are fetched /// during the installation of the Flutter SDK, in addition to those fetched /// from [storageBaseUrl]. /// /// By default the base URL is https://chrome-infra-packages.appspot.com/dl. /// However, if `FLUTTER_STORAGE_BASE_URL` environment variable is provided, /// then the following value is used: /// /// FLUTTER_STORAGE_BASE_URL/flutter_infra_release/cipd /// /// See also: /// /// * [storageBaseUrl], which determines how engine artifacts stored in the /// Google Cloud Storage buckets are fetched. /// * https://chromium.googlesource.com/infra/luci/luci-go/+/refs/heads/main/cipd, /// which contains information about CIPD. /// * [Cache] class-level dartdocs that explain how artifact mirrors work. String get cipdBaseUrl { final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL']; if (overrideUrl == null) { return 'https://chrome-infra-packages.appspot.com/dl'; } final Uri original; try { original = Uri.parse(overrideUrl); } on FormatException catch (err) { throwToolExit('"FLUTTER_STORAGE_BASE_URL" contains an invalid URI:\n$err'); } final String cipdOverride = original.replace( pathSegments: <String>[ ...original.pathSegments, 'flutter_infra_release', 'cipd', ], ).toString(); return cipdOverride; } bool _hasWarnedAboutStorageOverride = false; void _maybeWarnAboutStorageOverride(String overrideUrl) { if (_hasWarnedAboutStorageOverride) { return; } _logger.printError( 'Flutter assets will be downloaded from $overrideUrl. Make sure you trust this source!', emphasis: true, ); _hasWarnedAboutStorageOverride = true; } /// Return the top-level directory in the cache; this is `bin/cache`. Directory getRoot() { if (_rootOverride != null) { return _fileSystem.directory(_fileSystem.path.join(_rootOverride!.path, 'bin', 'cache')); } else { return _fileSystem.directory(_fileSystem.path.join(flutterRoot!, 'bin', 'cache')); } } String getHostPlatformArchName() { return getNameForHostPlatformArch(_osUtils.hostPlatform); } /// Return a directory in the cache dir. For `pkg`, this will return `bin/cache/pkg`. /// /// When [shouldCreate] is true, the cache directory at [name] will be created /// if it does not already exist. Directory getCacheDir(String name, { bool shouldCreate = true }) { final Directory dir = _fileSystem.directory(_fileSystem.path.join(getRoot().path, name)); if (!dir.existsSync() && shouldCreate) { dir.createSync(recursive: true); _osUtils.chmod(dir, '755'); } return dir; } /// Return the top-level directory for artifact downloads. Directory getDownloadDir() => getCacheDir('downloads'); /// Return the top-level mutable directory in the cache; this is `bin/cache/artifacts`. Directory getCacheArtifacts() => getCacheDir('artifacts'); /// Location of LICENSE file. File getLicenseFile() => _fileSystem.file(_fileSystem.path.join(flutterRoot!, 'LICENSE')); /// Get a named directory from with the cache's artifact directory; for example, /// `material_fonts` would return `bin/cache/artifacts/material_fonts`. Directory getArtifactDirectory(String name) { return getCacheArtifacts().childDirectory(name); } MapEntry<String, String> get dyLdLibEntry { if (_dyLdLibEntry != null) { return _dyLdLibEntry!; } final List<String> paths = <String>[]; for (final ArtifactSet artifact in _artifacts) { final Map<String, String> env = artifact.environment; if (env == null || !env.containsKey('DYLD_LIBRARY_PATH')) { continue; } final String path = env['DYLD_LIBRARY_PATH']!; if (path.isEmpty) { continue; } paths.add(path); } _dyLdLibEntry = MapEntry<String, String>('DYLD_LIBRARY_PATH', paths.join(':')); return _dyLdLibEntry!; } MapEntry<String, String>? _dyLdLibEntry; /// The web sdk has to be co-located with the dart-sdk so that they can share source /// code. Directory getWebSdkDirectory() { return getRoot().childDirectory('flutter_web_sdk'); } String? getVersionFor(String artifactName) { final File versionFile = _fileSystem.file(_fileSystem.path.join( _rootOverride?.path ?? flutterRoot!, 'bin', 'internal', '$artifactName.version', )); return versionFile.existsSync() ? versionFile.readAsStringSync().trim() : null; } /// Delete all stamp files maintained by the cache. void clearStampFiles() { try { getStampFileFor('flutter_tools').deleteSync(); for (final ArtifactSet artifact in _artifacts) { final File file = getStampFileFor(artifact.stampName); ErrorHandlingFileSystem.deleteIfExists(file); } } on FileSystemException catch (err) { _logger.printWarning('Failed to delete some stamp files: $err'); } } /// Read the stamp for [artifactName]. /// /// If the file is missing or cannot be parsed, returns `null`. String? getStampFor(String artifactName) { final File stampFile = getStampFileFor(artifactName); if (!stampFile.existsSync()) { return null; } try { return stampFile.readAsStringSync().trim(); } on FileSystemException { return null; } } void setStampFor(String artifactName, String version) { getStampFileFor(artifactName).writeAsStringSync(version); } File getStampFileFor(String artifactName) { return _fileSystem.file(_fileSystem.path.join(getRoot().path, '$artifactName.stamp')); } /// Returns `true` if either [entity] is older than the tools stamp or if /// [entity] doesn't exist. bool isOlderThanToolsStamp(FileSystemEntity entity) { final File flutterToolsStamp = getStampFileFor('flutter_tools'); return _fsUtils.isOlderThanReference( entity: entity, referenceFile: flutterToolsStamp, ); } Future<bool> isUpToDate() async { for (final ArtifactSet artifact in _artifacts) { if (!await artifact.isUpToDate(_fileSystem)) { return false; } } return true; } /// Update the cache to contain all `requiredArtifacts`. Future<void> updateAll(Set<DevelopmentArtifact> requiredArtifacts, {bool offline = false}) async { if (!_lockEnabled) { return; } for (final ArtifactSet artifact in _artifacts) { if (!requiredArtifacts.contains(artifact.developmentArtifact)) { _logger.printTrace('Artifact $artifact is not required, skipping update.'); continue; } if (await artifact.isUpToDate(_fileSystem)) { continue; } try { await artifact.update(_artifactUpdater, _logger, _fileSystem, _osUtils, offline: offline); } on SocketException catch (e) { if (_hostsBlockedInChina.contains(e.address?.host)) { _logger.printError( 'Failed to retrieve Flutter tool dependencies: ${e.message}.\n' "If you're in China, please see this page: " 'https://flutter.dev/community/china', emphasis: true, ); } rethrow; } } } Future<bool> areRemoteArtifactsAvailable({ String? engineVersion, bool includeAllPlatforms = true, }) async { final bool includeAllPlatformsState = this.includeAllPlatforms; bool allAvailable = true; this.includeAllPlatforms = includeAllPlatforms; for (final ArtifactSet cachedArtifact in _artifacts) { if (cachedArtifact is EngineCachedArtifact) { allAvailable &= await cachedArtifact.checkForArtifacts(engineVersion); } } this.includeAllPlatforms = includeAllPlatformsState; return allAvailable; } Future<bool> doesRemoteExist(String message, Uri url) async { final Status status = _logger.startProgress( message, ); bool exists; try { exists = await _net.doesRemoteFileExist(url); } finally { status.stop(); } return exists; } } /// Representation of a set of artifacts used by the tool. abstract class ArtifactSet { ArtifactSet(this.developmentArtifact) : assert(developmentArtifact != null); /// The development artifact. final DevelopmentArtifact developmentArtifact; /// [true] if the artifact is up to date. Future<bool> isUpToDate(FileSystem fileSystem); /// The environment variables (if any) required to consume the artifacts. Map<String, String> get environment { return const <String, String>{}; } /// Updates the artifact. Future<void> update( ArtifactUpdater artifactUpdater, Logger logger, FileSystem fileSystem, OperatingSystemUtils operatingSystemUtils, {bool offline = false} ); /// The canonical name of the artifact. String get name; // The name of the stamp file. Defaults to the same as the // artifact name. String get stampName => name; } /// An artifact set managed by the cache. abstract class CachedArtifact extends ArtifactSet { CachedArtifact( this.name, this.cache, DevelopmentArtifact developmentArtifact, ) : super(developmentArtifact); final Cache cache; @override final String name; @override String get stampName => name; Directory get location => cache.getArtifactDirectory(name); String? get version => cache.getVersionFor(name); // Whether or not to bypass normal platform filtering for this artifact. bool get ignorePlatformFiltering { return cache.includeAllPlatforms || (cache.platformOverrideArtifacts != null && cache.platformOverrideArtifacts!.contains(developmentArtifact.name)); } @override Future<bool> isUpToDate(FileSystem fileSystem) async { if (!location.existsSync()) { return false; } if (version != cache.getStampFor(stampName)) { return false; } return isUpToDateInner(fileSystem); } @override Future<void> update( ArtifactUpdater artifactUpdater, Logger logger, FileSystem fileSystem, OperatingSystemUtils operatingSystemUtils, {bool offline = false} ) async { if (!location.existsSync()) { try { location.createSync(recursive: true); } on FileSystemException catch (err) { logger.printError(err.toString()); throwToolExit( 'Failed to create directory for flutter cache at ${location.path}. ' 'Flutter may be missing permissions in its cache directory.' ); } } await updateInner(artifactUpdater, fileSystem, operatingSystemUtils); try { if (version == null) { logger.printWarning( 'No known version for the artifact name "$name". ' 'Flutter can continue, but the artifact may be re-downloaded on ' 'subsequent invocations until the problem is resolved.', ); } else { cache.setStampFor(stampName, version!); } } on FileSystemException catch (err) { logger.printWarning( 'The new artifact "$name" was downloaded, but Flutter failed to update ' 'its stamp file, receiving the error "$err". ' 'Flutter can continue, but the artifact may be re-downloaded on ' 'subsequent invocations until the problem is resolved.', ); } artifactUpdater.removeDownloadedFiles(); } /// Hook method for extra checks for being up-to-date. bool isUpToDateInner(FileSystem fileSystem) => true; Future<void> updateInner( ArtifactUpdater artifactUpdater, FileSystem fileSystem, OperatingSystemUtils operatingSystemUtils, ); } abstract class EngineCachedArtifact extends CachedArtifact { EngineCachedArtifact( this.stampName, Cache cache, DevelopmentArtifact developmentArtifact, ) : super('engine', cache, developmentArtifact); @override final String stampName; /// Return a list of (directory path, download URL path) tuples. List<List<String>> getBinaryDirs(); /// A list of cache directory paths to which the LICENSE file should be copied. List<String> getLicenseDirs(); /// A list of the dart package directories to download. List<String> getPackageDirs(); @override bool isUpToDateInner(FileSystem fileSystem) { final Directory pkgDir = cache.getCacheDir('pkg'); for (final String pkgName in getPackageDirs()) { final String pkgPath = fileSystem.path.join(pkgDir.path, pkgName); if (!fileSystem.directory(pkgPath).existsSync()) { return false; } } for (final List<String> toolsDir in getBinaryDirs()) { final Directory dir = fileSystem.directory(fileSystem.path.join(location.path, toolsDir[0])); if (!dir.existsSync()) { return false; } } for (final String licenseDir in getLicenseDirs()) { final File file = fileSystem.file(fileSystem.path.join(location.path, licenseDir, 'LICENSE')); if (!file.existsSync()) { return false; } } return true; } @override Future<void> updateInner( ArtifactUpdater artifactUpdater, FileSystem fileSystem, OperatingSystemUtils operatingSystemUtils, ) async { final String url = '${cache.storageBaseUrl}/flutter_infra_release/flutter/$version/'; final Directory pkgDir = cache.getCacheDir('pkg'); for (final String pkgName in getPackageDirs()) { await artifactUpdater.downloadZipArchive('Downloading package $pkgName...', Uri.parse('$url$pkgName.zip'), pkgDir); } for (final List<String> toolsDir in getBinaryDirs()) { final String cacheDir = toolsDir[0]; final String urlPath = toolsDir[1]; final Directory dir = fileSystem.directory(fileSystem.path.join(location.path, cacheDir)); // Avoid printing things like 'Downloading linux-x64 tools...' multiple times. final String friendlyName = urlPath.replaceAll('/artifacts.zip', '').replaceAll('.zip', ''); await artifactUpdater.downloadZipArchive('Downloading $friendlyName tools...', Uri.parse(url + urlPath), dir); _makeFilesExecutable(dir, operatingSystemUtils); final File frameworkZip = fileSystem.file(fileSystem.path.join(dir.path, 'FlutterMacOS.framework.zip')); if (frameworkZip.existsSync()) { final Directory framework = fileSystem.directory(fileSystem.path.join(dir.path, 'FlutterMacOS.framework')); ErrorHandlingFileSystem.deleteIfExists(framework, recursive: true); framework.createSync(); operatingSystemUtils.unzip(frameworkZip, framework); } } final File licenseSource = cache.getLicenseFile(); for (final String licenseDir in getLicenseDirs()) { final String licenseDestinationPath = fileSystem.path.join(location.path, licenseDir, 'LICENSE'); await licenseSource.copy(licenseDestinationPath); } } Future<bool> checkForArtifacts(String? engineVersion) async { engineVersion ??= version; final String url = '${cache.storageBaseUrl}/flutter_infra_release/flutter/$engineVersion/'; bool exists = false; for (final String pkgName in getPackageDirs()) { exists = await cache.doesRemoteExist('Checking package $pkgName is available...', Uri.parse('$url$pkgName.zip')); if (!exists) { return false; } } for (final List<String> toolsDir in getBinaryDirs()) { final String cacheDir = toolsDir[0]; final String urlPath = toolsDir[1]; exists = await cache.doesRemoteExist('Checking $cacheDir tools are available...', Uri.parse(url + urlPath)); if (!exists) { return false; } } return true; } void _makeFilesExecutable(Directory dir, OperatingSystemUtils operatingSystemUtils) { operatingSystemUtils.chmod(dir, 'a+r,a+x'); for (final File file in dir.listSync(recursive: true).whereType<File>()) { final FileStat stat = file.statSync(); final bool isUserExecutable = ((stat.mode >> 6) & 0x1) == 1; if (file.basename == 'flutter_tester' || isUserExecutable) { // Make the file readable and executable by all users. operatingSystemUtils.chmod(file, 'a+r,a+x'); } } } } /// An API for downloading and un-archiving artifacts, such as engine binaries or /// additional source code. class ArtifactUpdater { ArtifactUpdater({ required OperatingSystemUtils operatingSystemUtils, required Logger logger, required FileSystem fileSystem, required Directory tempStorage, required HttpClient httpClient, required Platform platform, required List<String> allowedBaseUrls, }) : _operatingSystemUtils = operatingSystemUtils, _httpClient = httpClient, _logger = logger, _fileSystem = fileSystem, _tempStorage = tempStorage, _platform = platform, _allowedBaseUrls = allowedBaseUrls; /// The number of times the artifact updater will repeat the artifact download loop. static const int _kRetryCount = 2; final Logger _logger; final OperatingSystemUtils _operatingSystemUtils; final FileSystem _fileSystem; final Directory _tempStorage; final HttpClient _httpClient; final Platform _platform; /// Artifacts should only be downloaded from URLs that use one of these /// prefixes. /// /// [ArtifactUpdater] will issue a warning if an attempt to download from a /// non-compliant URL is made. final List<String> _allowedBaseUrls; /// Keep track of the files we've downloaded for this execution so we /// can delete them after completion. We don't delete them right after /// extraction in case [update] is interrupted, so we can restart without /// starting from scratch. @visibleForTesting final List<File> downloadedFiles = <File>[]; /// Download a zip archive from the given [url] and unzip it to [location]. Future<void> downloadZipArchive( String message, Uri url, Directory location, ) { return _downloadArchive( message, url, location, _operatingSystemUtils.unzip, ); } /// Download a gzipped tarball from the given [url] and unpack it to [location]. Future<void> downloadZippedTarball(String message, Uri url, Directory location) { return _downloadArchive( message, url, location, _operatingSystemUtils.unpack, ); } /// Download an archive from the given [url] and unzip it to [location]. Future<void> _downloadArchive( String message, Uri url, Directory location, void Function(File, Directory) extractor, ) async { final String downloadPath = flattenNameSubdirs(url, _fileSystem); final File tempFile = _createDownloadFile(downloadPath); Status status; int retries = _kRetryCount; while (retries > 0) { status = _logger.startProgress( message, ); try { _ensureExists(tempFile.parent); if (tempFile.existsSync()) { tempFile.deleteSync(); } await _download(url, tempFile, status); if (!tempFile.existsSync()) { throw Exception('Did not find downloaded file ${tempFile.path}'); } } on Exception catch (err) { _logger.printTrace(err.toString()); retries -= 1; if (retries == 0) { throwToolExit( 'Failed to download $url. Ensure you have network connectivity and then try again.\n$err', ); } continue; } on ArgumentError catch (error) { final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL']; if (overrideUrl != null && url.toString().contains(overrideUrl)) { _logger.printError(error.toString()); throwToolExit( 'The value of FLUTTER_STORAGE_BASE_URL ($overrideUrl) could not be ' 'parsed as a valid url. Please see https://flutter.dev/community/china ' 'for an example of how to use it.\n' 'Full URL: $url', exitCode: kNetworkProblemExitCode, ); } // This error should not be hit if there was not a storage URL override, allow the // tool to crash. rethrow; } finally { status.stop(); } /// Unzipping multiple file into a directory will not remove old files /// from previous versions that are not present in the new bundle. final Directory destination = location.childDirectory( tempFile.fileSystem.path.basenameWithoutExtension(tempFile.path) ); try { ErrorHandlingFileSystem.deleteIfExists( destination, recursive: true, ); } on FileSystemException catch (error) { // Error that indicates another program has this file open and that it // cannot be deleted. For the cache, this is either the analyzer reading // the sky_engine package or a running flutter_tester device. const int kSharingViolation = 32; if (_platform.isWindows && error.osError?.errorCode == kSharingViolation) { throwToolExit( 'Failed to delete ${destination.path} because the local file/directory is in use ' 'by another process. Try closing any running IDEs or editors and trying ' 'again' ); } } _ensureExists(location); try { extractor(tempFile, location); } on Exception catch (err) { retries -= 1; if (retries == 0) { throwToolExit( 'Flutter could not download and/or extract $url. Ensure you have ' 'network connectivity and all of the required dependencies listed at ' 'flutter.dev/setup.\nThe original exception was: $err.' ); } _deleteIgnoringErrors(tempFile); continue; } return; } } /// Download bytes from [url], throwing non-200 responses as an exception. /// /// Validates that the md5 of the content bytes matches the provided /// `x-goog-hash` header, if present. This header should contain an md5 hash /// if the download source is Google cloud storage. /// /// See also: /// * https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooghash Future<void> _download(Uri url, File file, Status status) async { final bool isAllowedUrl = _allowedBaseUrls.any((String baseUrl) => url.toString().startsWith(baseUrl)); // In tests make this a hard failure. assert( isAllowedUrl, 'URL not allowed: $url\n' 'Allowed URLs must be based on one of: ${_allowedBaseUrls.join(', ')}', ); // In production, issue a warning but allow the download to proceed. if (!isAllowedUrl) { status.pause(); _logger.printWarning( 'Downloading an artifact that may not be reachable in some environments (e.g. firewalled environments): $url\n' 'This should not have happened. This is likely a Flutter SDK bug. Please file an issue at https://github.com/flutter/flutter/issues/new?template=1_activation.md' ); status.resume(); } final HttpClientRequest request = await _httpClient.getUrl(url); final HttpClientResponse response = await request.close(); if (response.statusCode != HttpStatus.ok) { throw Exception(response.statusCode); } final String? md5Hash = _expectedMd5(response.headers); ByteConversionSink? inputSink; late StreamController<Digest> digests; if (md5Hash != null) { _logger.printTrace('Content $url md5 hash: $md5Hash'); digests = StreamController<Digest>(); inputSink = md5.startChunkedConversion(digests); } final RandomAccessFile randomAccessFile = file.openSync(mode: FileMode.writeOnly); await response.forEach((List<int> chunk) { inputSink?.add(chunk); randomAccessFile.writeFromSync(chunk); }); randomAccessFile.closeSync(); if (inputSink != null) { inputSink.close(); final Digest digest = await digests.stream.last; final String rawDigest = base64.encode(digest.bytes); if (rawDigest != md5Hash) { throw Exception( 'Expected $url to have md5 checksum $md5Hash, but was $rawDigest. This ' 'may indicate a problem with your connection to the Flutter backend servers. ' 'Please re-try the download after confirming that your network connection is ' 'stable.' ); } } } String? _expectedMd5(HttpHeaders httpHeaders) { final List<String>? values = httpHeaders['x-goog-hash']; if (values == null) { return null; } String? rawMd5Hash; for (final String value in values) { if (value.startsWith('md5=')) { rawMd5Hash = value; break; } } if (rawMd5Hash == null) { return null; } final List<String> segments = rawMd5Hash.split('md5='); if (segments.length < 2) { return null; } final String md5Hash = segments[1]; if (md5Hash.isEmpty) { return null; } return md5Hash; } /// Create a temporary file and invoke [onTemporaryFile] with the file as /// argument, then add the temporary file to the [downloadedFiles]. File _createDownloadFile(String name) { final File tempFile = _fileSystem.file(_fileSystem.path.join(_tempStorage.path, name)); downloadedFiles.add(tempFile); return tempFile; } /// Create the given [directory] and parents, as necessary. void _ensureExists(Directory directory) { if (!directory.existsSync()) { directory.createSync(recursive: true); } } /// Clear any zip/gzip files downloaded. void removeDownloadedFiles() { for (final File file in downloadedFiles) { if (!file.existsSync()) { continue; } try { file.deleteSync(); } on FileSystemException catch (e) { _logger.printWarning('Failed to delete "${file.path}". Please delete manually. $e'); continue; } for (Directory directory = file.parent; directory.absolute.path != _tempStorage.absolute.path; directory = directory.parent) { if (directory.listSync().isNotEmpty) { break; } _deleteIgnoringErrors(directory); } } } static void _deleteIgnoringErrors(FileSystemEntity entity) { if (!entity.existsSync()) { return; } try { entity.deleteSync(); } on FileSystemException { // Ignore errors. } } } @visibleForTesting String flattenNameSubdirs(Uri url, FileSystem fileSystem) { final List<String> pieces = <String>[url.host, ...url.pathSegments]; final Iterable<String> convertedPieces = pieces.map<String>(_flattenNameNoSubdirs); return fileSystem.path.joinAll(convertedPieces); } /// Given a name containing slashes, colons, and backslashes, expand it into /// something that doesn't. String _flattenNameNoSubdirs(String fileName) { final List<int> replacedCodeUnits = <int>[ for (int codeUnit in fileName.codeUnits) ..._flattenNameSubstitutions[codeUnit] ?? <int>[codeUnit], ]; return String.fromCharCodes(replacedCodeUnits); } // Many characters are problematic in filenames, especially on Windows. final Map<int, List<int>> _flattenNameSubstitutions = <int, List<int>>{ r'@'.codeUnitAt(0): '@@'.codeUnits, r'/'.codeUnitAt(0): '@s@'.codeUnits, r'\'.codeUnitAt(0): '@bs@'.codeUnits, r':'.codeUnitAt(0): '@c@'.codeUnits, r'%'.codeUnitAt(0): '@per@'.codeUnits, r'*'.codeUnitAt(0): '@ast@'.codeUnits, r'<'.codeUnitAt(0): '@lt@'.codeUnits, r'>'.codeUnitAt(0): '@gt@'.codeUnits, r'"'.codeUnitAt(0): '@q@'.codeUnits, r'|'.codeUnitAt(0): '@pip@'.codeUnits, r'?'.codeUnitAt(0): '@ques@'.codeUnits, };