cache.dart 38.3 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7
import 'dart:async';

import 'package:crypto/crypto.dart';
8
import 'package:file/memory.dart';
9
import 'package:meta/meta.dart';
10
import 'package:process/process.dart';
11

12
import 'base/common.dart';
13
import 'base/error_handling_io.dart';
14
import 'base/file_system.dart';
15
import 'base/io.dart' show HttpClient, HttpClientRequest, HttpClientResponse, HttpHeaders, HttpStatus, SocketException;
16
import 'base/logger.dart';
17
import 'base/net.dart';
18
import 'base/os.dart' show OperatingSystemUtils;
19
import 'base/platform.dart';
20
import 'base/user_messages.dart';
21
import 'build_info.dart';
22
import 'convert.dart';
23
import 'features.dart';
24 25 26 27 28 29

const String kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo)
const String kFlutterEngineEnvironmentVariableName = 'FLUTTER_ENGINE'; // should point to //engine/src/ (root of flutter/engine repo)
const String kSnapshotFileName = 'flutter_tools.snapshot'; // in //flutter/bin/cache/
const String kFlutterToolsScriptFileName = 'flutter_tools.dart'; // in //flutter/packages/flutter_tools/bin/
const String kFlutterEnginePackageName = 'sky_engine';
30

31
/// A tag for a set of development artifacts that need to be cached.
32 33
class DevelopmentArtifact {

34
  const DevelopmentArtifact._(this.name, {this.feature});
35 36 37

  /// The name of the artifact.
  ///
38
  /// This should match the flag name in precache.dart.
39 40
  final String name;

41
  /// A feature to control the visibility of this artifact.
42
  final Feature? feature;
43

44
  /// Artifacts required for Android development.
45 46
  static const DevelopmentArtifact androidGenSnapshot = DevelopmentArtifact._('android_gen_snapshot', feature: flutterAndroidFeature);
  static const DevelopmentArtifact androidMaven = DevelopmentArtifact._('android_maven', feature: flutterAndroidFeature);
47

48
  // Artifacts used for internal builds.
49
  static const DevelopmentArtifact androidInternalBuild = DevelopmentArtifact._('android_internal_build', feature: flutterAndroidFeature);
50 51

  /// Artifacts required for iOS development.
52
  static const DevelopmentArtifact iOS = DevelopmentArtifact._('ios', feature: flutterIOSFeature);
53

54
  /// Artifacts required for web development.
55
  static const DevelopmentArtifact web = DevelopmentArtifact._('web', feature: flutterWebFeature);
56 57

  /// Artifacts required for desktop macOS.
58
  static const DevelopmentArtifact macOS = DevelopmentArtifact._('macos', feature: flutterMacOSDesktopFeature);
59 60

  /// Artifacts required for desktop Windows.
61
  static const DevelopmentArtifact windows = DevelopmentArtifact._('windows', feature: flutterWindowsDesktopFeature);
62

63
  /// Artifacts required for desktop Linux.
64
  static const DevelopmentArtifact linux = DevelopmentArtifact._('linux', feature: flutterLinuxDesktopFeature);
65 66

  /// Artifacts required for Fuchsia.
67
  static const DevelopmentArtifact fuchsia = DevelopmentArtifact._('fuchsia', feature: flutterFuchsiaFeature);
68

69
  /// Artifacts required for the Flutter Runner.
70
  static const DevelopmentArtifact flutterRunner = DevelopmentArtifact._('flutter_runner', feature: flutterFuchsiaFeature);
71

72 73 74
  /// Artifacts required for desktop Windows UWP.
  static const DevelopmentArtifact windowsUwp = DevelopmentArtifact._('winuwp', feature: windowsUwpEmbedding);

75
  /// Artifacts required for any development platform.
76 77 78
  ///
  /// This does not need to be explicitly returned from requiredArtifacts as
  /// it will always be downloaded.
79 80
  static const DevelopmentArtifact universal = DevelopmentArtifact._('universal');

81
  /// The values of DevelopmentArtifacts.
82
  static final List<DevelopmentArtifact> values = <DevelopmentArtifact>[
83 84 85
    androidGenSnapshot,
    androidMaven,
    androidInternalBuild,
86 87 88 89 90 91 92
    iOS,
    web,
    macOS,
    windows,
    linux,
    fuchsia,
    universal,
93
    flutterRunner,
94
    windowsUwp,
95
  ];
96 97

  @override
98
  String toString() => 'Artifact($name)';
99 100
}

101
/// A wrapper around the `bin/cache/` directory.
102 103 104
///
/// This does not provide any artifacts by default. See [FlutterCache] for the default
/// artifact set.
105
class Cache {
106
  /// [rootOverride] is configurable for testing.
107
  /// [artifacts] is configurable for testing.
108
  Cache({
109 110 111 112 113 114
    @protected Directory? rootOverride,
    @protected List<ArtifactSet>? artifacts,
    required Logger logger,
    required FileSystem fileSystem,
    required Platform platform,
    required OperatingSystemUtils osUtils,
115
  }) : _rootOverride = rootOverride,
116 117 118 119 120
       _logger = logger,
       _fileSystem = fileSystem,
       _platform = platform,
       _osUtils = osUtils,
      _net = Net(logger: logger, platform: platform),
121
      _fsUtils = FileSystemUtils(fileSystem: fileSystem, platform: platform),
122
      _artifacts = artifacts ?? <ArtifactSet>[];
123

124 125 126 127 128 129 130
  /// Create a [Cache] for testing.
  ///
  /// Defaults to a memory file system, fake platform,
  /// buffer logger, and no accessible artifacts.
  /// By default, the root cache directory path is "cache".
  @visibleForTesting
  factory Cache.test({
131 132 133 134 135 136
    Directory? rootOverride,
    List<ArtifactSet>? artifacts,
    Logger? logger,
    FileSystem? fileSystem,
    Platform? platform,
    required ProcessManager processManager,
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
  }) {
    fileSystem ??= rootOverride?.fileSystem ?? MemoryFileSystem.test();
    platform ??= FakePlatform(environment: <String, String>{});
    logger ??= BufferLogger.test();
    return Cache(
      rootOverride: rootOverride ??= fileSystem.directory('cache'),
      artifacts: artifacts ?? <ArtifactSet>[],
      logger: logger,
      fileSystem: fileSystem,
      platform: platform,
      osUtils: OperatingSystemUtils(
        fileSystem: fileSystem,
        logger: logger,
        platform: platform,
        processManager: processManager,
      ),
    );
  }

156 157 158 159
  final Logger _logger;
  final Platform _platform;
  final FileSystem _fileSystem;
  final OperatingSystemUtils _osUtils;
160
  final Directory? _rootOverride;
161 162 163
  final List<ArtifactSet> _artifacts;
  final Net _net;
  final FileSystemUtils _fsUtils;
164

165
  ArtifactUpdater get _artifactUpdater => __artifactUpdater ??= _createUpdater();
166
  ArtifactUpdater? __artifactUpdater;
167

168 169 170 171 172
  @protected
  void registerArtifact(ArtifactSet artifactSet) {
    _artifacts.add(artifactSet);
  }

173 174 175 176 177 178 179
  /// This has to be lazy because it requires FLUTTER_ROOT to be initialized.
  ArtifactUpdater _createUpdater() {
    return ArtifactUpdater(
      operatingSystemUtils: _osUtils,
      logger: _logger,
      fileSystem: _fileSystem,
      tempStorage: getDownloadDir(),
180 181
      platform: _platform,
      httpClient: HttpClient(),
182 183 184
    );
  }

185
  static const List<String> _hostsBlockedInChina = <String> [
186 187 188
    'storage.googleapis.com',
  ];

189
  // Initialized by FlutterCommandRunner on startup.
190 191
  // Explore making this field lazy to catch non-initialized access.
  static String? flutterRoot;
192

193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
  /// Determine the absolute and normalized path for the root of the current
  /// Flutter checkout.
  ///
  /// This method has a series of fallbacks for determining the repo location. The
  /// first success will immediately return the root without further checks.
  ///
  /// The order of these tests is:
  ///   1. FLUTTER_ROOT environment variable contains the path.
  ///   2. Platform script is a data URI scheme, returning `../..` to support
  ///      tests run from `packages/flutter_tools`.
  ///   3. Platform script is package URI scheme, returning the grandparent directory
  ///      of the package config file location from `packages/flutter_tools/.packages`.
  ///   4. Platform script file path is the snapshot path generated by `bin/flutter`,
  ///      returning the grandparent directory from `bin/cache`.
  ///   5. Platform script file name is the entrypoint in `packages/flutter_tools/bin/flutter_tools.dart`,
  ///      returning the 4th parent directory.
  ///   6. The current directory
  ///
  /// If an exception is thrown during any of these checks, an error message is
  /// printed and `.` is returned by default (6).
  static String defaultFlutterRoot({
214 215 216
    required Platform platform,
    required FileSystem fileSystem,
    required UserMessages userMessages,
217 218 219 220 221
  }) {
    String normalize(String path) {
      return fileSystem.path.normalize(fileSystem.path.absolute(path));
    }
    if (platform.environment.containsKey(kFlutterRootEnvironmentVariableName)) {
222
      return normalize(platform.environment[kFlutterRootEnvironmentVariableName]!);
223 224 225 226 227 228 229 230
    }
    try {
      if (platform.script.scheme == 'data') {
        return normalize('../..'); // The tool is running as a test.
      }
      final String Function(String) dirname = fileSystem.path.dirname;

      if (platform.script.scheme == 'package') {
231
        final String packageConfigPath = Uri.parse(platform.packageConfig!).toFilePath(
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
          windows: platform.isWindows,
        );
        return normalize(dirname(dirname(dirname(packageConfigPath))));
      }

      if (platform.script.scheme == 'file') {
        final String script = platform.script.toFilePath(
          windows: platform.isWindows,
        );
        if (fileSystem.path.basename(script) == kSnapshotFileName) {
          return normalize(dirname(dirname(fileSystem.path.dirname(script))));
        }
        if (fileSystem.path.basename(script) == kFlutterToolsScriptFileName) {
          return normalize(dirname(dirname(dirname(dirname(script)))));
        }
      }
    } on Exception catch (error) {
      // There is currently no logger attached since this is computed at startup.
      print(userMessages.runnerNoRoot('$error'));
    }
    return normalize('.');
  }

255 256 257 258
  // Whether to cache artifacts for all platforms. Defaults to only caching
  // artifacts for the current platform.
  bool includeAllPlatforms = false;

259 260
  // Names of artifacts which should be cached even if they would normally
  // be filtered out for the current platform.
261
  Set<String>? platformOverrideArtifacts;
262

263 264 265
  // Whether to cache the unsigned mac binaries. Defaults to caching the signed binaries.
  bool useUnsignedMacBinaries = false;

266
  static RandomAccessFile? _lock;
267 268
  static bool _lockEnabled = true;

269
  /// Turn off the [lock]/[releaseLock] mechanism.
270 271 272
  ///
  /// This is used by the tests since they run simultaneously and all in one
  /// process and so it would be a mess if they had to use the lock.
273
  @visibleForTesting
274 275 276 277
  static void disableLocking() {
    _lockEnabled = false;
  }

278
  /// Turn on the [lock]/[releaseLock] mechanism.
279 280 281 282 283 284 285
  ///
  /// This is used by the tests.
  @visibleForTesting
  static void enableLocking() {
    _lockEnabled = true;
  }

286 287 288 289 290 291 292 293
  /// Check if lock acquired, skipping FLUTTER_ALREADY_LOCKED reentrant checks.
  ///
  /// This is used by the tests.
  @visibleForTesting
  static bool isLocked() {
    return _lock != null;
  }

294 295
  /// Lock the cache directory.
  ///
296 297
  /// This happens while required artifacts are updated
  /// (see [FlutterCommandRunner.runCommand]).
298
  ///
299
  /// This uses normal POSIX flock semantics.
300
  Future<void> lock() async {
301
    if (!_lockEnabled) {
302
      return;
303
    }
304
    assert(_lock == null);
305
    final File lockFile =
306
      _fileSystem.file(_fileSystem.path.join(flutterRoot!, 'bin', 'cache', 'lockfile'));
307 308 309
    try {
      _lock = lockFile.openSync(mode: FileMode.write);
    } on FileSystemException catch (e) {
310 311
      _logger.printError('Failed to open or create the artifact cache lockfile: "$e"');
      _logger.printError('Please ensure you have permissions to create or open ${lockFile.path}');
312 313
      throwToolExit('Failed to open or create the lockfile');
    }
314 315 316 317
    bool locked = false;
    bool printed = false;
    while (!locked) {
      try {
318
        _lock!.lockSync();
319 320 321
        locked = true;
      } on FileSystemException {
        if (!printed) {
322
          _logger.printTrace('Waiting to be able to obtain lock of Flutter binary artifacts directory: ${_lock!.path}');
323
          _logger.printStatus('Waiting for another flutter command to release the startup lock...');
324 325
          printed = true;
        }
326
        await Future<void>.delayed(const Duration(milliseconds: 50));
327 328 329 330
      }
    }
  }

331 332 333 334
  /// Releases the lock.
  ///
  /// This happens automatically on startup (see [FlutterCommand.verifyThenRunCommand])
  /// after the command's required artifacts are updated.
335
  void releaseLock() {
336
    if (!_lockEnabled || _lock == null) {
337
      return;
338
    }
339
    _lock!.closeSync();
340 341 342
    _lock = null;
  }

343 344
  /// Checks if the current process owns the lock for the cache directory at
  /// this very moment; throws a [StateError] if it doesn't.
345 346
  void checkLockAcquired() {
    if (_lockEnabled && _lock == null && _platform.environment['FLUTTER_ALREADY_LOCKED'] != 'true') {
347
      throw StateError(
348 349 350 351 352
        'The current process does not own the lock for the cache directory. This is a bug in Flutter CLI tools.',
      );
    }
  }

353
  /// The current version of Dart used to build Flutter and run the tool.
354 355 356 357
  String get dartSdkVersion {
    if (_dartSdkVersion == null) {
      // Make the version string more customer-friendly.
      // Changes '2.1.0-dev.8.0.flutter-4312ae32' to '2.1.0 (build 2.1.0-dev.8.0 4312ae32)'
358
      final String justVersion = _platform.version.split(' ')[0];
359
      _dartSdkVersion = justVersion.replaceFirstMapped(RegExp(r'(\d+\.\d+\.\d+)(.+)'), (Match match) {
360
        final String noFlutter = match[2]!.replaceAll('.flutter-', ' ');
361 362 363
        return '${match[1]} (build ${match[1]}$noFlutter)';
      });
    }
364
    return _dartSdkVersion!;
365
  }
366
  String? _dartSdkVersion;
367

368
  /// The current version of the Flutter engine the flutter tool will download.
369 370
  String get engineRevision {
    _engineRevision ??= getVersionFor('engine');
371 372 373 374
    if (_engineRevision == null) {
      throwToolExit('Could not determine engine revision.');
    }
    return _engineRevision!;
375
  }
376
  String? _engineRevision;
377

378
  String get storageBaseUrl {
379
    final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL'];
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
    if (overrideUrl == null) {
      return 'https://storage.googleapis.com';
    }
    // verify that this is a valid URI.
    try {
      Uri.parse(overrideUrl);
    } on FormatException catch (err) {
      throwToolExit('"FLUTTER_STORAGE_BASE_URL" contains an invalid URI:\n$err');
    }
    _maybeWarnAboutStorageOverride(overrideUrl);
    return overrideUrl;
  }

  bool _hasWarnedAboutStorageOverride = false;

  void _maybeWarnAboutStorageOverride(String overrideUrl) {
    if (_hasWarnedAboutStorageOverride) {
      return;
    }
399
    _logger.printStatus(
400 401 402 403 404 405
      'Flutter assets will be downloaded from $overrideUrl. Make sure you trust this source!',
      emphasis: true,
    );
    _hasWarnedAboutStorageOverride = true;
  }

406
  /// Return the top-level directory in the cache; this is `bin/cache`.
407
  Directory getRoot() {
408
    if (_rootOverride != null) {
409
      return _fileSystem.directory(_fileSystem.path.join(_rootOverride!.path, 'bin', 'cache'));
410
    } else {
411
      return _fileSystem.directory(_fileSystem.path.join(flutterRoot!, 'bin', 'cache'));
412
    }
413
  }
414

415 416 417 418
  String getHostPlatformArchName() {
    return getNameForHostPlatformArch(_osUtils.hostPlatform);
  }

419 420
  /// Return a directory in the cache dir. For `pkg`, this will return `bin/cache/pkg`.
  Directory getCacheDir(String name) {
421
    final Directory dir = _fileSystem.directory(_fileSystem.path.join(getRoot().path, name));
422
    if (!dir.existsSync()) {
423
      dir.createSync(recursive: true);
424
      _osUtils.chmod(dir, '755');
425
    }
426
    return dir;
427 428
  }

429 430 431
  /// Return the top-level directory for artifact downloads.
  Directory getDownloadDir() => getCacheDir('downloads');

432 433 434
  /// Return the top-level mutable directory in the cache; this is `bin/cache/artifacts`.
  Directory getCacheArtifacts() => getCacheDir('artifacts');

435
  /// Location of LICENSE file.
436
  File getLicenseFile() => _fileSystem.file(_fileSystem.path.join(flutterRoot!, 'LICENSE'));
437

438 439 440
  /// Get a named directory from with the cache's artifact directory; for example,
  /// `material_fonts` would return `bin/cache/artifacts/material_fonts`.
  Directory getArtifactDirectory(String name) {
441
    return getCacheArtifacts().childDirectory(name);
442 443
  }

444 445
  MapEntry<String, String> get dyLdLibEntry {
    if (_dyLdLibEntry != null) {
446
      return _dyLdLibEntry!;
447 448
    }
    final List<String> paths = <String>[];
449
    for (final ArtifactSet artifact in _artifacts) {
450 451 452 453
      final Map<String, String> env = artifact.environment;
      if (env == null || !env.containsKey('DYLD_LIBRARY_PATH')) {
        continue;
      }
454
      final String path = env['DYLD_LIBRARY_PATH']!;
455 456
      if (path.isEmpty) {
        continue;
457
      }
458
      paths.add(path);
459 460
    }
    _dyLdLibEntry = MapEntry<String, String>('DYLD_LIBRARY_PATH', paths.join(':'));
461
    return _dyLdLibEntry!;
462
  }
463
  MapEntry<String, String>? _dyLdLibEntry;
464

465 466 467 468 469 470
  /// The web sdk has to be co-located with the dart-sdk so that they can share source
  /// code.
  Directory getWebSdkDirectory() {
    return getRoot().childDirectory('flutter_web_sdk');
  }

471
  String? getVersionFor(String artifactName) {
472
    final File versionFile = _fileSystem.file(_fileSystem.path.join(
473
      _rootOverride?.path ?? flutterRoot!,
474 475 476 477
      'bin',
      'internal',
      '$artifactName.version',
    ));
478 479 480
    return versionFile.existsSync() ? versionFile.readAsStringSync().trim() : null;
  }

481 482 483 484 485 486
    /// Delete all stamp files maintained by the cache.
  void clearStampFiles() {
    try {
      getStampFileFor('flutter_tools').deleteSync();
      for (final ArtifactSet artifact in _artifacts) {
        final File file = getStampFileFor(artifact.stampName);
487
        ErrorHandlingFileSystem.deleteIfExists(file);
488 489 490 491 492 493
      }
    } on FileSystemException catch (err) {
      _logger.printError('Failed to delete some stamp files: $err');
    }
  }

494 495 496
  /// Read the stamp for [artifactName].
  ///
  /// If the file is missing or cannot be parsed, returns `null`.
497
  String? getStampFor(String artifactName) {
498
    final File stampFile = getStampFileFor(artifactName);
499 500 501 502 503 504 505 506
    if (!stampFile.existsSync()) {
      return null;
    }
    try {
      return stampFile.readAsStringSync().trim();
    } on FileSystemException {
      return null;
    }
507 508
  }

509 510 511 512 513
  void setStampFor(String artifactName, String version) {
    getStampFileFor(artifactName).writeAsStringSync(version);
  }

  File getStampFileFor(String artifactName) {
514
    return _fileSystem.file(_fileSystem.path.join(getRoot().path, '$artifactName.stamp'));
515 516
  }

517 518 519
  /// Returns `true` if either [entity] is older than the tools stamp or if
  /// [entity] doesn't exist.
  bool isOlderThanToolsStamp(FileSystemEntity entity) {
520
    final File flutterToolsStamp = getStampFileFor('flutter_tools');
521
    return _fsUtils.isOlderThanReference(
522 523 524
      entity: entity,
      referenceFile: flutterToolsStamp,
    );
525 526
  }

527 528
  Future<bool> isUpToDate() async {
    for (final ArtifactSet artifact in _artifacts) {
529
      if (!await artifact.isUpToDate(_fileSystem)) {
530 531 532 533 534
        return false;
      }
    }
    return true;
  }
535

536 537 538
  /// Update the cache to contain all `requiredArtifacts`.
  Future<void> updateAll(Set<DevelopmentArtifact> requiredArtifacts) async {
    if (!_lockEnabled) {
539
      return;
540
    }
541
    for (final ArtifactSet artifact in _artifacts) {
542
      if (!requiredArtifacts.contains(artifact.developmentArtifact)) {
543
        _logger.printTrace('Artifact $artifact is not required, skipping update.');
544
        continue;
545
      }
546
      if (await artifact.isUpToDate(_fileSystem)) {
547 548 549
        continue;
      }
      try {
550
        await artifact.update(_artifactUpdater, _logger, _fileSystem, _osUtils);
551 552
      } on SocketException catch (e) {
        if (_hostsBlockedInChina.contains(e.address?.host)) {
553
          _logger.printError(
554
            'Failed to retrieve Flutter tool dependencies: ${e.message}.\n'
555
            "If you're in China, please see this page: "
556 557 558 559 560
            'https://flutter.dev/community/china',
            emphasis: true,
          );
        }
        rethrow;
561
      }
562 563
    }
  }
564 565

  Future<bool> areRemoteArtifactsAvailable({
566
    String? engineVersion,
567 568
    bool includeAllPlatforms = true,
  }) async {
569
    final bool includeAllPlatformsState = this.includeAllPlatforms;
570
    bool allAvailable = true;
571
    this.includeAllPlatforms = includeAllPlatforms;
572
    for (final ArtifactSet cachedArtifact in _artifacts) {
573
      if (cachedArtifact is EngineCachedArtifact) {
574
        allAvailable &= await cachedArtifact.checkForArtifacts(engineVersion);
575 576
      }
    }
577
    this.includeAllPlatforms = includeAllPlatformsState;
578
    return allAvailable;
579
  }
580 581

  Future<bool> doesRemoteExist(String message, Uri url) async {
582
    final Status status = _logger.startProgress(
583 584 585 586 587 588 589 590 591 592
      message,
    );
    bool exists;
    try {
      exists = await _net.doesRemoteFileExist(url);
    } finally {
      status.stop();
    }
    return exists;
  }
593 594
}

595 596 597 598 599 600 601 602
/// Representation of a set of artifacts used by the tool.
abstract class ArtifactSet {
  ArtifactSet(this.developmentArtifact) : assert(developmentArtifact != null);

  /// The development artifact.
  final DevelopmentArtifact developmentArtifact;

  /// [true] if the artifact is up to date.
603
  Future<bool> isUpToDate(FileSystem fileSystem);
604 605 606 607 608 609 610

  /// The environment variables (if any) required to consume the artifacts.
  Map<String, String> get environment {
    return const <String, String>{};
  }

  /// Updates the artifact.
611 612 613 614 615 616
  Future<void> update(
    ArtifactUpdater artifactUpdater,
    Logger logger,
    FileSystem fileSystem,
    OperatingSystemUtils operatingSystemUtils,
  );
617 618 619 620 621 622 623

  /// The canonical name of the artifact.
  String get name;

  // The name of the stamp file. Defaults to the same as the
  // artifact name.
  String get stampName => name;
624 625 626 627 628 629 630 631 632
}

/// An artifact set managed by the cache.
abstract class CachedArtifact extends ArtifactSet {
  CachedArtifact(
    this.name,
    this.cache,
    DevelopmentArtifact developmentArtifact,
  ) : super(developmentArtifact);
633

634
  final Cache cache;
635

636
  @override
637 638
  final String name;

639
  @override
640 641
  String get stampName => name;

642
  Directory get location => cache.getArtifactDirectory(name);
643 644

  String? get version => cache.getVersionFor(name);
645

646 647 648
  // Whether or not to bypass normal platform filtering for this artifact.
  bool get ignorePlatformFiltering {
    return cache.includeAllPlatforms ||
649
      (cache.platformOverrideArtifacts != null && cache.platformOverrideArtifacts!.contains(developmentArtifact.name));
650 651
  }

652
  @override
653
  Future<bool> isUpToDate(FileSystem fileSystem) async {
654
    if (!location.existsSync()) {
655
      return false;
656 657
    }
    if (version != cache.getStampFor(stampName)) {
658
      return false;
659
    }
660
    return isUpToDateInner(fileSystem);
661 662
  }

663
  @override
664 665 666 667 668 669
  Future<void> update(
    ArtifactUpdater artifactUpdater,
    Logger logger,
    FileSystem fileSystem,
    OperatingSystemUtils operatingSystemUtils,
  ) async {
670
    if (!location.existsSync()) {
671 672 673
      try {
        location.createSync(recursive: true);
      } on FileSystemException catch (err) {
674
        logger.printError(err.toString());
675 676 677 678 679
        throwToolExit(
          'Failed to create directory for flutter cache at ${location.path}. '
          'Flutter may be missing permissions in its cache directory.'
        );
      }
680
    }
681
    await updateInner(artifactUpdater, fileSystem, operatingSystemUtils);
682
    try {
683 684 685 686 687 688 689 690 691
      if (version == null) {
        logger.printError(
          'No known version for the artifact name "$name". '
          'Flutter can continue, but the artifact may be re-downloaded on '
          'subsequent invocations until the problem is resolved.',
        );
      } else {
        cache.setStampFor(stampName, version!);
      }
692
    } on FileSystemException catch (err) {
693
      logger.printError(
694 695 696 697 698 699
        'The new artifact "$name" was downloaded, but Flutter failed to update '
        'its stamp file, receiving the error "$err". '
        'Flutter can continue, but the artifact may be re-downloaded on '
        'subsequent invocations until the problem is resolved.',
      );
    }
700
    artifactUpdater.removeDownloadedFiles();
701 702
  }

703
  /// Hook method for extra checks for being up-to-date.
704
  bool isUpToDateInner(FileSystem fileSystem) => true;
705

706 707 708 709 710
  Future<void> updateInner(
    ArtifactUpdater artifactUpdater,
    FileSystem fileSystem,
    OperatingSystemUtils operatingSystemUtils,
  );
711
}
712

713

714 715 716 717
abstract class EngineCachedArtifact extends CachedArtifact {
  EngineCachedArtifact(
    this.stampName,
    Cache cache,
718 719
    DevelopmentArtifact developmentArtifact,
  ) : super('engine', cache, developmentArtifact);
720

721 722
  @override
  final String stampName;
723

724 725
  /// Return a list of (directory path, download URL path) tuples.
  List<List<String>> getBinaryDirs();
726

727 728
  /// A list of cache directory paths to which the LICENSE file should be copied.
  List<String> getLicenseDirs();
729

730 731
  /// A list of the dart package directories to download.
  List<String> getPackageDirs();
732

733
  @override
734
  bool isUpToDateInner(FileSystem fileSystem) {
735
    final Directory pkgDir = cache.getCacheDir('pkg');
736
    for (final String pkgName in getPackageDirs()) {
737 738
      final String pkgPath = fileSystem.path.join(pkgDir.path, pkgName);
      if (!fileSystem.directory(pkgPath).existsSync()) {
739
        return false;
740
      }
741
    }
742

743
    for (final List<String> toolsDir in getBinaryDirs()) {
744
      final Directory dir = fileSystem.directory(fileSystem.path.join(location.path, toolsDir[0]));
745
      if (!dir.existsSync()) {
746
        return false;
747
      }
748
    }
749

750
    for (final String licenseDir in getLicenseDirs()) {
751
      final File file = fileSystem.file(fileSystem.path.join(location.path, licenseDir, 'LICENSE'));
752
      if (!file.existsSync()) {
753
        return false;
754
      }
755 756
    }
    return true;
757 758
  }

759
  @override
760 761 762 763 764
  Future<void> updateInner(
    ArtifactUpdater artifactUpdater,
    FileSystem fileSystem,
    OperatingSystemUtils operatingSystemUtils,
  ) async {
765
    final String url = '${cache.storageBaseUrl}/flutter_infra_release/flutter/$version/';
766 767

    final Directory pkgDir = cache.getCacheDir('pkg');
768
    for (final String pkgName in getPackageDirs()) {
769
      await artifactUpdater.downloadZipArchive('Downloading package $pkgName...', Uri.parse(url + pkgName + '.zip'), pkgDir);
770 771
    }

772
    for (final List<String> toolsDir in getBinaryDirs()) {
773 774
      final String cacheDir = toolsDir[0];
      final String urlPath = toolsDir[1];
775
      final Directory dir = fileSystem.directory(fileSystem.path.join(location.path, cacheDir));
776 777 778

      // Avoid printing things like 'Downloading linux-x64 tools...' multiple times.
      final String friendlyName = urlPath.replaceAll('/artifacts.zip', '').replaceAll('.zip', '');
779
      await artifactUpdater.downloadZipArchive('Downloading $friendlyName tools...', Uri.parse(url + urlPath), dir);
780

781
      _makeFilesExecutable(dir, operatingSystemUtils);
782

783 784 785
      final File frameworkZip = fileSystem.file(fileSystem.path.join(dir.path, 'FlutterMacOS.framework.zip'));
      if (frameworkZip.existsSync()) {
        final Directory framework = fileSystem.directory(fileSystem.path.join(dir.path, 'FlutterMacOS.framework'));
786
        ErrorHandlingFileSystem.deleteIfExists(framework, recursive: true);
787 788
        framework.createSync();
        operatingSystemUtils.unzip(frameworkZip, framework);
Devon Carew's avatar
Devon Carew committed
789 790
      }
    }
791

792
    final File licenseSource = cache.getLicenseFile();
793
    for (final String licenseDir in getLicenseDirs()) {
794
      final String licenseDestinationPath = fileSystem.path.join(location.path, licenseDir, 'LICENSE');
795 796
      await licenseSource.copy(licenseDestinationPath);
    }
797 798
  }

799
  Future<bool> checkForArtifacts(String? engineVersion) async {
800
    engineVersion ??= version;
801
    final String url = '${cache.storageBaseUrl}/flutter_infra_release/flutter/$engineVersion/';
802

803
    bool exists = false;
804
    for (final String pkgName in getPackageDirs()) {
805
      exists = await cache.doesRemoteExist('Checking package $pkgName is available...', Uri.parse(url + pkgName + '.zip'));
806 807
      if (!exists) {
        return false;
808
      }
809
    }
810

811
    for (final List<String> toolsDir in getBinaryDirs()) {
812 813
      final String cacheDir = toolsDir[0];
      final String urlPath = toolsDir[1];
814
      exists = await cache.doesRemoteExist('Checking $cacheDir tools are available...',
815 816 817
          Uri.parse(url + urlPath));
      if (!exists) {
        return false;
818
      }
819
    }
820
    return true;
821 822
  }

823 824 825 826 827 828 829 830
  void _makeFilesExecutable(Directory dir, OperatingSystemUtils operatingSystemUtils) {
    operatingSystemUtils.chmod(dir, 'a+r,a+x');
    for (final File file in dir.listSync(recursive: true).whereType<File>()) {
      final FileStat stat = file.statSync();
      final bool isUserExecutable = ((stat.mode >> 6) & 0x1) == 1;
      if (file.basename == 'flutter_tester' || isUserExecutable) {
        // Make the file readable and executable by all users.
        operatingSystemUtils.chmod(file, 'a+r,a+x');
831 832 833
      }
    }
  }
834 835
}

836 837 838 839
/// An API for downloading and un-archiving artifacts, such as engine binaries or
/// additional source code.
class ArtifactUpdater {
  ArtifactUpdater({
840 841 842 843 844 845
    required OperatingSystemUtils operatingSystemUtils,
    required Logger logger,
    required FileSystem fileSystem,
    required Directory tempStorage,
    required HttpClient httpClient,
    required Platform platform,
846
  }) : _operatingSystemUtils = operatingSystemUtils,
847
       _httpClient = httpClient,
848 849
       _logger = logger,
       _fileSystem = fileSystem,
850 851 852 853 854
       _tempStorage = tempStorage,
       _platform = platform;

  /// The number of times the artifact updater will repeat the artifact download loop.
  static const int _kRetryCount = 2;
855 856 857 858 859

  final Logger _logger;
  final OperatingSystemUtils _operatingSystemUtils;
  final FileSystem _fileSystem;
  final Directory _tempStorage;
860 861
  final HttpClient _httpClient;
  final Platform _platform;
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902

  /// Keep track of the files we've downloaded for this execution so we
  /// can delete them after completion. We don't delete them right after
  /// extraction in case [update] is interrupted, so we can restart without
  /// starting from scratch.
  @visibleForTesting
  final List<File> downloadedFiles = <File>[];

  /// Download a zip archive from the given [url] and unzip it to [location].
  Future<void> downloadZipArchive(
    String message,
    Uri url,
    Directory location,
  ) {
    return _downloadArchive(
      message,
      url,
      location,
      _operatingSystemUtils.unzip,
    );
  }

  /// Download a gzipped tarball from the given [url] and unpack it to [location].
  Future<void> downloadZippedTarball(String message, Uri url, Directory location) {
    return _downloadArchive(
      message,
      url,
      location,
      _operatingSystemUtils.unpack,
    );
  }

  /// Download an archive from the given [url] and unzip it to [location].
  Future<void> _downloadArchive(
    String message,
    Uri url,
    Directory location,
    void Function(File, Directory) extractor,
  ) async {
    final String downloadPath = flattenNameSubdirs(url, _fileSystem);
    final File tempFile = _createDownloadFile(downloadPath);
903
    Status status;
904
    int retries = _kRetryCount;
905 906

    while (retries > 0) {
907 908 909
      status = _logger.startProgress(
        message,
      );
910 911
      try {
        _ensureExists(tempFile.parent);
912 913 914 915 916
        if (tempFile.existsSync()) {
          tempFile.deleteSync();
        }
        await _download(url, tempFile);

917 918 919
        if (!tempFile.existsSync()) {
          throw Exception('Did not find downloaded file ${tempFile.path}');
        }
920 921 922 923 924
      } on Exception catch (err) {
        _logger.printTrace(err.toString());
        retries -= 1;
        if (retries == 0) {
          throwToolExit(
925
            'Failed to download $url. Ensure you have network connectivity and then try again.\n$err',
926 927 928 929
          );
        }
        continue;
      } on ArgumentError catch (error) {
930
        final String? overrideUrl = _platform.environment['FLUTTER_STORAGE_BASE_URL'];
931 932 933 934 935 936 937 938 939 940 941 942 943
        if (overrideUrl != null && url.toString().contains(overrideUrl)) {
          _logger.printError(error.toString());
          throwToolExit(
            'The value of FLUTTER_STORAGE_BASE_URL ($overrideUrl) could not be '
            'parsed as a valid url. Please see https://flutter.dev/community/china '
            'for an example of how to use it.\n'
            'Full URL: $url',
            exitCode: kNetworkProblemExitCode,
          );
        }
        // This error should not be hit if there was not a storage URL override, allow the
        // tool to crash.
        rethrow;
944 945 946
      } finally {
        status.stop();
      }
947 948 949 950 951
      /// Unzipping multiple file into a directory will not remove old files
      /// from previous versions that are not present in the new bundle.
      final Directory destination = location.childDirectory(
        tempFile.fileSystem.path.basenameWithoutExtension(tempFile.path)
      );
952 953 954 955 956 957 958 959 960 961
      try {
        ErrorHandlingFileSystem.deleteIfExists(
          destination,
          recursive: true,
        );
      } on FileSystemException catch (error) {
        // Error that indicates another program has this file open and that it
        // cannot be deleted. For the cache, this is either the analyzer reading
        // the sky_engine package or a running flutter_tester device.
        const int kSharingViolation = 32;
962
        if (_platform.isWindows && error.osError?.errorCode == kSharingViolation) {
963 964 965 966 967 968 969
          throwToolExit(
            'Failed to delete ${destination.path} because the local file/directory is in use '
            'by another process. Try closing any running IDEs or editors and trying '
            'again'
          );
        }
      }
970 971 972 973
      _ensureExists(location);

      try {
        extractor(tempFile, location);
974
      } on Exception catch (err) {
975 976
        retries -= 1;
        if (retries == 0) {
977 978
          throwToolExit(
            'Flutter could not download and/or extract $url. Ensure you have '
979
            'network connectivity and all of the required dependencies listed at '
980 981
            'flutter.dev/setup.\nThe original exception was: $err.'
          );
982 983 984
        }
        _deleteIgnoringErrors(tempFile);
        continue;
985 986 987 988 989
      }
      return;
    }
  }

990
  /// Download bytes from [url], throwing non-200 responses as an exception.
991 992 993 994 995 996 997
  ///
  /// Validates that the md5 of the content bytes matches the provided
  /// `x-goog-hash` header, if present. This header should contain an md5 hash
  /// if the download source is Google cloud storage.
  ///
  /// See also:
  ///   * https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooghash
998
  Future<void> _download(Uri url, File file) async {
999 1000 1001 1002 1003
    final HttpClientRequest request = await _httpClient.getUrl(url);
    final HttpClientResponse response = await request.close();
    if (response.statusCode != HttpStatus.ok) {
      throw Exception(response.statusCode);
    }
1004

1005 1006 1007
    final String? md5Hash = _expectedMd5(response.headers);
    ByteConversionSink? inputSink;
    late StreamController<Digest> digests;
1008 1009 1010 1011 1012 1013
    if (md5Hash != null) {
      _logger.printTrace('Content $url md5 hash: $md5Hash');
      digests = StreamController<Digest>();
      inputSink = md5.startChunkedConversion(digests);
    }
    final RandomAccessFile randomAccessFile = file.openSync(mode: FileMode.writeOnly);
1014
    await response.forEach((List<int> chunk) {
1015 1016
      inputSink?.add(chunk);
      randomAccessFile.writeFromSync(chunk);
1017
    });
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
    randomAccessFile.closeSync();
    if (inputSink != null) {
      inputSink.close();
      final Digest digest = await digests.stream.last;
      final String rawDigest = base64.encode(digest.bytes);
      if (rawDigest != md5Hash) {
        throw Exception(''
          'Expected $url to have md5 checksum $md5Hash, but was $rawDigest. This '
          'may indicate a problem with your connection to the Flutter backend servers. '
          'Please re-try the download after confirming that your network connection is '
          'stable.'
        );
      }
    }
  }

1034 1035
  String? _expectedMd5(HttpHeaders httpHeaders) {
    final List<String>? values = httpHeaders['x-goog-hash'];
1036 1037 1038
    if (values == null) {
      return null;
    }
1039 1040 1041 1042 1043 1044 1045
    String? rawMd5Hash;
    for (final String value in values) {
      if (value.startsWith('md5=')) {
        rawMd5Hash = value;
        break;
      }
    }
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
    if (rawMd5Hash == null) {
      return null;
    }
    final List<String> segments = rawMd5Hash.split('md5=');
    if (segments.length < 2) {
      return null;
    }
    final String md5Hash = segments[1];
    if (md5Hash.isEmpty) {
      return null;
    }
    return md5Hash;
1058 1059
  }

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
  /// Create a temporary file and invoke [onTemporaryFile] with the file as
  /// argument, then add the temporary file to the [downloadedFiles].
  File _createDownloadFile(String name) {
    final File tempFile = _fileSystem.file(_fileSystem.path.join(_tempStorage.path, name));
    downloadedFiles.add(tempFile);
    return tempFile;
  }

  /// Create the given [directory] and parents, as necessary.
  void _ensureExists(Directory directory) {
    if (!directory.existsSync()) {
      directory.createSync(recursive: true);
    }
  }

    /// Clear any zip/gzip files downloaded.
  void removeDownloadedFiles() {
    for (final File file in downloadedFiles) {
      if (!file.existsSync()) {
        continue;
      }
      try {
        file.deleteSync();
      } on FileSystemException catch (e) {
1084
        _logger.printError('Failed to delete "${file.path}". Please delete manually. $e');
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
        continue;
      }
      for (Directory directory = file.parent; directory.absolute.path != _tempStorage.absolute.path; directory = directory.parent) {
        if (directory.listSync().isNotEmpty) {
          break;
        }
        _deleteIgnoringErrors(directory);
      }
    }
  }

  static void _deleteIgnoringErrors(FileSystemEntity entity) {
    if (!entity.existsSync()) {
      return;
    }
    try {
      entity.deleteSync();
    } on FileSystemException {
      // Ignore errors.
    }
  }
}

@visibleForTesting
String flattenNameSubdirs(Uri url, FileSystem fileSystem){
  final List<String> pieces = <String>[url.host, ...url.pathSegments];
  final Iterable<String> convertedPieces = pieces.map<String>(_flattenNameNoSubdirs);
  return fileSystem.path.joinAll(convertedPieces);
}
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138

/// Given a name containing slashes, colons, and backslashes, expand it into
/// something that doesn't.
String _flattenNameNoSubdirs(String fileName) {
  final List<int> replacedCodeUnits = <int>[
    for (int codeUnit in fileName.codeUnits)
      ..._flattenNameSubstitutions[codeUnit] ?? <int>[codeUnit],
  ];
  return String.fromCharCodes(replacedCodeUnits);
}

// Many characters are problematic in filenames, especially on Windows.
final Map<int, List<int>> _flattenNameSubstitutions = <int, List<int>>{
  r'@'.codeUnitAt(0): '@@'.codeUnits,
  r'/'.codeUnitAt(0): '@s@'.codeUnits,
  r'\'.codeUnitAt(0): '@bs@'.codeUnits,
  r':'.codeUnitAt(0): '@c@'.codeUnits,
  r'%'.codeUnitAt(0): '@per@'.codeUnits,
  r'*'.codeUnitAt(0): '@ast@'.codeUnits,
  r'<'.codeUnitAt(0): '@lt@'.codeUnits,
  r'>'.codeUnitAt(0): '@gt@'.codeUnits,
  r'"'.codeUnitAt(0): '@q@'.codeUnits,
  r'|'.codeUnitAt(0): '@pip@'.codeUnits,
  r'?'.codeUnitAt(0): '@ques@'.codeUnits,
};