Skip to content

F
Front-End
Unverified Commit 23032d77 authored by Jonah Williams's avatar Jonah Williams Committed by GitHub
Browse files
Options

[flutter_tools] refactor artifact downloading to retry zip exceptions. (#64512)

parent 39d7a019
Hide whitespace changes
Inline Side-by-side
Showing with 471 additions and 293 deletions
packages/flutter_tools/lib/src/base/error_handling_file_system.dart
View file @ 23032d77
......@@ -266,6 +266,16 @@ class ErrorHandlingDirectory
Link childLink(String basename) =>
wrapLink(fileSystem.directory(delegate).childLink(basename));
@override
void createSync({bool recursive = false}) {
return _runSync<void>(
() => delegate.createSync(recursive: recursive),
platform: _platform,
failureMessage:
'Flutter failed to create a directory at "${delegate.path}"',
);
}
@override
Future<Directory> createTemp([String prefix]) {
return _run<Directory>(
......
packages/flutter_tools/lib/src/cache.dart
View file @ 23032d77
......@@ -9,7 +9,7 @@ import 'package:meta/meta.dart';
import 'android/gradle_utils.dart';
import 'base/common.dart';
import 'base/file_system.dart';
import 'base/io.dart' show SocketException;
import 'base/io.dart' show ProcessException, SocketException;
import 'base/logger.dart';
import 'base/net.dart';
import 'base/os.dart' show OperatingSystemUtils;
......@@ -122,7 +122,7 @@ class Cache {
_artifacts.add(LinuxFuchsiaSDKArtifacts(this));
_artifacts.add(MacOSFuchsiaSDKArtifacts(this));
_artifacts.add(FlutterRunnerSDKArtifacts(this));
_artifacts.add(FlutterRunnerDebugSymbols(this));
_artifacts.add(FlutterRunnerDebugSymbols(this, platform: _platform));
for (final String artifactName in IosUsbArtifacts.artifactNames) {
_artifacts.add(IosUsbArtifacts(artifactName, this));
}
......@@ -137,6 +137,20 @@ class Cache {
final FileSystem _fileSystem;
final OperatingSystemUtils _osUtils;
ArtifactUpdater get _artifactUpdater => __artifactUpdater ??= _createUpdater();
ArtifactUpdater __artifactUpdater;
/// This has to be lazy because it requires FLUTTER_ROOT to be initialized.
ArtifactUpdater _createUpdater() {
return ArtifactUpdater(
operatingSystemUtils: _osUtils,
net: _net,
logger: _logger,
fileSystem: _fileSystem,
tempStorage: getDownloadDir(),
);
}
Net _net;
FileSystemUtils _fsUtils;
......@@ -410,34 +424,6 @@ class Cache {
bool isUpToDate() => _artifacts.every((ArtifactSet artifact) => artifact.isUpToDate());
Future<String> getThirdPartyFile(String urlStr, String serviceName) async {
final Uri url = Uri.parse(urlStr);
final Directory thirdPartyDir = getArtifactDirectory('third_party');
final Directory serviceDir = _fileSystem.directory(_fileSystem.path.join(
thirdPartyDir.path,
serviceName,
));
if (!serviceDir.existsSync()) {
serviceDir.createSync(recursive: true);
_osUtils.chmod(serviceDir, '755');
}
final File cachedFile = _fileSystem.file(_fileSystem.path.join(
serviceDir.path,
url.pathSegments.last,
));
if (!cachedFile.existsSync()) {
try {
await downloadFile(url, cachedFile);
} on Exception catch (e) {
throwToolExit('Failed to fetch third-party artifact $url: $e');
}
}
return cachedFile.path;
}
/// Update the cache to contain all `requiredArtifacts`.
Future<void> updateAll(Set<DevelopmentArtifact> requiredArtifacts) async {
if (!_lockEnabled) {
......@@ -452,7 +438,7 @@ class Cache {
continue;
}
try {
await artifact.update();
await artifact.update(_artifactUpdater);
} on SocketException catch (e) {
if (_hostsBlockedInChina.contains(e.address?.host)) {
_logger.printError(
......@@ -483,12 +469,6 @@ class Cache {
return allAvailible;
}
/// Download a file from the given [url] and write it to [location].
Future<void> downloadFile(Uri url, File location) async {
_ensureExists(location.parent);
await _net.fetchUrl(url, destFile: location);
}
Future<bool> doesRemoteExist(String message, Uri url) async {
final Status status = _logger.startProgress(
message,
......@@ -520,7 +500,7 @@ abstract class ArtifactSet {
}
/// Updates the artifact.
Future<void> update();
Future<void> update(ArtifactUpdater artifactUpdater);
/// The canonical name of the artifact.
String get name;
......@@ -549,13 +529,6 @@ abstract class CachedArtifact extends ArtifactSet {
Directory get location => cache.getArtifactDirectory(name);
String get version => cache.getVersionFor(name);
/// Keep track of the files we've downloaded for this execution so we
/// can delete them after completion. We don't delete them right after
/// extraction in case [update] is interrupted, so we can restart without
/// starting from scratch.
@visibleForTesting
final List<File> downloadedFiles = <File>[];
// Whether or not to bypass normal platform filtering for this artifact.
bool get ignorePlatformFiltering {
return cache.includeAllPlatforms ||
......@@ -574,7 +547,7 @@ abstract class CachedArtifact extends ArtifactSet {
}
@override
Future<void> update() async {
Future<void> update(ArtifactUpdater artifactUpdater) async {
if (!location.existsSync()) {
try {
location.createSync(recursive: true);
......@@ -586,7 +559,7 @@ abstract class CachedArtifact extends ArtifactSet {
);
}
}
await updateInner();
await updateInner(artifactUpdater);
try {
cache.setStampFor(stampName, version);
} on FileSystemException catch (err) {
......@@ -597,75 +570,16 @@ abstract class CachedArtifact extends ArtifactSet {
'subsequent invocations until the problem is resolved.',
);
}
_removeDownloadedFiles();
}
/// Clear any zip/gzip files downloaded.
void _removeDownloadedFiles() {
for (final File f in downloadedFiles) {
try {
f.deleteSync();
} on FileSystemException catch (e) {
globals.printError('Failed to delete "${f.path}". Please delete manually. $e');
continue;
}
for (Directory d = f.parent; d.absolute.path != cache.getDownloadDir().absolute.path; d = d.parent) {
if (d.listSync().isEmpty) {
d.deleteSync();
} else {
break;
}
}
}
artifactUpdater.removeDownloadedFiles();
}
/// Hook method for extra checks for being up-to-date.
bool isUpToDateInner() => true;
/// Template method to perform artifact update.
Future<void> updateInner();
Future<void> updateInner(ArtifactUpdater artifactUpdater);
Uri _toStorageUri(String path) => Uri.parse('${cache.storageBaseUrl}/$path');
/// Download an archive from the given [url] and unzip it to [location].
Future<void> _downloadArchive(String message, Uri url, Directory location, bool verifier(File f), void extractor(File f, Directory d)) {
return _withDownloadFile(flattenNameSubdirs(url), (File tempFile) async {
if (!verifier(tempFile)) {
final Status status = globals.logger.startProgress(message, timeout: timeoutConfiguration.slowOperation);
try {
await cache.downloadFile(url, tempFile);
status.stop();
// The exception is rethrown, so don't catch only Exceptions.
} catch (exception) { // ignore: avoid_catches_without_on_clauses
status.cancel();
rethrow;
}
} else {
globals.logger.printTrace('$message (cached)');
}
_ensureExists(location);
extractor(tempFile, location);
});
}
/// Download a zip archive from the given [url] and unzip it to [location].
Future<void> _downloadZipArchive(String message, Uri url, Directory location) {
return _downloadArchive(message, url, location, globals.os.verifyZip, globals.os.unzip);
}
/// Download a gzipped tarball from the given [url] and unpack it to [location].
Future<void> _downloadZippedTarball(String message, Uri url, Directory location) {
return _downloadArchive(message, url, location, globals.os.verifyGzip, globals.os.unpack);
}
/// Create a temporary file and invoke [onTemporaryFile] with the file as
/// argument, then add the temporary file to the [downloadedFiles].
Future<void> _withDownloadFile(String name, Future<void> onTemporaryFile(File file)) async {
final File tempFile = globals.fs.file(globals.fs.path.join(cache.getDownloadDir().path, name));
downloadedFiles.add(tempFile);
await onTemporaryFile(tempFile);
}
}
/// A cached artifact containing fonts used for Material Design.
......@@ -677,9 +591,9 @@ class MaterialFonts extends CachedArtifact {
);
@override
Future<void> updateInner() {
Future<void> updateInner(ArtifactUpdater artifactUpdater) {
final Uri archiveUri = _toStorageUri(version);
return _downloadZipArchive('Downloading Material fonts...', archiveUri, location);
return artifactUpdater.downloadZipArchive('Downloading Material fonts...', archiveUri, location);
}
}
......@@ -701,7 +615,7 @@ class FlutterWebSdk extends CachedArtifact {
String get version => cache.getVersionFor('engine');
@override
Future<void> updateInner() async {
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {
String platformName = 'flutter-web-sdk-';
if (globals.platform.isMacOS) {
platformName += 'darwin-x64';
......@@ -711,7 +625,7 @@ class FlutterWebSdk extends CachedArtifact {
platformName += 'windows-x64';
}
final Uri url = Uri.parse('${cache.storageBaseUrl}/flutter_infra/flutter/$version/$platformName.zip');
await _downloadZipArchive('Downloading Web SDK...', url, location);
await artifactUpdater.downloadZipArchive('Downloading Web SDK...', url, location);
// This is a temporary work-around for not being able to safely download into a shared directory.
for (final FileSystemEntity entity in location.listSync(recursive: true)) {
if (entity is File) {
......@@ -774,12 +688,12 @@ abstract class EngineCachedArtifact extends CachedArtifact {
}
@override
Future<void> updateInner() async {
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {
final String url = '${cache.storageBaseUrl}/flutter_infra/flutter/$version/';
final Directory pkgDir = cache.getCacheDir('pkg');
for (final String pkgName in getPackageDirs()) {
await _downloadZipArchive('Downloading package $pkgName...', Uri.parse(url + pkgName + '.zip'), pkgDir);
await artifactUpdater.downloadZipArchive('Downloading package $pkgName...', Uri.parse(url + pkgName + '.zip'), pkgDir);
}
for (final List<String> toolsDir in getBinaryDirs()) {
......@@ -789,7 +703,7 @@ abstract class EngineCachedArtifact extends CachedArtifact {
// Avoid printing things like 'Downloading linux-x64 tools...' multiple times.
final String friendlyName = urlPath.replaceAll('/artifacts.zip', '').replaceAll('.zip', '');
await _downloadZipArchive('Downloading $friendlyName tools...', Uri.parse(url + urlPath), dir);
await artifactUpdater.downloadZipArchive('Downloading $friendlyName tools...', Uri.parse(url + urlPath), dir);
_makeFilesExecutable(dir);
......@@ -851,7 +765,6 @@ abstract class EngineCachedArtifact extends CachedArtifact {
}
}
/// A cached artifact containing the dart:ui source code.
class FlutterSdk extends EngineCachedArtifact {
FlutterSdk(Cache cache) : super(
......@@ -1024,7 +937,7 @@ class AndroidMavenArtifacts extends ArtifactSet {
final Cache cache;
@override
Future<void> update() async {
Future<void> update(ArtifactUpdater artifactUpdater) async {
final Directory tempDir = cache.getRoot().createTempSync(
'flutter_gradle_wrapper.',
);
......@@ -1112,9 +1025,9 @@ class GradleWrapper extends CachedArtifact {
String get _gradleWrapper => globals.fs.path.join('gradle', 'wrapper', 'gradle-wrapper.jar');
@override
Future<void> updateInner() {
Future<void> updateInner(ArtifactUpdater artifactUpdater) {
final Uri archiveUri = _toStorageUri(version);
return _downloadZippedTarball('Downloading Gradle Wrapper...', archiveUri, location).then<void>((_) {
return artifactUpdater.downloadZippedTarball('Downloading Gradle Wrapper...', archiveUri, location).then<void>((_) {
// Delete property file, allowing templates to provide it.
globals.fs.file(globals.fs.path.join(location.path, 'gradle', 'wrapper', 'gradle-wrapper.properties')).deleteSync();
// Remove NOTICE file. Should not be part of the template.
......@@ -1160,9 +1073,9 @@ abstract class _FuchsiaSDKArtifacts extends CachedArtifact {
@override
Directory get location => cache.getArtifactDirectory('fuchsia');
Future<void> _doUpdate() {
Future<void> _doUpdate(ArtifactUpdater artifactUpdater) {
final String url = '$_cipdBaseUrl/$_path/+/$version';
return _downloadZipArchive('Downloading package fuchsia SDK...',
return artifactUpdater.downloadZipArchive('Downloading package fuchsia SDK...',
Uri.parse(url), location);
}
}
......@@ -1182,12 +1095,12 @@ class FlutterRunnerSDKArtifacts extends CachedArtifact {
String get version => cache.getVersionFor('engine');
@override
Future<void> updateInner() async {
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {
if (!globals.platform.isLinux && !globals.platform.isMacOS) {
return Future<void>.value();
}
final String url = '$_cipdBaseUrl/flutter/fuchsia/+/git_revision:$version';
await _downloadZipArchive('Downloading package flutter runner...',
await artifactUpdater.downloadZipArchive('Downloading package flutter runner...',
Uri.parse(url), location);
}
}
......@@ -1214,12 +1127,14 @@ class CipdArchiveResolver extends VersionedPackageResolver {
/// The debug symbols for flutter runner for Fuchsia development.
class FlutterRunnerDebugSymbols extends CachedArtifact {
FlutterRunnerDebugSymbols(Cache cache, {this.packageResolver = const CipdArchiveResolver(), this.dryRun = false})
: super('flutter_runner_debug_symbols', cache, DevelopmentArtifact.flutterRunner);
FlutterRunnerDebugSymbols(Cache cache, {
@required Platform platform,
this.packageResolver = const CipdArchiveResolver(),
}) : _platform = platform,
super('flutter_runner_debug_symbols', cache, DevelopmentArtifact.flutterRunner);
final VersionedPackageResolver packageResolver;
final bool dryRun;
final Platform _platform;
@override
Directory get location => cache.getArtifactDirectory(name);
......@@ -1227,24 +1142,23 @@ class FlutterRunnerDebugSymbols extends CachedArtifact {
@override
String get version => cache.getVersionFor('engine');
Future<void> _downloadDebugSymbols(String targetArch) async {
Future<void> _downloadDebugSymbols(String targetArch, ArtifactUpdater artifactUpdater) async {
final String packageName = 'fuchsia-debug-symbols-$targetArch';
final String url = packageResolver.resolveUrl(packageName, version);
if (!dryRun) {
await _downloadZipArchive(
'Downloading debug symbols for flutter runner - arch:$targetArch...',
Uri.parse(url),
location);
}
await artifactUpdater.downloadZipArchive(
'Downloading debug symbols for flutter runner - arch:$targetArch...',
Uri.parse(url),
location,
);
}
@override
Future<void> updateInner() async {
if (!globals.platform.isLinux && !globals.platform.isMacOS) {
return Future<void>.value();
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {
if (!_platform.isLinux && !_platform.isMacOS) {
return;
}
await _downloadDebugSymbols('x64');
await _downloadDebugSymbols('arm64');
await _downloadDebugSymbols('x64', artifactUpdater);
await _downloadDebugSymbols('arm64', artifactUpdater);
}
}
......@@ -1253,11 +1167,11 @@ class LinuxFuchsiaSDKArtifacts extends _FuchsiaSDKArtifacts {
LinuxFuchsiaSDKArtifacts(Cache cache) : super(cache, 'linux');
@override
Future<void> updateInner() {
Future<void> updateInner(ArtifactUpdater artifactUpdater) {
if (!globals.platform.isLinux) {
return Future<void>.value();
}
return _doUpdate();
return _doUpdate(artifactUpdater);
}
}
......@@ -1266,11 +1180,11 @@ class MacOSFuchsiaSDKArtifacts extends _FuchsiaSDKArtifacts {
MacOSFuchsiaSDKArtifacts(Cache cache) : super(cache, 'mac');
@override
Future<void> updateInner() async {
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {
if (!globals.platform.isMacOS) {
return Future<void>.value();
}
return _doUpdate();
return _doUpdate(artifactUpdater);
}
}
......@@ -1358,14 +1272,14 @@ class IosUsbArtifacts extends CachedArtifact {
}
@override
Future<void> updateInner() {
Future<void> updateInner(ArtifactUpdater artifactUpdater) {
if (!globals.platform.isMacOS && !ignorePlatformFiltering) {
return Future<void>.value();
}
if (location.existsSync()) {
location.deleteSync(recursive: true);
}
return _downloadZipArchive('Downloading $name...', archiveUri, location);
return artifactUpdater.downloadZipArchive('Downloading $name...', archiveUri, location);
}
@visibleForTesting
......@@ -1397,20 +1311,6 @@ String _flattenNameNoSubdirs(String fileName) {
return String.fromCharCodes(replacedCodeUnits);
}
@visibleForTesting
String flattenNameSubdirs(Uri url) {
final List<String> pieces = <String>[url.host, ...url.pathSegments];
final Iterable<String> convertedPieces = pieces.map<String>(_flattenNameNoSubdirs);
return globals.fs.path.joinAll(convertedPieces);
}
/// Create the given [directory] and parents, as necessary.
void _ensureExists(Directory directory) {
if (!directory.existsSync()) {
directory.createSync(recursive: true);
}
}
// TODO(jonahwilliams): upload debug desktop artifacts to host-debug and
// remove from existing host folder.
// https://github.com/flutter/flutter/issues/38935
......@@ -1487,3 +1387,148 @@ const List<List<String>> _dartSdks = <List<String>> [
<String>['linux-x64', 'dart-sdk-linux-x64.zip'],
<String>['windows-x64', 'dart-sdk-windows-x64.zip'],
];
/// An API for downloading and un-archiving artifacts, such as engine binaries or
/// additional source code.
class ArtifactUpdater {
ArtifactUpdater({
@required OperatingSystemUtils operatingSystemUtils,
@required Net net,
@required Logger logger,
@required FileSystem fileSystem,
@required Directory tempStorage
}) : _operatingSystemUtils = operatingSystemUtils,
_net = net,
_logger = logger,
_fileSystem = fileSystem,
_tempStorage = tempStorage;
final Logger _logger;
final Net _net;
final OperatingSystemUtils _operatingSystemUtils;
final FileSystem _fileSystem;
final Directory _tempStorage;
/// Keep track of the files we've downloaded for this execution so we
/// can delete them after completion. We don't delete them right after
/// extraction in case [update] is interrupted, so we can restart without
/// starting from scratch.
@visibleForTesting
final List<File> downloadedFiles = <File>[];
/// Download a zip archive from the given [url] and unzip it to [location].
Future<void> downloadZipArchive(
String message,
Uri url,
Directory location,
) {
return _downloadArchive(
message,
url,
location,
_operatingSystemUtils.unzip,
);
}
/// Download a gzipped tarball from the given [url] and unpack it to [location].
Future<void> downloadZippedTarball(String message, Uri url, Directory location) {
return _downloadArchive(
message,
url,
location,
_operatingSystemUtils.unpack,
);
}
/// Download an archive from the given [url] and unzip it to [location].
Future<void> _downloadArchive(
String message,
Uri url,
Directory location,
void Function(File, Directory) extractor,
) async {
final String downloadPath = flattenNameSubdirs(url, _fileSystem);
final File tempFile = _createDownloadFile(downloadPath);
final Status status = _logger.startProgress(
message,
timeout: null, // This will take a variable amount of time based on network connectivity.
);
int retries = 2;
while (retries > 0) {
try {
_ensureExists(tempFile.parent);
await _net.fetchUrl(url, destFile: tempFile, maxAttempts: 2);
} finally {
status.stop();
}
_ensureExists(location);
try {
extractor(tempFile, location);
} on ProcessException {
retries -= 1;
if (retries == 0) {
rethrow;
}
_deleteIgnoringErrors(tempFile);
continue;
}
return;
}
}
/// Create a temporary file and invoke [onTemporaryFile] with the file as
/// argument, then add the temporary file to the [downloadedFiles].
File _createDownloadFile(String name) {
final File tempFile = _fileSystem.file(_fileSystem.path.join(_tempStorage.path, name));
downloadedFiles.add(tempFile);
return tempFile;
}
/// Create the given [directory] and parents, as necessary.
void _ensureExists(Directory directory) {
if (!directory.existsSync()) {
directory.createSync(recursive: true);
}
}
/// Clear any zip/gzip files downloaded.
void removeDownloadedFiles() {
for (final File file in downloadedFiles) {
if (!file.existsSync()) {
continue;
}
try {
file.deleteSync();
} on FileSystemException catch (e) {
globals.printError('Failed to delete "${file.path}". Please delete manually. $e');
continue;
}
for (Directory directory = file.parent; directory.absolute.path != _tempStorage.absolute.path; directory = directory.parent) {
if (directory.listSync().isNotEmpty) {
break;
}
_deleteIgnoringErrors(directory);
}
}
}
static void _deleteIgnoringErrors(FileSystemEntity entity) {
if (!entity.existsSync()) {
return;
}
try {
entity.deleteSync();
} on FileSystemException {
// Ignore errors.
}
}
}
@visibleForTesting
String flattenNameSubdirs(Uri url, FileSystem fileSystem){
final List<String> pieces = <String>[url.host, ...url.pathSegments];
final Iterable<String> convertedPieces = pieces.map<String>(_flattenNameNoSubdirs);
return fileSystem.path.joinAll(convertedPieces);
}
packages/flutter_tools/test/general.shard/artifact_updater_test.dart 0 → 100644
View file @ 23032d77
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:file/memory.dart';
import 'package:file/src/interface/file.dart';
import 'package:file_testing/file_testing.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/base/io.dart';
import 'package:flutter_tools/src/base/logger.dart';
import 'package:flutter_tools/src/base/net.dart';
import 'package:flutter_tools/src/base/os.dart';
import 'package:flutter_tools/src/cache.dart';
import 'package:mockito/mockito.dart';
import '../src/common.dart';
void main() {
testWithoutContext('ArtifactUpdater can download a zip archive', () async {
final FakeNet net = FakeNet();
final MockOperatingSystemUtils operatingSystemUtils = MockOperatingSystemUtils();
final MemoryFileSystem fileSystem = MemoryFileSystem.test();
final BufferLogger logger = BufferLogger.test();
final ArtifactUpdater artifactUpdater = ArtifactUpdater(
fileSystem: fileSystem,
logger: logger,
net: net,
operatingSystemUtils: operatingSystemUtils,
tempStorage: fileSystem.currentDirectory.childDirectory('temp')
..createSync(),
);
await artifactUpdater.downloadZipArchive(
'test message',
Uri.parse('http:///test.zip'),
fileSystem.currentDirectory.childDirectory('out'),
);
expect(logger.statusText, contains('test message'));
expect(fileSystem.file('out/test'), exists);
});
testWithoutContext('ArtifactUpdater will de-download a file if unzipping fails', () async {
final FakeNet net = FakeNet();
final MockOperatingSystemUtils operatingSystemUtils = MockOperatingSystemUtils();
final MemoryFileSystem fileSystem = MemoryFileSystem.test();
final BufferLogger logger = BufferLogger.test();
final ArtifactUpdater artifactUpdater = ArtifactUpdater(
fileSystem: fileSystem,
logger: logger,
net: net,
operatingSystemUtils: operatingSystemUtils,
tempStorage: fileSystem.currentDirectory.childDirectory('temp')
..createSync(),
);
operatingSystemUtils.failures = 1;
await artifactUpdater.downloadZipArchive(
'test message',
Uri.parse('http:///test.zip'),
fileSystem.currentDirectory.childDirectory('out'),
);
expect(logger.statusText, contains('test message'));
expect(fileSystem.file('out/test'), exists);
expect(net.attempts, 2);
});
testWithoutContext('ArtifactUpdater will bail if unzipping fails more than twice', () async {
final FakeNet net = FakeNet();
final MockOperatingSystemUtils operatingSystemUtils = MockOperatingSystemUtils();
final MemoryFileSystem fileSystem = MemoryFileSystem.test();
final BufferLogger logger = BufferLogger.test();
final ArtifactUpdater artifactUpdater = ArtifactUpdater(
fileSystem: fileSystem,
logger: logger,
net: net,
operatingSystemUtils: operatingSystemUtils,
tempStorage: fileSystem.currentDirectory.childDirectory('temp')
..createSync(),
);
operatingSystemUtils.failures = 2;
expect(artifactUpdater.downloadZipArchive(
'test message',
Uri.parse('http:///test.zip'),
fileSystem.currentDirectory.childDirectory('out'),
), throwsA(isA<ProcessException>()));
expect(fileSystem.file('te,[/test'), isNot(exists));
expect(fileSystem.file('out/test'), isNot(exists));
});
testWithoutContext('ArtifactUpdater can download a tar archive', () async {
final FakeNet net = FakeNet();
final MockOperatingSystemUtils operatingSystemUtils = MockOperatingSystemUtils();
final MemoryFileSystem fileSystem = MemoryFileSystem.test();
final BufferLogger logger = BufferLogger.test();
final ArtifactUpdater artifactUpdater = ArtifactUpdater(
fileSystem: fileSystem,
logger: logger,
net: net,
operatingSystemUtils: operatingSystemUtils,
tempStorage: fileSystem.currentDirectory.childDirectory('temp')
..createSync(),
);
await artifactUpdater.downloadZippedTarball(
'test message',
Uri.parse('http:///test.zip'),
fileSystem.currentDirectory.childDirectory('out'),
);
expect(fileSystem.file('out/test'), exists);
});
testWithoutContext('ArtifactUpdater will delete downloaded files if they exist.', () async {
final FakeNet net = FakeNet();
final MockOperatingSystemUtils operatingSystemUtils = MockOperatingSystemUtils();
final MemoryFileSystem fileSystem = MemoryFileSystem.test();
final BufferLogger logger = BufferLogger.test();
final ArtifactUpdater artifactUpdater = ArtifactUpdater(
fileSystem: fileSystem,
logger: logger,
net: net,
operatingSystemUtils: operatingSystemUtils,
tempStorage: fileSystem.currentDirectory.childDirectory('temp')
..createSync(),
);
artifactUpdater.downloadedFiles.addAll(<File>[
fileSystem.file('a/b/c/d')..createSync(recursive: true),
fileSystem.file('d/e/f'),
]);
artifactUpdater.removeDownloadedFiles();
expect(fileSystem.file('a/b/c/d'), isNot(exists));
expect(logger.errorText, isEmpty);
});
}
class FakeNet implements Net {
int attempts = 0;
@override
Future<bool> doesRemoteFileExist(Uri url) async {
return true;
}
@override
Future<List<int>> fetchUrl(Uri url, {int maxAttempts, File destFile}) async {
attempts += 1;
if (destFile != null) {
destFile.createSync();
return null;
}
return <int>[];
}
}
class MockOperatingSystemUtils extends Mock implements OperatingSystemUtils {
int failures = 0;
@override
void unzip(File file, Directory targetDirectory) {
if (failures > 0) {
failures -= 1;
throw const ProcessException('zip', <String>[], 'Failed to unzip');
}
targetDirectory.childFile(file.fileSystem.path.basenameWithoutExtension(file.path))
.createSync();
}
@override
void unpack(File gzippedTarFile, Directory targetDirectory) {
if (failures > 0) {
failures -= 1;
throw const ProcessException('zip', <String>[], 'Failed to unzip');
}
targetDirectory.childFile(gzippedTarFile.fileSystem.path.basenameWithoutExtension(gzippedTarFile.path))
.createSync();
}
}
packages/flutter_tools/test/general.shard/base/error_handling_file_system_test.dart
View file @ 23032d77
......@@ -64,7 +64,7 @@ void setupWriteMocks({
)).thenThrow(FileSystemException('', '', OSError('', errorCode)));
}
void setupCreateTempMocks({
void setupDirectoryMocks({
FileSystem mockFileSystem,
ErrorHandlingFileSystem fs,
int errorCode,
......@@ -76,6 +76,8 @@ void setupCreateTempMocks({
});
when(mockDirectory.createTempSync(any))
.thenThrow(FileSystemException('', '', OSError('', errorCode)));
when(mockDirectory.createSync(recursive: anyNamed('recursive')))
.thenThrow(FileSystemException('', '', OSError('', errorCode)));
}
void main() {
......@@ -158,7 +160,7 @@ void main() {
});
testWithoutContext('when creating a temporary dir on a full device', () async {
setupCreateTempMocks(
setupDirectoryMocks(
mockFileSystem: mockFileSystem,
fs: fs,
errorCode: kDeviceFull,
......@@ -172,6 +174,20 @@ void main() {
expect(() => directory.createTempSync('prefix'),
throwsToolExit(message: expectedMessage));
});
testWithoutContext('when creating a directory with permission issues', () async {
setupDirectoryMocks(
mockFileSystem: mockFileSystem,
fs: fs,
errorCode: kUserPermissionDenied,
);
final Directory directory = fs.directory('directory');
const String expectedMessage = 'Flutter failed to create a directory at';
expect(() => directory.createSync(recursive: true),
throwsToolExit(message: expectedMessage));
});
});
group('throws ToolExit on Linux', () {
......@@ -209,7 +225,7 @@ void main() {
});
testWithoutContext('when creating a temporary dir on a full device', () async {
setupCreateTempMocks(
setupDirectoryMocks(
mockFileSystem: mockFileSystem,
fs: fs,
errorCode: enospc,
......
packages/flutter_tools/test/general.shard/cache_test.dart
View file @ 23032d77
......@@ -10,7 +10,6 @@ import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/base/io.dart' show InternetAddress, SocketException;
import 'package:flutter_tools/src/base/io.dart';
import 'package:flutter_tools/src/base/logger.dart';
import 'package:flutter_tools/src/base/net.dart';
import 'package:flutter_tools/src/base/os.dart';
import 'package:flutter_tools/src/base/platform.dart';
import 'package:flutter_tools/src/cache.dart';
......@@ -92,27 +91,6 @@ void main() {
memoryFileSystem = MemoryFileSystem();
});
testUsingContext('Continues on failed delete', () async {
final Directory artifactDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_artifact.');
final Directory downloadDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_download.');
when(mockCache.getArtifactDirectory(any)).thenReturn(artifactDir);
when(mockCache.getDownloadDir()).thenReturn(downloadDir);
final File mockFile = MockFile();
when(mockFile.deleteSync()).thenAnswer((_) {
throw const FileSystemException('delete failed');
});
final FakeDownloadedArtifact artifact = FakeDownloadedArtifact(
mockFile,
mockCache,
);
await artifact.update();
expect(testLogger.errorText, contains('delete failed'));
}, overrides: <Type, Generator>{
Cache: () => mockCache,
FileSystem: () => memoryFileSystem,
ProcessManager: () => FakeProcessManager.any(),
});
testUsingContext('Continues on failed stamp file update', () async {
final Directory artifactDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_artifact.');
final Directory downloadDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_download.');
......@@ -122,7 +100,7 @@ void main() {
throw const FileSystemException('stamp write failed');
});
final FakeSimpleArtifact artifact = FakeSimpleArtifact(mockCache);
await artifact.update();
await artifact.update(MockArtifactUpdater());
expect(testLogger.errorText, contains('stamp write failed'));
}, overrides: <Type, Generator>{
Cache: () => mockCache,
......@@ -159,23 +137,31 @@ void main() {
ProcessManager: () => FakeProcessManager.any(),
});
test('should not be up to date, if some cached artifact is not', () {
testUsingContext('should not be up to date, if some cached artifact is not', () {
final CachedArtifact artifact1 = MockCachedArtifact();
final CachedArtifact artifact2 = MockCachedArtifact();
when(artifact1.isUpToDate()).thenReturn(true);
when(artifact2.isUpToDate()).thenReturn(false);
final Cache cache = Cache(artifacts: <CachedArtifact>[artifact1, artifact2]);
expect(cache.isUpToDate(), isFalse);
}, overrides: <Type, Generator>{
ProcessManager: () => FakeProcessManager.any(),
FileSystem: () => MemoryFileSystem.test(),
});
test('should be up to date, if all cached artifacts are', () {
testUsingContext('should be up to date, if all cached artifacts are', () {
final CachedArtifact artifact1 = MockCachedArtifact();
final CachedArtifact artifact2 = MockCachedArtifact();
when(artifact1.isUpToDate()).thenReturn(true);
when(artifact2.isUpToDate()).thenReturn(true);
final Cache cache = Cache(artifacts: <CachedArtifact>[artifact1, artifact2]);
expect(cache.isUpToDate(), isTrue);
}, overrides: <Type, Generator>{
ProcessManager: () => FakeProcessManager.any(),
FileSystem: () => MemoryFileSystem.test(),
});
test('should update cached artifacts which are not up to date', () async {
testUsingContext('should update cached artifacts which are not up to date', () async {
final CachedArtifact artifact1 = MockCachedArtifact();
final CachedArtifact artifact2 = MockCachedArtifact();
when(artifact1.isUpToDate()).thenReturn(true);
......@@ -184,9 +170,13 @@ void main() {
await cache.updateAll(<DevelopmentArtifact>{
null,
});
verifyNever(artifact1.update());
verify(artifact2.update());
verifyNever(artifact1.update(any));
verify(artifact2.update(any));
}, overrides: <Type, Generator>{
ProcessManager: () => FakeProcessManager.any(),
FileSystem: () => MemoryFileSystem.test(),
});
testUsingContext("getter dyLdLibEntry concatenates the output of each artifact's dyLdLibEntry getter", () async {
final IosUsbArtifacts artifact1 = MockIosUsbArtifacts();
final IosUsbArtifacts artifact2 = MockIosUsbArtifacts();
......@@ -220,7 +210,7 @@ void main() {
when(artifact2.isUpToDate()).thenReturn(false);
final MockInternetAddress address = MockInternetAddress();
when(address.host).thenReturn('storage.googleapis.com');
when(artifact1.update()).thenThrow(SocketException(
when(artifact1.update(any)).thenThrow(SocketException(
'Connection reset by peer',
address: address,
));
......@@ -231,14 +221,17 @@ void main() {
});
fail('Mock thrown exception expected');
} on Exception {
verify(artifact1.update());
verify(artifact1.update(any));
// Don't continue when retrieval fails.
verifyNever(artifact2.update());
verifyNever(artifact2.update(any));
expect(
testLogger.errorText,
contains('https://flutter.dev/community/china'),
);
}
}, overrides: <Type, Generator>{
ProcessManager: () => FakeProcessManager.any(),
FileSystem: () => MemoryFileSystem.test(),
});
testUsingContext('Invalid URI for FLUTTER_STORAGE_BASE_URL throws ToolExit', () async {
......@@ -252,38 +245,33 @@ void main() {
});
});
testUsingContext('flattenNameSubdirs', () {
expect(flattenNameSubdirs(Uri.parse('http://flutter.dev/foo/bar')), 'flutter.dev/foo/bar');
expect(flattenNameSubdirs(Uri.parse('http://docs.flutter.io/foo/bar')), 'docs.flutter.io/foo/bar');
expect(flattenNameSubdirs(Uri.parse('https://www.flutter.dev')), 'www.flutter.dev');
}, overrides: <Type, Generator>{
FileSystem: () => MemoryFileSystem(),
ProcessManager: () => FakeProcessManager.any(),
testWithoutContext('flattenNameSubdirs', () {
expect(flattenNameSubdirs(Uri.parse('http://flutter.dev/foo/bar'), MemoryFileSystem()), 'flutter.dev/foo/bar');
expect(flattenNameSubdirs(Uri.parse('http://docs.flutter.io/foo/bar'), MemoryFileSystem()), 'docs.flutter.io/foo/bar');
expect(flattenNameSubdirs(Uri.parse('https://www.flutter.dev'), MemoryFileSystem()), 'www.flutter.dev');
});
group('EngineCachedArtifact', () {
FakeHttpClient fakeHttpClient;
FakePlatform fakePlatform;
MemoryFileSystem memoryFileSystem;
MemoryFileSystem fileSystem;
MockCache mockCache;
MockOperatingSystemUtils mockOperatingSystemUtils;
MockHttpClient mockHttpClient;
setUp(() {
fakeHttpClient = FakeHttpClient();
mockHttpClient = MockHttpClient();
fakePlatform = FakePlatform()..environment = const <String, String>{};
memoryFileSystem = MemoryFileSystem();
fakePlatform = FakePlatform(environment: const <String, String>{}, operatingSystem: 'linux');
mockCache = MockCache();
mockOperatingSystemUtils = MockOperatingSystemUtils();
fileSystem = MemoryFileSystem.test();
});
testUsingContext('makes binary dirs readable and executable by all', () async {
when(mockOperatingSystemUtils.verifyZip(any)).thenReturn(true);
final Directory artifactDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_artifact.');
final Directory downloadDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_download.');
final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync('flutter_cache_test_artifact.');
final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync('flutter_cache_test_download.');
when(mockCache.getArtifactDirectory(any)).thenReturn(artifactDir);
when(mockCache.getDownloadDir()).thenReturn(downloadDir);
artifactDir.childDirectory('bin_dir').createSync();
artifactDir.childFile('unused_url_path').createSync();
final FakeCachedArtifact artifact = FakeCachedArtifact(
cache: mockCache,
binaryDirs: <List<String>>[
......@@ -291,8 +279,8 @@ void main() {
],
requiredArtifacts: DevelopmentArtifact.universal,
);
await artifact.updateInner();
final Directory dir = memoryFileSystem.systemTempDirectory
await artifact.updateInner(MockArtifactUpdater());
final Directory dir = fileSystem.systemTempDirectory
.listSync(recursive: true)
.whereType<Directory>()
.singleWhere((Directory directory) => directory.basename == 'bin_dir', orElse: () => null);
......@@ -301,50 +289,8 @@ void main() {
verify(mockOperatingSystemUtils.chmod(argThat(hasPath(dir.path)), 'a+r,a+x'));
}, overrides: <Type, Generator>{
Cache: () => mockCache,
FileSystem: () => memoryFileSystem,
FileSystem: () => fileSystem,
ProcessManager: () => FakeProcessManager.any(),
HttpClientFactory: () => () => fakeHttpClient,
OperatingSystemUtils: () => mockOperatingSystemUtils,
Platform: () => fakePlatform,
});
testUsingContext('prints a friendly name when downloading', () async {
when(mockOperatingSystemUtils.verifyZip(any)).thenReturn(false);
final MockHttpClientRequest httpClientRequest = MockHttpClientRequest();
final MockHttpClientResponse httpClientResponse = MockHttpClientResponse();
when(httpClientResponse.statusCode).thenReturn(200);
when(httpClientRequest.close()).thenAnswer((_) async => httpClientResponse);
when(mockHttpClient.getUrl(any)).thenAnswer((_) async => httpClientRequest);
final Directory artifactDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_artifact.');
final Directory downloadDir = globals.fs.systemTempDirectory.createTempSync('flutter_cache_test_download.');
when(mockCache.getArtifactDirectory(any)).thenReturn(artifactDir);
when(mockCache.getDownloadDir()).thenReturn(downloadDir);
final FakeCachedArtifact artifact = FakeCachedArtifact(
cache: mockCache,
binaryDirs: <List<String>>[
<String>['bin_dir', 'darwin-x64/artifacts.zip'],
<String>['font-subset', 'darwin-x64/font-subset.zip'],
],
requiredArtifacts: DevelopmentArtifact.universal,
);
await artifact.updateInner();
expect(testLogger.statusText, isNotNull);
expect(testLogger.statusText, isNotEmpty);
expect(
testLogger.statusText.split('\n'),
<String>[
'Downloading darwin-x64 tools...',
'Downloading darwin-x64/font-subset tools...',
'',
],
);
}, overrides: <Type, Generator>{
Cache: () => mockCache,
FileSystem: () => memoryFileSystem,
ProcessManager: () => FakeProcessManager.any(),
HttpClientFactory: () => () => mockHttpClient,
OperatingSystemUtils: () => mockOperatingSystemUtils,
Platform: () => fakePlatform,
});
......@@ -392,7 +338,7 @@ void main() {
return Future<ProcessResult>.value(ProcessResult(0, 0, '', ''));
});
await mavenArtifacts.update();
await mavenArtifacts.update(MockArtifactUpdater());
expect(mavenArtifacts.isUpToDate(), isFalse);
}, overrides: <Type, Generator>{
......@@ -478,28 +424,23 @@ void main() {
});
});
group('Flutter runner debug symbols', () {
MockCache mockCache;
MockVersionedPackageResolver mockPackageResolver;
setUp(() {
mockCache = MockCache();
mockPackageResolver = MockVersionedPackageResolver();
});
testUsingContext('Downloads Flutter runner debug symbols', () async {
final FlutterRunnerDebugSymbols flutterRunnerDebugSymbols =
FlutterRunnerDebugSymbols(mockCache, packageResolver: mockPackageResolver, dryRun: true);
await flutterRunnerDebugSymbols.updateInner();
testWithoutContext('Downloads Flutter runner debug symbols', () async {
final MockCache mockCache = MockCache();
final MockVersionedPackageResolver mockPackageResolver = MockVersionedPackageResolver();
final FlutterRunnerDebugSymbols flutterRunnerDebugSymbols = FlutterRunnerDebugSymbols(
mockCache,
packageResolver: mockPackageResolver,
platform: FakePlatform(operatingSystem: 'linux'),
);
when(mockPackageResolver.resolveUrl(any, any)).thenReturn('');
verifyInOrder(<void>[
mockPackageResolver.resolveUrl('fuchsia-debug-symbols-x64', any),
mockPackageResolver.resolveUrl('fuchsia-debug-symbols-arm64', any),
]);
});
}, skip: !globals.platform.isLinux);
await flutterRunnerDebugSymbols.updateInner(MockArtifactUpdater());
verifyInOrder(<void>[
mockPackageResolver.resolveUrl('fuchsia-debug-symbols-x64', any),
mockPackageResolver.resolveUrl('fuchsia-debug-symbols-arm64', any),
]);
});
testUsingContext('FontSubset in univeral artifacts', () {
final MockCache mockCache = MockCache();
......@@ -727,7 +668,7 @@ class FakeSimpleArtifact extends CachedArtifact {
);
@override
Future<void> updateInner() async {
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {
// nop.
}
}
......@@ -742,16 +683,14 @@ class FakeDownloadedArtifact extends CachedArtifact {
final File downloadedFile;
@override
Future<void> updateInner() async {
downloadedFiles.add(downloadedFile);
}
Future<void> updateInner(ArtifactUpdater artifactUpdater) async {}
}
class MockArtifactUpdater extends Mock implements ArtifactUpdater {}
class MockProcessManager extends Mock implements ProcessManager {}
class MockFileSystem extends Mock implements FileSystem {}
class MockFile extends Mock implements File {}
class MockDirectory extends Mock implements Directory {}
class MockRandomAccessFile extends Mock implements RandomAccessFile {}
class MockCachedArtifact extends Mock implements CachedArtifact {}
class MockIosUsbArtifacts extends Mock implements IosUsbArtifacts {}
......@@ -759,6 +698,3 @@ class MockInternetAddress extends Mock implements InternetAddress {}
class MockCache extends Mock implements Cache {}
class MockOperatingSystemUtils extends Mock implements OperatingSystemUtils {}
class MockVersionedPackageResolver extends Mock implements VersionedPackageResolver {}
class MockHttpClientRequest extends Mock implements HttpClientRequest {}
class MockHttpClientResponse extends Mock implements HttpClientResponse {}
packages/flutter_tools/test/src/testbed.dart
View file @ 23032d77
......@@ -915,11 +915,6 @@ class FakeCache implements Cache {
throw UnsupportedError('Not supported in the fake Cache');
}
@override
Future<String> getThirdPartyFile(String urlStr, String serviceName) {
throw UnsupportedError('Not supported in the fake Cache');
}
@override
String getVersionFor(String artifactName) {
throw UnsupportedError('Not supported in the fake Cache');
......@@ -949,10 +944,6 @@ class FakeCache implements Cache {
Future<void> updateAll(Set<DevelopmentArtifact> requiredArtifacts) async {
}
@override
Future<void> downloadFile(Uri url, File location) async {
}
@override
Future<bool> doesRemoteExist(String message, Uri url) async {
return true;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

HIAST - GitLab