// Copyright 2016 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:convert' show base64, utf8; import 'package:json_rpc_2/json_rpc_2.dart' as rpc; import 'package:meta/meta.dart'; import 'asset.dart'; import 'base/context.dart'; import 'base/file_system.dart'; import 'base/io.dart'; import 'build_info.dart'; import 'bundle.dart'; import 'compile.dart'; import 'dart/package_map.dart'; import 'globals.dart'; import 'vmservice.dart'; class DevFSConfig { /// Should DevFS assume that symlink targets are stable? bool cacheSymlinks = false; /// Should DevFS assume that there are no symlinks to directories? bool noDirectorySymlinks = false; } DevFSConfig get devFSConfig => context[DevFSConfig]; /// Common superclass for content copied to the device. abstract class DevFSContent { bool _exists = true; /// Return true if this is the first time this method is called /// or if the entry has been modified since this method was last called. bool get isModified; /// Return true if this is the first time this method is called /// or if the entry has been modified after the given time /// or if the given time is null. bool isModifiedAfter(DateTime time); int get size; Future> contentsAsBytes(); Stream> contentsAsStream(); Stream> contentsAsCompressedStream() { return contentsAsStream().transform>(gzip.encoder); } /// Return the list of files this content depends on. List get fileDependencies => []; } // File content to be copied to the device. class DevFSFileContent extends DevFSContent { DevFSFileContent(this.file); static DevFSFileContent clone(DevFSFileContent fsFileContent) { final DevFSFileContent newFsFileContent = DevFSFileContent(fsFileContent.file); newFsFileContent._linkTarget = fsFileContent._linkTarget; newFsFileContent._fileStat = fsFileContent._fileStat; return newFsFileContent; } final FileSystemEntity file; FileSystemEntity _linkTarget; FileStat _fileStat; File _getFile() { if (_linkTarget != null) { return _linkTarget; } if (file is Link) { // The link target. return fs.file(file.resolveSymbolicLinksSync()); } return file; } void _stat() { if (_linkTarget != null) { // Stat the cached symlink target. final FileStat fileStat = _linkTarget.statSync(); if (fileStat.type == FileSystemEntityType.notFound) { _linkTarget = null; } else { _fileStat = fileStat; return; } } final FileStat fileStat = file.statSync(); _fileStat = fileStat.type == FileSystemEntityType.notFound ? null : fileStat; if (_fileStat != null && _fileStat.type == FileSystemEntityType.link) { // Resolve, stat, and maybe cache the symlink target. final String resolved = file.resolveSymbolicLinksSync(); final FileSystemEntity linkTarget = fs.file(resolved); // Stat the link target. final FileStat fileStat = linkTarget.statSync(); if (fileStat.type == FileSystemEntityType.notFound) { _fileStat = null; _linkTarget = null; } else if (devFSConfig.cacheSymlinks) { _linkTarget = linkTarget; } } if (_fileStat == null) { printError('Unable to get status of file "${file.path}": file not found.'); } } @override List get fileDependencies => [_getFile().path]; @override bool get isModified { final FileStat _oldFileStat = _fileStat; _stat(); if (_oldFileStat == null && _fileStat == null) return false; return _oldFileStat == null || _fileStat == null || _fileStat.modified.isAfter(_oldFileStat.modified); } @override bool isModifiedAfter(DateTime time) { final FileStat _oldFileStat = _fileStat; _stat(); if (_oldFileStat == null && _fileStat == null) return false; return time == null || _oldFileStat == null || _fileStat == null || _fileStat.modified.isAfter(time); } @override int get size { if (_fileStat == null) _stat(); // Can still be null if the file wasn't found. return _fileStat?.size ?? 0; } @override Future> contentsAsBytes() => _getFile().readAsBytes(); @override Stream> contentsAsStream() => _getFile().openRead(); } /// Byte content to be copied to the device. class DevFSByteContent extends DevFSContent { DevFSByteContent(this._bytes); List _bytes; bool _isModified = true; DateTime _modificationTime = DateTime.now(); List get bytes => _bytes; set bytes(List value) { _bytes = value; _isModified = true; _modificationTime = DateTime.now(); } /// Return true only once so that the content is written to the device only once. @override bool get isModified { final bool modified = _isModified; _isModified = false; return modified; } @override bool isModifiedAfter(DateTime time) { return time == null || _modificationTime.isAfter(time); } @override int get size => _bytes.length; @override Future> contentsAsBytes() async => _bytes; @override Stream> contentsAsStream() => Stream>.fromIterable(>[_bytes]); } /// String content to be copied to the device. class DevFSStringContent extends DevFSByteContent { DevFSStringContent(String string) : _string = string, super(utf8.encode(string)); String _string; String get string => _string; set string(String value) { _string = value; super.bytes = utf8.encode(_string); } @override set bytes(List value) { string = utf8.decode(value); } } /// Abstract DevFS operations interface. abstract class DevFSOperations { Future create(String fsName); Future destroy(String fsName); Future writeFile(String fsName, Uri deviceUri, DevFSContent content); Future deleteFile(String fsName, Uri deviceUri); } /// An implementation of [DevFSOperations] that speaks to the /// vm service. class ServiceProtocolDevFSOperations implements DevFSOperations { ServiceProtocolDevFSOperations(this.vmService); final VMService vmService; @override Future create(String fsName) async { final Map response = await vmService.vm.createDevFS(fsName); return Uri.parse(response['uri']); } @override Future destroy(String fsName) async { await vmService.vm.deleteDevFS(fsName); } @override Future writeFile(String fsName, Uri deviceUri, DevFSContent content) async { List bytes; try { bytes = await content.contentsAsBytes(); } catch (e) { return e; } final String fileContents = base64.encode(bytes); try { return await vmService.vm.invokeRpcRaw( '_writeDevFSFile', params: { 'fsName': fsName, 'uri': deviceUri.toString(), 'fileContents': fileContents }, ); } catch (error) { printTrace('DevFS: Failed to write $deviceUri: $error'); } } @override Future deleteFile(String fsName, Uri deviceUri) async { // TODO(johnmccutchan): Add file deletion to the devFS protocol. } } class DevFSException implements Exception { DevFSException(this.message, [this.error, this.stackTrace]); final String message; final dynamic error; final StackTrace stackTrace; } class _DevFSHttpWriter { _DevFSHttpWriter(this.fsName, VMService serviceProtocol) : httpAddress = serviceProtocol.httpAddress; final String fsName; final Uri httpAddress; static const int kMaxInFlight = 6; static const int kMaxRetries = 3; int _inFlight = 0; Map _outstanding; Completer _completer; HttpClient _client; Future write(Map entries) async { _client = HttpClient(); _client.maxConnectionsPerHost = kMaxInFlight; _completer = Completer(); _outstanding = Map.from(entries); _scheduleWrites(); await _completer.future; _client.close(); } void _scheduleWrites() { while (_inFlight < kMaxInFlight) { if (_outstanding.isEmpty) { // Finished. break; } final Uri deviceUri = _outstanding.keys.first; final DevFSContent content = _outstanding.remove(deviceUri); _scheduleWrite(deviceUri, content); _inFlight++; } } Future _scheduleWrite( Uri deviceUri, DevFSContent content, [ int retry = 0, ]) async { try { final HttpClientRequest request = await _client.putUrl(httpAddress); request.headers.removeAll(HttpHeaders.acceptEncodingHeader); request.headers.add('dev_fs_name', fsName); request.headers.add('dev_fs_uri_b64', base64.encode(utf8.encode(deviceUri.toString()))); final Stream> contents = content.contentsAsCompressedStream(); await request.addStream(contents); final HttpClientResponse response = await request.close(); await response.drain(); } on SocketException catch (socketException, stackTrace) { // We have one completer and can get up to kMaxInFlight errors. if (!_completer.isCompleted) _completer.completeError(socketException, stackTrace); return; } catch (e) { if (retry < kMaxRetries) { printTrace('Retrying writing "$deviceUri" to DevFS due to error: $e'); // Synchronization is handled by the _completer below. _scheduleWrite(deviceUri, content, retry + 1); // ignore: unawaited_futures return; } else { printError('Error writing "$deviceUri" to DevFS: $e'); } } _inFlight--; if ((_outstanding.isEmpty) && (_inFlight == 0)) { _completer.complete(); } else { _scheduleWrites(); } } } class DevFS { /// Create a [DevFS] named [fsName] for the local files in [rootDirectory]. DevFS(VMService serviceProtocol, this.fsName, this.rootDirectory, { String packagesFilePath }) : _operations = ServiceProtocolDevFSOperations(serviceProtocol), _httpWriter = _DevFSHttpWriter(fsName, serviceProtocol) { _packagesFilePath = packagesFilePath ?? fs.path.join(rootDirectory.path, kPackagesFileName); } DevFS.operations(this._operations, this.fsName, this.rootDirectory, { String packagesFilePath, }) : _httpWriter = null { _packagesFilePath = packagesFilePath ?? fs.path.join(rootDirectory.path, kPackagesFileName); } final DevFSOperations _operations; final _DevFSHttpWriter _httpWriter; final String fsName; final Directory rootDirectory; String _packagesFilePath; final Map _entries = {}; final Set assetPathsToEvict = Set(); final List>> _pendingOperations = >>[]; Uri _baseUri; Uri get baseUri => _baseUri; Uri deviceUriToHostUri(Uri deviceUri) { final String deviceUriString = deviceUri.toString(); final String baseUriString = baseUri.toString(); if (deviceUriString.startsWith(baseUriString)) { final String deviceUriSuffix = deviceUriString.substring(baseUriString.length); return rootDirectory.uri.resolve(deviceUriSuffix); } return deviceUri; } Future create() async { printTrace('DevFS: Creating new filesystem on the device ($_baseUri)'); try { _baseUri = await _operations.create(fsName); } on rpc.RpcException catch (rpcException) { // 1001 is kFileSystemAlreadyExists in //dart/runtime/vm/json_stream.h if (rpcException.code != 1001) rethrow; printTrace('DevFS: Creating failed. Destroying and trying again'); await destroy(); _baseUri = await _operations.create(fsName); } printTrace('DevFS: Created new filesystem on the device ($_baseUri)'); return _baseUri; } Future destroy() async { printTrace('DevFS: Deleting filesystem on the device ($_baseUri)'); await _operations.destroy(fsName); printTrace('DevFS: Deleted filesystem on the device ($_baseUri)'); } /// Updates files on the device. /// /// Returns the number of bytes synced. Future update({ @required String mainPath, String target, AssetBundle bundle, DateTime firstBuildTime, bool bundleFirstUpload = false, bool bundleDirty = false, Set fileFilter, @required ResidentCompiler generator, String dillOutputPath, @required bool trackWidgetCreation, bool fullRestart = false, String projectRootPath, @required String pathToReload, }) async { assert(trackWidgetCreation != null); assert(generator != null); // Mark all entries as possibly deleted. for (DevFSContent content in _entries.values) { content._exists = false; } // Scan workspace, packages, and assets printTrace('DevFS: Starting sync from $rootDirectory'); logger.printTrace('Scanning project files'); await _scanDirectory(rootDirectory, recursive: true, fileFilter: fileFilter); if (fs.isFileSync(_packagesFilePath)) { printTrace('Scanning package files'); await _scanPackages(fileFilter); } if (bundle != null) { printTrace('Scanning asset files'); bundle.entries.forEach((String archivePath, DevFSContent content) { _scanBundleEntry(archivePath, content); }); } // Handle deletions. printTrace('Scanning for deleted files'); final String assetBuildDirPrefix = _asUriPath(getAssetBuildDirectory()); final List toRemove = []; _entries.forEach((Uri deviceUri, DevFSContent content) { if (!content._exists) { final Future> operation = _operations.deleteFile(fsName, deviceUri) .then>((dynamic v) => v?.cast()); if (operation != null) _pendingOperations.add(operation); toRemove.add(deviceUri); if (deviceUri.path.startsWith(assetBuildDirPrefix)) { final String archivePath = deviceUri.path.substring(assetBuildDirPrefix.length); assetPathsToEvict.add(archivePath); } } }); if (toRemove.isNotEmpty) { printTrace('Removing deleted files'); toRemove.forEach(_entries.remove); await Future.wait>(_pendingOperations); _pendingOperations.clear(); } // Update modified files int numBytes = 0; final Map dirtyEntries = {}; _entries.forEach((Uri deviceUri, DevFSContent content) { String archivePath; if (deviceUri.path.startsWith(assetBuildDirPrefix)) archivePath = deviceUri.path.substring(assetBuildDirPrefix.length); // When doing full restart, copy content so that isModified does not // reset last check timestamp because we want to report all modified // files to incremental compiler next time user does hot reload. if (content.isModified || ((bundleDirty || bundleFirstUpload) && archivePath != null)) { dirtyEntries[deviceUri] = content; numBytes += content.size; if (archivePath != null && (!bundleFirstUpload || content.isModifiedAfter(firstBuildTime))) assetPathsToEvict.add(archivePath); } }); // We run generator even if [dirtyEntries] was empty because we want to // keep logic of accepting/rejecting generator's output simple: we must // accept/reject generator's output after every [update] call. Incremental // run with no changes is supposed to be fast (considering that it is // initiated by user key press). final List invalidatedFiles = []; final Set filesUris = Set(); for (Uri uri in dirtyEntries.keys.toList()) { if (!uri.path.startsWith(assetBuildDirPrefix)) { final DevFSContent content = dirtyEntries[uri]; if (content is DevFSFileContent) { filesUris.add(uri); invalidatedFiles.add(content.file.uri.toString()); numBytes -= content.size; } } } // No need to send source files because all compilation is done on the // host and result of compilation is single kernel file. filesUris.forEach(dirtyEntries.remove); printTrace('Compiling dart to kernel with ${invalidatedFiles.length} updated files'); if (fullRestart) { generator.reset(); } final CompilerOutput compilerOutput = await generator.recompile( mainPath, invalidatedFiles, outputPath: dillOutputPath ?? getDefaultApplicationKernelPath(trackWidgetCreation: trackWidgetCreation), packagesFilePath : _packagesFilePath, ); // Don't send full kernel file that would overwrite what VM already // started loading from. if (!bundleFirstUpload) { final String compiledBinary = compilerOutput?.outputFilename; if (compiledBinary != null && compiledBinary.isNotEmpty) { final Uri entryUri = fs.path.toUri(projectRootPath != null ? fs.path.relative(pathToReload, from: projectRootPath) : pathToReload, ); if (!dirtyEntries.containsKey(entryUri)) { final DevFSFileContent content = DevFSFileContent(fs.file(compiledBinary)); dirtyEntries[entryUri] = content; numBytes += content.size; } } } if (dirtyEntries.isNotEmpty) { printTrace('Updating files'); if (_httpWriter != null) { try { await _httpWriter.write(dirtyEntries); } on SocketException catch (socketException, stackTrace) { printTrace('DevFS sync failed. Lost connection to device: $socketException'); throw DevFSException('Lost connection to device.', socketException, stackTrace); } catch (exception, stackTrace) { printError('Could not update files on device: $exception'); throw DevFSException('Sync failed', exception, stackTrace); } } else { // Make service protocol requests for each. dirtyEntries.forEach((Uri deviceUri, DevFSContent content) { final Future> operation = _operations.writeFile(fsName, deviceUri, content) .then>((dynamic v) => v?.cast()); if (operation != null) _pendingOperations.add(operation); }); await Future.wait>(_pendingOperations, eagerError: true); _pendingOperations.clear(); } } printTrace('DevFS: Sync finished'); return numBytes; } void _scanFile(Uri deviceUri, FileSystemEntity file) { final DevFSContent content = _entries.putIfAbsent(deviceUri, () => DevFSFileContent(file)); content._exists = true; } void _scanBundleEntry(String archivePath, DevFSContent content) { // We write the assets into the AssetBundle working dir so that they // are in the same location in DevFS and the iOS simulator. final Uri deviceUri = fs.path.toUri(fs.path.join(getAssetBuildDirectory(), archivePath)); _entries[deviceUri] = content; content._exists = true; } bool _shouldIgnore(Uri deviceUri) { final List ignoredUriPrefixes = ['android/', _asUriPath(getBuildDirectory()), 'ios/', '.pub/']; for (String ignoredUriPrefix in ignoredUriPrefixes) { if (deviceUri.path.startsWith(ignoredUriPrefix)) return true; } return false; } bool _shouldSkip(FileSystemEntity file, String relativePath, Uri directoryUriOnDevice, { bool ignoreDotFiles = true, }) { if (file is Directory) { // Skip non-files. return true; } assert((file is Link) || (file is File)); final String basename = fs.path.basename(file.path); if (ignoreDotFiles && basename.startsWith('.')) { // Skip dot files, but not the '.packages' file (even though in dart1 // mode devfs['.packages'] will be overwritten with synthesized string content). return basename != '.packages'; } return false; } Uri _directoryUriOnDevice(Uri directoryUriOnDevice, Directory directory) { if (directoryUriOnDevice == null) { final String relativeRootPath = fs.path.relative(directory.path, from: rootDirectory.path); if (relativeRootPath == '.') { directoryUriOnDevice = Uri(); } else { directoryUriOnDevice = fs.path.toUri(relativeRootPath); } } return directoryUriOnDevice; } /// Scan all files from the [fileFilter] that are contained in [directory] and /// pass various filters (e.g. ignoreDotFiles). Future _scanFilteredDirectory(Set fileFilter, Directory directory, {Uri directoryUriOnDevice, bool ignoreDotFiles = true}) async { directoryUriOnDevice = _directoryUriOnDevice(directoryUriOnDevice, directory); try { final String absoluteDirectoryPath = canonicalizePath(directory.path); // For each file in the file filter. for (String filePath in fileFilter) { if (!filePath.startsWith(absoluteDirectoryPath)) { // File is not in this directory. Skip. continue; } final String relativePath = fs.path.relative(filePath, from: directory.path); final FileSystemEntity file = fs.file(filePath); if (_shouldSkip(file, relativePath, directoryUriOnDevice, ignoreDotFiles: ignoreDotFiles)) { continue; } final Uri deviceUri = directoryUriOnDevice.resolveUri(fs.path.toUri(relativePath)); if (!_shouldIgnore(deviceUri)) _scanFile(deviceUri, file); } } on FileSystemException catch (e) { _printScanDirectoryError(directory.path, e); return false; } return true; } /// Scan all files in [directory] that pass various filters (e.g. ignoreDotFiles). Future _scanDirectory(Directory directory, {Uri directoryUriOnDevice, bool recursive = false, bool ignoreDotFiles = true, Set fileFilter}) async { directoryUriOnDevice = _directoryUriOnDevice(directoryUriOnDevice, directory); if ((fileFilter != null) && fileFilter.isNotEmpty) { // When the fileFilter isn't empty, we can skip crawling the directory // tree and instead use the fileFilter as the source of potential files. return _scanFilteredDirectory(fileFilter, directory, directoryUriOnDevice: directoryUriOnDevice, ignoreDotFiles: ignoreDotFiles); } try { final Stream files = directory.list(recursive: recursive, followLinks: false); await for (FileSystemEntity file in files) { if (!devFSConfig.noDirectorySymlinks && (file is Link)) { // Check if this is a symlink to a directory and skip it. try { final FileSystemEntityType linkType = fs.statSync(file.resolveSymbolicLinksSync()).type; if (linkType == FileSystemEntityType.directory) continue; } on FileSystemException catch (e) { _printScanDirectoryError(file.path, e); continue; } } final String relativePath = fs.path.relative(file.path, from: directory.path); if (_shouldSkip(file, relativePath, directoryUriOnDevice, ignoreDotFiles: ignoreDotFiles)) { continue; } final Uri deviceUri = directoryUriOnDevice.resolveUri(fs.path.toUri(relativePath)); if (!_shouldIgnore(deviceUri)) _scanFile(deviceUri, file); } } on FileSystemException catch (e) { _printScanDirectoryError(directory.path, e); return false; } return true; } void _printScanDirectoryError(String path, Exception e) { printError( 'Error while scanning $path.\n' 'Hot Reload might not work until the following error is resolved:\n' '$e\n' ); } Future _scanPackages(Set fileFilter) async { StringBuffer sb; final PackageMap packageMap = PackageMap(_packagesFilePath); for (String packageName in packageMap.map.keys) { final Uri packageUri = packageMap.map[packageName]; final String packagePath = fs.path.fromUri(packageUri); final Directory packageDirectory = fs.directory(packageUri); Uri directoryUriOnDevice = fs.path.toUri(fs.path.join('packages', packageName) + fs.path.separator); bool packageExists = packageDirectory.existsSync(); if (!packageExists) { // If the package directory doesn't exist at all, we ignore it. continue; } if (fs.path.isWithin(rootDirectory.path, packagePath)) { // We already scanned everything under the root directory. directoryUriOnDevice = fs.path.toUri( fs.path.relative(packagePath, from: rootDirectory.path) + fs.path.separator ); } else { packageExists = await _scanDirectory(packageDirectory, directoryUriOnDevice: directoryUriOnDevice, recursive: true, fileFilter: fileFilter); } if (packageExists) { sb ??= StringBuffer(); sb.writeln('$packageName:$directoryUriOnDevice'); } } } } /// Converts a platform-specific file path to a platform-independent Uri path. String _asUriPath(String filePath) => fs.path.toUri(filePath).path + '/';