devfs.dart 20.9 KB
Newer Older
1 2 3 4 5 6 7
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert' show BASE64, UTF8;

8 9
import 'package:json_rpc_2/json_rpc_2.dart' as rpc;

10
import 'asset.dart';
11
import 'base/context.dart';
12
import 'base/file_system.dart';
13
import 'base/io.dart';
14
import 'build_info.dart';
15 16
import 'dart/package_map.dart';
import 'globals.dart';
17
import 'vmservice.dart';
18

19 20 21
class DevFSConfig {
  /// Should DevFS assume that symlink targets are stable?
  bool cacheSymlinks = false;
22 23
  /// Should DevFS assume that there are no symlinks to directories?
  bool noDirectorySymlinks = false;
24 25 26 27
}

DevFSConfig get devFSConfig => context[DevFSConfig];

28 29 30
/// Common superclass for content copied to the device.
abstract class DevFSContent {
  bool _exists = true;
31

32 33 34
  /// Return `true` if this is the first time this method is called
  /// or if the entry has been modified since this method was last called.
  bool get isModified;
35

36 37 38 39 40 41 42 43 44
  int get size;

  Future<List<int>> contentsAsBytes();

  Stream<List<int>> contentsAsStream();

  Stream<List<int>> contentsAsCompressedStream() {
    return contentsAsStream().transform(GZIP.encoder);
  }
45 46 47

  /// Return the list of files this content depends on.
  List<String> get fileDependencies => <String>[];
48 49 50 51 52
}

// File content to be copied to the device.
class DevFSFileContent extends DevFSContent {
  DevFSFileContent(this.file);
53

54
  final FileSystemEntity file;
55
  FileSystemEntity _linkTarget;
56
  FileStat _fileStat;
57

58 59 60
  File _getFile() {
    if (_linkTarget != null) {
      return _linkTarget;
61
    }
62 63 64
    if (file is Link) {
      // The link target.
      return fs.file(file.resolveSymbolicLinksSync());
65
    }
66
    return file;
67 68
  }

69
  void _stat() {
70 71 72 73 74
    if (_linkTarget != null) {
      // Stat the cached symlink target.
      _fileStat = _linkTarget.statSync();
      return;
    }
75
    _fileStat = file.statSync();
76
    if (_fileStat.type == FileSystemEntityType.LINK) {
77
      // Resolve, stat, and maybe cache the symlink target.
78 79
      final String resolved = file.resolveSymbolicLinksSync();
      final FileSystemEntity linkTarget = fs.file(resolved);
80 81 82 83 84
      // Stat the link target.
      _fileStat = linkTarget.statSync();
      if (devFSConfig.cacheSymlinks) {
        _linkTarget = linkTarget;
      }
85
    }
86
  }
87

88 89 90
  @override
  List<String> get fileDependencies => <String>[_getFile().path];

91 92
  @override
  bool get isModified {
93
    final FileStat _oldFileStat = _fileStat;
94 95
    _stat();
    return _oldFileStat == null || _fileStat.modified.isAfter(_oldFileStat.modified);
96 97
  }

98 99 100 101 102
  @override
  int get size {
    if (_fileStat == null)
      _stat();
    return _fileStat.size;
103
  }
104

105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  @override
  Future<List<int>> contentsAsBytes() => _getFile().readAsBytes();

  @override
  Stream<List<int>> contentsAsStream() => _getFile().openRead();
}

/// Byte content to be copied to the device.
class DevFSByteContent extends DevFSContent {
  DevFSByteContent(this._bytes);

  List<int> _bytes;

  bool _isModified = true;

  List<int> get bytes => _bytes;

122 123
  set bytes(List<int> value) {
    _bytes = value;
124
    _isModified = true;
125 126
  }

127 128 129
  /// Return `true` only once so that the content is written to the device only once.
  @override
  bool get isModified {
130
    final bool modified = _isModified;
131 132
    _isModified = false;
    return modified;
133
  }
134 135 136 137 138 139 140 141 142 143

  @override
  int get size => _bytes.length;

  @override
  Future<List<int>> contentsAsBytes() async => _bytes;

  @override
  Stream<List<int>> contentsAsStream() =>
      new Stream<List<int>>.fromIterable(<List<int>>[_bytes]);
144 145
}

146 147 148 149 150 151 152 153
/// String content to be copied to the device.
class DevFSStringContent extends DevFSByteContent {
  DevFSStringContent(String string) : _string = string, super(UTF8.encode(string));

  String _string;

  String get string => _string;

154 155
  set string(String value) {
    _string = value;
156 157 158 159
    super.bytes = UTF8.encode(_string);
  }

  @override
160 161
  set bytes(List<int> value) {
    string = UTF8.decode(value);
162 163
  }
}
164 165 166 167 168

/// Abstract DevFS operations interface.
abstract class DevFSOperations {
  Future<Uri> create(String fsName);
  Future<dynamic> destroy(String fsName);
169 170
  Future<dynamic> writeFile(String fsName, Uri deviceUri, DevFSContent content);
  Future<dynamic> deleteFile(String fsName, Uri deviceUri);
171 172 173
}

/// An implementation of [DevFSOperations] that speaks to the
174
/// vm service.
175
class ServiceProtocolDevFSOperations implements DevFSOperations {
176
  final VMService vmService;
177

178
  ServiceProtocolDevFSOperations(this.vmService);
179 180 181

  @override
  Future<Uri> create(String fsName) async {
182
    final Map<String, dynamic> response = await vmService.vm.createDevFS(fsName);
183 184 185 186 187
    return Uri.parse(response['uri']);
  }

  @override
  Future<dynamic> destroy(String fsName) async {
188
    await vmService.vm.deleteDevFS(fsName);
189 190 191
  }

  @override
192
  Future<dynamic> writeFile(String fsName, Uri deviceUri, DevFSContent content) async {
193 194
    List<int> bytes;
    try {
195
      bytes = await content.contentsAsBytes();
196 197 198
    } catch (e) {
      return e;
    }
199
    final String fileContents = BASE64.encode(bytes);
200
    try {
201 202 203 204
      return await vmService.vm.invokeRpcRaw(
        '_writeDevFSFile',
        params: <String, dynamic> {
          'fsName': fsName,
205
          'uri': deviceUri.toString(),
206 207 208 209
          'fileContents': fileContents
        },
      );
    } catch (error) {
210
      printTrace('DevFS: Failed to write $deviceUri: $error');
211
    }
212 213
  }

214
  @override
215
  Future<dynamic> deleteFile(String fsName, Uri deviceUri) async {
216 217
    // TODO(johnmccutchan): Add file deletion to the devFS protocol.
  }
218 219
}

220 221 222 223 224 225 226
class DevFSException implements Exception {
  DevFSException(this.message, [this.error, this.stackTrace]);
  final String message;
  final dynamic error;
  final StackTrace stackTrace;
}

227
class _DevFSHttpWriter {
228
  _DevFSHttpWriter(this.fsName, VMService serviceProtocol)
229 230 231 232 233 234
      : httpAddress = serviceProtocol.httpAddress;

  final String fsName;
  final Uri httpAddress;

  static const int kMaxInFlight = 6;
235
  static const int kMaxRetries = 3;
236 237

  int _inFlight = 0;
238
  Map<Uri, DevFSContent> _outstanding;
239
  Completer<Null> _completer;
240
  HttpClient _client;
241

242
  Future<Null> write(Map<Uri, DevFSContent> entries) async {
243
    _client = new HttpClient();
244 245
    _client.maxConnectionsPerHost = kMaxInFlight;
    _completer = new Completer<Null>();
246
    _outstanding = new Map<Uri, DevFSContent>.from(entries);
247
    _scheduleWrites();
248 249 250 251
    await _completer.future;
    _client.close();
  }

252
  void _scheduleWrites() {
253
    while (_inFlight < kMaxInFlight) {
254
      if (_outstanding.isEmpty) {
255 256 257
        // Finished.
        break;
      }
258 259
      final Uri deviceUri = _outstanding.keys.first;
      final DevFSContent content = _outstanding.remove(deviceUri);
260
      _scheduleWrite(deviceUri, content);
261 262 263 264
      _inFlight++;
    }
  }

265
  Future<Null> _scheduleWrite(
266
    Uri deviceUri,
267
    DevFSContent content, [
268 269
    int retry = 0,
  ]) async {
Ryan Macnak's avatar
Ryan Macnak committed
270
    try {
271
      final HttpClientRequest request = await _client.putUrl(httpAddress);
272
      request.headers.removeAll(HttpHeaders.ACCEPT_ENCODING);
Ryan Macnak's avatar
Ryan Macnak committed
273
      request.headers.add('dev_fs_name', fsName);
274
      request.headers.add('dev_fs_uri_b64',
275
          BASE64.encode(UTF8.encode(deviceUri.toString())));
276
      final Stream<List<int>> contents = content.contentsAsCompressedStream();
Ryan Macnak's avatar
Ryan Macnak committed
277
      await request.addStream(contents);
278
      final HttpClientResponse response = await request.close();
279
      await response.drain<Null>();
280 281 282 283 284
    } on SocketException catch (socketException, stackTrace) {
      // We have one completer and can get up to kMaxInFlight errors.
      if (!_completer.isCompleted)
        _completer.completeError(socketException, stackTrace);
      return;
285
    } catch (e) {
286
      if (retry < kMaxRetries) {
287
        printTrace('Retrying writing "$deviceUri" to DevFS due to error: $e');
288
        _scheduleWrite(deviceUri, content, retry + 1);
289 290
        return;
      } else {
291
        printError('Error writing "$deviceUri" to DevFS: $e');
292
      }
Ryan Macnak's avatar
Ryan Macnak committed
293
    }
294
    _inFlight--;
295
    if ((_outstanding.isEmpty) && (_inFlight == 0)) {
296 297
      _completer.complete(null);
    } else {
298
      _scheduleWrites();
299 300 301 302
    }
  }
}

303 304
class DevFS {
  /// Create a [DevFS] named [fsName] for the local files in [directory].
305
  DevFS(VMService serviceProtocol,
306
        this.fsName,
307 308 309
        this.rootDirectory, {
        String packagesFilePath
      })
310
    : _operations = new ServiceProtocolDevFSOperations(serviceProtocol),
311
      _httpWriter = new _DevFSHttpWriter(fsName, serviceProtocol) {
312
    _packagesFilePath =
313
        packagesFilePath ?? fs.path.join(rootDirectory.path, kPackagesFileName);
314
  }
315 316 317

  DevFS.operations(this._operations,
                   this.fsName,
318 319 320 321 322
                   this.rootDirectory, {
                   String packagesFilePath,
      })
    : _httpWriter = null {
    _packagesFilePath =
323
        packagesFilePath ?? fs.path.join(rootDirectory.path, kPackagesFileName);
324
  }
325 326

  final DevFSOperations _operations;
327
  final _DevFSHttpWriter _httpWriter;
328 329
  final String fsName;
  final Directory rootDirectory;
330
  String _packagesFilePath;
331
  final Map<Uri, DevFSContent> _entries = <Uri, DevFSContent>{};
332
  final Set<String> assetPathsToEvict = new Set<String>();
333

334
  final List<Future<Map<String, dynamic>>> _pendingOperations =
335
      <Future<Map<String, dynamic>>>[];
336

337 338 339 340
  Uri _baseUri;
  Uri get baseUri => _baseUri;

  Future<Uri> create() async {
341
    printTrace('DevFS: Creating new filesystem on the device ($_baseUri)');
342 343 344 345 346 347 348 349 350 351
    try {
      _baseUri = await _operations.create(fsName);
    } on rpc.RpcException catch (rpcException) {
      // 1001 is kFileSystemAlreadyExists in //dart/runtime/vm/json_stream.h
      if (rpcException.code != 1001)
        rethrow;
      printTrace('DevFS: Creating failed. Destroying and trying again');
      await destroy();
      _baseUri = await _operations.create(fsName);
    }
352 353 354 355
    printTrace('DevFS: Created new filesystem on the device ($_baseUri)');
    return _baseUri;
  }

356 357 358
  Future<Null> destroy() async {
    printTrace('DevFS: Deleting filesystem on the device ($_baseUri)');
    await _operations.destroy(fsName);
359 360 361
    printTrace('DevFS: Deleted filesystem on the device ($_baseUri)');
  }

362
  /// Update files on the device and return the number of bytes sync'd
363 364 365 366 367
  Future<int> update({
    AssetBundle bundle,
    bool bundleDirty: false,
    Set<String> fileFilter,
  }) async {
368 369 370 371 372 373
    // Mark all entries as possibly deleted.
    for (DevFSContent content in _entries.values) {
      content._exists = false;
    }

    // Scan workspace, packages, and assets
374
    printTrace('DevFS: Starting sync from $rootDirectory');
375
    logger.printTrace('Scanning project files');
376
    await _scanDirectory(rootDirectory,
377 378
                         recursive: true,
                         fileFilter: fileFilter);
379
    if (fs.isFileSync(_packagesFilePath)) {
380 381
      printTrace('Scanning package files');
      await _scanPackages(fileFilter);
382
    }
383
    if (bundle != null) {
384
      printTrace('Scanning asset files');
385 386 387
      bundle.entries.forEach((String archivePath, DevFSContent content) {
        _scanBundleEntry(archivePath, content, bundleDirty);
      });
388
    }
389

390
    // Handle deletions.
391
    printTrace('Scanning for deleted files');
392
    final String assetBuildDirPrefix = _asUriPath(getAssetBuildDirectory());
393
    final List<Uri> toRemove = <Uri>[];
394
    _entries.forEach((Uri deviceUri, DevFSContent content) {
395
      if (!content._exists) {
396
        final Future<Map<String, dynamic>> operation =
397
            _operations.deleteFile(fsName, deviceUri);
398 399
        if (operation != null)
          _pendingOperations.add(operation);
400 401
        toRemove.add(deviceUri);
        if (deviceUri.path.startsWith(assetBuildDirPrefix)) {
402
          final String archivePath = deviceUri.path.substring(assetBuildDirPrefix.length);
403 404
          assetPathsToEvict.add(archivePath);
        }
405
      }
406 407 408 409
    });
    if (toRemove.isNotEmpty) {
      printTrace('Removing deleted files');
      toRemove.forEach(_entries.remove);
410 411 412
      await Future.wait(_pendingOperations);
      _pendingOperations.clear();
    }
413

414 415
    // Update modified files
    int numBytes = 0;
416
    final Map<Uri, DevFSContent> dirtyEntries = <Uri, DevFSContent>{};
417
    _entries.forEach((Uri deviceUri, DevFSContent content) {
418
      String archivePath;
419 420
      if (deviceUri.path.startsWith(assetBuildDirPrefix))
        archivePath = deviceUri.path.substring(assetBuildDirPrefix.length);
421
      if (content.isModified || (bundleDirty && archivePath != null)) {
422
        dirtyEntries[deviceUri] = content;
423 424 425 426 427
        numBytes += content.size;
        if (archivePath != null)
          assetPathsToEvict.add(archivePath);
      }
    });
428
    if (dirtyEntries.isNotEmpty) {
429
      printTrace('Updating files');
430
      if (_httpWriter != null) {
431
        try {
432
          await _httpWriter.write(dirtyEntries);
433 434 435 436 437 438
        } on SocketException catch (socketException, stackTrace) {
          printTrace("DevFS sync failed. Lost connection to device: $socketException");
          throw new DevFSException('Lost connection to device.', socketException, stackTrace);
        } catch (exception, stackTrace) {
          printError("Could not update files on device: $exception");
          throw new DevFSException('Sync failed', exception, stackTrace);
439
        }
440 441
      } else {
        // Make service protocol requests for each.
442
        dirtyEntries.forEach((Uri deviceUri, DevFSContent content) {
443
          final Future<Map<String, dynamic>> operation =
444
              _operations.writeFile(fsName, deviceUri, content);
445 446
          if (operation != null)
            _pendingOperations.add(operation);
447
        });
448 449 450
        await Future.wait(_pendingOperations, eagerError: true);
        _pendingOperations.clear();
      }
451 452
    }

453
    printTrace('DevFS: Sync finished');
454
    return numBytes;
455 456
  }

457
  void _scanFile(Uri deviceUri, FileSystemEntity file) {
458
    final DevFSContent content = _entries.putIfAbsent(deviceUri, () => new DevFSFileContent(file));
459
    content._exists = true;
460 461
  }

462 463 464
  void _scanBundleEntry(String archivePath, DevFSContent content, bool bundleDirty) {
    // We write the assets into the AssetBundle working dir so that they
    // are in the same location in DevFS and the iOS simulator.
465
    final Uri deviceUri = fs.path.toUri(fs.path.join(getAssetBuildDirectory(), archivePath));
466

467
    _entries[deviceUri] = content;
468
    content._exists = true;
469 470
  }

471
  bool _shouldIgnore(Uri deviceUri) {
472
    final List<String> ignoredUriPrefixes = <String>['android/',
473 474 475 476 477
                                               _asUriPath(getBuildDirectory()),
                                               'ios/',
                                               '.pub/'];
    for (String ignoredUriPrefix in ignoredUriPrefixes) {
      if (deviceUri.path.startsWith(ignoredUriPrefix))
478 479 480 481 482
        return true;
    }
    return false;
  }

483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
  bool _shouldSkip(FileSystemEntity file,
                   String relativePath,
                   Uri directoryUriOnDevice, {
                   bool ignoreDotFiles: true,
                   }) {
    if (file is Directory) {
      // Skip non-files.
      return true;
    }
    assert((file is Link) || (file is File));
    if (ignoreDotFiles && fs.path.basename(file.path).startsWith('.')) {
      // Skip dot files.
      return true;
    }
    return false;
  }

  Uri _directoryUriOnDevice(Uri directoryUriOnDevice,
                            Directory directory) {
502
    if (directoryUriOnDevice == null) {
503
      final String relativeRootPath = fs.path.relative(directory.path, from: rootDirectory.path);
504 505 506 507 508
      if (relativeRootPath == '.') {
        directoryUriOnDevice = new Uri();
      } else {
        directoryUriOnDevice = fs.path.toUri(relativeRootPath);
      }
509
    }
510 511 512 513 514 515 516 517 518 519 520 521
    return directoryUriOnDevice;
  }

  /// Scan all files from the [fileFilter] that are contained in [directory] and
  /// pass various filters (e.g. ignoreDotFiles).
  Future<bool> _scanFilteredDirectory(Set<String> fileFilter,
                                      Directory directory,
                                      {Uri directoryUriOnDevice,
                                       bool ignoreDotFiles: true}) async {
    directoryUriOnDevice =
        _directoryUriOnDevice(directoryUriOnDevice, directory);
    try {
522
      final String absoluteDirectoryPath = canonicalizePath(directory.path);
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
      // For each file in the file filter.
      for (String filePath in fileFilter) {
        if (!filePath.startsWith(absoluteDirectoryPath)) {
          // File is not in this directory. Skip.
          continue;
        }
        final String relativePath =
          fs.path.relative(filePath, from: directory.path);
        final FileSystemEntity file = fs.file(filePath);
        if (_shouldSkip(file, relativePath, directoryUriOnDevice, ignoreDotFiles: ignoreDotFiles)) {
          continue;
        }
        final Uri deviceUri = directoryUriOnDevice.resolveUri(fs.path.toUri(relativePath));
        if (!_shouldIgnore(deviceUri))
          _scanFile(deviceUri, file);
      }
    } on FileSystemException catch (e) {
      _printScanDirectoryError(directory.path, e);
      return false;
    }
    return true;
  }

  /// Scan all files in [directory] that pass various filters (e.g. ignoreDotFiles).
  Future<bool> _scanDirectory(Directory directory,
                              {Uri directoryUriOnDevice,
                               bool recursive: false,
                               bool ignoreDotFiles: true,
                               Set<String> fileFilter}) async {
    directoryUriOnDevice = _directoryUriOnDevice(directoryUriOnDevice, directory);
    if ((fileFilter != null) && fileFilter.isNotEmpty) {
      // When the fileFilter isn't empty, we can skip crawling the directory
      // tree and instead use the fileFilter as the source of potential files.
      return _scanFilteredDirectory(fileFilter,
                                    directory,
                                    directoryUriOnDevice: directoryUriOnDevice,
                                    ignoreDotFiles: ignoreDotFiles);
    }
561
    try {
562
      final Stream<FileSystemEntity> files =
563 564
          directory.list(recursive: recursive, followLinks: false);
      await for (FileSystemEntity file in files) {
565 566
        if (!devFSConfig.noDirectorySymlinks && (file is Link)) {
          // Check if this is a symlink to a directory and skip it.
567 568 569 570 571 572 573
          try {
            final FileSystemEntityType linkType =
                fs.statSync(file.resolveSymbolicLinksSync()).type;
            if (linkType == FileSystemEntityType.DIRECTORY)
              continue;
          } on FileSystemException catch (e) {
            _printScanDirectoryError(file.path, e);
574 575 576
            continue;
          }
        }
577
        final String relativePath =
578
          fs.path.relative(file.path, from: directory.path);
579
        if (_shouldSkip(file, relativePath, directoryUriOnDevice, ignoreDotFiles: ignoreDotFiles)) {
580 581
          continue;
        }
582
        final Uri deviceUri = directoryUriOnDevice.resolveUri(fs.path.toUri(relativePath));
583 584
        if (!_shouldIgnore(deviceUri))
          _scanFile(deviceUri, file);
585
      }
586 587
    } on FileSystemException catch (e) {
      _printScanDirectoryError(directory.path, e);
588 589 590 591
      return false;
    }
    return true;
  }
592

593 594 595 596 597 598 599 600
  void _printScanDirectoryError(String path, Exception e) {
    printError(
        'Error while scanning $path.\n'
        'Hot Reload might not work until the following error is resolved:\n'
        '$e\n'
    );
  }

601 602
  Future<Null> _scanPackages(Set<String> fileFilter) async {
    StringBuffer sb;
603
    final PackageMap packageMap = new PackageMap(_packagesFilePath);
604 605

    for (String packageName in packageMap.map.keys) {
606
      final Uri packageUri = packageMap.map[packageName];
607
      final String packagePath = fs.path.fromUri(packageUri);
608
      final Directory packageDirectory = fs.directory(packageUri);
609
      Uri directoryUriOnDevice = fs.path.toUri(fs.path.join('packages', packageName) + fs.path.separator);
610 611 612 613 614 615
      bool packageExists = packageDirectory.existsSync();

      if (!packageExists) {
        // If the package directory doesn't exist at all, we ignore it.
        continue;
      }
616 617 618

      if (fs.path.isWithin(rootDirectory.path, packagePath)) {
        // We already scanned everything under the root directory.
619 620 621
        directoryUriOnDevice = fs.path.toUri(
            fs.path.relative(packagePath, from: rootDirectory.path) + fs.path.separator
        );
622 623 624
      } else {
        packageExists =
            await _scanDirectory(packageDirectory,
625
                                 directoryUriOnDevice: directoryUriOnDevice,
626 627 628
                                 recursive: true,
                                 fileFilter: fileFilter);
      }
629 630
      if (packageExists) {
        sb ??= new StringBuffer();
631
        sb.writeln('$packageName:$directoryUriOnDevice');
632 633 634
      }
    }
    if (sb != null) {
635
      final DevFSContent content = _entries[fs.path.toUri('.packages')];
636 637 638 639
      if (content is DevFSStringContent && content.string == sb.toString()) {
        content._exists = true;
        return;
      }
640
      _entries[fs.path.toUri('.packages')] = new DevFSStringContent(sb.toString());
641 642
    }
  }
643
}
644 645
/// Converts a platform-specific file path to a platform-independent Uri path.
String _asUriPath(String filePath) => fs.path.toUri(filePath).path + '/';