devfs.dart 16 KB
Newer Older
1 2 3 4 5
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
6

7
import 'package:json_rpc_2/json_rpc_2.dart' as rpc;
8
import 'package:meta/meta.dart';
9

10
import 'asset.dart';
11
import 'base/context.dart';
12
import 'base/file_system.dart';
13
import 'base/io.dart';
14
import 'build_info.dart';
15
import 'bundle.dart';
16
import 'compile.dart';
17
import 'convert.dart' show base64, utf8;
18 19
import 'dart/package_map.dart';
import 'globals.dart';
20
import 'vmservice.dart';
21

22 23 24
class DevFSConfig {
  /// Should DevFS assume that symlink targets are stable?
  bool cacheSymlinks = false;
25 26
  /// Should DevFS assume that there are no symlinks to directories?
  bool noDirectorySymlinks = false;
27 28
}

29
DevFSConfig get devFSConfig => context.get<DevFSConfig>();
30

31 32
/// Common superclass for content copied to the device.
abstract class DevFSContent {
33
  /// Return true if this is the first time this method is called
34 35
  /// or if the entry has been modified since this method was last called.
  bool get isModified;
36

37 38 39 40 41
  /// Return true if this is the first time this method is called
  /// or if the entry has been modified after the given time
  /// or if the given time is null.
  bool isModifiedAfter(DateTime time);

42 43
  int get size;

44
  Future<List<int>> contentsAsBytes();
45

46 47 48 49
  Stream<List<int>> contentsAsStream();

  Stream<List<int>> contentsAsCompressedStream() {
    return contentsAsStream().cast<List<int>>().transform<List<int>>(gzip.encoder);
50
  }
51

52 53
  /// Return the list of files this content depends on.
  List<String> get fileDependencies => <String>[];
54 55
}

56
// File content to be copied to the device.
57 58
class DevFSFileContent extends DevFSContent {
  DevFSFileContent(this.file);
59

60
  final FileSystemEntity file;
61
  FileSystemEntity _linkTarget;
62
  FileStat _fileStat;
63

64 65 66
  File _getFile() {
    if (_linkTarget != null) {
      return _linkTarget;
67
    }
68 69 70
    if (file is Link) {
      // The link target.
      return fs.file(file.resolveSymbolicLinksSync());
71
    }
72
    return file;
73 74
  }

75
  void _stat() {
76 77
    if (_linkTarget != null) {
      // Stat the cached symlink target.
78 79 80 81 82 83 84
      final FileStat fileStat = _linkTarget.statSync();
      if (fileStat.type == FileSystemEntityType.notFound) {
        _linkTarget = null;
      } else {
        _fileStat = fileStat;
        return;
      }
85
    }
86 87 88
    final FileStat fileStat = file.statSync();
    _fileStat = fileStat.type == FileSystemEntityType.notFound ? null : fileStat;
    if (_fileStat != null && _fileStat.type == FileSystemEntityType.link) {
89
      // Resolve, stat, and maybe cache the symlink target.
90 91
      final String resolved = file.resolveSymbolicLinksSync();
      final FileSystemEntity linkTarget = fs.file(resolved);
92
      // Stat the link target.
93 94 95 96 97
      final FileStat fileStat = linkTarget.statSync();
      if (fileStat.type == FileSystemEntityType.notFound) {
        _fileStat = null;
        _linkTarget = null;
      } else if (devFSConfig.cacheSymlinks) {
98 99
        _linkTarget = linkTarget;
      }
100
    }
101 102 103
    if (_fileStat == null) {
      printError('Unable to get status of file "${file.path}": file not found.');
    }
104
  }
105

106 107 108
  @override
  List<String> get fileDependencies => <String>[_getFile().path];

109 110
  @override
  bool get isModified {
111
    final FileStat _oldFileStat = _fileStat;
112
    _stat();
113
    if (_oldFileStat == null && _fileStat == null) {
114
      return false;
115
    }
116
    return _oldFileStat == null || _fileStat == null || _fileStat.modified.isAfter(_oldFileStat.modified);
117 118
  }

119 120 121 122
  @override
  bool isModifiedAfter(DateTime time) {
    final FileStat _oldFileStat = _fileStat;
    _stat();
123
    if (_oldFileStat == null && _fileStat == null) {
124
      return false;
125
    }
126 127 128 129
    return time == null
        || _oldFileStat == null
        || _fileStat == null
        || _fileStat.modified.isAfter(time);
130 131
  }

132 133
  @override
  int get size {
134
    if (_fileStat == null) {
135
      _stat();
136
    }
137 138
    // Can still be null if the file wasn't found.
    return _fileStat?.size ?? 0;
139
  }
140

141
  @override
142
  Future<List<int>> contentsAsBytes() => _getFile().readAsBytes();
143 144

  @override
145
  Stream<List<int>> contentsAsStream() => _getFile().openRead();
146 147 148 149 150 151 152 153 154
}

/// Byte content to be copied to the device.
class DevFSByteContent extends DevFSContent {
  DevFSByteContent(this._bytes);

  List<int> _bytes;

  bool _isModified = true;
155
  DateTime _modificationTime = DateTime.now();
156 157 158

  List<int> get bytes => _bytes;

159 160
  set bytes(List<int> value) {
    _bytes = value;
161
    _isModified = true;
162
    _modificationTime = DateTime.now();
163 164
  }

165
  /// Return true only once so that the content is written to the device only once.
166 167
  @override
  bool get isModified {
168
    final bool modified = _isModified;
169 170
    _isModified = false;
    return modified;
171
  }
172

173 174 175 176 177
  @override
  bool isModifiedAfter(DateTime time) {
    return time == null || _modificationTime.isAfter(time);
  }

178 179 180 181
  @override
  int get size => _bytes.length;

  @override
182
  Future<List<int>> contentsAsBytes() async => _bytes;
183 184

  @override
185 186
  Stream<List<int>> contentsAsStream() =>
      Stream<List<int>>.fromIterable(<List<int>>[_bytes]);
187 188
}

189
/// String content to be copied to the device.
190
class DevFSStringContent extends DevFSByteContent {
191 192 193
  DevFSStringContent(String string)
    : _string = string,
      super(utf8.encode(string));
194 195 196 197 198

  String _string;

  String get string => _string;

199 200
  set string(String value) {
    _string = value;
201
    super.bytes = utf8.encode(_string);
202 203 204
  }

  @override
205
  set bytes(List<int> value) {
206
    string = utf8.decode(value);
207 208
  }
}
209

210 211 212 213 214 215
/// Abstract DevFS operations interface.
abstract class DevFSOperations {
  Future<Uri> create(String fsName);
  Future<dynamic> destroy(String fsName);
  Future<dynamic> writeFile(String fsName, Uri deviceUri, DevFSContent content);
}
216

217 218 219 220
/// An implementation of [DevFSOperations] that speaks to the
/// vm service.
class ServiceProtocolDevFSOperations implements DevFSOperations {
  ServiceProtocolDevFSOperations(this.vmService);
221

222
  final VMService vmService;
223

224
  @override
225 226 227 228 229
  Future<Uri> create(String fsName) async {
    final Map<String, dynamic> response = await vmService.vm.createDevFS(fsName);
    return Uri.parse(response['uri']);
  }

230 231
  @override
  Future<dynamic> destroy(String fsName) async {
232 233
    await vmService.vm.deleteDevFS(fsName);
  }
234

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
  @override
  Future<dynamic> writeFile(String fsName, Uri deviceUri, DevFSContent content) async {
    List<int> bytes;
    try {
      bytes = await content.contentsAsBytes();
    } catch (e) {
      return e;
    }
    final String fileContents = base64.encode(bytes);
    try {
      return await vmService.vm.invokeRpcRaw(
        '_writeDevFSFile',
        params: <String, dynamic>{
          'fsName': fsName,
          'uri': deviceUri.toString(),
          'fileContents': fileContents,
        },
      );
    } catch (error) {
      printTrace('DevFS: Failed to write $deviceUri: $error');
    }
  }
}

class DevFSException implements Exception {
  DevFSException(this.message, [this.error, this.stackTrace]);
  final String message;
  final dynamic error;
  final StackTrace stackTrace;
}

class _DevFSHttpWriter {
  _DevFSHttpWriter(this.fsName, VMService serviceProtocol)
    : httpAddress = serviceProtocol.httpAddress;

  final String fsName;
  final Uri httpAddress;

  static const int kMaxInFlight = 6;

  int _inFlight = 0;
  Map<Uri, DevFSContent> _outstanding;
  Completer<void> _completer;
  final HttpClient _client = HttpClient();

280
  Future<void> write(Map<Uri, DevFSContent> entries) async {
281
    _client.maxConnectionsPerHost = kMaxInFlight;
282
    _completer = Completer<void>();
283
    _outstanding = Map<Uri, DevFSContent>.from(entries);
284
    _scheduleWrites();
285 286 287
    await _completer.future;
  }

288
  void _scheduleWrites() {
289
    while ((_inFlight < kMaxInFlight) && (!_completer.isCompleted) && _outstanding.isNotEmpty) {
290 291
      final Uri deviceUri = _outstanding.keys.first;
      final DevFSContent content = _outstanding.remove(deviceUri);
292
      _startWrite(deviceUri, content, retry: 10);
293
      _inFlight += 1;
294
    }
295
    if ((_inFlight == 0) && (!_completer.isCompleted) && _outstanding.isEmpty) {
296
      _completer.complete();
297
    }
298 299
  }

300
  Future<void> _startWrite(
301
    Uri deviceUri,
302
    DevFSContent content, {
303
    int retry = 0,
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
  }) async {
    while(true) {
      try {
        final HttpClientRequest request = await _client.putUrl(httpAddress);
        request.headers.removeAll(HttpHeaders.acceptEncodingHeader);
        request.headers.add('dev_fs_name', fsName);
        request.headers.add('dev_fs_uri_b64', base64.encode(utf8.encode('$deviceUri')));
        final Stream<List<int>> contents = content.contentsAsCompressedStream();
        await request.addStream(contents);
        final HttpClientResponse response = await request.close();
        response.listen((_) => null,
            onError: (dynamic error) { printTrace('error: $error'); },
            cancelOnError: true);
        break;
      } catch (error, trace) {
        if (!_completer.isCompleted) {
          printTrace('Error writing "$deviceUri" to DevFS: $error');
          if (retry > 0) {
            retry--;
            printTrace('trying again in a few - $retry more attempts left');
            await Future<void>.delayed(const Duration(milliseconds: 500));
            continue;
          }
          _completer.completeError(error, trace);
        }
329
      }
Ryan Macnak's avatar
Ryan Macnak committed
330
    }
331 332
    _inFlight -= 1;
    _scheduleWrites();
333 334 335
  }
}

336 337
// Basic statistics for DevFS update operation.
class UpdateFSReport {
338 339 340 341 342
  UpdateFSReport({
    bool success = false,
    int invalidatedSourcesCount = 0,
    int syncedBytes = 0,
  }) {
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
    _success = success;
    _invalidatedSourcesCount = invalidatedSourcesCount;
    _syncedBytes = syncedBytes;
  }

  bool get success => _success;
  int get invalidatedSourcesCount => _invalidatedSourcesCount;
  int get syncedBytes => _syncedBytes;

  void incorporateResults(UpdateFSReport report) {
    if (!report._success) {
      _success = false;
    }
    _invalidatedSourcesCount += report._invalidatedSourcesCount;
    _syncedBytes += report._syncedBytes;
  }

  bool _success;
  int _invalidatedSourcesCount;
  int _syncedBytes;
}

365
class DevFS {
366
  /// Create a [DevFS] named [fsName] for the local files in [rootDirectory].
367 368 369 370
  DevFS(
    VMService serviceProtocol,
    this.fsName,
    this.rootDirectory, {
371
    String packagesFilePath,
372 373
  }) : _operations = ServiceProtocolDevFSOperations(serviceProtocol),
       _httpWriter = _DevFSHttpWriter(fsName, serviceProtocol),
374
       _packagesFilePath = packagesFilePath ?? fs.path.join(rootDirectory.path, kPackagesFileName);
375

376 377 378 379 380
  DevFS.operations(
    this._operations,
    this.fsName,
    this.rootDirectory, {
    String packagesFilePath,
381 382
  }) : _httpWriter = null,
       _packagesFilePath = packagesFilePath ?? fs.path.join(rootDirectory.path, kPackagesFileName);
383 384

  final DevFSOperations _operations;
385
  final _DevFSHttpWriter _httpWriter;
386 387
  final String fsName;
  final Directory rootDirectory;
388
  final String _packagesFilePath;
389
  final Set<String> assetPathsToEvict = <String>{};
390 391
  List<Uri> sources = <Uri>[];
  DateTime lastCompiled;
392

393 394 395
  Uri _baseUri;
  Uri get baseUri => _baseUri;

396 397 398 399 400 401 402 403 404 405
  Uri deviceUriToHostUri(Uri deviceUri) {
    final String deviceUriString = deviceUri.toString();
    final String baseUriString = baseUri.toString();
    if (deviceUriString.startsWith(baseUriString)) {
      final String deviceUriSuffix = deviceUriString.substring(baseUriString.length);
      return rootDirectory.uri.resolve(deviceUriSuffix);
    }
    return deviceUri;
  }

406
  Future<Uri> create() async {
407
    printTrace('DevFS: Creating new filesystem on the device ($_baseUri)');
408 409 410 411
    try {
      _baseUri = await _operations.create(fsName);
    } on rpc.RpcException catch (rpcException) {
      // 1001 is kFileSystemAlreadyExists in //dart/runtime/vm/json_stream.h
412
      if (rpcException.code != 1001) {
413
        rethrow;
414
      }
415 416 417 418
      printTrace('DevFS: Creating failed. Destroying and trying again');
      await destroy();
      _baseUri = await _operations.create(fsName);
    }
419 420 421 422
    printTrace('DevFS: Created new filesystem on the device ($_baseUri)');
    return _baseUri;
  }

423
  Future<void> destroy() async {
424 425
    printTrace('DevFS: Deleting filesystem on the device ($_baseUri)');
    await _operations.destroy(fsName);
426 427 428
    printTrace('DevFS: Deleted filesystem on the device ($_baseUri)');
  }

429 430 431
  /// Updates files on the device.
  ///
  /// Returns the number of bytes synced.
432
  Future<UpdateFSReport> update({
433
    @required String mainPath,
434
    String target,
435
    AssetBundle bundle,
436
    DateTime firstBuildTime,
437
    bool bundleFirstUpload = false,
438
    @required ResidentCompiler generator,
439
    String dillOutputPath,
440
    @required bool trackWidgetCreation,
441
    bool fullRestart = false,
442
    String projectRootPath,
443
    @required String pathToReload,
444
    @required List<Uri> invalidatedFiles,
445
  }) async {
446 447
    assert(trackWidgetCreation != null);
    assert(generator != null);
448

449 450 451 452 453
    // Update modified files
    final String assetBuildDirPrefix = _asUriPath(getAssetBuildDirectory());
    final Map<Uri, DevFSContent> dirtyEntries = <Uri, DevFSContent>{};

    int syncedBytes = 0;
454
    if (bundle != null) {
455
      printTrace('Scanning asset files');
456 457 458
      // We write the assets into the AssetBundle working dir so that they
      // are in the same location in DevFS and the iOS simulator.
      final String assetDirectory = getAssetBuildDirectory();
459
      bundle.entries.forEach((String archivePath, DevFSContent content) {
460
        final Uri deviceUri = fs.path.toUri(fs.path.join(assetDirectory, archivePath));
461 462 463 464 465 466 467 468 469 470 471 472
        if (deviceUri.path.startsWith(assetBuildDirPrefix)) {
          archivePath = deviceUri.path.substring(assetBuildDirPrefix.length);
        }
        // Only update assets if they have been modified, or if this is the
        // first upload of the asset bundle.
        if (content.isModified || (bundleFirstUpload && archivePath != null)) {
          dirtyEntries[deviceUri] = content;
          syncedBytes += content.size;
          if (archivePath != null && !bundleFirstUpload) {
            assetPathsToEvict.add(archivePath);
          }
        }
473
      });
474
    }
475 476 477
    if (fullRestart) {
      generator.reset();
    }
478
    printTrace('Compiling dart to kernel with ${invalidatedFiles.length} updated files');
479
    lastCompiled = DateTime.now();
480 481 482
    final CompilerOutput compilerOutput = await generator.recompile(
      mainPath,
      invalidatedFiles,
483
      outputPath:  dillOutputPath ?? getDefaultApplicationKernelPath(trackWidgetCreation: trackWidgetCreation),
484 485
      packagesFilePath : _packagesFilePath,
    );
486
    if (compilerOutput == null || compilerOutput.errorCount > 0) {
487 488
      return UpdateFSReport(success: false);
    }
489
    // list of sources that needs to be monitored are in [compilerOutput.sources]
490
    sources = compilerOutput.sources;
491
    //
492 493 494 495 496 497 498 499 500
    // Don't send full kernel file that would overwrite what VM already
    // started loading from.
    if (!bundleFirstUpload) {
      final String compiledBinary = compilerOutput?.outputFilename;
      if (compiledBinary != null && compiledBinary.isNotEmpty) {
        final Uri entryUri = fs.path.toUri(projectRootPath != null
          ? fs.path.relative(pathToReload, from: projectRootPath)
          : pathToReload,
        );
501 502 503
        final DevFSFileContent content = DevFSFileContent(fs.file(compiledBinary));
        syncedBytes += content.size;
        dirtyEntries[entryUri] = content;
504
      }
505
    }
506
    printTrace('Updating files');
507
    if (dirtyEntries.isNotEmpty) {
508
      try {
509
        await _httpWriter.write(dirtyEntries);
510 511 512 513 514 515
      } on SocketException catch (socketException, stackTrace) {
        printTrace('DevFS sync failed. Lost connection to device: $socketException');
        throw DevFSException('Lost connection to device.', socketException, stackTrace);
      } catch (exception, stackTrace) {
        printError('Could not update files on device: $exception');
        throw DevFSException('Sync failed', exception, stackTrace);
516
      }
517
    }
518
    printTrace('DevFS: Sync finished');
519
    return UpdateFSReport(success: true, syncedBytes: syncedBytes,
520
         invalidatedSourcesCount: invalidatedFiles.length);
521
  }
522
}
523

524 525
/// Converts a platform-specific file path to a platform-independent Uri path.
String _asUriPath(String filePath) => fs.path.toUri(filePath).path + '/';