file_store.dart 7.75 KB
Newer Older
Ian Hickson's avatar
Ian Hickson committed
1
// Copyright 2014 The Flutter Authors. All rights reserved.
2 3 4 5 6 7 8 9 10
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:collection';
import 'dart:typed_data';

import 'package:crypto/crypto.dart';

import '../base/file_system.dart';
11
import '../base/logger.dart';
12
import '../base/utils.dart';
13
import '../convert.dart';
14
import 'hash.dart';
15

16 17 18 19 20
/// An encoded representation of all file hashes.
class FileStorage {
  FileStorage(this.version, this.files);

  factory FileStorage.fromBuffer(Uint8List buffer) {
21 22 23 24
    final Map<String, dynamic>? json = castStringKeyedMap(jsonDecode(utf8.decode(buffer)));
    if (json == null) {
      throw Exception('File storage format invalid');
    }
25
    final int version = json['version'] as int;
26
    final List<Map<String, dynamic>> rawCachedFiles = (json['files'] as List<dynamic>).cast<Map<String, dynamic>>();
27 28
    final List<FileHash> cachedFiles = <FileHash>[
      for (final Map<String, dynamic> rawFile in rawCachedFiles) FileHash._fromJson(rawFile),
29 30 31 32 33
    ];
    return FileStorage(version, cachedFiles);
  }

  final int version;
34
  final List<FileHash> files;
35 36 37 38 39

  List<int> toBuffer() {
    final Map<String, Object> json = <String, Object>{
      'version': version,
      'files': <Object>[
40
        for (final FileHash file in files) file.toJson(),
41 42 43 44 45 46 47
      ],
    };
    return utf8.encode(jsonEncode(json));
  }
}

/// A stored file hash and path.
48 49
class FileHash {
  FileHash(this.path, this.hash);
50

51
  factory FileHash._fromJson(Map<String, dynamic> json) {
52 53 54
    if (!json.containsKey('path') || !json.containsKey('hash')) {
      throw Exception('File storage format invalid');
    }
55
    return FileHash(json['path']! as String, json['hash']! as String);
56 57 58 59 60 61 62 63 64 65 66 67
  }

  final String path;
  final String hash;

  Object toJson() {
    return <String, Object>{
      'path': path,
      'hash': hash,
    };
  }
}
68

69 70 71 72 73 74 75 76 77 78 79 80
/// The strategy used by [FileStore] to determine if a file has been
/// invalidated.
enum FileStoreStrategy {
  /// The [FileStore] will compute an md5 hash of the file contents.
  hash,

  /// The [FileStore] will check for differences in the file's last modified
  /// timestamp.
  timestamp,
}

/// A globally accessible cache of files.
81 82 83
///
/// In cases where multiple targets read the same source files as inputs, we
/// avoid recomputing or storing multiple copies of hashes by delegating
84 85 86 87 88 89
/// through this class.
///
/// This class uses either timestamps or file hashes depending on the
/// provided [FileStoreStrategy]. All information  is held in memory during
/// a build operation, and may be persisted to cache in the root build
/// directory.
90 91
///
/// The format of the file store is subject to change and not part of its API.
92 93
class FileStore {
  FileStore({
94 95
    required File cacheFile,
    required Logger logger,
96 97 98
    FileStoreStrategy strategy = FileStoreStrategy.hash,
  }) : _logger = logger,
       _strategy = strategy,
99
       _cacheFile = cacheFile;
100

101
  final File _cacheFile;
102
  final Logger _logger;
103
  final FileStoreStrategy _strategy;
104

105 106
  final HashMap<String, String> previousAssetKeys = HashMap<String, String>();
  final HashMap<String, String> currentAssetKeys = HashMap<String, String>();
107 108

  // The name of the file which stores the file hashes.
109
  static const String kFileCache = '.filecache';
110 111

  // The current version of the file cache storage format.
112
  static const int _kVersion = 2;
113 114 115

  /// Read file hashes from disk.
  void initialize() {
116
    _logger.printTrace('Initializing file store');
117
    if (!_cacheFile.existsSync()) {
118 119
      return;
    }
120
    Uint8List data;
121
    try {
122
      data = _cacheFile.readAsBytesSync();
123
    } on FileSystemException catch (err) {
124
      _logger.printError(
125
        'Failed to read file store at ${_cacheFile.path} due to $err.\n'
126 127 128 129 130 131
        'Build artifacts will not be cached. Try clearing the cache directories '
        'with "flutter clean"',
      );
      return;
    }

132 133 134
    FileStorage fileStorage;
    try {
      fileStorage = FileStorage.fromBuffer(data);
135 136
    } on Exception catch (err) {
      _logger.printTrace('Filestorage format changed: $err');
137
      _cacheFile.deleteSync();
138 139
      return;
    }
140
    if (fileStorage.version != _kVersion) {
141
      _logger.printTrace('file cache format updating, clearing old hashes.');
142
      _cacheFile.deleteSync();
143 144
      return;
    }
145
    for (final FileHash fileHash in fileStorage.files) {
146
      previousAssetKeys[fileHash.path] = fileHash.hash;
147
    }
148
    _logger.printTrace('Done initializing file store');
149 150
  }

151
  /// Persist file marks to disk for a non-incremental build.
152
  void persist() {
153
    _logger.printTrace('Persisting file store');
154 155
    if (!_cacheFile.existsSync()) {
      _cacheFile.createSync(recursive: true);
156
    }
157
    final List<FileHash> fileHashes = <FileHash>[];
158
    for (final MapEntry<String, String> entry in currentAssetKeys.entries) {
159
      fileHashes.add(FileHash(entry.key, entry.value));
160
    }
161 162 163 164
    final FileStorage fileStorage = FileStorage(
      _kVersion,
      fileHashes,
    );
165
    final List<int> buffer = fileStorage.toBuffer();
166
    try {
167
      _cacheFile.writeAsBytesSync(buffer);
168
    } on FileSystemException catch (err) {
169
      _logger.printError(
170
        'Failed to persist file store at ${_cacheFile.path} due to $err.\n'
171 172 173 174
        'Build artifacts will not be cached. Try clearing the cache directories '
        'with "flutter clean"',
      );
    }
175
    _logger.printTrace('Done persisting file store');
176 177
  }

178 179 180 181 182 183 184 185
  /// Reset `previousMarks` for an incremental build.
  void persistIncremental() {
    previousAssetKeys.clear();
    previousAssetKeys.addAll(currentAssetKeys);
    currentAssetKeys.clear();
  }

  /// Computes a diff of the provided files and returns a list of files
186
  /// that were dirty.
187
  List<File> diffFileList(List<File> files) {
188
    final List<File> dirty = <File>[];
189 190
    switch (_strategy) {
      case FileStoreStrategy.hash:
191 192 193
        for (final File file in files) {
          _hashFile(file, dirty);
        }
194 195 196 197 198 199 200
        break;
      case FileStoreStrategy.timestamp:
        for (final File file in files) {
          _checkModification(file, dirty);
        }
        break;
    }
201 202
    return dirty;
  }
203

204 205
  void _checkModification(File file, List<File> dirty) {
    final String absolutePath = file.path;
206
    final String? previousTime = previousAssetKeys[absolutePath];
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221

    // If the file is missing it is assumed to be dirty.
    if (!file.existsSync()) {
      currentAssetKeys.remove(absolutePath);
      previousAssetKeys.remove(absolutePath);
      dirty.add(file);
      return;
    }
    final String modifiedTime = file.lastModifiedSync().toString();
    if (modifiedTime != previousTime) {
      dirty.add(file);
    }
    currentAssetKeys[absolutePath] = modifiedTime;
  }

222 223 224 225 226
  // 64k is the same sized buffer used by dart:io for `File.openRead`.
  static final Uint8List _readBuffer = Uint8List(64 * 1024);

  void _hashFile(File file, List<File> dirty) {
    final String absolutePath = file.path;
227
    final String? previousHash = previousAssetKeys[absolutePath];
228 229 230 231 232 233 234 235 236
    // If the file is missing it is assumed to be dirty.
    if (!file.existsSync()) {
      currentAssetKeys.remove(absolutePath);
      previousAssetKeys.remove(absolutePath);
      dirty.add(file);
      return;
    }
    final int fileBytes = file.lengthSync();
    final Md5Hash hash = Md5Hash();
237
    RandomAccessFile? openFile;
238
    try {
239
      openFile = file.openSync();
240 241 242 243 244
      int bytes = 0;
      while (bytes < fileBytes) {
        final int bytesRead = openFile.readIntoSync(_readBuffer);
        hash.addChunk(_readBuffer, bytesRead);
        bytes += bytesRead;
245
      }
246
    } finally {
247 248 249 250 251 252
      openFile?.closeSync();
    }
    final Digest digest = Digest(hash.finalize().buffer.asUint8List());
    final String currentHash = digest.toString();
    if (currentHash != previousHash) {
      dirty.add(file);
253
    }
254
    currentAssetKeys[absolutePath] = currentHash;
255 256
  }
}