[flutter_tools] improve hash performance in build_system (#70065)

packages/flutter_tools/lib/src/build_system/build_system.dart

@@ -772,7 +772,7 @@ class _BuildInstance {
       // If we're missing a depfile, wait until after evaluating the target to
       // compute changes.
       final bool canSkip = !node.missingDepfile &&
-        await node.computeChanges(environment, fileCache, fileSystem, logger);
+        node.computeChanges(environment, fileCache, fileSystem, logger);
 
       if (canSkip) {
         skipped = true;
@@ -802,11 +802,11 @@ class _BuildInstance {
       // If we were missing the depfile, resolve input files after executing the
       // target so that all file hashes are up to date on the next run.
       if (node.missingDepfile) {
-        await fileCache.diffFileList(node.inputs);
+        fileCache.diffFileList(node.inputs);
       }
 
       // Always update hashes for output files.
-      await fileCache.diffFileList(node.outputs);
+      fileCache.diffFileList(node.outputs);
       node.target._writeStamp(node.inputs, node.outputs, environment);
       updateGraph();
 
@@ -1001,12 +1001,12 @@ class Node {
   /// Collect hashes for all inputs to determine if any have changed.
   ///
   /// Returns whether this target can be skipped.
-  Future<bool> computeChanges(
+  bool computeChanges(
     Environment environment,
     FileStore fileStore,
     FileSystem fileSystem,
     Logger logger,
-  ) async {
+  ) {
     final Set<String> currentOutputPaths = <String>{
       for (final File file in outputs) file.path,
     };
@@ -1075,7 +1075,7 @@ class Node {
     // If we have files to diff, compute them asynchronously and then
     // update the result.
     if (sourcesToDiff.isNotEmpty) {
-      final List<File> dirty = await fileStore.diffFileList(sourcesToDiff);
+      final List<File> dirty = fileStore.diffFileList(sourcesToDiff);
       if (dirty.isNotEmpty) {
         invalidatedReasons.add(InvalidatedReason.inputChanged);
         _dirty = true;

packages/flutter_tools/lib/src/build_system/file_store.dart

@@ -2,22 +2,17 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:async';
 import 'dart:collection';
 import 'dart:typed_data';
 
 import 'package:crypto/crypto.dart';
 import 'package:meta/meta.dart';
-import 'package:pool/pool.dart';
 
 import '../base/file_system.dart';
 import '../base/logger.dart';
 import '../base/utils.dart';
 import '../convert.dart';
-import 'build_system.dart';
-
-/// The default threshold for file chunking is 250 KB, or about the size of `framework.dart`.
-const int kDefaultFileChunkThresholdBytes = 250000;
+import 'hash.dart';
 
 /// An encoded representation of all file hashes.
 class FileStorage {
@@ -94,16 +89,13 @@ class FileStore {
     @required File cacheFile,
     @required Logger logger,
     FileStoreStrategy strategy = FileStoreStrategy.hash,
-    int fileChunkThreshold = kDefaultFileChunkThresholdBytes,
   }) : _logger = logger,
        _strategy = strategy,
-       _cacheFile = cacheFile,
-       _fileChunkThreshold = fileChunkThreshold;
+       _cacheFile = cacheFile;
 
   final File _cacheFile;
   final Logger _logger;
   final FileStoreStrategy _strategy;
-  final int _fileChunkThreshold;
 
   final HashMap<String, String> previousAssetKeys = HashMap<String, String>();
   final HashMap<String, String> currentAssetKeys = HashMap<String, String>();
@@ -187,14 +179,13 @@ class FileStore {
 
   /// Computes a diff of the provided files and returns a list of files
   /// that were dirty.
-  Future<List<File>> diffFileList(List<File> files) async {
+  List<File> diffFileList(List<File> files) {
     final List<File> dirty = <File>[];
     switch (_strategy) {
       case FileStoreStrategy.hash:
-        final Pool openFiles = Pool(kMaxOpenFiles);
-        await Future.wait(<Future<void>>[
-          for (final File file in files) _hashFile(file, dirty, openFiles)
-        ]);
+        for (final File file in files) {
+          _hashFile(file, dirty);
+        }
         break;
       case FileStoreStrategy.timestamp:
         for (final File file in files) {
@@ -223,37 +214,38 @@ class FileStore {
     currentAssetKeys[absolutePath] = modifiedTime;
   }
 
-  Future<void> _hashFile(File file, List<File> dirty, Pool pool) async {
-    final PoolResource resource = await pool.request();
+  // 64k is the same sized buffer used by dart:io for `File.openRead`.
+  static final Uint8List _readBuffer = Uint8List(64 * 1024);
+
+  void _hashFile(File file, List<File> dirty) {
+    final String absolutePath = file.path;
+    final String previousHash = previousAssetKeys[absolutePath];
+    // If the file is missing it is assumed to be dirty.
+    if (!file.existsSync()) {
+      currentAssetKeys.remove(absolutePath);
+      previousAssetKeys.remove(absolutePath);
+      dirty.add(file);
+      return;
+    }
+    final int fileBytes = file.lengthSync();
+    final Md5Hash hash = Md5Hash();
+    RandomAccessFile openFile;
     try {
-      final String absolutePath = file.path;
-      final String previousHash = previousAssetKeys[absolutePath];
-      // If the file is missing it is assumed to be dirty.
-      if (!file.existsSync()) {
-        currentAssetKeys.remove(absolutePath);
-        previousAssetKeys.remove(absolutePath);
-        dirty.add(file);
-        return;
+      openFile = file.openSync(mode: FileMode.read);
+      int bytes = 0;
+      while (bytes < fileBytes) {
+        final int bytesRead = openFile.readIntoSync(_readBuffer);
+        hash.addChunk(_readBuffer, bytesRead);
+        bytes += bytesRead;
       }
-      Digest digest;
-      final int fileBytes = file.lengthSync();
-      // For files larger than a given threshold, chunk the conversion.
-      if (fileBytes > _fileChunkThreshold) {
-        final StreamController<Digest> digests = StreamController<Digest>();
-        final ByteConversionSink inputSink = md5.startChunkedConversion(digests);
-        await file.openRead().forEach(inputSink.add);
-        inputSink.close();
-        digest = await digests.stream.last;
-      } else {
-        digest = md5.convert(await file.readAsBytes());
-      }
-      final String currentHash = digest.toString();
-      if (currentHash != previousHash) {
-        dirty.add(file);
-      }
-      currentAssetKeys[absolutePath] = currentHash;
     } finally {
-      resource.release();
+      openFile?.closeSync();
+    }
+    final Digest digest = Digest(hash.finalize().buffer.asUint8List());
+    final String currentHash = digest.toString();
+    if (currentHash != previousHash) {
+      dirty.add(file);
     }
+    currentAssetKeys[absolutePath] = currentHash;
   }
 }

packages/flutter_tools/lib/src/build_system/hash.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:typed_data';
+
+/// Data from a non-linear mathematical function that functions as
+/// reproducible noise.
+final Uint32List _noise = Uint32List.fromList(<int>[
+  0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a,
+  0xa8304613, 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
+  0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340,
+  0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
+  0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, 0xa9e3e905, 0xfcefa3f8,
+  0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
+  0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa,
+  0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
+  0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92,
+  0xffeff47d, 0x85845dd1, 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
+  0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391
+]);
+
+/// Per-round shift amounts.
+const List<int> _shiftAmounts = <int>[
+  07, 12, 17, 22, 07, 12, 17, 22, 07, 12, 17, 22, 07, 12, 17, 22, 05, 09, 14,
+  20, 05, 09, 14, 20, 05, 09, 14, 20, 05, 09, 14, 20, 04, 11, 16, 23, 04, 11,
+  16, 23, 04, 11, 16, 23, 04, 11, 16, 23, 06, 10, 15, 21, 06, 10, 15, 21, 06,
+  10, 15, 21, 06, 10, 15, 21
+];
+
+/// A bitmask that limits an integer to 32 bits.
+const int _mask32 = 0xFFFFFFFF;
+
+/// An incremental hash computation of md5.
+class Md5Hash {
+  Md5Hash() {
+    _digest[0] = 0x67452301;
+    _digest[1] = 0xefcdab89;
+    _digest[2] = 0x98badcfe;
+    _digest[3] = 0x10325476;
+  }
+
+  // 64 bytes is 512 bits.
+  static const int _kChunkSize = 64;
+
+  /// The current hash digest.
+  final Uint32List _digest = Uint32List(4);
+  final Uint8List _scratchSpace = Uint8List(_kChunkSize);
+  int _remainingLength = 0;
+  int _contentLength = 0;
+
+  void addChunk(Uint8List data, [int stop]) {
+    assert(_remainingLength == 0);
+    stop ??= data.length;
+    int i = 0;
+    for (; i <= stop - _kChunkSize; i += _kChunkSize) {
+      final Uint32List view = Uint32List.view(data.buffer, i, 16);
+      _writeChunk(view);
+    }
+    if (i != stop) {
+      // The data must be copied so that the provided buffer can be reused.
+      int j = 0;
+      for (; i < stop; i += 1) {
+        _scratchSpace[j] = data[i];
+        j += 1;
+      }
+      _remainingLength = j;
+    }
+    _contentLength += stop;
+  }
+
+  void _writeChunk(Uint32List chunk) {
+    // help dart remove bounds checks
+    // ignore: unnecessary_statements
+    chunk[15];
+    // ignore: unnecessary_statements
+    _shiftAmounts[63];
+    // ignore: unnecessary_statements
+    _noise[63];
+
+    int d = _digest[3];
+    int c = _digest[2];
+    int b = _digest[1];
+    int a = _digest[0];
+    int e = 0;
+    int f = 0;
+    int i = 0;
+    for (; i < 16; i += 1) {
+      e = (b & c) | ((~b & _mask32) & d);
+      f = i;
+      final int temp = d;
+      d = c;
+      c = b;
+      b = _add32(
+          b,
+          _rotl32(_add32(_add32(a, e), _add32(_noise[i], chunk[f])),
+              _shiftAmounts[i]));
+      a = temp;
+    }
+    for (; i < 32; i += 1) {
+      e = (d & b) | ((~d & _mask32) & c);
+      f = ((5 * i) + 1) % 16;
+      final int temp = d;
+      d = c;
+      c = b;
+      b = _add32(
+          b,
+          _rotl32(_add32(_add32(a, e), _add32(_noise[i], chunk[f])),
+              _shiftAmounts[i]));
+      a = temp;
+    }
+    for (; i < 48; i += 1) {
+      e = b ^ c ^ d;
+      f = ((3 * i) + 5) % 16;
+      final int temp = d;
+      d = c;
+      c = b;
+      b = _add32(
+          b,
+          _rotl32(_add32(_add32(a, e), _add32(_noise[i], chunk[f])),
+              _shiftAmounts[i]));
+      a = temp;
+    }
+    for (; i < 64; i+= 1) {
+      e = c ^ (b | (~d & _mask32));
+      f = (7 * i) % 16;
+      final int temp = d;
+      d = c;
+      c = b;
+      b = _add32(
+          b,
+          _rotl32(_add32(_add32(a, e), _add32(_noise[i], chunk[f])),
+              _shiftAmounts[i]));
+      a = temp;
+    }
+
+    _digest[0] += a;
+    _digest[1] += b;
+    _digest[2] += c;
+    _digest[3] += d;
+  }
+
+  Uint32List finalize() {
+    // help dart remove bounds checks
+    // ignore: unnecessary_statements
+    _scratchSpace[63];
+    _scratchSpace[_remainingLength] = 0x80;
+    _remainingLength += 1;
+
+    final int zeroes = 56 - _remainingLength;
+    for (int i = _remainingLength; i < zeroes; i += 1) {
+      _scratchSpace[i] = 0;
+    }
+    final int bitLength = _contentLength * 8;
+    _scratchSpace.buffer.asByteData().setUint64(56, bitLength, Endian.little);
+    _writeChunk(Uint32List.view(_scratchSpace.buffer, 0, 16));
+    return _digest;
+  }
+
+  /// Adds [x] and [y] with 32-bit overflow semantics.
+  int _add32(int x, int y) => (x + y) & _mask32;
+
+  /// Bitwise rotates [val] to the left by [shift], obeying 32-bit overflow
+  /// semantics.
+  int _rotl32(int val, int shift) {
+    final int modShift = shift & 31;
+    return ((val << modShift) & _mask32) | ((val & _mask32) >> (32 - modShift));
+  }
+}

packages/flutter_tools/test/general.shard/build_system/file_store_test.dart

@@ -4,7 +4,6 @@
 
 import 'dart:typed_data';
 
-import 'package:crypto/crypto.dart';
 import 'package:file/memory.dart';
 import 'package:file_testing/file_testing.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -40,27 +39,28 @@ void main() {
     final FileStore fileCache = FileStore(
       cacheFile: cacheFile,
       logger: BufferLogger.test(),
+      strategy: FileStoreStrategy.timestamp,
     );
     fileCache.initialize();
     final File file = fileSystem.file('test')..createSync();
 
     // Initial run does not contain any timestamps for file.
-    expect(await fileCache.diffFileList(<File>[file]), hasLength(1));
+    expect(fileCache.diffFileList(<File>[file]), hasLength(1));
 
     // Swap current timestamps to previous timestamps.
     fileCache.persistIncremental();
 
     // timestamp matches previous timestamp.
-    expect(await fileCache.diffFileList(<File>[file]), isEmpty);
+    expect(fileCache.diffFileList(<File>[file]), isEmpty);
 
     // clear current timestamp list.
     fileCache.persistIncremental();
 
     // modify the time stamp.
-    file.writeAsStringSync('foo');
+    file.setLastModifiedSync(DateTime(1991));
 
     // verify the file is marked as dirty again.
-    expect(await fileCache.diffFileList(<File>[file]), hasLength(1));
+    expect(fileCache.diffFileList(<File>[file]), hasLength(1));
   });
 
   testWithoutContext('FileStore saves and restores to file cache', () async {
@@ -75,7 +75,7 @@ void main() {
       ..writeAsStringSync('hello');
 
     fileCache.initialize();
-    await fileCache.diffFileList(<File>[file]);
+    fileCache.diffFileList(<File>[file]);
     fileCache.persist();
     final String currentHash =  fileCache.currentAssetKeys[file.path];
     final Uint8List buffer = cacheFile
@@ -119,7 +119,7 @@ void main() {
 
     cacheFile.parent.deleteSync(recursive: true);
 
-    await fileCache.diffFileList(<File>[file]);
+    fileCache.diffFileList(<File>[file]);
 
     expect(fileCache.persist, returnsNormally);
   });
@@ -133,7 +133,7 @@ void main() {
     );
     fileCache.initialize();
 
-    final List<File> results = await fileCache.diffFileList(<File>[fileSystem.file('hello.dart')]);
+    final List<File> results = fileCache.diffFileList(<File>[fileSystem.file('hello.dart')]);
 
     expect(results, hasLength(1));
     expect(results.single.path, 'hello.dart');
@@ -183,7 +183,6 @@ void main() {
     final FileStore fileCache = FileStore(
       cacheFile: cacheFile,
       logger: BufferLogger.test(),
-      fileChunkThreshold: 1, // Chunk files larger than 1 byte.
     );
     final File file = fileSystem.file('foo.dart')
       ..createSync()
@@ -192,11 +191,9 @@ void main() {
 
     cacheFile.parent.deleteSync(recursive: true);
 
-    await fileCache.diffFileList(<File>[file]);
+    fileCache.diffFileList(<File>[file]);
 
-    // Validate that chunked hash is the same as non-chunked.
-    expect(fileCache.currentAssetKeys['foo.dart'],
-      md5.convert(file.readAsBytesSync()).toString());
+    expect(fileCache.currentAssetKeys['foo.dart'], '5d41402abc4b2a76b9719d911017c592');
   });
 }
 

packages/flutter_tools/test/general.shard/build_system/hash_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:typed_data';
+
+import 'package:convert/convert.dart';
+import 'package:flutter_tools/src/build_system/hash.dart';
+
+import '../../src/common.dart';
+
+void main() {
+
+  // Examples taken from https://en.wikipedia.org/wiki/MD5
+  testWithoutContext('md5 control test zero length string', () {
+    final Md5Hash hash = Md5Hash();
+
+    expect(hex.encode(hash.finalize().buffer.asUint8List()), 'd41d8cd98f00b204e9800998ecf8427e');
+  });
+
+  testWithoutContext('md5 control test fox test', () {
+    final Md5Hash hash = Md5Hash();
+    hash.addChunk(ascii.encode('The quick brown fox jumps over the lazy dog'));
+
+    expect(hex.encode(hash.finalize().buffer.asUint8List()), '9e107d9d372bb6826bd81d3542a419d6');
+  });
+
+  testWithoutContext('md5 control test fox test with period', () {
+    final Md5Hash hash = Md5Hash();
+    hash.addChunk(ascii.encode('The quick brown fox jumps over the lazy dog.'));
+
+    expect(hex.encode(hash.finalize().buffer.asUint8List()), 'e4d909c290d0fb1ca068ffaddf22cbd0');
+  });
+
+  testWithoutContext('Can hash bytes less than 64 length', () {
+    final Uint8List bytes = Uint8List.fromList(<int>[1, 2, 3, 4, 5, 6]);
+    final Md5Hash hashA = Md5Hash();
+    hashA.addChunk(bytes);
+
+    expect(hashA.finalize(), <int>[1810219370, 268668871, 3900423769, 1277973076]);
+
+    final Md5Hash hashB = Md5Hash();
+    hashB.addChunk(bytes);
+
+    expect(hashB.finalize(), <int>[1810219370, 268668871, 3900423769, 1277973076]);
+  });
+
+  testWithoutContext('Can hash bytes exactly 64 length', () {
+    final Uint8List bytes = Uint8List.fromList(List<int>.filled(64, 2));
+    final Md5Hash hashA = Md5Hash();
+    hashA.addChunk(bytes);
+
+    expect(hashA.finalize(), <int>[260592333, 2557619848, 2729912077, 812879060]);
+
+    final Md5Hash hashB = Md5Hash();
+    hashB.addChunk(bytes);
+
+    expect(hashB.finalize(), <int>[260592333, 2557619848, 2729912077, 812879060]);
+  });
+
+  testWithoutContext('Can hash bytes more than 64 length', () {
+    final Uint8List bytes = Uint8List.fromList(List<int>.filled(514, 2));
+    final Md5Hash hashA = Md5Hash();
+    hashA.addChunk(bytes);
+
+    expect(hashA.finalize(), <int>[387658779, 2003142991, 243395797, 1487291259]);
+
+    final Md5Hash hashB = Md5Hash();
+    hashB.addChunk(bytes);
+
+    expect(hashB.finalize(), <int>[387658779, 2003142991, 243395797, 1487291259]);
+  });
+}

packages/flutter_tools/test/general.shard/resident_runner_test.dart

@@ -889,10 +889,10 @@ void main() {
 
   testUsingContext('ResidentRunner can send target platform to analytics from hot reload', () => testbed.run(() async {
     fakeVmServiceHost = FakeVmServiceHost(requests: <VmServiceExpectation>[
-      listViews,
       listViews,
       listViews,
       setAssetBundlePath,
+      listViews,
       FakeVmServiceRequest(
         method: 'getVM',
         jsonResponse: vm_service.VM.parse(<String, Object>{
@@ -945,6 +945,7 @@ void main() {
       appStartedCompleter: onAppStart,
       connectionInfoCompleter: onConnectionInfo,
     ));
+    await onAppStart.future;
 
     final OperationResult result = await residentRunner.restart(fullRestart: false);
     expect(result.fatal, false);
@@ -966,12 +967,12 @@ void main() {
         jsonResponse: fakeVM.toJson(),
       ),
       listViews,
+      setAssetBundlePath,
       listViews,
       FakeVmServiceRequest(
         method: 'getVM',
         jsonResponse: fakeVM.toJson(),
       ),
-      setAssetBundlePath,
       const FakeVmServiceRequest(
         method: 'reloadSources',
         args: <String, Object>{
@@ -1054,6 +1055,7 @@ void main() {
       connectionInfoCompleter: onConnectionInfo,
     ));
 
+    await onAppStart.future;
     final OperationResult result = await residentRunner.restart(fullRestart: false);
 
     expect(result.fatal, false);