[tool] Generate a binary version of the asset manifest (#117233)

* initial

* update asset_bundle_package_test

* Update asset_bundle_test.dart

* Update asset_bundle_package_fonts_test.dart

* update pubspec checksum for smc dependency

* flutter update-packages --force-upgrade

* prefer += 1 over ++
Co-authored-by: Jonah Williams <jonahwilliams@google.com>

* add regexp comment

* rescope int list comparison function

* update packages
Co-authored-by: Jonah Williams <jonahwilliams@google.com>

dev/benchmarks/multiple_flutters/module/pubspec.yaml

@@ -41,7 +41,7 @@ dependencies:
   typed_data: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   vector_math: 2.1.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   win32: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  xdg_directories: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  xdg_directories: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
 flutter:
   uses-material-design: true
@@ -51,4 +51,4 @@ flutter:
     androidPackage: com.example.multiple_flutters_module
     iosBundleIdentifier: com.example.multipleFluttersModule
 
-# PUBSPEC CHECKSUM: e9fc
+# PUBSPEC CHECKSUM: eafd

dev/integration_tests/android_views/pubspec.yaml

@@ -47,7 +47,7 @@ dependencies:
   vm_service: 9.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   webdriver: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   win32: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  xdg_directories: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  xdg_directories: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
 dev_dependencies:
   flutter_test:
@@ -94,4 +94,4 @@ dev_dependencies:
 flutter:
   uses-material-design: true
 
-# PUBSPEC CHECKSUM: 11b0
+# PUBSPEC CHECKSUM: 28b1

dev/integration_tests/hybrid_android_views/pubspec.yaml

@@ -45,7 +45,7 @@ dependencies:
   vm_service: 9.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   webdriver: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   win32: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  xdg_directories: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  xdg_directories: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
 dev_dependencies:
   flutter_test:
@@ -92,4 +92,4 @@ dev_dependencies:
 flutter:
   uses-material-design: true
 
-# PUBSPEC CHECKSUM: 11b0
+# PUBSPEC CHECKSUM: 28b1

packages/flutter_tools/lib/src/asset.dart

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:typed_data';
+
 import 'package:meta/meta.dart';
 import 'package:package_config/package_config.dart';
+import 'package:standard_message_codec/standard_message_codec.dart';
 
 import 'base/context.dart';
 import 'base/deferred_component.dart';
@@ -162,7 +165,11 @@ class ManifestAssetBundle implements AssetBundle {
 
   DateTime? _lastBuildTimestamp;
 
-  static const String _kAssetManifestJson = 'AssetManifest.json';
+  // We assume the main asset is designed for a device pixel ratio of 1.0.
+  static const double _defaultResolution = 1.0;
+  static const String _kAssetManifestJsonFilename = 'AssetManifest.json';
+  static const String _kAssetManifestBinFilename = 'AssetManifest.bin';
+
   static const String _kNoticeFile = 'NOTICES';
   // Comically, this can't be name with the more common .gz file extension
   // because when it's part of an AAR and brought into another APK via gradle,
@@ -230,8 +237,15 @@ class ManifestAssetBundle implements AssetBundle {
     // device.
     _lastBuildTimestamp = DateTime.now();
     if (flutterManifest.isEmpty) {
-      entries[_kAssetManifestJson] = DevFSStringContent('{}');
-      entryKinds[_kAssetManifestJson] = AssetKind.regular;
+      entries[_kAssetManifestJsonFilename] = DevFSStringContent('{}');
+      entryKinds[_kAssetManifestJsonFilename] = AssetKind.regular;
+      entries[_kAssetManifestJsonFilename] = DevFSStringContent('{}');
+      entryKinds[_kAssetManifestJsonFilename] = AssetKind.regular;
+      final ByteData emptyAssetManifest =
+        const StandardMessageCodec().encodeMessage(<dynamic, dynamic>{})!;
+      entries[_kAssetManifestBinFilename] =
+        DevFSByteContent(emptyAssetManifest.buffer.asUint8List(0, emptyAssetManifest.lengthInBytes));
+      entryKinds[_kAssetManifestBinFilename] = AssetKind.regular;
       return 0;
     }
 
@@ -428,7 +442,10 @@ class ManifestAssetBundle implements AssetBundle {
       _wildcardDirectories[uri] ??= _fileSystem.directory(uri);
     }
 
-    final DevFSStringContent assetManifest  = _createAssetManifest(assetVariants, deferredComponentsAssetVariants);
+    final Map<String, List<String>> assetManifest =
+      _createAssetManifest(assetVariants, deferredComponentsAssetVariants);
+    final DevFSStringContent assetManifestJson = DevFSStringContent(json.encode(assetManifest));
+    final DevFSByteContent assetManifestBinary = _createAssetManifestBinary(assetManifest);
     final DevFSStringContent fontManifest = DevFSStringContent(json.encode(fonts));
     final LicenseResult licenseResult = _licenseCollector.obtainLicenses(packageConfig, additionalLicenseFiles);
     if (licenseResult.errorMessages.isNotEmpty) {
@@ -452,7 +469,8 @@ class ManifestAssetBundle implements AssetBundle {
         _fileSystem.file('DOES_NOT_EXIST_RERUN_FOR_WILDCARD$suffix').absolute);
     }
 
-    _setIfChanged(_kAssetManifestJson, assetManifest, AssetKind.regular);
+    _setIfChanged(_kAssetManifestJsonFilename, assetManifestJson, AssetKind.regular);
+    _setIfChanged(_kAssetManifestBinFilename, assetManifestBinary, AssetKind.regular);
     _setIfChanged(kFontManifestJson, fontManifest, AssetKind.regular);
     _setLicenseIfChanged(licenseResult.combinedLicenses, targetPlatform);
     return 0;
@@ -460,18 +478,31 @@ class ManifestAssetBundle implements AssetBundle {
 
   @override
   List<File> additionalDependencies = <File>[];
-
-  void _setIfChanged(String key, DevFSStringContent content, AssetKind assetKind) {
-    if (!entries.containsKey(key)) {
-      entries[key] = content;
-      entryKinds[key] = assetKind;
+  void _setIfChanged(String key, DevFSContent content, AssetKind assetKind) {
+    final DevFSContent? oldContent = entries[key];
+    // In the case that the content is unchanged, we want to avoid an overwrite
+    // as the isModified property may be reset to true,
+    if (oldContent is DevFSByteContent && content is DevFSByteContent &&
+        _compareIntLists(oldContent.bytes, content.bytes)) {
       return;
     }
-    final DevFSStringContent? oldContent = entries[key] as DevFSStringContent?;
-    if (oldContent?.string != content.string) {
+
     entries[key] = content;
     entryKinds[key] = assetKind;
   }
+
+  static bool _compareIntLists(List<int> o1, List<int> o2) {
+    if (o1.length != o2.length) {
+      return false;
+    }
+
+    for (int index = 0; index < o1.length; index++) {
+      if (o1[index] != o2[index]) {
+        return false;
+      }
+    }
+
+    return true;
   }
 
   void _setLicenseIfChanged(
@@ -623,14 +654,14 @@ class ManifestAssetBundle implements AssetBundle {
     return deferredComponentsAssetVariants;
   }
 
-  DevFSStringContent _createAssetManifest(
+  Map<String, List<String>> _createAssetManifest(
     Map<_Asset, List<_Asset>> assetVariants,
     Map<String, Map<_Asset, List<_Asset>>> deferredComponentsAssetVariants
   ) {
-    final Map<String, List<String>> jsonObject = <String, List<String>>{};
-    final Map<_Asset, List<String>> jsonEntries = <_Asset, List<String>>{};
+    final Map<String, List<String>> manifest = <String, List<String>>{};
+    final Map<_Asset, List<String>> entries = <_Asset, List<String>>{};
     assetVariants.forEach((_Asset main, List<_Asset> variants) {
-      jsonEntries[main] = <String>[
+      entries[main] = <String>[
         for (final _Asset variant in variants)
           variant.entryUri.path,
       ];
@@ -638,24 +669,69 @@ class ManifestAssetBundle implements AssetBundle {
     if (deferredComponentsAssetVariants != null) {
       for (final Map<_Asset, List<_Asset>> componentAssets in deferredComponentsAssetVariants.values) {
         componentAssets.forEach((_Asset main, List<_Asset> variants) {
-          jsonEntries[main] = <String>[
+          entries[main] = <String>[
             for (final _Asset variant in variants)
               variant.entryUri.path,
           ];
         });
       }
     }
-    final List<_Asset> sortedKeys = jsonEntries.keys.toList()
+    final List<_Asset> sortedKeys = entries.keys.toList()
         ..sort((_Asset left, _Asset right) => left.entryUri.path.compareTo(right.entryUri.path));
     for (final _Asset main in sortedKeys) {
       final String decodedEntryPath = Uri.decodeFull(main.entryUri.path);
-      final List<String> rawEntryVariantsPaths = jsonEntries[main]!;
+      final List<String> rawEntryVariantsPaths = entries[main]!;
       final List<String> decodedEntryVariantPaths = rawEntryVariantsPaths
         .map((String value) => Uri.decodeFull(value))
         .toList();
-      jsonObject[decodedEntryPath] = decodedEntryVariantPaths;
+      manifest[decodedEntryPath] = decodedEntryVariantPaths;
+    }
+    return manifest;
   }
-    return DevFSStringContent(json.encode(jsonObject));
+
+  // Matches path-like strings ending in a number followed by an 'x'.
+  // Example matches include "assets/animals/2.0x", "plants/3x", and "2.7x".
+  static final RegExp _extractPixelRatioFromKeyRegExp = RegExp(r'/?(\d+(\.\d*)?)x$');
+
+  DevFSByteContent _createAssetManifestBinary(
+    Map<String, List<String>> assetManifest
+  ) {
+    double parseScale(String key) {
+      final Uri assetUri = Uri.parse(key);
+      String directoryPath = '';
+      if (assetUri.pathSegments.length > 1) {
+        directoryPath = assetUri.pathSegments[assetUri.pathSegments.length - 2];
+      }
+
+      final Match? match = _extractPixelRatioFromKeyRegExp.firstMatch(directoryPath);
+      if (match != null && match.groupCount > 0) {
+        return double.parse(match.group(1)!);
+      }
+      return _defaultResolution;
+    }
+
+    final Map<String, dynamic> result = <String, dynamic>{};
+
+    for (final MapEntry<String, dynamic> manifestEntry in assetManifest.entries) {
+      final List<dynamic> resultVariants = <dynamic>[];
+      final List<String> entries = (manifestEntry.value as List<dynamic>).cast<String>();
+      for (final String variant in entries) {
+        if (variant == manifestEntry.key) {
+          // With the newer binary format, don't include the main asset in it's
+          // list of variants. This reduces parsing time at runtime.
+          continue;
+        }
+        final Map<String, dynamic> resultVariant = <String, dynamic>{};
+        final double variantDevicePixelRatio = parseScale(variant);
+        resultVariant['asset'] = variant;
+        resultVariant['dpr'] = variantDevicePixelRatio;
+        resultVariants.add(resultVariant);
+      }
+      result[manifestEntry.key] = resultVariants;
+    }
+
+    final ByteData message = const StandardMessageCodec().encodeMessage(result)!;
+    return DevFSByteContent(message.buffer.asUint8List(0, message.lengthInBytes));
   }
 
   /// Prefixes family names and asset paths of fonts included from packages with

packages/flutter_tools/pubspec.yaml

@@ -57,6 +57,8 @@ dependencies:
 
   vm_service: 9.4.0
 
+  standard_message_codec: 0.0.1+3
+
   _fe_analyzer_shared: 52.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   analyzer: 5.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -102,4 +104,4 @@ dartdoc:
   # Exclude this package from the hosted API docs.
   nodoc: true
 
-# PUBSPEC CHECKSUM: 02f9
+# PUBSPEC CHECKSUM: 899b

packages/flutter_tools/test/general.shard/asset_bundle_package_fonts_test.dart

@@ -111,8 +111,8 @@ $fontsSection
 
       final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
       await bundle.build(packagesPath: '.packages');
-      expect(bundle.entries.length, 3); // LICENSE, AssetManifest, FontManifest
-      expect(bundle.entries.containsKey('FontManifest.json'), isTrue);
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.bin',
+        'AssetManifest.json', 'FontManifest.json', 'NOTICES.Z']));
     }, overrides: <Type, Generator>{
       FileSystem: () => testFileSystem,
       ProcessManager: () => FakeProcessManager.any(),

packages/flutter_tools/test/general.shard/asset_bundle_test.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:convert';
+import 'dart:typed_data';
 
 import 'package:file/memory.dart';
 import 'package:flutter_tools/src/artifacts.dart';
@@ -13,6 +14,7 @@ import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/bundle_builder.dart';
 import 'package:flutter_tools/src/devfs.dart';
 import 'package:flutter_tools/src/globals.dart' as globals;
+import 'package:standard_message_codec/standard_message_codec.dart';
 
 import '../src/common.dart';
 import '../src/context.dart';
@@ -48,12 +50,20 @@ void main() {
 
       final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
       await bundle.build(packagesPath: '.packages');
-      expect(bundle.entries.length, 1);
-      const String expectedAssetManifest = '{}';
+      expect(bundle.entries.keys,
+        unorderedEquals(<String>['AssetManifest.json', 'AssetManifest.bin'])
+      );
+      const String expectedJsonAssetManifest = '{}';
+      const Map<Object, Object> expectedBinAssetManifest = <Object, Object>{};
       expect(
         utf8.decode(await bundle.entries['AssetManifest.json']!.contentsAsBytes()),
-        expectedAssetManifest,
+        expectedJsonAssetManifest,
+      );
+      expect(
+        const StandardMessageCodec().decodeMessage(ByteData.sublistView(Uint8List.fromList(await bundle.entries['AssetManifest.bin']!.contentsAsBytes()))),
+        expectedBinAssetManifest
       );
+
     }, overrides: <Type, Generator>{
       FileSystem: () => testFileSystem,
       ProcessManager: () => FakeProcessManager.any(),
@@ -72,14 +82,8 @@ flutter:
 ''');
       final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
       await bundle.build(packagesPath: '.packages');
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 4);
-      expect(bundle.needsBuild(), false);
-
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.json',
+        'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z', 'assets/foo/bar.txt']));
       // Simulate modifying the files by updating the filestat time manually.
       globals.fs.file(globals.fs.path.join('assets', 'foo', 'fizz.txt'))
         ..createSync(recursive: true)
@@ -87,13 +91,9 @@ flutter:
 
       expect(bundle.needsBuild(), true);
       await bundle.build(packagesPath: '.packages');
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      //  - assets/foo/fizz.txt
-      expect(bundle.entries.length, 5);
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.json',
+          'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z', 'assets/foo/bar.txt',
+          'assets/foo/fizz.txt']));
     }, overrides: <Type, Generator>{
       FileSystem: () => testFileSystem,
       ProcessManager: () => FakeProcessManager.any(),
@@ -112,12 +112,8 @@ flutter:
       globals.fs.file('.packages').createSync();
       final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
       await bundle.build(packagesPath: '.packages');
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 4);
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.json',
+        'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z', 'assets/foo/bar.txt']));
       expect(bundle.needsBuild(), false);
 
       // Delete the wildcard directory and update pubspec file.
@@ -138,12 +134,8 @@ name: example''')
       // supporting file deletion.
       expect(bundle.needsBuild(), true);
       await bundle.build(packagesPath: '.packages');
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 4);
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.json',
+        'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z', 'assets/foo/bar.txt']));
     }, overrides: <Type, Generator>{
       FileSystem: () => testFileSystem,
       ProcessManager: () => FakeProcessManager.any(),
@@ -166,12 +158,8 @@ flutter:
       globals.fs.file('.packages').createSync();
       final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
       await bundle.build(packagesPath: '.packages');
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 4);
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.json',
+        'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z', 'assets/foo/bar.txt']));
       expect(bundle.needsBuild(), false);
     }, overrides: <Type, Generator>{
       FileSystem: () => testFileSystem,
@@ -203,12 +191,8 @@ flutter:
         splitDeferredAssets: true,
       ).createBundle();
       await bundle.build(packagesPath: '.packages', deferredComponentsEnabled: true);
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 4);
+      expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.json',
+        'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z', 'assets/foo/bar.txt']));
       expect(bundle.deferredComponentsEntries.length, 1);
       expect(bundle.deferredComponentsEntries['component1']!.length, 2);
       expect(bundle.needsBuild(), false);
@@ -237,12 +221,9 @@ flutter:
 ''');
       final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
       await bundle.build(packagesPath: '.packages');
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 6);
+      expect(bundle.entries.keys, unorderedEquals(<String>['assets/foo/bar.txt',
+        'assets/bar/barbie.txt', 'assets/wild/dash.txt', 'AssetManifest.json',
+        'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z']));
       expect(bundle.deferredComponentsEntries.isEmpty, true);
       expect(bundle.needsBuild(), false);
     }, overrides: <Type, Generator>{
@@ -275,12 +256,8 @@ flutter:
         splitDeferredAssets: true,
       ).createBundle();
       await bundle.build(packagesPath: '.packages', deferredComponentsEnabled: true);
-      // Expected assets:
-      //  - asset manifest
-      //  - font manifest
-      //  - license file
-      //  - assets/foo/bar.txt
-      expect(bundle.entries.length, 4);
+      expect(bundle.entries.keys, unorderedEquals(<String>['assets/foo/bar.txt',
+        'AssetManifest.json', 'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z']));
       expect(bundle.deferredComponentsEntries.length, 1);
       expect(bundle.deferredComponentsEntries['component1']!.length, 2);
       expect(bundle.needsBuild(), false);
@@ -293,7 +270,8 @@ flutter:
       expect(bundle.needsBuild(), true);
       await bundle.build(packagesPath: '.packages', deferredComponentsEnabled: true);
 
-      expect(bundle.entries.length, 4);
+      expect(bundle.entries.keys, unorderedEquals(<String>['assets/foo/bar.txt',
+        'AssetManifest.json', 'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z']));
       expect(bundle.deferredComponentsEntries.length, 1);
       expect(bundle.deferredComponentsEntries['component1']!.length, 3);
     }, overrides: <Type, Generator>{
@@ -641,7 +619,8 @@ flutter:
 
     await bundle.build(packagesPath: '.packages');
 
-    expect(bundle.entries, hasLength(4));
+      expect(bundle.entries.keys, unorderedEquals(<String>['packages/foo/bar/fizz.txt',
+        'AssetManifest.json', 'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z']));
     expect(bundle.needsBuild(), false);
 
     // Does not track dependency's wildcard directories.
@@ -776,7 +755,8 @@ flutter:
     final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
 
     expect(await bundle.build(packagesPath: '.packages'), 0);
-    expect(bundle.entries.length, 4);
+    expect(bundle.entries.keys, unorderedEquals(<String>['assets/foo.txt',
+      'AssetManifest.json', 'AssetManifest.bin', 'FontManifest.json', 'NOTICES.Z']));
   }, overrides: <Type, Generator>{
     FileSystem: () => MemoryFileSystem.test(),
     ProcessManager: () => FakeProcessManager.any(),