Reland double gzip wrapping NOTICES to reduce on-disk installed space (#80897)

dev/bots/pubspec.yaml

@@ -18,6 +18,7 @@ dependencies:
 
   _fe_analyzer_shared: 21.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   analyzer: 1.5.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  archive: 3.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   async: 2.5.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   charcode: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -61,4 +62,4 @@ dependencies:
 dev_dependencies:
   test_api: 0.3.0
 
-# PUBSPEC CHECKSUM: 4292
+# PUBSPEC CHECKSUM: a6c4

dev/devicelab/bin/tasks/module_test.dart

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
+import 'dart:typed_data';
 
+import 'package:archive/archive.dart';
 import 'package:flutter_devicelab/framework/apk_utils.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
@@ -314,6 +317,24 @@ Future<void> main() async {
         'lib/armeabi-v7a/libflutter.so',
       ], await getFilesInApk(releaseHostApk));
 
+      section('Check the NOTICE file is correct');
+
+      await inDirectory(hostApp, () async {
+        final File apkFile = File(releaseHostApk);
+        final Archive apk = ZipDecoder().decodeBytes(apkFile.readAsBytesSync());
+        // Shouldn't be missing since we already checked it exists above.
+        final ArchiveFile noticesFile = apk.findFile('assets/flutter_assets/NOTICES.Z');
+
+        final Uint8List licenseData = noticesFile.content as Uint8List;
+        if (licenseData == null) {
+          return TaskResult.failure('Invalid license file.');
+        }
+        final String licenseString = utf8.decode(gzip.decode(licenseData));
+        if (!licenseString.contains('skia') || !licenseString.contains('Flutter Authors')) {
+          return TaskResult.failure('License content missing.');
+        }
+      });
+
       section('Check release AndroidManifest.xml');
 
       final String androidManifestRelease = await getAndroidManifest(debugHostApk);

dev/devicelab/bin/tasks/module_test_ios.dart

@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
+import 'dart:typed_data';
 
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/host_agent.dart';
@@ -214,6 +216,8 @@ Future<void> main() async {
 
       final File objectiveCAnalyticsOutputFile = File(path.join(tempDir.path, 'analytics-objc.log'));
       final Directory objectiveCBuildDirectory = Directory(path.join(tempDir.path, 'build-objc'));
+
+      section('Build iOS Objective-C host app');
       await inDirectory(objectiveCHostApp, () async {
         await exec(
           'pod',
@@ -282,6 +286,28 @@ Future<void> main() async {
         'isolate_snapshot_data',
       ));
 
+      section('Check the NOTICE file is correct');
+
+      final String licenseFilePath = path.join(
+        objectiveCBuildDirectory.path,
+        'Host.app',
+        'Frameworks',
+        'App.framework',
+        'flutter_assets',
+        'NOTICES.Z',
+      );
+      checkFileExists(licenseFilePath);
+
+      await inDirectory(objectiveCBuildDirectory, () async {
+        final Uint8List licenseData = File(licenseFilePath).readAsBytesSync();
+        final String licenseString = utf8.decode(gzip.decode(licenseData));
+        if (!licenseString.contains('skia') || !licenseString.contains('Flutter Authors')) {
+          return TaskResult.failure('License content missing');
+        }
+      });
+
+      section('Check that the host build sends the correct analytics');
+
       final String objectiveCAnalyticsOutput = objectiveCAnalyticsOutputFile.readAsStringSync();
       if (!objectiveCAnalyticsOutput.contains('cd24: ios')
           || !objectiveCAnalyticsOutput.contains('cd25: true')

dev/devicelab/lib/framework/apk_utils.dart

@@ -13,7 +13,7 @@ final String platformLineSep = Platform.isWindows ? '\r\n' : '\n';
 
 final List<String> flutterAssets = <String>[
   'assets/flutter_assets/AssetManifest.json',
-  'assets/flutter_assets/NOTICES',
+  'assets/flutter_assets/NOTICES.Z',
   'assets/flutter_assets/fonts/MaterialIcons-Regular.otf',
   'assets/flutter_assets/packages/cupertino_icons/assets/CupertinoIcons.ttf',
 ];

dev/devicelab/lib/tasks/perf_tests.dart

@@ -1186,7 +1186,7 @@ class CompileTest {
 
     final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
     final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so'];
-    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES'];
+    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES.Z'];
 
     return <String, dynamic>{
       'libflutter_uncompressed_bytes': libflutter.uncompressedSize,

dev/devicelab/pubspec.yaml

@@ -7,6 +7,7 @@ environment:
   sdk: ">=2.3.0 <3.0.0"
 
 dependencies:
+  archive: 3.1.2
   args: 2.1.0
   file: 6.1.0
   http: 0.13.1
@@ -20,6 +21,7 @@ dependencies:
 
   charcode: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   collection: 1.15.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  crypto: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   http_parser: 4.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pedantic: 1.11.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   source_span: 1.8.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -37,7 +39,6 @@ dev_dependencies:
   cli_util: 0.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   convert: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   coverage: 1.0.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  crypto: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   glob: 2.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   http_multi_server: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   io: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -63,4 +64,4 @@ dev_dependencies:
   webkit_inspection_protocol: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   yaml: 3.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 4292
+# PUBSPEC CHECKSUM: a6c4

packages/flutter/lib/src/services/binding.dart

@@ -4,6 +4,8 @@
 
 
 import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
 import 'dart:typed_data';
 import 'dart:ui' as ui;
 
@@ -104,7 +106,21 @@ mixin ServicesBinding on BindingBase, SchedulerBinding {
     // TODO(ianh): Remove this complexity once these bugs are fixed.
     final Completer<String> rawLicenses = Completer<String>();
     scheduleTask(() async {
-      rawLicenses.complete(await rootBundle.loadString('NOTICES', cache: false));
+      rawLicenses.complete(
+        kIsWeb
+            // NOTICES for web isn't compressed since we don't have access to
+            // dart:io on the client side and it's already compressed between
+            // the server and client.
+            ? rootBundle.loadString('NOTICES', cache: false)
+            : () async {
+              // The compressed version doesn't have a more common .gz extension
+              // because gradle for Android non-transparently manipulates .gz files.
+              final ByteData licenseBytes = await rootBundle.load('NOTICES.Z');
+              List<int> bytes = licenseBytes.buffer.asUint8List();
+              bytes = gzip.decode(bytes);
+              return utf8.decode(bytes);
+            }()
+      );
     }, Priority.animation);
     await rawLicenses.future;
     final Completer<List<LicenseEntry>> parsedLicenses = Completer<List<LicenseEntry>>();

packages/flutter/test/services/asset_bundle_test.dart

@@ -15,10 +15,10 @@ class TestAssetBundle extends CachingAssetBundle {
 
   @override
   Future<ByteData> load(String key) async {
+    loadCallCount[key] = loadCallCount[key] ?? 0 + 1;
     if (key == 'AssetManifest.json')
       return ByteData.view(Uint8List.fromList(const Utf8Encoder().convert('{"one": ["one"]}')).buffer);
 
-    loadCallCount[key] = loadCallCount[key] ?? 0 + 1;
     if (key == 'one')
       return ByteData(1)..setInt8(0, 49);
     throw FlutterError('key not found');

packages/flutter/test/services/binding_test.dart

@@ -2,6 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
+import 'dart:io';
+import 'dart:typed_data';
+
 import 'package:flutter/foundation.dart';
 import 'package:flutter/scheduler.dart';
 import 'package:flutter/services.dart';
@@ -45,6 +49,15 @@ class TestBinding extends BindingBase with SchedulerBinding, ServicesBinding {
           return const StringCodec().encodeMessage(licenses);
         }
         return null;
+      })
+      ..setMockMessageHandler('flutter/assets', (ByteData? message) async {
+        if (const StringCodec().decodeMessage(message) == 'NOTICES.Z' && !kIsWeb) {
+          return Uint8List.fromList(gzip.encode(utf8.encode(licenses))).buffer.asByteData();
+        }
+        if (const StringCodec().decodeMessage(message) == 'NOTICES' && kIsWeb) {
+          return const StringCodec().encodeMessage(licenses);
+        }
+        return null;
       });
   }
 }

packages/flutter_tools/bin/fuchsia_asset_builder.dart

@@ -10,6 +10,7 @@ import 'package:flutter_tools/src/base/context.dart';
 import 'package:flutter_tools/src/base/file_system.dart' as libfs;
 import 'package:flutter_tools/src/base/io.dart';
 import 'package:flutter_tools/src/bundle.dart';
+import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/cache.dart';
 import 'package:flutter_tools/src/context_runner.dart';
 import 'package:flutter_tools/src/devfs.dart';
@@ -61,6 +62,7 @@ Future<void> run(List<String> args) async {
     manifestPath: argResults[_kOptionManifest] as String ?? defaultManifestPath,
     assetDirPath: assetDir,
     packagesPath: argResults[_kOptionPackages] as String,
+    targetPlatform: TargetPlatform.fuchsia_arm64 // This is not arch specific.
   );
 
   if (assets == null) {

packages/flutter_tools/lib/src/asset.dart

@@ -82,6 +82,7 @@ abstract class AssetBundle {
     String assetDirPath,
     @required String packagesPath,
     bool deferredComponentsEnabled = false,
+    TargetPlatform targetPlatform,
   });
 }
 
@@ -141,6 +142,13 @@ class ManifestAssetBundle implements AssetBundle {
 
   static const String _kAssetManifestJson = 'AssetManifest.json';
   static const String _kNoticeFile = 'NOTICES';
+  // Comically, this can't be name with the more common .gz file extension
+  // because when it's part of an AAR and brought into another APK via gradle,
+  // gradle individually traverses all the files of the AAR and unzips .gz
+  // files (b/37117906). A less common .Z extension still describes how the
+  // file is formatted if users want to manually inspect the application
+  // bundle and is recognized by default file handlers on OS such as macOS.˚
+  static const String _kNoticeZippedFile = 'NOTICES.Z';
 
   @override
   bool wasBuiltOnce() => _lastBuildTimestamp != null;
@@ -180,6 +188,7 @@ class ManifestAssetBundle implements AssetBundle {
     String assetDirPath,
     @required String packagesPath,
     bool deferredComponentsEnabled = false,
+    TargetPlatform targetPlatform,
   }) async {
     assetDirPath ??= getAssetBuildDirectory();
     FlutterProject flutterProject;
@@ -407,7 +416,6 @@ class ManifestAssetBundle implements AssetBundle {
       return 1;
     }
 
-    final DevFSStringContent licenses = DevFSStringContent(licenseResult.combinedLicenses);
     additionalDependencies = licenseResult.dependencies;
 
     if (wildcardDirectories.isNotEmpty) {
@@ -425,7 +433,7 @@ class ManifestAssetBundle implements AssetBundle {
 
     _setIfChanged(_kAssetManifestJson, assetManifest);
     _setIfChanged(kFontManifestJson, fontManifest);
-    _setIfChanged(_kNoticeFile, licenses);
+    _setLicenseIfChanged(licenseResult.combinedLicenses, targetPlatform);
     return 0;
   }
 
@@ -443,6 +451,35 @@ class ManifestAssetBundle implements AssetBundle {
     }
   }
 
+  void _setLicenseIfChanged(
+    String combinedLicenses,
+    TargetPlatform targetPlatform,
+  ) {
+    // On the web, don't compress the NOTICES file since the client doesn't have
+    // dart:io to decompress it. So use the standard _setIfChanged to check if
+    // the strings still match.
+    if (targetPlatform == TargetPlatform.web_javascript) {
+      _setIfChanged(_kNoticeFile, DevFSStringContent(combinedLicenses));
+      return;
+    }
+
+    // On other platforms, let the NOTICES file be compressed. But use a
+    // specialized DevFSStringCompressingBytesContent class to compare
+    // the uncompressed strings to not incur decompression/decoding while making
+    // the comparison.
+    if (!entries.containsKey(_kNoticeZippedFile) ||
+        !(entries[_kNoticeZippedFile] as DevFSStringCompressingBytesContent)
+            .equals(combinedLicenses)) {
+      entries[_kNoticeZippedFile] = DevFSStringCompressingBytesContent(
+        combinedLicenses,
+        // A zlib dictionary is a hinting string sequence with the most
+        // likely string occurrences at the end. This ends up just being
+        // common English words with domain specific words like copyright.
+        hintString: 'copyrightsoftwaretothisinandorofthe',
+      );
+    }
+  }
+
   List<_Asset> _getMaterialAssets() {
     final List<_Asset> result = <_Asset>[];
     for (final Map<String, Object> family in kMaterialFonts) {

packages/flutter_tools/lib/src/base/io.dart

@@ -102,7 +102,8 @@ export 'dart:io'
         systemEncoding,
         WebSocket,
         WebSocketException,
-        WebSocketTransformer;
+        WebSocketTransformer,
+        ZLibEncoder;
 
 /// Exits the process with the given [exitCode].
 typedef ExitFunction = void Function(int exitCode);

packages/flutter_tools/lib/src/build_system/targets/assets.dart

@@ -58,6 +58,7 @@ Future<Depfile> copyAssets(Environment environment, Directory outputDirectory, {
     packagesPath: environment.projectDir.childFile('.packages').path,
     assetDirPath: null,
     deferredComponentsEnabled: environment.defines[kDeferredComponents] == 'true',
+    targetPlatform: targetPlatform,
   );
   if (resultCode != 0) {
     throw Exception('Failed to bundle asset files.');

packages/flutter_tools/lib/src/bundle.dart

@@ -182,6 +182,7 @@ Future<AssetBundle> buildAssets({
   String manifestPath,
   String assetDirPath,
   @required String packagesPath,
+  TargetPlatform targetPlatform,
 }) async {
   assetDirPath ??= getAssetBuildDirectory();
   packagesPath ??= globals.fs.path.absolute(packagesPath);
@@ -192,6 +193,7 @@ Future<AssetBundle> buildAssets({
     manifestPath: manifestPath,
     assetDirPath: assetDirPath,
     packagesPath: packagesPath,
+    targetPlatform: targetPlatform,
   );
   if (result != 0) {
     return null;

packages/flutter_tools/lib/src/devfs.dart

@@ -203,6 +203,61 @@ class DevFSStringContent extends DevFSByteContent {
   }
 }
 
+/// A string compressing DevFSContent.
+///
+/// A specialized DevFSContent similar to DevFSByteContent where the contents
+/// are the compressed bytes of a string. Its difference is that the original
+/// uncompressed string can be compared with directly without the indirection
+/// of a compute-expensive uncompress/decode and compress/encode to compare
+/// the strings.
+///
+/// The `hintString` parameter is a zlib dictionary hinting mechanism to suggest
+/// the most common string occurrences to potentially assist with compression.
+class DevFSStringCompressingBytesContent extends DevFSContent {
+  DevFSStringCompressingBytesContent(this._string, { String hintString })
+    : _compressor = ZLibEncoder(
+      dictionary: hintString == null
+          ? null
+          : utf8.encode(hintString),
+      gzip: true,
+      level: 9,
+    );
+
+  final String _string;
+  final ZLibEncoder _compressor;
+  final DateTime _modificationTime = DateTime.now();
+
+  List<int> _bytes;
+  bool _isModified = true;
+
+  List<int> get bytes => _bytes ??= _compressor.convert(utf8.encode(_string));
+
+  /// Return true only once so that the content is written to the device only once.
+  @override
+  bool get isModified {
+    final bool modified = _isModified;
+    _isModified = false;
+    return modified;
+  }
+
+  @override
+  bool isModifiedAfter(DateTime time) {
+    return time == null || _modificationTime.isAfter(time);
+  }
+
+  @override
+  int get size => bytes.length;
+
+  @override
+  Future<List<int>> contentsAsBytes() async => bytes;
+
+  @override
+  Stream<List<int>> contentsAsStream() => Stream<List<int>>.value(bytes);
+
+  /// This checks the source string with another string.
+  bool equals(String string) => _string == string;
+}
+
 class DevFSException implements Exception {
   DevFSException(this.message, [this.error, this.stackTrace]);
   final String message;

packages/flutter_tools/lib/src/isolated/resident_web_runner.dart

@@ -483,7 +483,10 @@ class ResidentWebRunner extends ResidentRunner {
     final bool rebuildBundle = assetBundle.needsBuild();
     if (rebuildBundle) {
       _logger.printTrace('Updating assets');
-      final int result = await assetBundle.build(packagesPath: debuggingOptions.buildInfo.packagesPath);
+      final int result = await assetBundle.build(
+        packagesPath: debuggingOptions.buildInfo.packagesPath,
+        targetPlatform: TargetPlatform.web_javascript,
+      );
       if (result != 0) {
         return UpdateFSReport(success: false);
       }

packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart

@@ -88,7 +88,7 @@ flutter:
 
     expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/AssetManifest.json'), exists);
     expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/FontManifest.json'), exists);
-    expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/NOTICES'), exists);
+    expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/NOTICES.Z'), exists);
     // See https://github.com/flutter/flutter/issues/35293
     expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/assets/foo/bar.png'), exists);
     // See https://github.com/flutter/flutter/issues/46163

packages/flutter_tools/test/general.shard/devfs_test.dart

@@ -111,6 +111,16 @@ void main() {
     expect(content.isModified, isFalse);
   });
 
+  testWithoutContext('DevFSStringCompressingBytesContent', () {
+    final DevFSStringCompressingBytesContent content =
+        DevFSStringCompressingBytesContent('uncompressed string');
+
+    expect(content.equals('uncompressed string'), isTrue);
+    expect(content.bytes, isNotNull);
+    expect(content.isModified, isTrue);
+    expect(content.isModified, isFalse);
+  });
+
   testWithoutContext('DevFS create throws a DevFSException when vmservice disconnects unexpectedly', () async {
     final FileSystem fileSystem = MemoryFileSystem.test();
     final OperatingSystemUtils osUtils = MockOperatingSystemUtils();

packages/flutter_tools/test/general.shard/windows/install_manifest_test.dart

@@ -132,7 +132,7 @@ flutter:
       'C:/build/flutter_assets/assets/foo.png',
       'C:/build/flutter_assets/AssetManifest.json',
       'C:/build/flutter_assets/FontManifest.json',
-      'C:/build/flutter_assets/NOTICES',
+      'C:/build/flutter_assets/NOTICES.Z',
       'C:/winuwp/flutter/ephemeral/flutter_windows_winuwp.dll',
       'C:/winuwp/flutter/ephemeral/flutter_windows_winuwp.dll.pdb',
       'C:/winuwp/flutter/ephemeral/icudtl.dat'