Enable asset transformation feature in hot reload workflow (excluding Web) (#144161)

Partial implementation of https://github.com/flutter/flutter/issues/143348

This enables asset transformation during hot reload (except for web, because that has its own implementation of `DevFS` 🙃). Asset transformers will be reapplied after changing any asset and performing a hot reload during `flutter run`.

packages/flutter_tools/lib/src/asset.dart

@@ -218,8 +218,8 @@ class ManifestAssetBundle implements AssetBundle {
       return true;
     }
 
-    final FileStat stat = _fileSystem.file(manifestPath).statSync();
-    if (stat.type == FileSystemEntityType.notFound) {
+    final FileStat manifestStat = _fileSystem.file(manifestPath).statSync();
+    if (manifestStat.type == FileSystemEntityType.notFound) {
       return true;
     }
 
@@ -235,7 +235,7 @@ class ManifestAssetBundle implements AssetBundle {
       }
     }
 
-    return stat.modified.isAfter(lastBuildTimestamp);
+    return manifestStat.modified.isAfter(lastBuildTimestamp);
   }
 
   @override

packages/flutter_tools/lib/src/build_system/tools/asset_transformer.dart

@@ -3,11 +3,16 @@
 // found in the LICENSE file.
 
 
+import 'dart:typed_data';
+
+import 'package:pool/pool.dart';
 import 'package:process/process.dart';
 
 import '../../base/error_handling_io.dart';
 import '../../base/file_system.dart';
 import '../../base/io.dart';
+import '../../base/logger.dart';
+import '../../devfs.dart';
 import '../../flutter_manifest.dart';
 import '../build_system.dart';
 
@@ -52,6 +57,7 @@ final class AssetTransformer {
     File tempInputFile = _fileSystem.systemTempDirectory.childFile(getTempFilePath(0));
     await asset.copy(tempInputFile.path);
     File tempOutputFile = _fileSystem.systemTempDirectory.childFile(getTempFilePath(1));
+    ErrorHandlingFileSystem.deleteIfExists(tempOutputFile);
 
     try {
       for (final (int i, AssetTransformerEntry transformer) in transformerEntries.indexed) {
@@ -71,10 +77,12 @@ final class AssetTransformer {
 
         ErrorHandlingFileSystem.deleteIfExists(tempInputFile);
         if (i == transformerEntries.length - 1) {
+          await _fileSystem.file(outputPath).create(recursive: true);
           await tempOutputFile.copy(outputPath);
         } else {
           tempInputFile = tempOutputFile;
           tempOutputFile = _fileSystem.systemTempDirectory.childFile(getTempFilePath(i+2));
+          ErrorHandlingFileSystem.deleteIfExists(tempOutputFile);
         }
       }
     } finally {
@@ -136,6 +144,69 @@ final class AssetTransformer {
   }
 }
 
+
+// A wrapper around [AssetTransformer] to support hot reload of transformed assets.
+final class DevelopmentAssetTransformer {
+  DevelopmentAssetTransformer({
+    required FileSystem fileSystem,
+    required AssetTransformer transformer,
+    required Logger logger,
+  })  : _fileSystem = fileSystem,
+        _transformer = transformer,
+        _logger = logger;
+
+  final AssetTransformer _transformer;
+  final FileSystem _fileSystem;
+  final Pool _transformationPool = Pool(4);
+  final Logger _logger;
+
+  /// Re-transforms an asset and returns a [DevFSContent] that should be synced
+  /// to the attached device in its place.
+  ///
+  /// Returns `null` if any of the transformation subprocesses failed.
+  Future<DevFSContent?> retransformAsset({
+    required String inputAssetKey,
+    required DevFSContent inputAssetContent,
+    required List<AssetTransformerEntry> transformerEntries,
+    required String workingDirectory,
+  }) async {
+    final File output = _fileSystem.systemTempDirectory.childFile('retransformerInput-$inputAssetKey');
+    ErrorHandlingFileSystem.deleteIfExists(output);
+    File? inputFile;
+    bool cleanupInput = false;
+    Uint8List result;
+    PoolResource? resource;
+    try {
+      resource = await _transformationPool.request();
+      if (inputAssetContent is DevFSFileContent) {
+        inputFile = inputAssetContent.file as File;
+      } else {
+        inputFile = _fileSystem.systemTempDirectory.childFile('retransformerInput-$inputAssetKey');
+        inputFile.writeAsBytesSync(await inputAssetContent.contentsAsBytes());
+        cleanupInput = true;
+      }
+      final AssetTransformationFailure? failure = await _transformer.transformAsset(
+        asset: inputFile,
+        outputPath: output.path,
+        transformerEntries: transformerEntries,
+        workingDirectory: workingDirectory,
+      );
+      if (failure != null) {
+        _logger.printError(failure.message);
+        return null;
+      }
+      result = output.readAsBytesSync();
+    } finally {
+      resource?.release();
+      ErrorHandlingFileSystem.deleteIfExists(output);
+      if (cleanupInput && inputFile != null) {
+        ErrorHandlingFileSystem.deleteIfExists(inputFile);
+      }
+    }
+    return DevFSByteContent(result);
+  }
+}
+
 final class AssetTransformationFailure {
   const AssetTransformationFailure(this.message);
 

packages/flutter_tools/lib/src/devfs.dart

@@ -5,8 +5,10 @@
 import 'dart:async';
 
 import 'package:package_config/package_config.dart';
+import 'package:process/process.dart';
 import 'package:vm_service/vm_service.dart' as vm_service;
 
+import 'artifacts.dart';
 import 'asset.dart';
 import 'base/config.dart';
 import 'base/context.dart';
@@ -16,6 +18,7 @@ import 'base/logger.dart';
 import 'base/net.dart';
 import 'base/os.dart';
 import 'build_info.dart';
+import 'build_system/tools/asset_transformer.dart';
 import 'build_system/tools/scene_importer.dart';
 import 'build_system/tools/shader_compiler.dart';
 import 'compile.dart';
@@ -454,6 +457,8 @@ class DevFS {
     required OperatingSystemUtils osUtils,
     required Logger logger,
     required FileSystem fileSystem,
+    required ProcessManager processManager,
+    required Artifacts artifacts,
     HttpClient? httpClient,
     Duration? uploadRetryThrottle,
     StopwatchFactory stopwatchFactory = const StopwatchFactory(),
@@ -471,7 +476,16 @@ class DevFS {
           ? HttpClient()
           : context.get<HttpClientFactory>()!())),
        _stopwatchFactory = stopwatchFactory,
-       _config = config;
+       _config = config,
+       _assetTransformer = DevelopmentAssetTransformer(
+          transformer: AssetTransformer(
+            processManager: processManager,
+            fileSystem: fileSystem,
+            dartBinaryPath: artifacts.getArtifactPath(Artifact.engineDartBinary),
+          ),
+          fileSystem: fileSystem,
+          logger: logger,
+        );
 
   final FlutterVmService _vmService;
   final _DevFSHttpWriter _httpWriter;
@@ -479,9 +493,10 @@ class DevFS {
   final FileSystem _fileSystem;
   final StopwatchFactory _stopwatchFactory;
   final Config? _config;
+  final DevelopmentAssetTransformer _assetTransformer;
 
   final String fsName;
-  final Directory? rootDirectory;
+  final Directory rootDirectory;
   final Set<String> assetPathsToEvict = <String>{};
   final Set<String> shaderPathsToEvict = <String>{};
   final Set<String> scenePathsToEvict = <String>{};
@@ -505,7 +520,7 @@ class DevFS {
     final String baseUriString = baseUri.toString();
     if (deviceUriString.startsWith(baseUriString)) {
       final String deviceUriSuffix = deviceUriString.substring(baseUriString.length);
-      return rootDirectory!.uri.resolve(deviceUriSuffix);
+      return rootDirectory.uri.resolve(deviceUriSuffix);
     }
     return deviceUri;
   }
@@ -600,7 +615,7 @@ class DevFS {
       invalidatedFiles,
       outputPath: dillOutputPath,
       fs: _fileSystem,
-      projectRootPath: rootDirectory?.path,
+      projectRootPath: rootDirectory.path,
       packageConfig: packageConfig,
       checkDartPluginRegistry: true, // The entry point is assumed not to have changed.
       dartPluginRegistrant: dartPluginRegistrant,
@@ -636,8 +651,8 @@ class DevFS {
         if (archivePath == _kFontManifest) {
           didUpdateFontManifest = true;
         }
-
-        switch (bundle.entries[archivePath]?.kind) {
+        final AssetKind? kind = bundle.entries[archivePath]?.kind;
+        switch (kind) {
           case AssetKind.shader:
             final Future<DevFSContent?> pending = shaderCompiler.recompileShader(entry.content);
             pendingAssetBuilds.add(pending);
@@ -672,11 +687,30 @@ class DevFS {
           case AssetKind.regular:
           case AssetKind.font:
           case null:
-            dirtyEntries[deviceUri] = entry.content;
-            syncedBytes += entry.content.size;
-            if (!bundleFirstUpload) {
-              assetPathsToEvict.add(archivePath);
-            }
+            final Future<DevFSContent?> pending = (() async {
+              if (entry.transformers.isEmpty || kind != AssetKind.regular) {
+                return entry.content;
+              }
+              return _assetTransformer.retransformAsset(
+                inputAssetKey: archivePath,
+                inputAssetContent: entry.content,
+                transformerEntries: entry.transformers,
+                workingDirectory: rootDirectory.path,
+              );
+            })();
+
+            pendingAssetBuilds.add(pending);
+            pending.then((DevFSContent? content) {
+              if (content == null) {
+                assetBuildFailed = true;
+                return;
+              }
+              dirtyEntries[deviceUri] = content;
+              syncedBytes += content.size;
+              if (!bundleFirstUpload) {
+                assetPathsToEvict.add(archivePath);
+              }
+            });
         }
       });
 

packages/flutter_tools/lib/src/isolated/devfs_web.dart

@@ -701,6 +701,7 @@ class WebDevFS implements DevFS {
     required this.nullSafetyMode,
     required this.ddcModuleSystem,
     required this.webRenderer,
+    required this.rootDirectory,
     this.testMode = false,
   }) : _port = port;
 
@@ -857,7 +858,7 @@ class WebDevFS implements DevFS {
   String get fsName => 'web_asset';
 
   @override
-  Directory? get rootDirectory => null;
+  final Directory rootDirectory;
 
   @override
   Future<UpdateFSReport> update({

packages/flutter_tools/lib/src/isolated/resident_web_runner.dart

@@ -311,6 +311,7 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
           nativeNullAssertions: debuggingOptions.nativeNullAssertions,
           ddcModuleSystem: debuggingOptions.buildInfo.ddcModuleFormat == DdcModuleFormat.ddc,
           webRenderer: debuggingOptions.webRenderer,
+          rootDirectory: fileSystem.directory(projectRootPath),
         );
         Uri url = await device!.devFS!.create();
         if (debuggingOptions.tlsCertKeyPath != null && debuggingOptions.tlsCertPath != null) {

packages/flutter_tools/lib/src/resident_runner.dart

@@ -385,6 +385,8 @@ class FlutterDevice {
       osUtils: globals.os,
       fileSystem: globals.fs,
       logger: globals.logger,
+      processManager: globals.processManager,
+      artifacts: globals.artifacts!,
     );
     return devFS!.create();
   }

packages/flutter_tools/test/general.shard/build_system/targets/asset_transformer_test.dart

@@ -74,6 +74,7 @@ void main() {
     expect(transformationFailure, isNull, reason: logger.errorText);
     expect(processManager, hasNoRemainingExpectations);
     expect(fileSystem.file(outputPath).readAsStringSync(), 'hello world');
+    expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
   });
 
   testWithoutContext('logs useful error information when transformation process returns a nonzero exit code', () async {
@@ -138,6 +139,7 @@ stdout:
 Beginning transformation
 stderr:
 Something went wrong''');
+    expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
   });
 
   testWithoutContext('prints error message when the transformer does not produce an output file', () async {
@@ -197,6 +199,7 @@ stdout:
 stderr:
 Transformation failed, but I forgot to exit with a non-zero code.'''
     );
+    expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
   });
 
   testWithoutContext('correctly chains transformations when there are multiple of them', () async {
@@ -283,6 +286,7 @@ Transformation failed, but I forgot to exit with a non-zero code.'''
     expect(processManager, hasNoRemainingExpectations);
     expect(failure, isNull);
     expect(fileSystem.file(outputPath).readAsStringSync(), '012');
+    expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
   });
 
   testWithoutContext('prints an error when a transformer in a chain (thats not the first) does not produce an output', () async {
@@ -368,6 +372,6 @@ Transformation failed, but I forgot to exit with a non-zero code.'''
     );
     expect(processManager, hasNoRemainingExpectations);
     expect(fileSystem.file(outputPath), isNot(exists));
-    expect(fileSystem.systemTempDirectory.listSync(), isEmpty);
+    expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
   });
 }

packages/flutter_tools/test/general.shard/web/devfs_web_ddc_modules_test.dart

@@ -909,6 +909,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.unsound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.html,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.flutterJs.createSync(recursive: true);
@@ -1044,6 +1045,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.sound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.html,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.flutterJs.createSync(recursive: true);
@@ -1179,6 +1181,7 @@ void main() {
                 nullSafetyMode: NullSafetyMode.sound,
                 ddcModuleSystem: usesDdcModuleSystem,
                 webRenderer: WebRendererMode.canvaskit,
+                rootDirectory: globals.fs.currentDirectory,
               );
               webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
               webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1251,6 +1254,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.sound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.canvaskit,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1299,6 +1303,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.sound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.canvaskit,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1349,6 +1354,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.sound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.auto,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1401,6 +1407,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.unsound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.canvaskit,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1571,6 +1578,7 @@ void main() {
               nullSafetyMode: NullSafetyMode.unsound,
               ddcModuleSystem: usesDdcModuleSystem,
               webRenderer: WebRendererMode.canvaskit,
+              rootDirectory: globals.fs.currentDirectory,
             );
             webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
             webDevFS.stackTraceMapper.createSync(recursive: true);

packages/flutter_tools/test/general.shard/web/devfs_web_test.dart

@@ -697,6 +697,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.unsound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.html,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.flutterJs.createSync(recursive: true);
@@ -808,6 +809,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.sound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.html,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.flutterJs.createSync(recursive: true);
@@ -918,6 +920,7 @@ void main() {
         nullSafetyMode: NullSafetyMode.sound,
         ddcModuleSystem: usesDdcModuleSystem,
         webRenderer: WebRendererMode.canvaskit,
+        rootDirectory: globals.fs.currentDirectory,
       );
       webDevFS.requireJS.createSync(recursive: true);
       webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -981,6 +984,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.sound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.canvaskit,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1028,6 +1032,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.sound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.canvaskit,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1076,6 +1081,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.sound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.auto,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1125,6 +1131,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.unsound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.canvaskit,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.stackTraceMapper.createSync(recursive: true);
@@ -1278,6 +1285,7 @@ void main() {
       nullSafetyMode: NullSafetyMode.unsound,
       ddcModuleSystem: usesDdcModuleSystem,
       webRenderer: WebRendererMode.canvaskit,
+      rootDirectory: globals.fs.currentDirectory,
     );
     webDevFS.requireJS.createSync(recursive: true);
     webDevFS.stackTraceMapper.createSync(recursive: true);