Unverified Commit ff3b6dc0 authored by Andrew Kolos's avatar Andrew Kolos Committed by GitHub

Enable asset transformation feature in hot reload workflow (excluding Web) (#144161)

Partial implementation of https://github.com/flutter/flutter/issues/143348

This enables asset transformation during hot reload (except for web, because that has its own implementation of `DevFS` 🙃). Asset transformers will be reapplied after changing any asset and performing a hot reload during `flutter run`.
parent 15e8d324
......@@ -218,8 +218,8 @@ class ManifestAssetBundle implements AssetBundle {
return true;
}
final FileStat stat = _fileSystem.file(manifestPath).statSync();
if (stat.type == FileSystemEntityType.notFound) {
final FileStat manifestStat = _fileSystem.file(manifestPath).statSync();
if (manifestStat.type == FileSystemEntityType.notFound) {
return true;
}
......@@ -235,7 +235,7 @@ class ManifestAssetBundle implements AssetBundle {
}
}
return stat.modified.isAfter(lastBuildTimestamp);
return manifestStat.modified.isAfter(lastBuildTimestamp);
}
@override
......
......@@ -3,11 +3,16 @@
// found in the LICENSE file.
import 'dart:typed_data';
import 'package:pool/pool.dart';
import 'package:process/process.dart';
import '../../base/error_handling_io.dart';
import '../../base/file_system.dart';
import '../../base/io.dart';
import '../../base/logger.dart';
import '../../devfs.dart';
import '../../flutter_manifest.dart';
import '../build_system.dart';
......@@ -52,6 +57,7 @@ final class AssetTransformer {
File tempInputFile = _fileSystem.systemTempDirectory.childFile(getTempFilePath(0));
await asset.copy(tempInputFile.path);
File tempOutputFile = _fileSystem.systemTempDirectory.childFile(getTempFilePath(1));
ErrorHandlingFileSystem.deleteIfExists(tempOutputFile);
try {
for (final (int i, AssetTransformerEntry transformer) in transformerEntries.indexed) {
......@@ -71,10 +77,12 @@ final class AssetTransformer {
ErrorHandlingFileSystem.deleteIfExists(tempInputFile);
if (i == transformerEntries.length - 1) {
await _fileSystem.file(outputPath).create(recursive: true);
await tempOutputFile.copy(outputPath);
} else {
tempInputFile = tempOutputFile;
tempOutputFile = _fileSystem.systemTempDirectory.childFile(getTempFilePath(i+2));
ErrorHandlingFileSystem.deleteIfExists(tempOutputFile);
}
}
} finally {
......@@ -136,6 +144,69 @@ final class AssetTransformer {
}
}
// A wrapper around [AssetTransformer] to support hot reload of transformed assets.
final class DevelopmentAssetTransformer {
DevelopmentAssetTransformer({
required FileSystem fileSystem,
required AssetTransformer transformer,
required Logger logger,
}) : _fileSystem = fileSystem,
_transformer = transformer,
_logger = logger;
final AssetTransformer _transformer;
final FileSystem _fileSystem;
final Pool _transformationPool = Pool(4);
final Logger _logger;
/// Re-transforms an asset and returns a [DevFSContent] that should be synced
/// to the attached device in its place.
///
/// Returns `null` if any of the transformation subprocesses failed.
Future<DevFSContent?> retransformAsset({
required String inputAssetKey,
required DevFSContent inputAssetContent,
required List<AssetTransformerEntry> transformerEntries,
required String workingDirectory,
}) async {
final File output = _fileSystem.systemTempDirectory.childFile('retransformerInput-$inputAssetKey');
ErrorHandlingFileSystem.deleteIfExists(output);
File? inputFile;
bool cleanupInput = false;
Uint8List result;
PoolResource? resource;
try {
resource = await _transformationPool.request();
if (inputAssetContent is DevFSFileContent) {
inputFile = inputAssetContent.file as File;
} else {
inputFile = _fileSystem.systemTempDirectory.childFile('retransformerInput-$inputAssetKey');
inputFile.writeAsBytesSync(await inputAssetContent.contentsAsBytes());
cleanupInput = true;
}
final AssetTransformationFailure? failure = await _transformer.transformAsset(
asset: inputFile,
outputPath: output.path,
transformerEntries: transformerEntries,
workingDirectory: workingDirectory,
);
if (failure != null) {
_logger.printError(failure.message);
return null;
}
result = output.readAsBytesSync();
} finally {
resource?.release();
ErrorHandlingFileSystem.deleteIfExists(output);
if (cleanupInput && inputFile != null) {
ErrorHandlingFileSystem.deleteIfExists(inputFile);
}
}
return DevFSByteContent(result);
}
}
final class AssetTransformationFailure {
const AssetTransformationFailure(this.message);
......
......@@ -5,8 +5,10 @@
import 'dart:async';
import 'package:package_config/package_config.dart';
import 'package:process/process.dart';
import 'package:vm_service/vm_service.dart' as vm_service;
import 'artifacts.dart';
import 'asset.dart';
import 'base/config.dart';
import 'base/context.dart';
......@@ -16,6 +18,7 @@ import 'base/logger.dart';
import 'base/net.dart';
import 'base/os.dart';
import 'build_info.dart';
import 'build_system/tools/asset_transformer.dart';
import 'build_system/tools/scene_importer.dart';
import 'build_system/tools/shader_compiler.dart';
import 'compile.dart';
......@@ -454,6 +457,8 @@ class DevFS {
required OperatingSystemUtils osUtils,
required Logger logger,
required FileSystem fileSystem,
required ProcessManager processManager,
required Artifacts artifacts,
HttpClient? httpClient,
Duration? uploadRetryThrottle,
StopwatchFactory stopwatchFactory = const StopwatchFactory(),
......@@ -471,7 +476,16 @@ class DevFS {
? HttpClient()
: context.get<HttpClientFactory>()!())),
_stopwatchFactory = stopwatchFactory,
_config = config;
_config = config,
_assetTransformer = DevelopmentAssetTransformer(
transformer: AssetTransformer(
processManager: processManager,
fileSystem: fileSystem,
dartBinaryPath: artifacts.getArtifactPath(Artifact.engineDartBinary),
),
fileSystem: fileSystem,
logger: logger,
);
final FlutterVmService _vmService;
final _DevFSHttpWriter _httpWriter;
......@@ -479,9 +493,10 @@ class DevFS {
final FileSystem _fileSystem;
final StopwatchFactory _stopwatchFactory;
final Config? _config;
final DevelopmentAssetTransformer _assetTransformer;
final String fsName;
final Directory? rootDirectory;
final Directory rootDirectory;
final Set<String> assetPathsToEvict = <String>{};
final Set<String> shaderPathsToEvict = <String>{};
final Set<String> scenePathsToEvict = <String>{};
......@@ -505,7 +520,7 @@ class DevFS {
final String baseUriString = baseUri.toString();
if (deviceUriString.startsWith(baseUriString)) {
final String deviceUriSuffix = deviceUriString.substring(baseUriString.length);
return rootDirectory!.uri.resolve(deviceUriSuffix);
return rootDirectory.uri.resolve(deviceUriSuffix);
}
return deviceUri;
}
......@@ -600,7 +615,7 @@ class DevFS {
invalidatedFiles,
outputPath: dillOutputPath,
fs: _fileSystem,
projectRootPath: rootDirectory?.path,
projectRootPath: rootDirectory.path,
packageConfig: packageConfig,
checkDartPluginRegistry: true, // The entry point is assumed not to have changed.
dartPluginRegistrant: dartPluginRegistrant,
......@@ -636,8 +651,8 @@ class DevFS {
if (archivePath == _kFontManifest) {
didUpdateFontManifest = true;
}
switch (bundle.entries[archivePath]?.kind) {
final AssetKind? kind = bundle.entries[archivePath]?.kind;
switch (kind) {
case AssetKind.shader:
final Future<DevFSContent?> pending = shaderCompiler.recompileShader(entry.content);
pendingAssetBuilds.add(pending);
......@@ -672,11 +687,30 @@ class DevFS {
case AssetKind.regular:
case AssetKind.font:
case null:
dirtyEntries[deviceUri] = entry.content;
syncedBytes += entry.content.size;
final Future<DevFSContent?> pending = (() async {
if (entry.transformers.isEmpty || kind != AssetKind.regular) {
return entry.content;
}
return _assetTransformer.retransformAsset(
inputAssetKey: archivePath,
inputAssetContent: entry.content,
transformerEntries: entry.transformers,
workingDirectory: rootDirectory.path,
);
})();
pendingAssetBuilds.add(pending);
pending.then((DevFSContent? content) {
if (content == null) {
assetBuildFailed = true;
return;
}
dirtyEntries[deviceUri] = content;
syncedBytes += content.size;
if (!bundleFirstUpload) {
assetPathsToEvict.add(archivePath);
}
});
}
});
......
......@@ -701,6 +701,7 @@ class WebDevFS implements DevFS {
required this.nullSafetyMode,
required this.ddcModuleSystem,
required this.webRenderer,
required this.rootDirectory,
this.testMode = false,
}) : _port = port;
......@@ -857,7 +858,7 @@ class WebDevFS implements DevFS {
String get fsName => 'web_asset';
@override
Directory? get rootDirectory => null;
final Directory rootDirectory;
@override
Future<UpdateFSReport> update({
......
......@@ -311,6 +311,7 @@ Please provide a valid TCP port (an integer between 0 and 65535, inclusive).
nativeNullAssertions: debuggingOptions.nativeNullAssertions,
ddcModuleSystem: debuggingOptions.buildInfo.ddcModuleFormat == DdcModuleFormat.ddc,
webRenderer: debuggingOptions.webRenderer,
rootDirectory: fileSystem.directory(projectRootPath),
);
Uri url = await device!.devFS!.create();
if (debuggingOptions.tlsCertKeyPath != null && debuggingOptions.tlsCertPath != null) {
......
......@@ -385,6 +385,8 @@ class FlutterDevice {
osUtils: globals.os,
fileSystem: globals.fs,
logger: globals.logger,
processManager: globals.processManager,
artifacts: globals.artifacts!,
);
return devFS!.create();
}
......
......@@ -74,6 +74,7 @@ void main() {
expect(transformationFailure, isNull, reason: logger.errorText);
expect(processManager, hasNoRemainingExpectations);
expect(fileSystem.file(outputPath).readAsStringSync(), 'hello world');
expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
});
testWithoutContext('logs useful error information when transformation process returns a nonzero exit code', () async {
......@@ -138,6 +139,7 @@ stdout:
Beginning transformation
stderr:
Something went wrong''');
expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
});
testWithoutContext('prints error message when the transformer does not produce an output file', () async {
......@@ -197,6 +199,7 @@ stdout:
stderr:
Transformation failed, but I forgot to exit with a non-zero code.'''
);
expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
});
testWithoutContext('correctly chains transformations when there are multiple of them', () async {
......@@ -283,6 +286,7 @@ Transformation failed, but I forgot to exit with a non-zero code.'''
expect(processManager, hasNoRemainingExpectations);
expect(failure, isNull);
expect(fileSystem.file(outputPath).readAsStringSync(), '012');
expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
});
testWithoutContext('prints an error when a transformer in a chain (thats not the first) does not produce an output', () async {
......@@ -368,6 +372,6 @@ Transformation failed, but I forgot to exit with a non-zero code.'''
);
expect(processManager, hasNoRemainingExpectations);
expect(fileSystem.file(outputPath), isNot(exists));
expect(fileSystem.systemTempDirectory.listSync(), isEmpty);
expect(fileSystem.directory('.tmp_rand0').listSync(), isEmpty, reason: 'Transformer did not clean up after itself.');
});
}
......@@ -909,6 +909,7 @@ void main() {
nullSafetyMode: NullSafetyMode.unsound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.html,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.flutterJs.createSync(recursive: true);
......@@ -1044,6 +1045,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.html,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.flutterJs.createSync(recursive: true);
......@@ -1179,6 +1181,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1251,6 +1254,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1299,6 +1303,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1349,6 +1354,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.auto,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1401,6 +1407,7 @@ void main() {
nullSafetyMode: NullSafetyMode.unsound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1571,6 +1578,7 @@ void main() {
nullSafetyMode: NullSafetyMode.unsound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.ddcModuleLoaderJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......
......@@ -697,6 +697,7 @@ void main() {
nullSafetyMode: NullSafetyMode.unsound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.html,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.flutterJs.createSync(recursive: true);
......@@ -808,6 +809,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.html,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.flutterJs.createSync(recursive: true);
......@@ -918,6 +920,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -981,6 +984,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1028,6 +1032,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1076,6 +1081,7 @@ void main() {
nullSafetyMode: NullSafetyMode.sound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.auto,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1125,6 +1131,7 @@ void main() {
nullSafetyMode: NullSafetyMode.unsound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......@@ -1278,6 +1285,7 @@ void main() {
nullSafetyMode: NullSafetyMode.unsound,
ddcModuleSystem: usesDdcModuleSystem,
webRenderer: WebRendererMode.canvaskit,
rootDirectory: globals.fs.currentDirectory,
);
webDevFS.requireJS.createSync(recursive: true);
webDevFS.stackTraceMapper.createSync(recursive: true);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment