// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:typed_data'; import 'package:meta/meta.dart'; import 'package:package_config/package_config.dart'; import 'package:process/process.dart'; import 'package:usage/uuid/uuid.dart'; import 'artifacts.dart'; import 'base/common.dart'; import 'base/file_system.dart'; import 'base/io.dart'; import 'base/logger.dart'; import 'base/platform.dart'; import 'build_info.dart'; import 'convert.dart'; /// The target model describes the set of core libraries that are available within /// the SDK. class TargetModel { /// Parse a [TargetModel] from a raw string. /// /// Throws an exception if passed a value other than 'flutter', /// 'flutter_runner', 'vm', or 'dartdevc'. factory TargetModel(String rawValue) { switch (rawValue) { case 'flutter': return flutter; case 'flutter_runner': return flutterRunner; case 'vm': return vm; case 'dartdevc': return dartdevc; } throw Exception('Unexpected target model $rawValue'); } const TargetModel._(this._value); /// The Flutter patched Dart SDK. static const TargetModel flutter = TargetModel._('flutter'); /// The Fuchsia patched SDK. static const TargetModel flutterRunner = TargetModel._('flutter_runner'); /// The Dart VM. static const TargetModel vm = TargetModel._('vm'); /// The development compiler for JavaScript. static const TargetModel dartdevc = TargetModel._('dartdevc'); final String _value; @override String toString() => _value; } class CompilerOutput { const CompilerOutput(this.outputFilename, this.errorCount, this.sources, {this.expressionData}); final String outputFilename; final int errorCount; final List sources; /// This field is only non-null for expression compilation requests. final Uint8List? expressionData; } enum StdoutState { CollectDiagnostic, CollectDependencies } /// Handles stdin/stdout communication with the frontend server. class StdoutHandler { StdoutHandler({ required Logger logger, required FileSystem fileSystem, }) : _logger = logger, _fileSystem = fileSystem { reset(); } final Logger _logger; final FileSystem _fileSystem; String? boundaryKey; StdoutState state = StdoutState.CollectDiagnostic; Completer? compilerOutput; final List sources = []; bool _suppressCompilerMessages = false; bool _expectSources = true; bool _readFile = false; void handler(String message) { const String kResultPrefix = 'result '; if (boundaryKey == null && message.startsWith(kResultPrefix)) { boundaryKey = message.substring(kResultPrefix.length); return; } final String? messageBoundaryKey = boundaryKey; if (messageBoundaryKey != null && message.startsWith(messageBoundaryKey)) { if (_expectSources) { if (state == StdoutState.CollectDiagnostic) { state = StdoutState.CollectDependencies; return; } } if (message.length <= messageBoundaryKey.length) { compilerOutput?.complete(null); return; } final int spaceDelimiter = message.lastIndexOf(' '); final String fileName = message.substring(messageBoundaryKey.length + 1, spaceDelimiter); final int errorCount = int.parse(message.substring(spaceDelimiter + 1).trim()); Uint8List? expressionData; if (_readFile) { expressionData = _fileSystem.file(fileName).readAsBytesSync(); } final CompilerOutput output = CompilerOutput( fileName, errorCount, sources, expressionData: expressionData, ); compilerOutput?.complete(output); return; } if (state == StdoutState.CollectDiagnostic) { if (!_suppressCompilerMessages) { _logger.printError(message); } else { _logger.printTrace(message); } } else { assert(state == StdoutState.CollectDependencies); switch (message[0]) { case '+': sources.add(Uri.parse(message.substring(1))); break; case '-': sources.remove(Uri.parse(message.substring(1))); break; default: _logger.printTrace('Unexpected prefix for $message uri - ignoring'); } } } // This is needed to get ready to process next compilation result output, // with its own boundary key and new completer. void reset({ bool suppressCompilerMessages = false, bool expectSources = true, bool readFile = false }) { boundaryKey = null; compilerOutput = Completer(); _suppressCompilerMessages = suppressCompilerMessages; _expectSources = expectSources; _readFile = readFile; state = StdoutState.CollectDiagnostic; } } /// List the preconfigured build options for a given build mode. List buildModeOptions(BuildMode mode, List dartDefines) { switch (mode) { case BuildMode.debug: return [ '-Ddart.vm.profile=false', // This allows the CLI to override the value of this define for unit // testing the framework. if (!dartDefines.any((String define) => define.startsWith('dart.vm.product'))) '-Ddart.vm.product=false', '--enable-asserts', ]; case BuildMode.profile: return [ '-Ddart.vm.profile=true', '-Ddart.vm.product=false', ]; case BuildMode.release: return [ '-Ddart.vm.profile=false', '-Ddart.vm.product=true', ]; } throw Exception('Unknown BuildMode: $mode'); } /// A compiler interface for producing single (non-incremental) kernel files. class KernelCompiler { KernelCompiler({ required FileSystem fileSystem, required Logger logger, required ProcessManager processManager, required Artifacts artifacts, required List fileSystemRoots, required String fileSystemScheme, @visibleForTesting StdoutHandler? stdoutHandler, }) : _logger = logger, _fileSystem = fileSystem, _artifacts = artifacts, _processManager = processManager, _fileSystemScheme = fileSystemScheme, _fileSystemRoots = fileSystemRoots, _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger, fileSystem: fileSystem); final FileSystem _fileSystem; final Artifacts _artifacts; final ProcessManager _processManager; final Logger _logger; final String _fileSystemScheme; final List _fileSystemRoots; final StdoutHandler _stdoutHandler; Future compile({ required String sdkRoot, String? mainPath, String? outputFilePath, String? depFilePath, TargetModel targetModel = TargetModel.flutter, bool linkPlatformKernelIn = false, bool aot = false, List? extraFrontEndOptions, List? fileSystemRoots, String? fileSystemScheme, String? initializeFromDill, String? platformDill, Directory? buildDir, bool checkDartPluginRegistry = false, required String? packagesPath, required BuildMode buildMode, required bool trackWidgetCreation, required List dartDefines, required PackageConfig packageConfig, }) async { final String frontendServer = _artifacts.getArtifactPath( Artifact.frontendServerSnapshotForEngineDartSdk ); // This is a URI, not a file path, so the forward slash is correct even on Windows. if (!sdkRoot.endsWith('/')) { sdkRoot = '$sdkRoot/'; } final String engineDartPath = _artifacts.getHostArtifact(HostArtifact.engineDartBinary).path; if (!_processManager.canRun(engineDartPath)) { throwToolExit('Unable to find Dart binary at $engineDartPath'); } String? mainUri; final File mainFile = _fileSystem.file(mainPath); final Uri mainFileUri = mainFile.uri; if (packagesPath != null) { mainUri = packageConfig.toPackageUri(mainFileUri)?.toString(); } mainUri ??= toMultiRootPath(mainFileUri, _fileSystemScheme, _fileSystemRoots, _fileSystem.path.separator == r'\'); if (outputFilePath != null && !_fileSystem.isFileSync(outputFilePath)) { _fileSystem.file(outputFilePath).createSync(recursive: true); } if (buildDir != null && checkDartPluginRegistry) { // Check if there's a Dart plugin registrant. // This is contained in the file `generated_main.dart` under `.dart_tools/flutter_build/`. final File newMainDart = buildDir.parent.childFile('generated_main.dart'); if (newMainDart.existsSync()) { mainUri = newMainDart.path; } } final List command = [ engineDartPath, '--disable-dart-dev', frontendServer, '--sdk-root', sdkRoot, '--target=$targetModel', '--no-print-incremental-dependencies', for (final Object dartDefine in dartDefines) '-D$dartDefine', ...buildModeOptions(buildMode, dartDefines), if (trackWidgetCreation) '--track-widget-creation', if (!linkPlatformKernelIn) '--no-link-platform', if (aot) ...[ '--aot', '--tfa', ], if (packagesPath != null) ...[ '--packages', packagesPath, ], if (outputFilePath != null) ...[ '--output-dill', outputFilePath, ], if (depFilePath != null && (fileSystemRoots == null || fileSystemRoots.isEmpty)) ...[ '--depfile', depFilePath, ], if (fileSystemRoots != null) for (final String root in fileSystemRoots) ...[ '--filesystem-root', root, ], if (fileSystemScheme != null) ...[ '--filesystem-scheme', fileSystemScheme, ], if (initializeFromDill != null) ...[ '--initialize-from-dill', initializeFromDill, ], if (platformDill != null) ...[ '--platform', platformDill, ], ...?extraFrontEndOptions, mainUri, ]; _logger.printTrace(command.join(' ')); final Process server = await _processManager.start(command); server.stderr .transform(utf8.decoder) .listen(_logger.printError); server.stdout .transform(utf8.decoder) .transform(const LineSplitter()) .listen(_stdoutHandler.handler); final int exitCode = await server.exitCode; if (exitCode == 0) { return _stdoutHandler.compilerOutput?.future; } return null; } } /// Class that allows to serialize compilation requests to the compiler. abstract class _CompilationRequest { _CompilationRequest(this.completer); Completer completer; Future _run(DefaultResidentCompiler compiler); Future run(DefaultResidentCompiler compiler) async { completer.complete(await _run(compiler)); } } class _RecompileRequest extends _CompilationRequest { _RecompileRequest( Completer completer, this.mainUri, this.invalidatedFiles, this.outputPath, this.packageConfig, this.suppressErrors, ) : super(completer); Uri mainUri; List? invalidatedFiles; String outputPath; PackageConfig packageConfig; bool suppressErrors; @override Future _run(DefaultResidentCompiler compiler) async => compiler._recompile(this); } class _CompileExpressionRequest extends _CompilationRequest { _CompileExpressionRequest( Completer completer, this.expression, this.definitions, this.typeDefinitions, this.libraryUri, this.klass, this.isStatic, ) : super(completer); String expression; List? definitions; List? typeDefinitions; String? libraryUri; String? klass; bool isStatic; @override Future _run(DefaultResidentCompiler compiler) async => compiler._compileExpression(this); } class _CompileExpressionToJsRequest extends _CompilationRequest { _CompileExpressionToJsRequest( Completer completer, this.libraryUri, this.line, this.column, this.jsModules, this.jsFrameValues, this.moduleName, this.expression, ) : super(completer); final String? libraryUri; final int line; final int column; final Map? jsModules; final Map? jsFrameValues; final String? moduleName; final String? expression; @override Future _run(DefaultResidentCompiler compiler) async => compiler._compileExpressionToJs(this); } class _RejectRequest extends _CompilationRequest { _RejectRequest(Completer completer) : super(completer); @override Future _run(DefaultResidentCompiler compiler) async => compiler._reject(); } /// Wrapper around incremental frontend server compiler, that communicates with /// server via stdin/stdout. /// /// The wrapper is intended to stay resident in memory as user changes, reloads, /// restarts the Flutter app. abstract class ResidentCompiler { factory ResidentCompiler(String sdkRoot, { required BuildMode buildMode, required Logger logger, required ProcessManager processManager, required Artifacts artifacts, required Platform platform, required FileSystem fileSystem, bool testCompilation, bool trackWidgetCreation, String packagesPath, List fileSystemRoots, String? fileSystemScheme, String initializeFromDill, TargetModel targetModel, bool unsafePackageSerialization, List extraFrontEndOptions, String platformDill, List? dartDefines, String librariesSpec, }) = DefaultResidentCompiler; // TODO(jonahwilliams): find a better way to configure additional file system // roots from the runner. // See: https://github.com/flutter/flutter/issues/50494 void addFileSystemRoot(String root); /// If invoked for the first time, it compiles Dart script identified by /// [mainPath], [invalidatedFiles] list is ignored. /// On successive runs [invalidatedFiles] indicates which files need to be /// recompiled. If [mainPath] is [null], previously used [mainPath] entry /// point that is used for recompilation. /// Binary file name is returned if compilation was successful, otherwise /// null is returned. Future recompile( Uri mainUri, List? invalidatedFiles, { required String outputPath, required PackageConfig packageConfig, required String projectRootPath, required FileSystem fs, bool suppressErrors = false, }); Future compileExpression( String expression, List? definitions, List? typeDefinitions, String? libraryUri, String? klass, bool isStatic, ); /// Compiles [expression] in [libraryUri] at [line]:[column] to JavaScript /// in [moduleName]. /// /// Values listed in [jsFrameValues] are substituted for their names in the /// [expression]. /// /// Ensures that all [jsModules] are loaded and accessible inside the /// expression. /// /// Example values of parameters: /// [moduleName] is of the form '/packages/hello_world_main.dart' /// [jsFrameValues] is a map from js variable name to its primitive value /// or another variable name, for example /// { 'x': '1', 'y': 'y', 'o': 'null' } /// [jsModules] is a map from variable name to the module name, where /// variable name is the name originally used in JavaScript to contain the /// module object, for example: /// { 'dart':'dart_sdk', 'main': '/packages/hello_world_main.dart' } /// Returns a [CompilerOutput] including the name of the file containing the /// compilation result and a number of errors. Future compileExpressionToJs( String libraryUri, int line, int column, Map jsModules, Map jsFrameValues, String moduleName, String expression, ); /// Should be invoked when results of compilation are accepted by the client. /// /// Either [accept] or [reject] should be called after every [recompile] call. void accept(); /// Should be invoked when results of compilation are rejected by the client. /// /// Either [accept] or [reject] should be called after every [recompile] call. Future reject(); /// Should be invoked when frontend server compiler should forget what was /// accepted previously so that next call to [recompile] produces complete /// kernel file. void reset(); Future shutdown(); } @visibleForTesting class DefaultResidentCompiler implements ResidentCompiler { DefaultResidentCompiler( String sdkRoot, { required this.buildMode, required Logger logger, required ProcessManager processManager, required Artifacts artifacts, required Platform platform, required FileSystem fileSystem, this.testCompilation = false, this.trackWidgetCreation = true, this.packagesPath, this.fileSystemRoots = const [], this.fileSystemScheme, this.initializeFromDill, this.targetModel = TargetModel.flutter, this.unsafePackageSerialization = false, this.extraFrontEndOptions, this.platformDill, List? dartDefines, this.librariesSpec, @visibleForTesting StdoutHandler? stdoutHandler, }) : assert(sdkRoot != null), _logger = logger, _processManager = processManager, _artifacts = artifacts, _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger, fileSystem: fileSystem), _platform = platform, dartDefines = dartDefines ?? const [], // This is a URI, not a file path, so the forward slash is correct even on Windows. sdkRoot = sdkRoot.endsWith('/') ? sdkRoot : '$sdkRoot/'; final Logger _logger; final ProcessManager _processManager; final Artifacts _artifacts; final Platform _platform; final bool testCompilation; final BuildMode buildMode; final bool trackWidgetCreation; final String? packagesPath; final TargetModel targetModel; final List fileSystemRoots; final String? fileSystemScheme; final String? initializeFromDill; final bool unsafePackageSerialization; final List? extraFrontEndOptions; final List dartDefines; final String? librariesSpec; @override void addFileSystemRoot(String root) { fileSystemRoots.add(root); } /// The path to the root of the Dart SDK used to compile. /// /// This is used to resolve the [platformDill]. final String sdkRoot; /// The path to the platform dill file. /// /// This does not need to be provided for the normal Flutter workflow. final String? platformDill; Process? _server; final StdoutHandler _stdoutHandler; bool _compileRequestNeedsConfirmation = false; final StreamController<_CompilationRequest> _controller = StreamController<_CompilationRequest>(); @override Future recompile( Uri mainUri, List? invalidatedFiles, { required String outputPath, required PackageConfig packageConfig, bool suppressErrors = false, String? projectRootPath, FileSystem? fs, }) async { assert(outputPath != null); if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } // `generated_main.dart` contains the Dart plugin registry. if (projectRootPath != null && fs != null) { final File generatedMainDart = fs.file( fs.path.join( projectRootPath, '.dart_tool', 'flutter_build', 'generated_main.dart', ), ); if (generatedMainDart != null && generatedMainDart.existsSync()) { mainUri = generatedMainDart.uri; } } final Completer completer = Completer(); _controller.add( _RecompileRequest(completer, mainUri, invalidatedFiles, outputPath, packageConfig, suppressErrors) ); return completer.future; } Future _recompile(_RecompileRequest request) async { _stdoutHandler.reset(); _compileRequestNeedsConfirmation = true; _stdoutHandler._suppressCompilerMessages = request.suppressErrors; final String mainUri = request.packageConfig.toPackageUri(request.mainUri)?.toString() ?? toMultiRootPath(request.mainUri, fileSystemScheme, fileSystemRoots, _platform.isWindows); final Process? server = _server; if (server == null) { return _compile(mainUri, request.outputPath); } final String inputKey = Uuid().generateV4(); server.stdin.writeln('recompile $mainUri $inputKey'); _logger.printTrace('<- recompile $mainUri $inputKey'); final List? invalidatedFiles = request.invalidatedFiles; if (invalidatedFiles != null) { for (final Uri fileUri in invalidatedFiles) { String message; if (fileUri.scheme == 'package') { message = fileUri.toString(); } else { message = request.packageConfig.toPackageUri(fileUri)?.toString() ?? toMultiRootPath(fileUri, fileSystemScheme, fileSystemRoots, _platform.isWindows); } server.stdin.writeln(message); _logger.printTrace(message); } } server.stdin.writeln(inputKey); _logger.printTrace('<- $inputKey'); return _stdoutHandler.compilerOutput?.future; } final List<_CompilationRequest> _compilationQueue = <_CompilationRequest>[]; Future _handleCompilationRequest(_CompilationRequest request) async { final bool isEmpty = _compilationQueue.isEmpty; _compilationQueue.add(request); // Only trigger processing if queue was empty - i.e. no other requests // are currently being processed. This effectively enforces "one // compilation request at a time". if (isEmpty) { while (_compilationQueue.isNotEmpty) { final _CompilationRequest request = _compilationQueue.first; await request.run(this); _compilationQueue.removeAt(0); } } } Future _compile( String scriptUri, String? outputPath, ) async { final String frontendServer = _artifacts.getArtifactPath( Artifact.frontendServerSnapshotForEngineDartSdk ); final List command = [ _artifacts.getHostArtifact(HostArtifact.engineDartBinary).path, '--disable-dart-dev', frontendServer, '--sdk-root', sdkRoot, '--incremental', if (testCompilation) '--no-print-incremental-dependencies', '--target=$targetModel', // TODO(jonahwilliams): remove once this becomes the default behavior // in the frontend_server. // https://github.com/flutter/flutter/issues/52693 '--debugger-module-names', // TODO(annagrin): remove once this becomes the default behavior // in the frontend_server. // https://github.com/flutter/flutter/issues/59902 '--experimental-emit-debug-metadata', for (final Object dartDefine in dartDefines) '-D$dartDefine', if (outputPath != null) ...[ '--output-dill', outputPath, ], if (librariesSpec != null) ...[ '--libraries-spec', librariesSpec!, ], if (packagesPath != null) ...[ '--packages', packagesPath!, ], ...buildModeOptions(buildMode, dartDefines), if (trackWidgetCreation) '--track-widget-creation', if (fileSystemRoots != null) for (final String root in fileSystemRoots) ...[ '--filesystem-root', root, ], if (fileSystemScheme != null) ...[ '--filesystem-scheme', fileSystemScheme!, ], if (initializeFromDill != null) ...[ '--initialize-from-dill', initializeFromDill!, ], if (platformDill != null) ...[ '--platform', platformDill!, ], if (unsafePackageSerialization == true) '--unsafe-package-serialization', ...?extraFrontEndOptions, ]; _logger.printTrace(command.join(' ')); _server = await _processManager.start(command); _server?.stdout .transform(utf8.decoder) .transform(const LineSplitter()) .listen( _stdoutHandler.handler, onDone: () { // when outputFilename future is not completed, but stdout is closed // process has died unexpectedly. if (_stdoutHandler.compilerOutput?.isCompleted == false) { _stdoutHandler.compilerOutput?.complete(null); throwToolExit('the Dart compiler exited unexpectedly.'); } }); _server?.stderr .transform(utf8.decoder) .transform(const LineSplitter()) .listen(_logger.printError); unawaited(_server?.exitCode.then((int code) { if (code != 0) { throwToolExit('the Dart compiler exited unexpectedly.'); } })); _server?.stdin.writeln('compile $scriptUri'); _logger.printTrace('<- compile $scriptUri'); return _stdoutHandler.compilerOutput?.future; } @override Future compileExpression( String expression, List? definitions, List? typeDefinitions, String? libraryUri, String? klass, bool isStatic, ) async { if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } final Completer completer = Completer(); final _CompileExpressionRequest request = _CompileExpressionRequest( completer, expression, definitions, typeDefinitions, libraryUri, klass, isStatic); _controller.add(request); return completer.future; } Future _compileExpression(_CompileExpressionRequest request) async { _stdoutHandler.reset(suppressCompilerMessages: true, expectSources: false, readFile: true); // 'compile-expression' should be invoked after compiler has been started, // program was compiled. final Process? server = _server; if (server == null) { return null; } final String inputKey = Uuid().generateV4(); server.stdin ..writeln('compile-expression $inputKey') ..writeln(request.expression); request.definitions?.forEach(server.stdin.writeln); server.stdin.writeln(inputKey); request.typeDefinitions?.forEach(server.stdin.writeln); server.stdin ..writeln(inputKey) ..writeln(request.libraryUri ?? '') ..writeln(request.klass ?? '') ..writeln(request.isStatic); return _stdoutHandler.compilerOutput?.future; } @override Future compileExpressionToJs( String libraryUri, int line, int column, Map jsModules, Map jsFrameValues, String moduleName, String expression, ) { if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } final Completer completer = Completer(); _controller.add( _CompileExpressionToJsRequest( completer, libraryUri, line, column, jsModules, jsFrameValues, moduleName, expression) ); return completer.future; } Future _compileExpressionToJs(_CompileExpressionToJsRequest request) async { _stdoutHandler.reset(suppressCompilerMessages: true, expectSources: false); // 'compile-expression-to-js' should be invoked after compiler has been started, // program was compiled. final Process? server = _server; if (server == null) { return null; } final String inputKey = Uuid().generateV4(); server.stdin ..writeln('compile-expression-to-js $inputKey') ..writeln(request.libraryUri ?? '') ..writeln(request.line) ..writeln(request.column); request.jsModules?.forEach((String k, String v) { server.stdin.writeln('$k:$v'); }); server.stdin.writeln(inputKey); request.jsFrameValues?.forEach((String k, String v) { server.stdin.writeln('$k:$v'); }); server.stdin ..writeln(inputKey) ..writeln(request.moduleName ?? '') ..writeln(request.expression ?? ''); return _stdoutHandler.compilerOutput?.future; } @override void accept() { if (_compileRequestNeedsConfirmation) { _server?.stdin.writeln('accept'); _logger.printTrace('<- accept'); } _compileRequestNeedsConfirmation = false; } @override Future reject() { if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } final Completer completer = Completer(); _controller.add(_RejectRequest(completer)); return completer.future; } Future _reject() async { if (!_compileRequestNeedsConfirmation) { return Future.value(null); } _stdoutHandler.reset(expectSources: false); _server?.stdin.writeln('reject'); _logger.printTrace('<- reject'); _compileRequestNeedsConfirmation = false; return _stdoutHandler.compilerOutput?.future; } @override void reset() { _server?.stdin.writeln('reset'); _logger.printTrace('<- reset'); } @override Future shutdown() async { // Server was never successfully created. final Process? server = _server; if (server == null) { return 0; } _logger.printTrace('killing pid ${server.pid}'); server.kill(); return server.exitCode; } } /// Convert a file URI into a multi-root scheme URI if provided, otherwise /// return unmodified. @visibleForTesting String toMultiRootPath(Uri fileUri, String? scheme, List fileSystemRoots, bool windows) { if (scheme == null || fileSystemRoots.isEmpty || fileUri.scheme != 'file') { return fileUri.toString(); } final String filePath = fileUri.toFilePath(windows: windows); for (final String fileSystemRoot in fileSystemRoots) { if (filePath.startsWith(fileSystemRoot)) { return scheme + '://' + filePath.substring(fileSystemRoot.length); } } return fileUri.toString(); }