// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // @dart = 2.8 import 'dart:async'; import 'dart:typed_data'; import 'package:meta/meta.dart'; import 'package:package_config/package_config.dart'; import 'package:process/process.dart'; import 'package:usage/uuid/uuid.dart'; import 'artifacts.dart'; import 'base/common.dart'; import 'base/file_system.dart'; import 'base/io.dart'; import 'base/logger.dart'; import 'base/platform.dart'; import 'build_info.dart'; import 'convert.dart'; /// The target model describes the set of core libraries that are available within /// the SDK. class TargetModel { /// Parse a [TargetModel] from a raw string. /// /// Throws an [AssertionError] if passed a value other than 'flutter' or /// 'flutter_runner'. factory TargetModel(String rawValue) { switch (rawValue) { case 'flutter': return flutter; case 'flutter_runner': return flutterRunner; case 'vm': return vm; case 'dartdevc': return dartdevc; } assert(false); return null; } const TargetModel._(this._value); /// The Flutter patched Dart SDK. static const TargetModel flutter = TargetModel._('flutter'); /// The Fuchsia patched SDK. static const TargetModel flutterRunner = TargetModel._('flutter_runner'); /// The Dart VM. static const TargetModel vm = TargetModel._('vm'); /// The development compiler for JavaScript. static const TargetModel dartdevc = TargetModel._('dartdevc'); final String _value; @override String toString() => _value; } class CompilerOutput { const CompilerOutput(this.outputFilename, this.errorCount, this.sources, {this.expressionData}); final String outputFilename; final int errorCount; final List sources; /// This field is only non-null for expression compilation requests. final Uint8List expressionData; } enum StdoutState { CollectDiagnostic, CollectDependencies } /// Handles stdin/stdout communication with the frontend server. class StdoutHandler { StdoutHandler({ @required Logger logger, @required FileSystem fileSystem, }) : _logger = logger, _fileSystem = fileSystem { reset(); } final Logger _logger; final FileSystem _fileSystem; String boundaryKey; StdoutState state = StdoutState.CollectDiagnostic; Completer compilerOutput; final List sources = []; bool _suppressCompilerMessages; bool _expectSources; bool _readFile; void handler(String message) { const String kResultPrefix = 'result '; if (boundaryKey == null && message.startsWith(kResultPrefix)) { boundaryKey = message.substring(kResultPrefix.length); return; } if (message.startsWith(boundaryKey)) { if (_expectSources) { if (state == StdoutState.CollectDiagnostic) { state = StdoutState.CollectDependencies; return; } } if (message.length <= boundaryKey.length) { compilerOutput.complete(null); return; } final int spaceDelimiter = message.lastIndexOf(' '); final String fileName = message.substring(boundaryKey.length + 1, spaceDelimiter); final int errorCount = int.parse(message.substring(spaceDelimiter + 1).trim()); Uint8List expressionData; if (_readFile) { expressionData = _fileSystem.file(fileName).readAsBytesSync(); } final CompilerOutput output = CompilerOutput( fileName, errorCount, sources, expressionData: expressionData, ); compilerOutput.complete(output); return; } if (state == StdoutState.CollectDiagnostic) { if (!_suppressCompilerMessages) { _logger.printError(message); } else { _logger.printTrace(message); } } else { assert(state == StdoutState.CollectDependencies); switch (message[0]) { case '+': sources.add(Uri.parse(message.substring(1))); break; case '-': sources.remove(Uri.parse(message.substring(1))); break; default: _logger.printTrace('Unexpected prefix for $message uri - ignoring'); } } } // This is needed to get ready to process next compilation result output, // with its own boundary key and new completer. void reset({ bool suppressCompilerMessages = false, bool expectSources = true, bool readFile = false }) { boundaryKey = null; compilerOutput = Completer(); _suppressCompilerMessages = suppressCompilerMessages; _expectSources = expectSources; _readFile = readFile; state = StdoutState.CollectDiagnostic; } } /// List the preconfigured build options for a given build mode. List buildModeOptions(BuildMode mode, List dartDefines) { switch (mode) { case BuildMode.debug: return [ '-Ddart.vm.profile=false', // This allows the CLI to override the value of this define for unit // testing the framework. if (!dartDefines.any((String define) => define.startsWith('dart.vm.product'))) '-Ddart.vm.product=false', '--enable-asserts', ]; case BuildMode.profile: return [ '-Ddart.vm.profile=true', '-Ddart.vm.product=false', ]; case BuildMode.release: return [ '-Ddart.vm.profile=false', '-Ddart.vm.product=true', ]; } throw Exception('Unknown BuildMode: $mode'); } /// A compiler interface for producing single (non-incremental) kernel files. class KernelCompiler { KernelCompiler({ @required FileSystem fileSystem, @required Logger logger, @required ProcessManager processManager, @required Artifacts artifacts, @required List fileSystemRoots, @required String fileSystemScheme, @visibleForTesting StdoutHandler stdoutHandler, }) : _logger = logger, _fileSystem = fileSystem, _artifacts = artifacts, _processManager = processManager, _fileSystemScheme = fileSystemScheme, _fileSystemRoots = fileSystemRoots, _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger, fileSystem: fileSystem); final FileSystem _fileSystem; final Artifacts _artifacts; final ProcessManager _processManager; final Logger _logger; final String _fileSystemScheme; final List _fileSystemRoots; final StdoutHandler _stdoutHandler; Future compile({ String sdkRoot, String mainPath, String outputFilePath, String depFilePath, TargetModel targetModel = TargetModel.flutter, bool linkPlatformKernelIn = false, bool aot = false, List extraFrontEndOptions, List fileSystemRoots, String fileSystemScheme, String initializeFromDill, String platformDill, Directory buildDir, bool checkDartPluginRegistry = false, @required String packagesPath, @required BuildMode buildMode, @required bool trackWidgetCreation, @required List dartDefines, @required PackageConfig packageConfig, }) async { final String frontendServer = _artifacts.getArtifactPath( Artifact.frontendServerSnapshotForEngineDartSdk ); // This is a URI, not a file path, so the forward slash is correct even on Windows. if (!sdkRoot.endsWith('/')) { sdkRoot = '$sdkRoot/'; } final String engineDartPath = _artifacts.getHostArtifact(HostArtifact.engineDartBinary).path; if (!_processManager.canRun(engineDartPath)) { throwToolExit('Unable to find Dart binary at $engineDartPath'); } String mainUri; final File mainFile = _fileSystem.file(mainPath); final Uri mainFileUri = mainFile.uri; if (packagesPath != null) { mainUri = packageConfig.toPackageUri(mainFileUri)?.toString(); } mainUri ??= toMultiRootPath(mainFileUri, _fileSystemScheme, _fileSystemRoots, _fileSystem.path.separator == r'\'); if (outputFilePath != null && !_fileSystem.isFileSync(outputFilePath)) { _fileSystem.file(outputFilePath).createSync(recursive: true); } if (buildDir != null && checkDartPluginRegistry) { // Check if there's a Dart plugin registrant. // This is contained in the file `generated_main.dart` under `.dart_tools/flutter_build/`. final File newMainDart = buildDir.parent.childFile('generated_main.dart'); if (newMainDart.existsSync()) { mainUri = newMainDart.path; } } final List command = [ engineDartPath, '--disable-dart-dev', frontendServer, '--sdk-root', sdkRoot, '--target=$targetModel', '--no-print-incremental-dependencies', for (final Object dartDefine in dartDefines) '-D$dartDefine', ...buildModeOptions(buildMode, dartDefines), if (trackWidgetCreation) '--track-widget-creation', if (!linkPlatformKernelIn) '--no-link-platform', if (aot) ...[ '--aot', '--tfa', ], if (packagesPath != null) ...[ '--packages', packagesPath, ], if (outputFilePath != null) ...[ '--output-dill', outputFilePath, ], if (depFilePath != null && (fileSystemRoots == null || fileSystemRoots.isEmpty)) ...[ '--depfile', depFilePath, ], if (fileSystemRoots != null) for (final String root in fileSystemRoots) ...[ '--filesystem-root', root, ], if (fileSystemScheme != null) ...[ '--filesystem-scheme', fileSystemScheme, ], if (initializeFromDill != null) ...[ '--initialize-from-dill', initializeFromDill, ], if (platformDill != null) ...[ '--platform', platformDill, ], ...?extraFrontEndOptions, mainUri ?? mainPath, ]; _logger.printTrace(command.join(' ')); final Process server = await _processManager.start(command); server.stderr .transform(utf8.decoder) .listen(_logger.printError); server.stdout .transform(utf8.decoder) .transform(const LineSplitter()) .listen(_stdoutHandler.handler); final int exitCode = await server.exitCode; if (exitCode == 0) { return _stdoutHandler.compilerOutput.future; } return null; } } /// Class that allows to serialize compilation requests to the compiler. abstract class _CompilationRequest { _CompilationRequest(this.completer); Completer completer; Future _run(DefaultResidentCompiler compiler); Future run(DefaultResidentCompiler compiler) async { completer.complete(await _run(compiler)); } } class _RecompileRequest extends _CompilationRequest { _RecompileRequest( Completer completer, this.mainUri, this.invalidatedFiles, this.outputPath, this.packageConfig, this.suppressErrors, ) : super(completer); Uri mainUri; List invalidatedFiles; String outputPath; PackageConfig packageConfig; bool suppressErrors; @override Future _run(DefaultResidentCompiler compiler) async => compiler._recompile(this); } class _CompileExpressionRequest extends _CompilationRequest { _CompileExpressionRequest( Completer completer, this.expression, this.definitions, this.typeDefinitions, this.libraryUri, this.klass, this.isStatic, ) : super(completer); String expression; List definitions; List typeDefinitions; String libraryUri; String klass; bool isStatic; @override Future _run(DefaultResidentCompiler compiler) async => compiler._compileExpression(this); } class _CompileExpressionToJsRequest extends _CompilationRequest { _CompileExpressionToJsRequest( Completer completer, this.libraryUri, this.line, this.column, this.jsModules, this.jsFrameValues, this.moduleName, this.expression, ) : super(completer); final String libraryUri; final int line; final int column; final Map jsModules; final Map jsFrameValues; final String moduleName; final String expression; @override Future _run(DefaultResidentCompiler compiler) async => compiler._compileExpressionToJs(this); } class _RejectRequest extends _CompilationRequest { _RejectRequest(Completer completer) : super(completer); @override Future _run(DefaultResidentCompiler compiler) async => compiler._reject(); } /// Wrapper around incremental frontend server compiler, that communicates with /// server via stdin/stdout. /// /// The wrapper is intended to stay resident in memory as user changes, reloads, /// restarts the Flutter app. abstract class ResidentCompiler { factory ResidentCompiler(String sdkRoot, { @required BuildMode buildMode, @required Logger logger, @required ProcessManager processManager, @required Artifacts artifacts, @required Platform platform, @required FileSystem fileSystem, bool testCompilation, bool trackWidgetCreation, String packagesPath, List fileSystemRoots, String fileSystemScheme, String initializeFromDill, TargetModel targetModel, bool unsafePackageSerialization, List extraFrontEndOptions, String platformDill, List dartDefines, String librariesSpec, }) = DefaultResidentCompiler; // TODO(jonahwilliams): find a better way to configure additional file system // roots from the runner. // See: https://github.com/flutter/flutter/issues/50494 void addFileSystemRoot(String root); /// If invoked for the first time, it compiles Dart script identified by /// [mainPath], [invalidatedFiles] list is ignored. /// On successive runs [invalidatedFiles] indicates which files need to be /// recompiled. If [mainPath] is [null], previously used [mainPath] entry /// point that is used for recompilation. /// Binary file name is returned if compilation was successful, otherwise /// null is returned. Future recompile( Uri mainUri, List invalidatedFiles, { @required String outputPath, @required PackageConfig packageConfig, @required String projectRootPath, @required FileSystem fs, bool suppressErrors = false, }); Future compileExpression( String expression, List definitions, List typeDefinitions, String libraryUri, String klass, bool isStatic, ); /// Compiles [expression] in [libraryUri] at [line]:[column] to JavaScript /// in [moduleName]. /// /// Values listed in [jsFrameValues] are substituted for their names in the /// [expression]. /// /// Ensures that all [jsModules] are loaded and accessible inside the /// expression. /// /// Example values of parameters: /// [moduleName] is of the form '/packages/hello_world_main.dart' /// [jsFrameValues] is a map from js variable name to its primitive value /// or another variable name, for example /// { 'x': '1', 'y': 'y', 'o': 'null' } /// [jsModules] is a map from variable name to the module name, where /// variable name is the name originally used in JavaScript to contain the /// module object, for example: /// { 'dart':'dart_sdk', 'main': '/packages/hello_world_main.dart' } /// Returns a [CompilerOutput] including the name of the file containing the /// compilation result and a number of errors. Future compileExpressionToJs( String libraryUri, int line, int column, Map jsModules, Map jsFrameValues, String moduleName, String expression, ); /// Should be invoked when results of compilation are accepted by the client. /// /// Either [accept] or [reject] should be called after every [recompile] call. void accept(); /// Should be invoked when results of compilation are rejected by the client. /// /// Either [accept] or [reject] should be called after every [recompile] call. Future reject(); /// Should be invoked when frontend server compiler should forget what was /// accepted previously so that next call to [recompile] produces complete /// kernel file. void reset(); Future shutdown(); } @visibleForTesting class DefaultResidentCompiler implements ResidentCompiler { DefaultResidentCompiler( String sdkRoot, { @required this.buildMode, @required Logger logger, @required ProcessManager processManager, @required Artifacts artifacts, @required Platform platform, @required FileSystem fileSystem, this.testCompilation = false, this.trackWidgetCreation = true, this.packagesPath, this.fileSystemRoots, this.fileSystemScheme, this.initializeFromDill, this.targetModel = TargetModel.flutter, this.unsafePackageSerialization, this.extraFrontEndOptions, this.platformDill, List dartDefines, this.librariesSpec, @visibleForTesting StdoutHandler stdoutHandler, }) : assert(sdkRoot != null), _logger = logger, _processManager = processManager, _artifacts = artifacts, _stdoutHandler = stdoutHandler ?? StdoutHandler(logger: logger, fileSystem: fileSystem), _platform = platform, dartDefines = dartDefines ?? const [], // This is a URI, not a file path, so the forward slash is correct even on Windows. sdkRoot = sdkRoot.endsWith('/') ? sdkRoot : '$sdkRoot/'; final Logger _logger; final ProcessManager _processManager; final Artifacts _artifacts; final Platform _platform; final bool testCompilation; final BuildMode buildMode; final bool trackWidgetCreation; final String packagesPath; final TargetModel targetModel; final List fileSystemRoots; final String fileSystemScheme; final String initializeFromDill; final bool unsafePackageSerialization; final List extraFrontEndOptions; final List dartDefines; final String librariesSpec; @override void addFileSystemRoot(String root) { fileSystemRoots.add(root); } /// The path to the root of the Dart SDK used to compile. /// /// This is used to resolve the [platformDill]. final String sdkRoot; /// The path to the platform dill file. /// /// This does not need to be provided for the normal Flutter workflow. final String platformDill; Process _server; final StdoutHandler _stdoutHandler; bool _compileRequestNeedsConfirmation = false; final StreamController<_CompilationRequest> _controller = StreamController<_CompilationRequest>(); @override Future recompile( Uri mainUri, List invalidatedFiles, { @required String outputPath, @required PackageConfig packageConfig, bool suppressErrors = false, String projectRootPath, FileSystem fs, }) async { assert(outputPath != null); if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } // `generated_main.dart` contains the Dart plugin registry. if (projectRootPath != null && fs != null) { final File generatedMainDart = fs.file( fs.path.join( projectRootPath, '.dart_tool', 'flutter_build', 'generated_main.dart', ), ); if (generatedMainDart != null && generatedMainDart.existsSync()) { mainUri = generatedMainDart.uri; } } final Completer completer = Completer(); _controller.add( _RecompileRequest(completer, mainUri, invalidatedFiles, outputPath, packageConfig, suppressErrors) ); return completer.future; } Future _recompile(_RecompileRequest request) async { _stdoutHandler.reset(); _compileRequestNeedsConfirmation = true; _stdoutHandler._suppressCompilerMessages = request.suppressErrors; final String mainUri = request.packageConfig.toPackageUri(request.mainUri)?.toString() ?? toMultiRootPath(request.mainUri, fileSystemScheme, fileSystemRoots, _platform.isWindows); if (_server == null) { return _compile(mainUri, request.outputPath); } final String inputKey = Uuid().generateV4(); _server.stdin.writeln('recompile $mainUri $inputKey'); _logger.printTrace('<- recompile $mainUri $inputKey'); for (final Uri fileUri in request.invalidatedFiles) { String message; if (fileUri.scheme == 'package') { message = fileUri.toString(); } else { message = request.packageConfig.toPackageUri(fileUri)?.toString() ?? toMultiRootPath(fileUri, fileSystemScheme, fileSystemRoots, _platform.isWindows); } _server.stdin.writeln(message); _logger.printTrace(message); } _server.stdin.writeln(inputKey); _logger.printTrace('<- $inputKey'); return _stdoutHandler.compilerOutput.future; } final List<_CompilationRequest> _compilationQueue = <_CompilationRequest>[]; Future _handleCompilationRequest(_CompilationRequest request) async { final bool isEmpty = _compilationQueue.isEmpty; _compilationQueue.add(request); // Only trigger processing if queue was empty - i.e. no other requests // are currently being processed. This effectively enforces "one // compilation request at a time". if (isEmpty) { while (_compilationQueue.isNotEmpty) { final _CompilationRequest request = _compilationQueue.first; await request.run(this); _compilationQueue.removeAt(0); } } } Future _compile( String scriptUri, String outputPath, ) async { final String frontendServer = _artifacts.getArtifactPath( Artifact.frontendServerSnapshotForEngineDartSdk ); final List command = [ _artifacts.getHostArtifact(HostArtifact.engineDartBinary).path, '--disable-dart-dev', frontendServer, '--sdk-root', sdkRoot, '--incremental', if (testCompilation) '--no-print-incremental-dependencies', '--target=$targetModel', // TODO(jonahwilliams): remove once this becomes the default behavior // in the frontend_server. // https://github.com/flutter/flutter/issues/52693 '--debugger-module-names', // TODO(annagrin): remove once this becomes the default behavior // in the frontend_server. // https://github.com/flutter/flutter/issues/59902 '--experimental-emit-debug-metadata', for (final Object dartDefine in dartDefines) '-D$dartDefine', if (outputPath != null) ...[ '--output-dill', outputPath, ], if (librariesSpec != null) ...[ '--libraries-spec', librariesSpec, ], if (packagesPath != null) ...[ '--packages', packagesPath, ], ...buildModeOptions(buildMode, dartDefines), if (trackWidgetCreation) '--track-widget-creation', if (fileSystemRoots != null) for (final String root in fileSystemRoots) ...[ '--filesystem-root', root, ], if (fileSystemScheme != null) ...[ '--filesystem-scheme', fileSystemScheme, ], if (initializeFromDill != null) ...[ '--initialize-from-dill', initializeFromDill, ], if (platformDill != null) ...[ '--platform', platformDill, ], if (unsafePackageSerialization == true) '--unsafe-package-serialization', ...?extraFrontEndOptions, ]; _logger.printTrace(command.join(' ')); _server = await _processManager.start(command); _server.stdout .transform(utf8.decoder) .transform(const LineSplitter()) .listen( _stdoutHandler.handler, onDone: () { // when outputFilename future is not completed, but stdout is closed // process has died unexpectedly. if (!_stdoutHandler.compilerOutput.isCompleted) { _stdoutHandler.compilerOutput.complete(null); throwToolExit('the Dart compiler exited unexpectedly.'); } }); _server.stderr .transform(utf8.decoder) .transform(const LineSplitter()) .listen(_logger.printError); unawaited(_server.exitCode.then((int code) { if (code != 0) { throwToolExit('the Dart compiler exited unexpectedly.'); } })); _server.stdin.writeln('compile $scriptUri'); _logger.printTrace('<- compile $scriptUri'); return _stdoutHandler.compilerOutput.future; } @override Future compileExpression( String expression, List definitions, List typeDefinitions, String libraryUri, String klass, bool isStatic, ) async { if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } final Completer completer = Completer(); final _CompileExpressionRequest request = _CompileExpressionRequest( completer, expression, definitions, typeDefinitions, libraryUri, klass, isStatic); _controller.add(request); return completer.future; } Future _compileExpression(_CompileExpressionRequest request) async { _stdoutHandler.reset(suppressCompilerMessages: true, expectSources: false, readFile: true); // 'compile-expression' should be invoked after compiler has been started, // program was compiled. if (_server == null) { return null; } final String inputKey = Uuid().generateV4(); _server.stdin ..writeln('compile-expression $inputKey') ..writeln(request.expression); request.definitions?.forEach(_server.stdin.writeln); _server.stdin.writeln(inputKey); request.typeDefinitions?.forEach(_server.stdin.writeln); _server.stdin ..writeln(inputKey) ..writeln(request.libraryUri ?? '') ..writeln(request.klass ?? '') ..writeln(request.isStatic ?? false); return _stdoutHandler.compilerOutput.future; } @override Future compileExpressionToJs( String libraryUri, int line, int column, Map jsModules, Map jsFrameValues, String moduleName, String expression, ) { if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } final Completer completer = Completer(); _controller.add( _CompileExpressionToJsRequest( completer, libraryUri, line, column, jsModules, jsFrameValues, moduleName, expression) ); return completer.future; } Future _compileExpressionToJs(_CompileExpressionToJsRequest request) async { _stdoutHandler.reset(suppressCompilerMessages: true, expectSources: false); // 'compile-expression-to-js' should be invoked after compiler has been started, // program was compiled. if (_server == null) { return null; } final String inputKey = Uuid().generateV4(); _server.stdin ..writeln('compile-expression-to-js $inputKey') ..writeln(request.libraryUri ?? '') ..writeln(request.line) ..writeln(request.column); request.jsModules?.forEach((String k, String v) { _server.stdin.writeln('$k:$v'); }); _server.stdin.writeln(inputKey); request.jsFrameValues?.forEach((String k, String v) { _server.stdin.writeln('$k:$v'); }); _server.stdin ..writeln(inputKey) ..writeln(request.moduleName ?? '') ..writeln(request.expression ?? ''); return _stdoutHandler.compilerOutput.future; } @override void accept() { if (_compileRequestNeedsConfirmation) { _server.stdin.writeln('accept'); _logger.printTrace('<- accept'); } _compileRequestNeedsConfirmation = false; } @override Future reject() { if (!_controller.hasListener) { _controller.stream.listen(_handleCompilationRequest); } final Completer completer = Completer(); _controller.add(_RejectRequest(completer)); return completer.future; } Future _reject() { if (!_compileRequestNeedsConfirmation) { return Future.value(null); } _stdoutHandler.reset(expectSources: false); _server.stdin.writeln('reject'); _logger.printTrace('<- reject'); _compileRequestNeedsConfirmation = false; return _stdoutHandler.compilerOutput.future; } @override void reset() { _server?.stdin?.writeln('reset'); _logger.printTrace('<- reset'); } @override Future shutdown() async { // Server was never successfully created. if (_server == null) { return 0; } _logger.printTrace('killing pid ${_server.pid}'); _server.kill(); return _server.exitCode; } } /// Convert a file URI into a multi-root scheme URI if provided, otherwise /// return unmodified. @visibleForTesting String toMultiRootPath(Uri fileUri, String scheme, List fileSystemRoots, bool windows) { if (scheme == null || fileSystemRoots.isEmpty || fileUri.scheme != 'file') { return fileUri.toString(); } final String filePath = fileUri.toFilePath(windows: windows); for (final String fileSystemRoot in fileSystemRoots) { if (filePath.startsWith(fileSystemRoot)) { return scheme + '://' + filePath.substring(fileSystemRoot.length); } } return fileUri.toString(); }