Teach Linux to use local engine (#31631)

packages/flutter_tools/bin/tool_backend.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io'; // ignore: dart_io_import.
+import 'package:path/path.dart' as path; // ignore: package_path_import.
+
+/// Executes the required flutter tasks for a linux build.
+Future<void> main(List<String> arguments) async {
+  final String targetPlatform = arguments[0];
+  final String buildMode = arguments[1];
+
+  final String projectDirectory = Platform.environment['PROJECT_DIR'];
+  final bool verbose = Platform.environment['VERBOSE_SCRIPT_LOGGING'] != null;
+  final bool trackWidgetCreation = Platform.environment['TRACK_WIDGET_CREATION'] != null;
+  final String flutterTarget = Platform.environment['FLUTTER_TARGET'] ?? 'lib/main.dart';
+  final String flutterEngine = Platform.environment['FLUTTER_ENGINE'];
+  final String localEngine = Platform.environment['LOCAL_ENGINE'];
+  final String flutterRoot = Platform.environment['FLUTTER_ROOT'];
+
+  Directory.current = projectDirectory;
+
+  if (localEngine != null && !localEngine.contains(buildMode)) {
+    stderr.write('''
+ERROR: Requested build with Flutter local engine at '$localEngine'
+This engine is not compatible with FLUTTER_BUILD_MODE: '$buildMode'.
+You can fix this by updating the LOCAL_ENGINE environment variable, or
+by running:
+  flutter build linux --local-engine=host_$buildMode
+or
+  flutter build linux --local-engine=host_${buildMode}_unopt
+========================================================================
+''');
+    exit(1);
+  }
+
+  String cacheDirectory;
+  switch (targetPlatform) {
+    case 'linux-x64':
+      cacheDirectory = 'linux/flutter';
+      break;
+    default:
+      stderr.write('Unsupported target platform $targetPlatform');
+      exit(1);
+  }
+
+  final String flutterExecutable = path.join(
+      flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
+  final ProcessResult unpackResult = await Process.run(
+    flutterExecutable,
+    <String>[
+      '--suppress-analytics',
+      if (verbose) '--verbose',
+      'unpack',
+      '--target-platform=$targetPlatform',
+      '--cache-dir=$cacheDirectory',
+      if (flutterEngine != null) '--local-engine-src-path=$flutterEngine',
+      if (localEngine != null) '--local-engine=$localEngine',
+    ]);
+  if (unpackResult.exitCode != 0) {
+    stderr.write(unpackResult.stderr);
+    exit(1);
+  }
+  final ProcessResult buildResult = await Process.run(
+    flutterExecutable,
+    <String>[
+      '--suppress-analytics',
+      if (verbose) '--verbose',
+      'build',
+      'bundle',
+      '--target=$flutterTarget',
+      '--target-platform=$targetPlatform',
+      if (trackWidgetCreation) '--track-widget-creation',
+      if (flutterEngine != null) '--local-engine-src-path=$flutterEngine',
+      if (localEngine != null) '--local-engine=$localEngine',
+    ]);
+  if (buildResult.exitCode != 0) {
+    stderr.write(buildResult.stderr);
+    exit(1);
+  }
+  exit(0);
+}

packages/flutter_tools/bin/tool_backend.sh

+#!/usr/bin/env bash
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+readonly flutter_bin_dir="${FLUTTER_ROOT}/bin"
+readonly dart_bin_dir="${flutter_bin_dir}/cache/dart-sdk/bin"
+
+exec "${dart_bin_dir}/dart" "${FLUTTER_ROOT}/packages/flutter_tools/bin/tool_backend.dart" "${@:1}"

packages/flutter_tools/lib/executable.dart

@@ -37,6 +37,7 @@ import 'src/commands/screenshot.dart';
 import 'src/commands/shell_completion.dart';
 import 'src/commands/test.dart';
 import 'src/commands/train.dart';
+import 'src/commands/unpack.dart';
 import 'src/commands/update_packages.dart';
 import 'src/commands/upgrade.dart';
 import 'src/commands/version.dart';
@@ -82,6 +83,7 @@ Future<void> main(List<String> args) async {
     ShellCompletionCommand(),
     TestCommand(verboseHelp: verboseHelp),
     TrainingCommand(),
+    UnpackCommand(),
     UpdatePackagesCommand(hidden: !verboseHelp),
     UpgradeCommand(),
     VersionCommand(),

packages/flutter_tools/lib/src/commands/unpack.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import '../artifacts.dart';
+import '../base/common.dart';
+import '../base/file_system.dart';
+import '../base/io.dart';
+import '../base/process_manager.dart';
+import '../build_info.dart';
+import '../cache.dart';
+import '../globals.dart';
+import '../runner/flutter_command.dart';
+
+/// The directory in the Flutter cache for each platform's artifacts.
+const Map<TargetPlatform, String> flutterArtifactPlatformDirectory =
+    <TargetPlatform, String>{
+  TargetPlatform.linux_x64: 'linux-x64',
+  TargetPlatform.darwin_x64: 'darwin-x64',
+  TargetPlatform.windows_x64: 'windows-x64',
+};
+
+// TODO(jonahwilliams): this should come from a configuration in each build
+// directory.
+const Map<TargetPlatform, List<String>> artifactFilesByPlatform = <TargetPlatform, List<String>>{
+  TargetPlatform.linux_x64: <String>[
+    'libflutter_linux.so',
+    'flutter_export.h',
+    'flutter_messenger.h',
+    'flutter_plugin_registrar.h',
+    'flutter_glfw.h',
+    'icudtl.dat',
+    'cpp_client_wrapper/',
+  ],
+  TargetPlatform.darwin_x64: <String>[
+    'FlutterMacOS.framework',
+  ],
+  TargetPlatform.windows_x64: <String>[
+    'flutter_windows.dll',
+    'flutter_windows.dll.exp',
+    'flutter_windows.dll.lib',
+    'flutter_windows.dll.pdb',
+    'flutter_export.h',
+    'flutter_messenger.h',
+    'flutter_plugin_registrar.h',
+    'flutter_glfw.h',
+    'icudtl.dat',
+    'cpp_client_wrapper/',
+  ],
+};
+
+/// Copies desktop artifacts to local cache directories.
+class UnpackCommand extends FlutterCommand {
+  UnpackCommand() {
+    argParser.addOption(
+      'target-platform',
+      allowed: <String>['darwin-x64', 'linux-x64', 'windows-x64'],
+    );
+    argParser.addOption('cache-dir',
+        help: 'Location to output platform specific artifacts.');
+  }
+
+  @override
+  String get description => 'unpack desktop artifacts';
+
+  @override
+  String get name => 'unpack';
+
+  @override
+  bool get hidden => true;
+
+  @override
+  bool get isExperimental => true;
+
+  @override
+  Future<FlutterCommandResult> runCommand() async {
+    final String targetName = argResults['target-platform'];
+    final String targetDirectory = argResults['cache-dir'];
+    if (!fs.directory(targetDirectory).existsSync()) {
+      fs.directory(targetDirectory).createSync(recursive: true);
+    }
+    final TargetPlatform targetPlatform = getTargetPlatformForName(targetName);
+    final ArtifactUnpacker flutterArtifactFetcher = ArtifactUnpacker(targetPlatform);
+    bool success = true;
+    if (artifacts is LocalEngineArtifacts) {
+      final LocalEngineArtifacts localEngineArtifacts = artifacts;
+      success = flutterArtifactFetcher.copyLocalBuildArtifacts(
+        localEngineArtifacts.engineOutPath,
+        targetDirectory,
+      );
+    } else {
+      success = flutterArtifactFetcher.copyCachedArtifacts(
+        targetDirectory,
+      );
+    }
+    if (!success) {
+      throwToolExit('Failed to unpack desktop artifacts.');
+    }
+    return null;
+  }
+}
+
+/// Manages the copying of cached or locally built Flutter artifacts, including
+/// tracking the last-copied versions and updating only if necessary.
+class ArtifactUnpacker {
+  /// Creates a new fetcher for the given configuration.
+  const ArtifactUnpacker(this.platform);
+
+  /// The platform to copy artifacts for.
+  final TargetPlatform platform;
+
+  /// Checks [targetDirectory] to see if artifacts have already been copied for
+  /// the current hash, and if not, copies the artifacts for [platform] from the
+  /// Flutter cache (after ensuring that the cache is present).
+  ///
+  /// Returns true if the artifacts were successfully copied, or were already
+  /// present with the correct hash.
+  bool copyCachedArtifacts(String targetDirectory) {
+    String cacheStamp;
+    switch (platform) {
+      case TargetPlatform.linux_x64:
+        cacheStamp = 'linux-sdk';
+        break;
+      case TargetPlatform.windows_x64:
+        cacheStamp = 'windows-sdk';
+        break;
+      case TargetPlatform.darwin_x64:
+        cacheStamp = 'macos-sdk';
+        break;
+      default:
+        throwToolExit('Unsupported target platform: $platform');
+    }
+    final String targetHash =
+        readHashFileIfPossible(Cache.instance.getStampFileFor(cacheStamp));
+    if (targetHash == null) {
+      printError('Failed to find engine stamp file');
+      return false;
+    }
+
+    try {
+      final String currentHash = _lastCopiedHash(targetDirectory);
+      if (currentHash == null || targetHash != currentHash) {
+        // Copy them to the target directory.
+        final String flutterCacheDirectory = fs.path.join(
+          Cache.flutterRoot,
+          'bin',
+          'cache',
+          'artifacts',
+          'engine',
+          flutterArtifactPlatformDirectory[platform],
+        );
+        if (!_copyArtifactFiles(flutterCacheDirectory, targetDirectory)) {
+          return false;
+        }
+        _setLastCopiedHash(targetDirectory, targetHash);
+        printTrace('Copied artifacts for version $targetHash.');
+      } else {
+        printTrace('Artifacts for version $targetHash already present.');
+      }
+    } catch (error, stackTrace) {
+      printError(stackTrace.toString());
+      printError(error.toString());
+      return false;
+    }
+    return true;
+  }
+
+  /// Acts like [copyCachedArtifacts], replacing the artifacts and updating
+  /// the version stamp, except that it pulls the artifact from a local engine
+  /// build with the given [buildConfiguration] (e.g., host_debug_unopt) whose
+  /// checkout is rooted at [engineRoot].
+  bool copyLocalBuildArtifacts(String buildOutput, String targetDirectory) {
+    if (!_copyArtifactFiles(buildOutput, targetDirectory)) {
+      return false;
+    }
+
+    // Update the hash file to indicate that it's a local build, so that it's
+    // obvious where it came from.
+    _setLastCopiedHash(targetDirectory, 'local build: $buildOutput');
+
+    return true;
+  }
+
+  /// Copies the artifact files for [platform] from [sourceDirectory] to
+  /// [targetDirectory].
+  bool _copyArtifactFiles(String sourceDirectory, String targetDirectory) {
+    final List<String> artifactFiles = artifactFilesByPlatform[platform];
+    if (artifactFiles == null) {
+      printError('Unsupported platform: $platform.');
+      return false;
+    }
+
+    try {
+      fs.directory(targetDirectory).createSync(recursive: true);
+
+      // On macOS, delete the existing framework if any before copying in the
+      // new one, since it's a directory. On the other platforms, where files
+      // are just individual files, this isn't necessary since copying over
+      // existing files will do the right thing.
+      if (platform == TargetPlatform.darwin_x64) {
+        _copyMacOSFramework(
+            fs.path.join(sourceDirectory, artifactFiles[0]), targetDirectory);
+      } else {
+        for (final String entityName in artifactFiles) {
+          final String sourcePath = fs.path.join(sourceDirectory, entityName);
+          final String targetPath = fs.path.join(targetDirectory, entityName);
+          if (entityName.endsWith('/')) {
+            copyDirectorySync(
+              fs.directory(sourcePath),
+              fs.directory(targetPath),
+            );
+          } else {
+            fs.file(sourcePath)
+              .copySync(fs.path.join(targetDirectory, entityName));
+          }
+        }
+      }
+
+      printTrace('Copied artifacts from $sourceDirectory.');
+    } catch (e, stackTrace) {
+      printError(stackTrace.toString());
+      printError(e.message);
+      return false;
+    }
+    return true;
+  }
+
+  /// Returns a File object for the file containing the last copied hash
+  /// in [directory].
+  File _lastCopiedHashFile(String directory) {
+    return fs.file(fs.path.join(directory, '.last_artifact_version'));
+  }
+
+  /// Returns the hash of the artifacts last copied to [directory], or null if
+  /// they haven't been copied.
+  String _lastCopiedHash(String directory) {
+    // Sanity check that at least one file is present; this won't catch every
+    // case, but handles someone deleting all the non-hidden cached files to
+    // force fresh copy.
+    final String artifactFilePath = fs.path.join(
+      directory,
+      artifactFilesByPlatform[platform].first,
+    );
+    if (!fs.file(artifactFilePath).existsSync()) {
+      return null;
+    }
+    final File hashFile = _lastCopiedHashFile(directory);
+    return readHashFileIfPossible(hashFile);
+  }
+
+  /// Writes [hash] to the file that stores the last copied hash for
+  /// in [directory].
+  void _setLastCopiedHash(String directory, String hash) {
+    _lastCopiedHashFile(directory).writeAsStringSync(hash);
+  }
+
+  /// Copies the framework at [frameworkPath] to [targetDirectory]
+  /// by invoking 'cp -R'.
+  ///
+  /// The shelling out is done to avoid complications with preserving special
+  /// files (e.g., symbolic links) in the framework structure.
+  ///
+  /// Removes any previous version of the framework that already exists in the
+  /// target directory.
+  void _copyMacOSFramework(String frameworkPath, String targetDirectory) {
+    _deleteFrameworkIfPresent(
+        fs.path.join(targetDirectory, fs.path.basename(frameworkPath)));
+
+    final ProcessResult result = processManager
+        .runSync(<String>['cp', '-R', frameworkPath, targetDirectory]);
+    if (result.exitCode != 0) {
+      throw Exception(
+        'Failed to copy framework (exit ${result.exitCode}:\n'
+        '${result.stdout}\n---\n${result.stderr}',
+      );
+    }
+  }
+
+  /// Recursively deletes the framework at [frameworkPath], if it exists.
+  void _deleteFrameworkIfPresent(String frameworkPath) {
+    // Ensure that the path is a framework, to minimize the potential for
+    // catastrophic deletion bugs with bad arguments.
+    if (fs.path.extension(frameworkPath) != '.framework') {
+      throw Exception(
+          'Attempted to delete a non-framework directory: $frameworkPath');
+    }
+
+    final Directory directory = fs.directory(frameworkPath);
+    if (directory.existsSync()) {
+      directory.deleteSync(recursive: true);
+    }
+  }
+
+  /// Returns the engine hash from [file] as a String, or null.
+  ///
+  /// If the file is missing, or cannot be read, returns null.
+  String readHashFileIfPossible(File file) {
+    if (!file.existsSync()) {
+      return null;
+    }
+    try {
+      return file.readAsStringSync().trim();
+    } on FileSystemException {
+      // If the file can't be read for any reason, just treat it as missing.
+      return null;
+    }
+  }
+}

packages/flutter_tools/lib/src/linux/build_linux.dart

@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import '../artifacts.dart';
 import '../base/common.dart';
+import '../base/file_system.dart';
 import '../base/io.dart';
 import '../base/logger.dart';
 import '../base/process_manager.dart';
@@ -13,21 +15,32 @@ import '../globals.dart';
 import '../project.dart';
 
 /// Builds the Linux project through the Makefile.
-Future<void> buildLinux(LinuxProject linuxProject, BuildInfo buildInfo) async {
-  /// Cache flutter root in linux directory.
-  linuxProject.editableHostAppDirectory.childFile('.generated_flutter_root')
+Future<void> buildLinux(LinuxProject linuxProject, BuildInfo buildInfo, {String target = 'lib/main.dart'}) async {
+  final String buildFlag = buildInfo?.isDebug == true ? 'debug' : 'release';
+  final StringBuffer buffer = StringBuffer('''
+# Generated code do not commit.
+export FLUTTER_ROOT=${Cache.flutterRoot}
+export BUILD=$buildFlag
+export TRACK_WIDGET_CREATION=${buildInfo?.trackWidgetCreation == true}
+export FLUTTER_TARGET=$target
+export PROJECT_DIR=${linuxProject.project.directory.path}
+''');
+  if (artifacts is LocalEngineArtifacts) {
+    final LocalEngineArtifacts localEngineArtifacts = artifacts;
+    final String engineOutPath = localEngineArtifacts.engineOutPath;
+    buffer.writeln('export FLUTTER_ENGINE=${fs.path.dirname(fs.path.dirname(engineOutPath))}');
+    buffer.writeln('export LOCAL_ENGINE=${fs.path.basename(engineOutPath)}');
+  }
+
+  /// Cache flutter configuration files in the linux directory.
+  linuxProject.cacheDirectory.childFile('generated_config')
     ..createSync(recursive: true)
-    ..writeAsStringSync(Cache.flutterRoot);
+    ..writeAsStringSync(buffer.toString());
 
-  final String buildFlag = buildInfo?.isDebug == true ? 'debug' : 'release';
-  final String bundleFlags = buildInfo?.trackWidgetCreation == true ? '--track-widget-creation' : '';
   final Process process = await processManager.start(<String>[
     'make',
     '-C',
     linuxProject.editableHostAppDirectory.path,
-    'BUILD=$buildFlag',
-    'FLUTTER_ROOT=${Cache.flutterRoot}',
-    'FLUTTER_BUNDLE_FLAGS=$bundleFlags',
   ], runInShell: true);
   final Status status = logger.startProgress(
     'Building Linux application...',

packages/flutter_tools/lib/src/project.dart

@@ -599,6 +599,8 @@ class LinuxProject {
 
   Directory get editableHostAppDirectory => project.directory.childDirectory('linux');
 
+  Directory get cacheDirectory => editableHostAppDirectory.childDirectory('flutter');
+
   bool existsSync() => editableHostAppDirectory.existsSync();
 
   /// The Linux project makefile.

packages/flutter_tools/pubspec.yaml

@@ -5,7 +5,7 @@ author: Flutter Authors <flutter-dev@googlegroups.com>
 
 environment:
   # The pub client defaults to an <2.0.0 sdk constraint which we need to explicitly overwrite.
-  sdk: ">=2.2.0 <3.0.0"
+  sdk: ">=2.2.2 <3.0.0"
 
 dependencies:
   # To update these, use "flutter update-packages --force-upgrade".

packages/flutter_tools/test/commands/build_linux_test.dart

@@ -21,7 +21,6 @@ import '../src/mocks.dart';
 void main() {
   Cache.disableLocking();
   final MockProcessManager mockProcessManager = MockProcessManager();
-  final MemoryFileSystem memoryFilesystem = MemoryFileSystem();
   final MockProcess mockProcess = MockProcess();
   final MockPlatform linuxPlatform = MockPlatform();
   final MockPlatform notLinuxPlatform = MockPlatform();
@@ -46,7 +45,7 @@ void main() {
     ), throwsA(isInstanceOf<ToolExit>()));
   }, overrides: <Type, Generator>{
     Platform: () => linuxPlatform,
-    FileSystem: () => memoryFilesystem,
+    FileSystem: () => MemoryFileSystem(),
   });
 
   testUsingContext('Linux build fails on non-linux platform', () async {
@@ -61,22 +60,20 @@ void main() {
     ), throwsA(isInstanceOf<ToolExit>()));
   }, overrides: <Type, Generator>{
     Platform: () => notLinuxPlatform,
-    FileSystem: () => memoryFilesystem,
+    FileSystem: () => MemoryFileSystem(),
   });
 
-  testUsingContext('Linux build invokes make', () async {
+  testUsingContext('Linux build invokes make and writes temporary files', () async {
     final BuildCommand command = BuildCommand();
     applyMocksToCommand(command);
     fs.file('linux/build.sh').createSync(recursive: true);
     fs.file('pubspec.yaml').createSync();
     fs.file('.packages').createSync();
+
     when(mockProcessManager.start(<String>[
       'make',
       '-C',
       '/linux',
-      'BUILD=release',
-      'FLUTTER_ROOT=/',
-      'FLUTTER_BUNDLE_FLAGS=',
     ], runInShell: true)).thenAnswer((Invocation invocation) async {
       return mockProcess;
     });
@@ -84,8 +81,9 @@ void main() {
     await createTestCommandRunner(command).run(
       const <String>['build', 'linux']
     );
+    expect(fs.file('linux/flutter/generated_config').existsSync(), true);
   }, overrides: <Type, Generator>{
-    FileSystem: () => memoryFilesystem,
+    FileSystem: () => MemoryFileSystem(),
     ProcessManager: () => mockProcessManager,
     Platform: () => linuxPlatform,
   });