build bundle with assemble (#37508)

dev/devicelab/bin/tasks/gradle_plugin_light_apk_test.dart

@@ -141,7 +141,7 @@ Future<void> main() async {
       await runProjectTest((FlutterProject project) async {
         section('gradlew assembleDebug');
         await project.runGradleTask('assembleDebug');
-        final String errorMessage = validateSnapshotDependency(project, 'build/app.dill');
+        final String errorMessage = validateSnapshotDependency(project, 'kernel_blob.bin');
         if (errorMessage != null) {
           throw TaskResult.failure(errorMessage);
         }

dev/devicelab/lib/framework/apk_utils.dart

@@ -382,15 +382,19 @@ Future<ProcessResult> _resultOfGradleTask({String workingDirectory, String task,
 
 class _Dependencies {
   _Dependencies(String depfilePath) {
-    final RegExp _separatorExpr = RegExp(r'([^\\]) ');
-    final RegExp _escapeExpr = RegExp(r'\\(.)');
-
     // Depfile format:
     // outfile1 outfile2 : file1.dart file2.dart file3.dart file\ 4.dart
     final String contents = File(depfilePath).readAsStringSync();
-    final List<String> colonSeparated = contents.split(': ');
-    target = colonSeparated[0].trim();
-    dependencies = colonSeparated[1]
+    final List<String> colonSeparated = contents.split(':');
+    targets = _processList(colonSeparated[0].trim());
+    dependencies = _processList(colonSeparated[1].trim());
+  }
+
+  final RegExp _separatorExpr = RegExp(r'([^\\]) ');
+  final RegExp _escapeExpr = RegExp(r'\\(.)');
+
+  Set<String> _processList(String rawText) {
+    return rawText
     // Put every file on right-hand side on the separate line
         .replaceAllMapped(_separatorExpr, (Match match) => '${match.group(1)}\n')
         .split('\n')
@@ -400,7 +404,7 @@ class _Dependencies {
         .toSet();
   }
 
-  String target;
+  Set<String> targets;
   Set<String> dependencies;
 }
 
@@ -409,6 +413,6 @@ String validateSnapshotDependency(FlutterProject project, String expectedTarget)
   final _Dependencies deps = _Dependencies(
       path.join(project.rootPath, 'build', 'app', 'intermediates',
           'flutter', 'debug', 'android-arm', 'snapshot_blob.bin.d'));
-  return deps.target == expectedTarget ? null :
-    'Dependency file should have $expectedTarget as target. Instead has ${deps.target}';
+  return deps.targets.any((String target) => target.contains(expectedTarget)) ? null :
+  'Dependency file should have $expectedTarget as target. Instead has ${deps.targets}';
 }

packages/flutter_tools/lib/src/artifacts.dart

@@ -92,6 +92,9 @@ String _artifactToFileName(Artifact artifact, [ TargetPlatform platform, BuildMo
     case Artifact.frontendServerSnapshotForEngineDartSdk:
       return 'frontend_server.dart.snapshot';
     case Artifact.engineDartBinary:
+      if (platform == TargetPlatform.windows_x64) {
+        return 'dart.exe';
+      }
       return 'dart';
     case Artifact.dart2jsSnapshot:
       return 'dart2js.dart.snapshot';
@@ -265,7 +268,7 @@ class CachedArtifacts extends Artifacts {
       case Artifact.engineDartSdkPath:
         return dartSdkPath;
       case Artifact.engineDartBinary:
-        return fs.path.join(dartSdkPath, 'bin', _artifactToFileName(artifact));
+        return fs.path.join(dartSdkPath, 'bin', _artifactToFileName(artifact, platform));
       case Artifact.platformKernelDill:
         return fs.path.join(_getFlutterPatchedSdkPath(mode), _artifactToFileName(artifact));
       case Artifact.platformLibrariesJson:

packages/flutter_tools/lib/src/build_system/build_system.dart

@@ -10,6 +10,7 @@ import 'package:crypto/crypto.dart';
 import 'package:meta/meta.dart';
 import 'package:pool/pool.dart';
 
+import '../base/context.dart';
 import '../base/file_system.dart';
 import '../base/platform.dart';
 import '../cache.dart';
@@ -21,6 +22,9 @@ import 'source.dart';
 
 export 'source.dart';
 
+/// The [BuildSystem] instance.
+BuildSystem get buildSystem => context.get<BuildSystem>();
+
 /// Configuration for the build system itself.
 class BuildSystemConfig {
   /// Create a new [BuildSystemConfig].
@@ -497,14 +501,14 @@ class _BuildInstance {
       }
       if (canSkip) {
         skipped = true;
-        printStatus('Skipping target: ${node.target.name}');
+        printTrace('Skipping target: ${node.target.name}');
         for (File output in node.outputs) {
           outputFiles[output.path] = output;
         }
       } else {
-        printStatus('${node.target.name}: Starting');
+        printTrace('${node.target.name}: Starting');
         await node.target.build(environment);
-        printStatus('${node.target.name}: Complete');
+        printTrace('${node.target.name}: Complete');
 
         // Update hashes for output files.
         await fileCache.hashFiles(node.outputs);

packages/flutter_tools/lib/src/build_system/targets/assets.dart

@@ -48,6 +48,43 @@ class AssetBehavior extends SourceBehavior {
   }
 }
 
+/// A specific asset behavior for building bundles.
+class AssetOutputBehavior extends SourceBehavior {
+  const AssetOutputBehavior();
+
+  @override
+  List<File> inputs(Environment environment) {
+    final AssetBundle assetBundle = AssetBundleFactory.instance.createBundle();
+    assetBundle.build(
+      manifestPath: environment.projectDir.childFile('pubspec.yaml').path,
+      packagesPath: environment.projectDir.childFile('.packages').path,
+    );
+    // Filter the file type to remove the files that are generated by this
+    // command as inputs.
+    final List<File> results = <File>[];
+    final Iterable<DevFSFileContent> files = assetBundle.entries.values.whereType<DevFSFileContent>();
+    for (DevFSFileContent devFsContent in files) {
+      results.add(fs.file(devFsContent.file.path));
+    }
+    return results;
+  }
+
+  @override
+  List<File> outputs(Environment environment) {
+    final AssetBundle assetBundle = AssetBundleFactory.instance.createBundle();
+    assetBundle.build(
+      manifestPath: environment.projectDir.childFile('pubspec.yaml').path,
+      packagesPath: environment.projectDir.childFile('.packages').path,
+    );
+    final List<File> results = <File>[];
+    for (String key in assetBundle.entries.keys) {
+      final File file = fs.file(fs.path.join(environment.outputDir.path, key));
+      results.add(file);
+    }
+    return results;
+  }
+}
+
 /// Copy the assets defined in the flutter manifest into a build directory.
 class CopyAssets extends Target {
   const CopyAssets();

packages/flutter_tools/lib/src/build_system/targets/dart.dart

@@ -2,17 +2,22 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'package:pool/pool.dart';
+
 import '../../artifacts.dart';
+import '../../asset.dart';
 import '../../base/build.dart';
 import '../../base/file_system.dart';
 import '../../base/platform.dart';
 import '../../build_info.dart';
 import '../../compile.dart';
 import '../../dart/package_map.dart';
+import '../../devfs.dart';
 import '../../globals.dart';
 import '../../project.dart';
 import '../build_system.dart';
 import '../exceptions.dart';
+import 'assets.dart';
 
 /// The define to pass a [BuildMode].
 const String kBuildMode= 'BuildMode';
@@ -55,6 +60,111 @@ List<File> listDartSources(Environment environment) {
   return dartFiles;
 }
 
+/// Copies the prebuilt flutter bundle.
+// This is a one-off rule for implementing build bundle in terms of assemble.
+class CopyFlutterBundle extends Target {
+  const CopyFlutterBundle();
+
+  @override
+  String get name => 'copy_flutter_bundle';
+
+  @override
+  List<Source> get inputs => const <Source>[
+    Source.artifact(Artifact.vmSnapshotData, mode: BuildMode.debug),
+    Source.artifact(Artifact.isolateSnapshotData, mode: BuildMode.debug),
+    Source.pattern('{BUILD_DIR}/app.dill'),
+    Source.behavior(AssetOutputBehavior())
+  ];
+
+  @override
+  List<Source> get outputs => const <Source>[
+    Source.pattern('{OUTPUT_DIR}/vm_snapshot_data'),
+    Source.pattern('{OUTPUT_DIR}/isolate_snapshot_data'),
+    Source.pattern('{OUTPUT_DIR}/kernel_blob.bin'),
+    Source.pattern('{OUTPUT_DIR}/AssetManifest.json'),
+    Source.pattern('{OUTPUT_DIR}/FontManifest.json'),
+    Source.pattern('{OUTPUT_DIR}/LICENSE'),
+    Source.behavior(AssetOutputBehavior())
+  ];
+
+  @override
+  Future<void> build(Environment environment) async {
+    if (environment.defines[kBuildMode] == null) {
+      throw MissingDefineException(kBuildMode, 'copy_flutter_bundle');
+    }
+    final BuildMode buildMode = getBuildModeForName(environment.defines[kBuildMode]);
+
+    // We're not smart enough to only remove assets that are removed. If
+    // anything changes blow away the whole directory.
+    if (environment.outputDir.existsSync()) {
+      environment.outputDir.deleteSync(recursive: true);
+    }
+    environment.outputDir.createSync(recursive: true);
+
+    // Only copy the prebuilt runtimes and kernel blob in debug mode.
+    if (buildMode == BuildMode.debug) {
+      final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData, mode: BuildMode.debug);
+      final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData, mode: BuildMode.debug);
+      environment.buildDir.childFile('app.dill')
+          .copySync(environment.outputDir.childFile('kernel_blob.bin').path);
+      fs.file(vmSnapshotData)
+          .copySync(environment.outputDir.childFile('vm_snapshot_data').path);
+      fs.file(isolateSnapshotData)
+          .copySync(environment.outputDir.childFile('isolate_snapshot_data').path);
+    }
+
+    final AssetBundle assetBundle = AssetBundleFactory.instance.createBundle();
+    await assetBundle.build();
+    final Pool pool = Pool(64);
+    await Future.wait<void>(
+      assetBundle.entries.entries.map<Future<void>>((MapEntry<String, DevFSContent> entry) async {
+        final PoolResource resource = await pool.request();
+        try {
+          final File file = fs.file(fs.path.join(environment.outputDir.path, entry.key));
+          file.parent.createSync(recursive: true);
+          final DevFSContent content = entry.value;
+          if (content is DevFSFileContent && content.file is File) {
+            await (content.file as File).copy(file.path);
+          } else {
+            await file.writeAsBytes(await entry.value.contentsAsBytes());
+          }
+        } finally {
+          resource.release();
+        }
+      }));
+  }
+
+  @override
+  List<Target> get dependencies => const <Target>[
+    KernelSnapshot(),
+  ];
+}
+
+/// Copies the prebuilt flutter bundle for release mode.
+class ReleaseCopyFlutterBundle extends CopyFlutterBundle {
+  const ReleaseCopyFlutterBundle();
+
+  @override
+  String get name => 'release_flutter_bundle';
+
+  @override
+  List<Source> get inputs => const <Source>[
+    Source.behavior(AssetOutputBehavior())
+  ];
+
+  @override
+  List<Source> get outputs => const <Source>[
+    Source.pattern('{OUTPUT_DIR}/AssetManifest.json'),
+    Source.pattern('{OUTPUT_DIR}/FontManifest.json'),
+    Source.pattern('{OUTPUT_DIR}/LICENSE'),
+    Source.behavior(AssetOutputBehavior())
+  ];
+
+  @override
+  List<Target> get dependencies => const <Target>[];
+}
+
+
 /// Generate a snapshot of the dart code used in the program.
 class KernelSnapshot extends Target {
   const KernelSnapshot();
@@ -91,6 +201,7 @@ class KernelSnapshot extends Target {
     final BuildMode buildMode = getBuildModeForName(environment.defines[kBuildMode]);
     final String targetFile = environment.defines[kTargetFile] ?? fs.path.join('lib', 'main.dart');
     final String packagesPath = environment.projectDir.childFile('.packages').path;
+    // TODO(jonahwilliams): use a full file uri to remove Dart complaint about entrypoint.
     final PackageUriMapper packageUriMapper = PackageUriMapper(targetFile,
         packagesPath, null, null);
 
@@ -101,7 +212,6 @@ class KernelSnapshot extends Target {
       targetModel: TargetModel.flutter,
       targetProductVm: buildMode == BuildMode.release,
       outputFilePath: environment.buildDir.childFile('app.dill').path,
-      depFilePath: null,
       packagesPath: packagesPath,
       linkPlatformKernelIn: buildMode == BuildMode.release,
       mainPath: packageUriMapper.map(targetFile)?.toString() ?? targetFile,

packages/flutter_tools/lib/src/bundle.dart

@@ -12,6 +12,8 @@ import 'asset.dart';
 import 'base/common.dart';
 import 'base/file_system.dart';
 import 'build_info.dart';
+import 'build_system/build_system.dart';
+import 'build_system/targets/dart.dart';
 import 'compile.dart';
 import 'dart/package_map.dart';
 import 'devfs.dart';
@@ -69,6 +71,7 @@ class BundleBuilder {
     List<String> extraGenSnapshotOptions = const <String>[],
     List<String> fileSystemRoots,
     String fileSystemScheme,
+    bool shouldBuildWithAssemble = false,
   }) async {
     mainPath ??= defaultMainPath;
     depfilePath ??= defaultDepfilePath;
@@ -77,6 +80,18 @@ class BundleBuilder {
     applicationKernelFilePath ??= getDefaultApplicationKernelPath(trackWidgetCreation: trackWidgetCreation);
     final FlutterProject flutterProject = FlutterProject.current();
 
+    if (shouldBuildWithAssemble) {
+      await buildWithAssemble(
+        buildMode: buildMode ?? BuildMode.debug,
+        targetPlatform: platform,
+        mainPath: mainPath,
+        flutterProject: flutterProject,
+        outputDir: assetDirPath,
+        depfilePath: depfilePath,
+      );
+      return;
+    }
+
     DevFSContent kernelContent;
     if (!precompiledSnapshot) {
       if ((extraFrontEndOptions != null) && extraFrontEndOptions.isNotEmpty)
@@ -122,6 +137,56 @@ class BundleBuilder {
   }
 }
 
+/// Build an application bundle using flutter assemble.
+///
+/// This is a temporary shim to migrate the build implementations.
+Future<void> buildWithAssemble({
+  @required FlutterProject flutterProject,
+  @required BuildMode buildMode,
+  @required TargetPlatform targetPlatform,
+  @required String mainPath,
+  @required String outputDir,
+  @required String depfilePath,
+}) async {
+  final Environment environment = Environment(
+    projectDir: flutterProject.directory,
+    outputDir: fs.directory(outputDir),
+    buildDir: flutterProject.dartTool.childDirectory('flutter_build'),
+    defines: <String, String>{
+      kTargetFile: mainPath,
+      kBuildMode: getNameForBuildMode(buildMode),
+      kTargetPlatform: getNameForTargetPlatform(targetPlatform),
+    }
+  );
+  final Target target = buildMode == BuildMode.debug
+    ? const CopyFlutterBundle()
+    : const ReleaseCopyFlutterBundle();
+  final BuildResult result = await buildSystem.build(target, environment);
+
+  if (!result.success) {
+    for (ExceptionMeasurement measurement in result.exceptions.values) {
+      printError(measurement.exception.toString());
+      printError(measurement.stackTrace.toString());
+    }
+    throwToolExit('Failed to build bundle.');
+  }
+
+  // Output depfile format:
+  final StringBuffer buffer = StringBuffer();
+  for (File outputFile in result.outputFiles) {
+    buffer.write('${outputFile.path} ');
+  }
+  buffer.write(':');
+  for (File inputFile in result.inputFiles) {
+    buffer.write('${inputFile.path} ');
+  }
+  final File depfile = fs.file(depfilePath);
+  if (!depfile.parent.existsSync()) {
+    depfile.parent.createSync(recursive: true);
+  }
+  depfile.writeAsStringSync(buffer.toString());
+}
+
 Future<AssetBundle> buildAssets({
   String manifestPath,
   String assetDirPath,

packages/flutter_tools/lib/src/commands/assemble.dart

@@ -5,7 +5,6 @@
 import 'package:meta/meta.dart';
 
 import '../base/common.dart';
-import '../base/context.dart';
 import '../base/file_system.dart';
 import '../build_system/build_system.dart';
 import '../build_system/targets/assets.dart';
@@ -18,9 +17,6 @@ import '../globals.dart';
 import '../project.dart';
 import '../runner/flutter_command.dart';
 
-/// The [BuildSystem] instance.
-BuildSystem get buildSystem => context.get<BuildSystem>();
-
 /// All currently implemented targets.
 const List<Target> _kDefaultTargets = <Target>[
   UnpackLinux(),

packages/flutter_tools/lib/src/commands/build_bundle.dart

@@ -131,6 +131,7 @@ class BuildBundleCommand extends BuildSubCommand {
       extraGenSnapshotOptions: argResults[FlutterOptions.kExtraGenSnapshotOptions],
       fileSystemScheme: argResults['filesystem-scheme'],
       fileSystemRoots: argResults['filesystem-root'],
+      shouldBuildWithAssemble: true,
     );
     return null;
   }

packages/flutter_tools/test/general.shard/build_system/targets/dart_test.dart

@@ -81,6 +81,7 @@ flutter_tools:lib/''');
         fs.path.join('bin', 'cache', 'pkg', 'sky_engine', 'sdk_ext',
             'vmservice_io.dart'),
         fs.path.join('bin', 'cache', 'dart-sdk', 'bin', 'dart'),
+        fs.path.join('bin', 'cache', 'dart-sdk', 'bin', 'dart.exe'),
         fs.path.join(engineArtifacts, getNameForHostPlatform(hostPlatform),
             'frontend_server.dart.snapshot'),
         fs.path.join(engineArtifacts, 'android-arm-profile',

packages/flutter_tools/test/general.shard/bundle_shim_test.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_tools/src/base/common.dart';
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/build_info.dart';
+import 'package:flutter_tools/src/build_system/build_system.dart';
+import 'package:flutter_tools/src/bundle.dart';
+import 'package:flutter_tools/src/project.dart';
+import 'package:mockito/mockito.dart';
+
+import '../src/common.dart';
+import '../src/testbed.dart';
+
+// Tests for the temporary flutter assemble/bundle shim.
+void main() {
+  Testbed testbed;
+
+  setUp(() {
+    testbed = Testbed(overrides: <Type, Generator>{
+      BuildSystem: () => MockBuildSystem(),
+    });
+  });
+
+  test('Copies assets to expected directory after building', () => testbed.run(() async {
+    when(buildSystem.build(any, any)).thenAnswer((Invocation invocation) async {
+      final Environment environment = invocation.positionalArguments[1];
+      environment.outputDir.childFile('kernel_blob.bin').createSync(recursive: true);
+      environment.outputDir.childFile('isolate_snapshot_data').createSync();
+      environment.outputDir.childFile('vm_snapshot_data').createSync();
+      environment.outputDir.childFile('LICENSE').createSync(recursive: true);
+      return BuildResult(success: true);
+    });
+    await buildWithAssemble(
+      buildMode: BuildMode.debug,
+      flutterProject: FlutterProject.current(),
+      mainPath: fs.path.join('lib', 'main.dart'),
+      outputDir: 'example',
+      targetPlatform: TargetPlatform.ios,
+      depfilePath: 'example.d',
+    );
+    expect(fs.file(fs.path.join('example', 'kernel_blob.bin')).existsSync(), true);
+    expect(fs.file(fs.path.join('example', 'LICENSE')).existsSync(), true);
+    expect(fs.file(fs.path.join('example.d')).existsSync(), true);
+  }));
+
+  test('Handles build system failure', () => testbed.run(() {
+    when(buildSystem.build(any, any)).thenAnswer((Invocation _) async {
+      return BuildResult(
+        success: false,
+        exceptions: <String, ExceptionMeasurement>{},
+      );
+    });
+
+    expect(() => buildWithAssemble(
+      buildMode: BuildMode.debug,
+      flutterProject: FlutterProject.current(),
+      mainPath: 'lib/main.dart',
+      outputDir: 'example',
+      targetPlatform: TargetPlatform.linux_x64,
+      depfilePath: 'example.d',
+    ), throwsA(isInstanceOf<ToolExit>()));
+  }));
+}
+
+class MockBuildSystem extends Mock implements BuildSystem {}