Add flutter_build package for codegen and version sync with tool (#26978)

dev/devicelab/pubspec.yaml

@@ -18,7 +18,7 @@ dependencies:
   stack_trace: 1.9.3
   vm_service_client: 0.2.6
 
-  archive: 2.0.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  archive: 2.0.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   async: 2.0.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   charcode: 1.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   collection: 1.14.11 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -74,4 +74,4 @@ dev_dependencies:
   watcher: 0.9.7+10 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   yaml: 2.1.15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: ba20
+# PUBSPEC CHECKSUM: 3f21

dev/integration_tests/android_views/pubspec.yaml

@@ -7,7 +7,7 @@ dependencies:
     sdk: flutter
   flutter_driver:
     sdk: flutter
-  path_provider: 0.4.1
+  path_provider: 0.5.0
   collection: 1.14.11
   assets_for_android_views:
     git:
@@ -79,4 +79,4 @@ dev_dependencies:
 flutter:
   uses-material-design: true
 
-# PUBSPEC CHECKSUM: fc8f
+# PUBSPEC CHECKSUM: 018f

dev/integration_tests/ui/pubspec.yaml

@@ -14,7 +14,7 @@ dependencies:
   test: 1.5.2
 
   analyzer: 0.33.6+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  archive: 2.0.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  archive: 2.0.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   args: 1.5.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   async: 2.0.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 1.0.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -80,4 +80,4 @@ dev_dependencies:
 flutter:
   uses-material-design: true
 
-# PUBSPEC CHECKSUM: 13f1
+# PUBSPEC CHECKSUM: 9bf2

dev/tools/pubspec.yaml

@@ -6,7 +6,7 @@ environment:
   sdk: ">=2.0.0-dev.68.0 <3.0.0"
 
 dependencies:
-  archive: 2.0.7
+  archive: 2.0.8
   args: 1.5.1
   http: 0.12.0+1
   intl: 0.15.7
@@ -37,4 +37,4 @@ dev_dependencies:
   stack_trace: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   stream_channel: 1.6.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 566f
+# PUBSPEC CHECKSUM: 5e70

examples/flutter_gallery/pubspec.yaml

@@ -8,13 +8,13 @@ dependencies:
   flutter:
     sdk: flutter
   collection: 1.14.11
-  device_info: 0.3.0
+  device_info: 0.4.0
   intl: 0.15.7
-  connectivity: 0.3.2
+  connectivity: 0.4.0
   string_scanner: 1.0.4
-  url_launcher: 4.1.0+1
+  url_launcher: 4.2.0+1
   cupertino_icons: 0.1.2
-  video_player: 0.8.0
+  video_player: 0.10.0
   scoped_model: 1.0.1
   shrine_images: 1.1.1
 
@@ -242,4 +242,4 @@ flutter:
         - asset: packages/flutter_gallery_assets/fonts/private/googlesans/GoogleSansDisplay-Regular.ttf
           weight: 400
 
-# PUBSPEC CHECKSUM: 1640
+# PUBSPEC CHECKSUM: cd6a

packages/flutter_build/analysis_options.yaml

+# Use the analysis options settings from the top level of the repo (not
+# the ones from above, which include the `public_member_api_docs` rule).
+
+include: ../../analysis_options.yaml

packages/flutter_build/build.yaml

+# Read about `build.yaml` at https://pub.dartlang.org/packages/build_config
+builders:
+  kernel:
+    import: "package:flutter_build/flutter_build.dart"
+    builder_factories:
+      - "flutterKernelBuilder"
+    build_extensions:
+      ".dart":
+        - ".app.dill"
+        - ".packages"
+    auto_apply: root_package
+    build_to: cache
+    defaults:
+      generate_for:
+        include:
+          - lib/**

packages/flutter_build/lib/flutter_build.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:build/build.dart';
+
+import 'src/kernel_builder.dart';
+
+/// Utility method called by build_runner to generate a build script.
+///
+/// This method does not need to be invoked manually, and instead should
+/// be configured via the `build.yaml` file.
+Builder flutterKernelBuilder(BuilderOptions builderOptions) {
+  final Map<String, Object> config = builderOptions.config;
+  final List<Object> extraFrontEndOptions = config['extraFrontEndOptions'];
+  return FlutterKernelBuilder(
+    aot: config['aot'],
+    disabled: config['disabled'],
+    engineDartBinaryPath: config['engineDartBinaryPath'],
+    frontendServerPath: config['frontendServerPath'],
+    incrementalCompilerByteStorePath: config['incrementalCompilerByteStorePath'],
+    linkPlatformKernelIn: config['linkPlatformKernelIn'],
+    mainPath: config['mainPath'],
+    packagesPath: config['packagesPath'],
+    sdkRoot: config['sdkRoot'],
+    targetProductVm: config['targetProductVm'],
+    trackWidgetCreation: config['trackWidgetCreation'],
+    extraFrontEndOptions: extraFrontEndOptions.cast<String>(),
+  );
+}

packages/flutter_build/lib/src/kernel_builder.dart

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:build/build.dart';
+import 'package:package_config/packages_file.dart' as packages_file;
+import 'package:meta/meta.dart';
+import 'package:path/path.dart' as path;
+
+const String _kFlutterDillOutputExtension = '.app.dill';
+const String _kPackagesExtension = '.packages';
+const String multiRootScheme = 'org-dartlang-app';
+
+/// A builder which creates a kernel and packages file for a Flutter app.
+///
+/// Unlike the package:build kernel builders, this creates a single kernel from
+/// dart source using the frontend server binary. The newly created .package
+/// file replaces the relative root of the current package with a multi-root
+/// which includes the generated directory.
+class FlutterKernelBuilder implements Builder {
+  const FlutterKernelBuilder({
+    @required this.disabled,
+    @required this.mainPath,
+    @required this.aot,
+    @required this.trackWidgetCreation,
+    @required this.targetProductVm,
+    @required this.linkPlatformKernelIn,
+    @required this.extraFrontEndOptions,
+    @required this.sdkRoot,
+    @required this.packagesPath,
+    @required this.incrementalCompilerByteStorePath,
+    @required this.frontendServerPath,
+    @required this.engineDartBinaryPath,
+  });
+
+  /// The path to the entrypoint that will be compiled.
+  final String mainPath;
+
+  /// The path to the pub generated .packages file.
+  final String packagesPath;
+
+  /// The path to the root of the flutter patched SDK.
+  final String sdkRoot;
+
+  /// The path to the frontend server snapshot.
+  final String frontendServerPath;
+
+  /// The path to the dart executable to use to run the frontend server
+  /// snapshot.
+  final String engineDartBinaryPath;
+
+  /// Whether to build an ahead of time build.
+  final bool aot;
+
+  /// Whether to disable this builder entirely.
+  final bool disabled;
+
+  /// Whether the `trackWidgetCreation` flag is provided to the frontend
+  /// server.
+  final bool trackWidgetCreation;
+
+  /// Whether to provide the Dart product define to the frontend server.
+  final bool targetProductVm;
+
+  /// When in batch mode, link platform kernel file into result kernel file.
+  final bool linkPlatformKernelIn;
+
+  /// Whether to compile incrementally.
+  final String incrementalCompilerByteStorePath;
+
+  /// Additional arguments to pass to the frontend server.
+  final List<String> extraFrontEndOptions;
+
+  @override
+  Map<String, List<String>> get buildExtensions => const <String, List<String>>{
+    '.dart': <String>[_kFlutterDillOutputExtension, _kPackagesExtension],
+  };
+
+  @override
+  Future<void> build(BuildStep buildStep) async {
+    // Do not run builder if it has been disabled or if this asset does not
+    // correspond to the current entrypoint.
+    if (disabled || !mainPath.contains(buildStep.inputId.path)) {
+      return;
+    }
+    final AssetId outputId = buildStep.inputId.changeExtension(_kFlutterDillOutputExtension);
+    final AssetId packagesOutputId = buildStep.inputId.changeExtension(_kPackagesExtension);
+
+    // Create a scratch space file that can be read/written by the frontend server.
+    // It is okay to hard-code these file names because we will copy them back
+    // from the temp directory at the end of the build step.
+    final Directory tempDirecory = await Directory.systemTemp.createTemp('_flutter_build');
+    final File packagesFile = File(path.join(tempDirecory.path, _kPackagesExtension));
+    final File outputFile =  File(path.join(tempDirecory.path, 'main.app.dill'));
+    await outputFile.create();
+    await packagesFile.create();
+
+    final Directory projectDir = File(packagesPath).parent;
+    final String packageName = buildStep.inputId.package;
+    final String oldPackagesContents = await File(packagesPath).readAsString();
+    // Note: currently we only replace the root package with a multiroot
+    // scheme. To support codegen on arbitrary packages we will need to do
+    // this for each dependency.
+    final String newPackagesContents = oldPackagesContents.replaceFirst('$packageName:lib/', '$packageName:$multiRootScheme:///lib/');
+    await packagesFile.writeAsString(newPackagesContents);
+
+    // start up the frontend server with configuration.
+    final List<String> arguments = <String>[
+      frontendServerPath,
+      '--sdk-root',
+      sdkRoot,
+      '--strong',
+      '--target=flutter',
+    ];
+    if (trackWidgetCreation) {
+      arguments.add('--track-widget-creation');
+    }
+    if (!linkPlatformKernelIn) {
+      arguments.add('--no-link-platform');
+    }
+    if (aot) {
+      arguments.add('--aot');
+      arguments.add('--tfa');
+    }
+    if (targetProductVm) {
+      arguments.add('-Ddart.vm.product=true');
+    }
+    if (incrementalCompilerByteStorePath != null) {
+      arguments.add('--incremental');
+    }
+    final String generatedRoot = path.join(projectDir.absolute.path, '.dart_tool', 'build', 'generated', '$packageName');
+    arguments.addAll(<String>[
+      '--packages',
+      packagesFile.path,
+      '--output-dill',
+      outputFile.path,
+      '--filesystem-root',
+      projectDir.absolute.path,
+      '--filesystem-root',
+      generatedRoot,
+      '--filesystem-scheme',
+      multiRootScheme,
+    ]);
+    if (extraFrontEndOptions != null) {
+      arguments.addAll(extraFrontEndOptions);
+    }
+    final Uri mainUri = _PackageUriMapper.findUri(
+      mainPath,
+      packagesFile.path,
+      multiRootScheme,
+      <String>[projectDir.absolute.path, generatedRoot],
+    );
+    arguments.add(mainUri.toString());
+    // Invoke the frontend server and copy the dill back to the output
+    // directory.
+    try {
+      final Process server = await Process.start(engineDartBinaryPath, arguments);
+      final _StdoutHandler _stdoutHandler = _StdoutHandler();
+      server.stderr
+        .transform<String>(utf8.decoder)
+        .listen(log.shout);
+      server.stdout
+        .transform<String>(utf8.decoder)
+        .transform<String>(const LineSplitter())
+        .listen(_stdoutHandler.handler);
+      await server.exitCode;
+      await _stdoutHandler.compilerOutput.future;
+      await buildStep.writeAsBytes(outputId, await outputFile.readAsBytes());
+      await buildStep.writeAsBytes(packagesOutputId, await packagesFile.readAsBytes());
+    } catch (err, stackTrace) {
+      log.shout('frontend server failed to start: $err, $stackTrace');
+    }
+  }
+}
+
+class _StdoutHandler {
+  _StdoutHandler() {
+    reset();
+  }
+
+  bool compilerMessageReceived = false;
+  String boundaryKey;
+  Completer<_CompilerOutput> compilerOutput;
+
+  bool _suppressCompilerMessages;
+
+  void handler(String message) {
+    log.info(message);
+    const String kResultPrefix = 'result ';
+    if (boundaryKey == null) {
+      if (message.startsWith(kResultPrefix))
+        boundaryKey = message.substring(kResultPrefix.length);
+    } else if (message.startsWith(boundaryKey)) {
+      if (message.length <= boundaryKey.length) {
+        compilerOutput.complete(null);
+        return;
+      }
+      final int spaceDelimiter = message.lastIndexOf(' ');
+      compilerOutput.complete(
+        _CompilerOutput(
+          message.substring(boundaryKey.length + 1, spaceDelimiter),
+          int.parse(message.substring(spaceDelimiter + 1).trim())));
+    } else if (!_suppressCompilerMessages) {
+      if (compilerMessageReceived == false) {
+        log.info('\nCompiler message:');
+        compilerMessageReceived = true;
+      }
+      log.info(message);
+    }
+  }
+
+  // This is needed to get ready to process next compilation result output,
+  // with its own boundary key and new completer.
+  void reset({bool suppressCompilerMessages = false}) {
+    boundaryKey = null;
+    compilerMessageReceived = false;
+    compilerOutput = Completer<_CompilerOutput>();
+    _suppressCompilerMessages = suppressCompilerMessages;
+  }
+}
+
+/// Converts filesystem paths to package URIs.
+class _PackageUriMapper {
+  _PackageUriMapper(String scriptPath, String packagesPath, this.fileSystemScheme, this.fileSystemRoots) {
+    final List<int> bytes = File(path.absolute(packagesPath)).readAsBytesSync();
+    final Map<String, Uri> packageMap = packages_file.parse(bytes, Uri.file(packagesPath, windows: Platform.isWindows));
+    final String scriptUri = Uri.file(scriptPath, windows: Platform.isWindows).toString();
+    for (String packageName in packageMap.keys) {
+      final String prefix = packageMap[packageName].toString();
+      if (fileSystemScheme != null && fileSystemRoots != null && prefix.contains(fileSystemScheme)) {
+        _packageName = packageName;
+        _uriPrefix = fileSystemRoots.map((String name) => Uri.file('$name/lib/', windows: Platform.isWindows).toString()).toList();
+        return;
+      }
+      if (scriptUri.startsWith(prefix)) {
+        _packageName = packageName;
+        _uriPrefix = <String>[prefix];
+        return;
+      }
+    }
+  }
+
+  final String fileSystemScheme;
+  final List<String> fileSystemRoots;
+
+  String _packageName;
+  List<String> _uriPrefix;
+
+  Uri map(String scriptPath) {
+    if (_packageName == null) {
+      return null;
+    }
+    final String scriptUri = Uri.file(scriptPath, windows: Platform.isWindows).toString();
+    for (String uriPrefix in _uriPrefix) {
+      if (scriptUri.startsWith(uriPrefix)) {
+        return Uri.parse('package:$_packageName/${scriptUri.substring(uriPrefix.length)}');
+      }
+    }
+    return null;
+  }
+
+  static Uri findUri(String scriptPath, String packagesPath, String fileSystemScheme, List<String> fileSystemRoots) {
+    return _PackageUriMapper(scriptPath, packagesPath, fileSystemScheme, fileSystemRoots).map(scriptPath);
+  }
+}
+
+class _CompilerOutput {
+  const _CompilerOutput(this.outputFilename, this.errorCount);
+
+  final String outputFilename;
+  final int errorCount;
+}

packages/flutter_build/pubspec.yaml

+name: flutter_build
+
+environment:
+  # The pub client defaults to an <2.0.0 sdk constraint which we need to explicitly overwrite.
+  sdk: ">=2.0.0-dev.68.0 <3.0.0"
+
+dependencies:
+  # To update these, use "flutter update-packages --force-upgrade".
+  build: 1.1.0
+  package_config: 1.0.5
+  path: 1.6.2
+
+  analyzer: 0.33.6+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  args: 1.5.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  async: 2.0.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  charcode: 1.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  collection: 1.14.11 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  convert: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  crypto: 2.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  csslib: 0.14.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  front_end: 0.1.6+9 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  glob: 1.1.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  html: 0.13.3+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  kernel: 0.3.6+9 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  logging: 0.11.3+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  meta: 1.1.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  plugin: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pub_semver: 1.4.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_span: 1.5.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  string_scanner: 1.0.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  term_glyph: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  typed_data: 1.1.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  utf: 0.9.0+5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  watcher: 0.9.7+10 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  yaml: 2.1.15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+
+dartdoc:
+  # Exclude this package from the hosted API docs.
+  nodoc: true
+
+# PUBSPEC CHECKSUM: 329a

packages/flutter_tools/lib/src/commands/update_packages.dart

@@ -119,7 +119,7 @@ class UpdatePackagesCommand extends FlutterCommand {
         );
       }
       // Only retain flutter, flutter_test, flutter_driver, and flutter_localizations.
-      const List<String> consumerPackages = <String>['flutter', 'flutter_test', 'flutter_driver', 'flutter_localizations'];
+      const List<String> consumerPackages = <String>['flutter', 'flutter_test', 'flutter_driver', 'flutter_localizations', 'flutter_build'];
       // ensure we only get flutter/packages
       packages.retainWhere((Directory directory) {
         return consumerPackages.any((String package) {

packages/flutter_tools/pubspec.yaml

@@ -10,7 +10,7 @@ environment:
 dependencies:
   # To update these, use "flutter update-packages --force-upgrade".
   analyzer: 0.33.6+1
-  archive: 2.0.7
+  archive: 2.0.8
   args: 1.5.1
   cli_util: 0.1.3+2
   completion: 0.2.1+1
@@ -96,4 +96,4 @@ dartdoc:
   # Exclude this package from the hosted API docs.
   nodoc: true
 
-# PUBSPEC CHECKSUM: 78f0
+# PUBSPEC CHECKSUM: b4f1