Support deferred imports on profile/release builds of Flutter Web (#41222)

packages/flutter_tools/lib/src/build_runner/build_script.dart

@@ -11,6 +11,7 @@ import 'dart:isolate';
 import 'package:analyzer/dart/analysis/results.dart';
 import 'package:analyzer/dart/analysis/utilities.dart';
 import 'package:analyzer/dart/ast/ast.dart';
+import 'package:archive/archive.dart';
 import 'package:build/build.dart';
 import 'package:build_config/build_config.dart';
 import 'package:build_modules/build_modules.dart';
@@ -26,6 +27,7 @@ import 'package:build_web_compilers/build_web_compilers.dart';
 import 'package:build_web_compilers/builders.dart';
 import 'package:build_web_compilers/src/dev_compiler_bootstrap.dart';
 import 'package:crypto/crypto.dart';
+import 'package:glob/glob.dart';
 import 'package:path/path.dart' as path; // ignore: package_path_import
 import 'package:scratch_space/scratch_space.dart';
 import 'package:test_core/backend.dart';
@@ -461,8 +463,32 @@ Future<void> bootstrapDart2Js(BuildStep buildStep, String flutterWebSdk, bool pr
   final AssetId jsOutputId = dartEntrypointId.changeExtension(jsEntrypointExtension);
   final File jsOutputFile = scratchSpace.fileFor(jsOutputId);
   if (result.succeeded && jsOutputFile.existsSync()) {
-    log.info(result.output);
-    // Explicitly write out the original js file and sourcemap.
+    final String rootDir = path.dirname(jsOutputFile.path);
+    final String dartFile = path.basename(dartEntrypointId.path);
+    final Glob fileGlob = Glob('$dartFile.js*');
+    final Archive archive = Archive();
+    await for (FileSystemEntity jsFile in fileGlob.list(root: rootDir)) {
+      if (jsFile.path.endsWith(jsEntrypointExtension) ||
+          jsFile.path.endsWith(jsEntrypointSourceMapExtension)) {
+        // These are explicitly output, and are not part of the archive.
+        continue;
+      }
+      if (jsFile is File) {
+        final String fileName = path.relative(jsFile.path, from: rootDir);
+        final FileStat fileStats = jsFile.statSync();
+        archive.addFile(
+            ArchiveFile(fileName, fileStats.size, await jsFile.readAsBytes())
+              ..mode = fileStats.mode
+              ..lastModTime = fileStats.modified.millisecondsSinceEpoch);
+      }
+    }
+    if (archive.isNotEmpty) {
+      final AssetId archiveId = dartEntrypointId.changeExtension(jsEntrypointArchiveExtension);
+      await buildStep.writeAsBytes(archiveId, TarEncoder().encode(archive));
+    }
+
+    // Explicitly write out the original js file and sourcemap - we can't output
+    // these as part of the archive because they already have asset nodes.
     await scratchSpace.copyOutput(jsOutputId, buildStep);
     final AssetId jsSourceMapId =
         dartEntrypointId.changeExtension(jsEntrypointSourceMapExtension);

packages/flutter_tools/lib/src/build_runner/web_fs.dart

@@ -4,10 +4,9 @@
 
 import 'dart:async';
 
+import 'package:archive/archive.dart';
 import 'package:build_daemon/client.dart';
-import 'package:build_daemon/constants.dart';
-import 'package:build_daemon/constants.dart' hide BuildMode;
-import 'package:build_daemon/constants.dart' as daemon show BuildMode;
+import 'package:build_daemon/constants.dart' as daemon;
 import 'package:build_daemon/data/build_status.dart';
 import 'package:build_daemon/data/build_target.dart';
 import 'package:build_daemon/data/server_log.dart';
@@ -89,6 +88,7 @@ class WebFs {
     this._server,
     this._dwds,
     this.uri,
+    this._assetServer,
   );
 
   /// The server uri.
@@ -97,6 +97,7 @@ class WebFs {
   final HttpServer _server;
   final Dwds _dwds;
   final BuildDaemonClient _client;
+  final AssetServer _assetServer;
   StreamSubscription<void> _connectedApps;
 
   static const String _kHostName = 'localhost';
@@ -106,6 +107,7 @@ class WebFs {
     await _dwds?.stop();
     await _server.close(force: true);
     await _connectedApps?.cancel();
+    _assetServer?.dispose();
   }
 
   Future<DebugConnection> _cachedExtensionFuture;
@@ -180,21 +182,22 @@ class WebFs {
     await assetBundle.build();
     await writeBundle(fs.directory(getAssetBuildDirectory()), assetBundle.entries);
 
-    // Initialize the dwds server.
-    final int hostPort = port == null ? await os.findFreePort() : int.tryParse(port);
-    // Map the bootstrap files to the correct package directory.
     final String targetBaseName = fs.path
-      .withoutExtension(target).replaceFirst('lib${fs.path.separator}', '');
+        .withoutExtension(target).replaceFirst('lib${fs.path.separator}', '');
     final Map<String, String> mappedUrls = <String, String>{
       'main.dart.js': 'packages/${flutterProject.manifest.appName}/'
-        '${targetBaseName}_web_entrypoint.dart.js',
+          '${targetBaseName}_web_entrypoint.dart.js',
       '${targetBaseName}_web_entrypoint.dart.js.map': 'packages/${flutterProject.manifest.appName}/'
-        '${targetBaseName}_web_entrypoint.dart.js.map',
+          '${targetBaseName}_web_entrypoint.dart.js.map',
       '${targetBaseName}_web_entrypoint.dart.bootstrap.js': 'packages/${flutterProject.manifest.appName}/'
-        '${targetBaseName}_web_entrypoint.dart.bootstrap.js',
+          '${targetBaseName}_web_entrypoint.dart.bootstrap.js',
       '${targetBaseName}_web_entrypoint.digests': 'packages/${flutterProject.manifest.appName}/'
-        '${targetBaseName}_web_entrypoint.digests',
+          '${targetBaseName}_web_entrypoint.digests',
     };
+
+    // Initialize the dwds server.
+    final int hostPort = port == null ? await os.findFreePort() : int.tryParse(port);
+
     final Pipeline pipeline = const Pipeline().addMiddleware((Handler innerHandler) {
       return (Request request) async {
         // Redirect the main.dart.js to the target file we decided to serve.
@@ -237,9 +240,10 @@ class WebFs {
     } else {
       handler = pipeline.addHandler(proxyHandler('http://localhost:$daemonAssetPort/web/'));
     }
+    final AssetServer assetServer = AssetServer(flutterProject, targetBaseName);
     Cascade cascade = Cascade();
     cascade = cascade.add(handler);
-    cascade = cascade.add(_assetHandler(flutterProject));
+    cascade = cascade.add(assetServer.handle);
     final HttpServer server = await httpMultiServerFactory(hostname ?? _kHostName, hostPort);
     shelf_io.serveRequests(server, cascade.handler);
     return WebFs(
@@ -247,98 +251,136 @@ class WebFs {
       server,
       dwds,
       'http://$_kHostName:$hostPort/',
+      assetServer,
     );
   }
+}
 
-  static Future<Response> Function(Request request) _assetHandler(FlutterProject flutterProject) {
-    final PackageMap packageMap = PackageMap(PackageMap.globalPackagesPath);
-    return (Request request) async {
-      if (request.url.path.contains('stack_trace_mapper')) {
-        final File file = fs.file(fs.path.join(
-          artifacts.getArtifactPath(Artifact.engineDartSdkPath),
-          'lib',
-          'dev_compiler',
-          'web',
-          'dart_stack_trace_mapper.js',
-        ));
-        return Response.ok(file.readAsBytesSync(), headers: <String, String>{
-          'Content-Type': 'text/javascript',
-        });
-      } else if (request.url.path.contains('require.js')) {
-        final File file = fs.file(fs.path.join(
-          artifacts.getArtifactPath(Artifact.engineDartSdkPath),
+class AssetServer {
+  AssetServer(this.flutterProject, this.targetBaseName);
+
+  final FlutterProject flutterProject;
+  final String targetBaseName;
+  final PackageMap packageMap = PackageMap(PackageMap.globalPackagesPath);
+  Directory partFiles;
+
+  Future<Response> handle(Request request) async {
+    if (request.url.path.contains('stack_trace_mapper')) {
+      final File file = fs.file(fs.path.join(
+        artifacts.getArtifactPath(Artifact.engineDartSdkPath),
+        'lib',
+        'dev_compiler',
+        'web',
+        'dart_stack_trace_mapper.js',
+      ));
+      return Response.ok(file.readAsBytesSync(), headers: <String, String>{
+        'Content-Type': 'text/javascript',
+      });
+    } else if (request.url.path.endsWith('part.js')) {
+      // Lazily unpack any deferred imports in release/profile mode. These are
+      // placed into an archive by build_runner, and are named based on the main
+      // entrypoint + a "part" suffix (Though the actual names are arbitrary).
+      // To make this easier to deal with they are copied into a temp directory.
+      if (partFiles == null) {
+        final File dart2jsArchive = fs.file(fs.path.join(
+          flutterProject.dartTool.path,
+          'build',
+          'flutter_web',
+          '${flutterProject.manifest.appName}',
           'lib',
-          'dev_compiler',
-          'kernel',
-          'amd',
-          'require.js',
-        ));
-        return Response.ok(file.readAsBytesSync(), headers: <String, String>{
-          'Content-Type': 'text/javascript',
-        });
-      } else if (request.url.path.endsWith('dart_sdk.js')) {
-        final File file = fs.file(fs.path.join(
-          artifacts.getArtifactPath(Artifact.flutterWebSdk),
-          'kernel',
-          'amd',
-          'dart_sdk.js',
+          '${targetBaseName}_web_entrypoint.dart.js.tar.gz',
         ));
-        return Response.ok(file.readAsBytesSync(), headers: <String, String>{
-          'Content-Type': 'text/javascript',
-        });
-      } else if (request.url.path.endsWith('dart_sdk.js.map')) {
-        final File file = fs.file(fs.path.join(
-          artifacts.getArtifactPath(Artifact.flutterWebSdk),
-          'kernel',
-          'amd',
-          'dart_sdk.js.map',
-        ));
-        return Response.ok(file.readAsBytesSync());
-      } else if (request.url.path.endsWith('.dart')) {
-        // This is likely a sourcemap request. The first segment is the
-        // package name, and the rest is the path to the file relative to
-        // the package uri. For example, `foo/bar.dart` would represent a
-        // file at a path like `foo/lib/bar.dart`. If there is no leading
-        // segment, then we assume it is from the current package.
-
-        // Handle sdk requests that have mangled urls from engine build.
-        if (request.url.path.contains('flutter_web_sdk')) {
-          // Note: the request is a uri and not a file path, so they always use `/`.
-          final String sdkPath = fs.path.joinAll(request.url.path.split('flutter_web_sdk/').last.split('/'));
-          final String webSdkPath = artifacts.getArtifactPath(Artifact.flutterWebSdk);
-          return Response.ok(fs.file(fs.path.join(webSdkPath, sdkPath)).readAsBytesSync());
+        if (dart2jsArchive.existsSync()) {
+          final Archive archive = TarDecoder().decodeBytes(dart2jsArchive.readAsBytesSync());
+          partFiles = fs.systemTempDirectory.createTempSync('_flutter_tool')
+            ..createSync();
+          for (ArchiveFile file in archive) {
+            partFiles.childFile(file.name).writeAsBytesSync(file.content);
+          }
         }
+      }
+      final String fileName = fs.path.basename(request.url.path);
+      return Response.ok(partFiles.childFile(fileName).readAsBytesSync(), headers: <String, String>{
+        'Content-Type': 'text/javascript',
+      });
+    } else if (request.url.path.contains('require.js')) {
+      final File file = fs.file(fs.path.join(
+        artifacts.getArtifactPath(Artifact.engineDartSdkPath),
+        'lib',
+        'dev_compiler',
+        'kernel',
+        'amd',
+        'require.js',
+      ));
+      return Response.ok(file.readAsBytesSync(), headers: <String, String>{
+        'Content-Type': 'text/javascript',
+      });
+    } else if (request.url.path.endsWith('dart_sdk.js')) {
+      final File file = fs.file(fs.path.join(
+        artifacts.getArtifactPath(Artifact.flutterWebSdk),
+        'kernel',
+        'amd',
+        'dart_sdk.js',
+      ));
+      return Response.ok(file.readAsBytesSync(), headers: <String, String>{
+        'Content-Type': 'text/javascript',
+      });
+    } else if (request.url.path.endsWith('dart_sdk.js.map')) {
+      final File file = fs.file(fs.path.join(
+        artifacts.getArtifactPath(Artifact.flutterWebSdk),
+        'kernel',
+        'amd',
+        'dart_sdk.js.map',
+      ));
+      return Response.ok(file.readAsBytesSync());
+    } else if (request.url.path.endsWith('.dart')) {
+      // This is likely a sourcemap request. The first segment is the
+      // package name, and the rest is the path to the file relative to
+      // the package uri. For example, `foo/bar.dart` would represent a
+      // file at a path like `foo/lib/bar.dart`. If there is no leading
+      // segment, then we assume it is from the current package.
+
+      // Handle sdk requests that have mangled urls from engine build.
+      if (request.url.path.contains('flutter_web_sdk')) {
+        // Note: the request is a uri and not a file path, so they always use `/`.
+        final String sdkPath = fs.path.joinAll(request.url.path.split('flutter_web_sdk/').last.split('/'));
+        final String webSdkPath = artifacts.getArtifactPath(Artifact.flutterWebSdk);
+        return Response.ok(fs.file(fs.path.join(webSdkPath, sdkPath)).readAsBytesSync());
+      }
 
-        final String packageName = request.url.pathSegments.length == 1
+      final String packageName = request.url.pathSegments.length == 1
           ? flutterProject.manifest.appName
           : request.url.pathSegments.first;
-        String filePath = fs.path.joinAll(request.url.pathSegments.length == 1
+      String filePath = fs.path.joinAll(request.url.pathSegments.length == 1
           ? request.url.pathSegments
           : request.url.pathSegments.skip(1));
-        String packagePath = packageMap.map[packageName]?.toFilePath(windows: platform.isWindows);
-        // If the package isn't found, then we have an issue with relative
-        // paths within the main project.
-        if (packagePath == null) {
-          packagePath = packageMap.map[flutterProject.manifest.appName]
+      String packagePath = packageMap.map[packageName]?.toFilePath(windows: platform.isWindows);
+      // If the package isn't found, then we have an issue with relative
+      // paths within the main project.
+      if (packagePath == null) {
+        packagePath = packageMap.map[flutterProject.manifest.appName]
             .toFilePath(windows: platform.isWindows);
-          filePath = request.url.path;
-        }
-        final File file = fs.file(fs.path.join(packagePath, filePath));
-        if (file.existsSync()) {
-          return Response.ok(file.readAsBytesSync());
-        }
-        return Response.notFound('');
-      } else if (request.url.path.contains('assets')) {
-        final String assetPath = request.url.path.replaceFirst('assets/', '');
-        final File file = fs.file(fs.path.join(getAssetBuildDirectory(), assetPath));
-        if (file.existsSync()) {
-          return Response.ok(file.readAsBytesSync());
-        } else {
-          return Response.notFound('');
-        }
+        filePath = request.url.path;
+      }
+      final File file = fs.file(fs.path.join(packagePath, filePath));
+      if (file.existsSync()) {
+        return Response.ok(file.readAsBytesSync());
       }
       return Response.notFound('');
-    };
+    } else if (request.url.path.contains('assets')) {
+      final String assetPath = request.url.path.replaceFirst('assets/', '');
+      final File file = fs.file(fs.path.join(getAssetBuildDirectory(), assetPath));
+      if (file.existsSync()) {
+        return Response.ok(file.readAsBytesSync());
+      } else {
+        return Response.notFound('');
+      }
+    }
+    return Response.notFound('');
+  }
+
+  void dispose() {
+    partFiles?.deleteSync(recursive: true);
   }
 }
 
@@ -459,7 +501,7 @@ class BuildDaemonCreator {
 
   /// Retrieve the asset server port for the current daemon.
   int assetServerPort(Directory workingDirectory) {
-    final String portFilePath = fs.path.join(daemonWorkspace(workingDirectory.path), '.asset_server_port');
+    final String portFilePath = fs.path.join(daemon.daemonWorkspace(workingDirectory.path), '.asset_server_port');
     return int.tryParse(fs.file(portFilePath).readAsStringSync());
   }
 }

packages/flutter_tools/lib/src/web/compile.dart

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'package:archive/archive.dart';
 import 'package:meta/meta.dart';
 
 import '../asset.dart';
@@ -48,9 +49,23 @@ Future<void> buildWeb(FlutterProject flutterProject, String target, BuildInfo bu
         flutterProject.manifest.appName,
         '${fs.path.withoutExtension(target)}_web_entrypoint.dart.js',
       );
+      // Check for deferred import outputs.
+      final File dart2jsArchive = fs.file(fs.path.join(
+        flutterProject.dartTool.path,
+        'build',
+        'flutter_web',
+        '${flutterProject.manifest.appName}',
+        '${fs.path.withoutExtension(target)}_web_entrypoint.dart.js.tar.gz'),
+      );
       fs.file(outputPath).copySync(fs.path.join(outputDir.path, 'main.dart.js'));
       fs.file('$outputPath.map').copySync(fs.path.join(outputDir.path, 'main.dart.js.map'));
       flutterProject.web.indexFile.copySync(fs.path.join(outputDir.path, 'index.html'));
+      if (dart2jsArchive.existsSync()) {
+        final Archive archive = TarDecoder().decodeBytes(dart2jsArchive.readAsBytesSync());
+        for (ArchiveFile file in archive.files) {
+          outputDir.childFile(file.name).writeAsBytesSync(file.content);
+        }
+      }
     }
   } catch (err) {
     printError(err.toString());

packages/flutter_tools/test/general.shard/commands/build_web_test.dart

@@ -2,7 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
+
+import 'package:archive/archive.dart';
 import 'package:args/command_runner.dart';
+import 'package:file_testing/file_testing.dart';
 import 'package:flutter_tools/src/base/common.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/platform.dart';
@@ -25,6 +29,7 @@ void main() {
   MockWebCompilationProxy mockWebCompilationProxy;
   Testbed testbed;
   MockPlatform mockPlatform;
+  bool addArchive = false;
 
   setUpAll(() {
     Cache.flutterRoot = '';
@@ -32,6 +37,7 @@ void main() {
   });
 
   setUp(() {
+    addArchive = false;
     mockWebCompilationProxy = MockWebCompilationProxy();
     testbed = Testbed(setup: () {
       fs.file('pubspec.yaml')
@@ -46,9 +52,18 @@ void main() {
         mode: anyNamed('mode'),
         initializePlatform: anyNamed('initializePlatform'),
       )).thenAnswer((Invocation invocation) {
-        final String path = fs.path.join('.dart_tool', 'build', 'flutter_web', 'foo', 'lib', 'main_web_entrypoint.dart.js');
+        final String prefix = fs.path.join('.dart_tool', 'build', 'flutter_web', 'foo', 'lib');
+        final String path = fs.path.join(prefix, 'main_web_entrypoint.dart.js');
         fs.file(path).createSync(recursive: true);
         fs.file('$path.map').createSync();
+        if (addArchive) {
+          final List<int> bytes = utf8.encode('void main() {}');
+          final TarEncoder encoder = TarEncoder();
+          final Archive archive = Archive()
+            ..addFile(ArchiveFile.noCompress('main_web_entrypoint.1.dart.js', bytes.length, bytes));
+          fs.file(fs.path.join(prefix, 'main_web_entrypoint.dart.js.tar.gz'))
+            ..writeAsBytes(encoder.encode(archive));
+        }
         return Future<bool>.value(true);
       });
     }, overrides: <Type, Generator>{
@@ -59,6 +74,19 @@ void main() {
     });
   });
 
+  test('Copies generated part files out of build directory', () => testbed.run(() async {
+    addArchive = true;
+    await buildWeb(
+      FlutterProject.current(),
+      fs.path.join('lib', 'main.dart'),
+      BuildInfo.release,
+      false,
+    );
+
+    expect(fs.file(fs.path.join('build', 'web', 'main_web_entrypoint.1.dart.js')), exists);
+    expect(fs.file(fs.path.join('build', 'web', 'main.dart.js')), exists);
+  }));
+
   test('Refuses to build for web when missing index.html', () => testbed.run(() async {
     fs.file(fs.path.join('web', 'index.html')).deleteSync();
 
@@ -82,16 +110,6 @@ void main() {
     expect(await runner.run(), 1);
   }));
 
-  test('Can build for web', () => testbed.run(() async {
-
-    await buildWeb(
-      FlutterProject.current(),
-      fs.path.join('lib', 'main.dart'),
-      BuildInfo.debug,
-      false,
-    );
-  }));
-
   test('Refuses to build a debug build for web', () => testbed.run(() async {
     final CommandRunner<void> runner = createTestCommandRunner(BuildCommand());