Added web compiler debug metadata (#61265)

Made flutter tools serve metadata produced by the frontend server:

pass --experimental-emit-metadata flag to flutter engine
store and serve metadata from web asset server
store and serve merged metadata from web asset server
add tests to verify that metadata is served from memory
update dwds version so it can read metadata
configure dwds to read metadata from file
Prerequisite changes (landed):

sdk: https://dart-review.googlesource.com/c/sdk/+/150181
flutter engine: flutter/engine#19168
webdev:dart-lang/webdev#1064

packages/flutter_tools/lib/src/build_runner/devfs_web.dart

@@ -46,6 +46,7 @@ typedef DwdsLauncher = Future<Dwds> Function({
   void Function(Level, String) logWriter,
   bool verbose,
   UrlEncoder urlEncoder,
+  bool useFileProvider,
   ExpressionCompiler expressionCompiler,
 });
 
@@ -245,6 +246,7 @@ class WebAssetServer implements AssetReader {
           serverPathForModule,
           serverPathForAppUri,
         ),
+        useFileProvider: true,
         expressionCompiler: expressionCompiler
       );
       shelf.Pipeline pipeline = const shelf.Pipeline();
@@ -271,6 +273,8 @@ class WebAssetServer implements AssetReader {
   // RandomAccessFile and read on demand.
   final Map<String, Uint8List> _files = <String, Uint8List>{};
   final Map<String, Uint8List> _sourcemaps = <String, Uint8List>{};
+  final Map<String, Uint8List> _metadataFiles = <String, Uint8List>{};
+  String _mergedMetadata;
   final PackageConfig _packages;
   final InternetAddress internetAddress;
   /* late final */ Dwds dwds;
@@ -282,6 +286,9 @@ class WebAssetServer implements AssetReader {
   @visibleForTesting
   Uint8List getSourceMap(String path) => _sourcemaps[path];
 
+  @visibleForTesting
+  Uint8List getMetadata(String path) => _metadataFiles[path];
+
   // handle requests for JavaScript source, dart sources maps, or asset files.
   @visibleForTesting
   Future<shelf.Response> handleRequest(shelf.Request request) async {
@@ -341,6 +348,20 @@ class WebAssetServer implements AssetReader {
       return shelf.Response.ok(bytes, headers: headers);
     }
 
+    // If this is a metadata file, then it might be in the in-memory cache.
+    // Attempt to lookup the file by URI.
+    if (_metadataFiles.containsKey(requestPath)) {
+      final List<int> bytes = getMetadata(requestPath);
+      final String etag = bytes.hashCode.toString();
+      if (ifNoneMatch == etag) {
+        return shelf.Response.notModified();
+      }
+      headers[HttpHeaders.contentLengthHeader] = bytes.length.toString();
+      headers[HttpHeaders.contentTypeHeader] = 'application/json';
+      headers[HttpHeaders.etagHeader] = etag;
+      return shelf.Response.ok(bytes, headers: headers);
+    }
+
     File file = _resolveDartFile(requestPath);
 
     // If all of the lookups above failed, the file might have been an asset.
@@ -405,10 +426,15 @@ class WebAssetServer implements AssetReader {
   /// Update the in-memory asset server with the provided source and manifest files.
   ///
   /// Returns a list of updated modules.
-  List<String> write(File codeFile, File manifestFile, File sourcemapFile) {
+  List<String> write(
+      File codeFile,
+      File manifestFile,
+      File sourcemapFile,
+      File metadataFile) {
     final List<String> modules = <String>[];
     final Uint8List codeBytes = codeFile.readAsBytesSync();
     final Uint8List sourcemapBytes = sourcemapFile.readAsBytesSync();
+    final Uint8List metadataBytes = metadataFile.readAsBytesSync();
     final Map<String, dynamic> manifest = castStringKeyedMap(json.decode(manifestFile.readAsStringSync()));
     for (final String filePath in manifest.keys) {
       if (filePath == null) {
@@ -418,7 +444,10 @@ class WebAssetServer implements AssetReader {
       final Map<String, dynamic> offsets = castStringKeyedMap(manifest[filePath]);
       final List<int> codeOffsets = (offsets['code'] as List<dynamic>).cast<int>();
       final List<int> sourcemapOffsets = (offsets['sourcemap'] as List<dynamic>).cast<int>();
-      if (codeOffsets.length != 2 || sourcemapOffsets.length != 2) {
+      final List<int> metadataOffsets = (offsets['metadata'] as List<dynamic>).cast<int>();
+      if (codeOffsets.length != 2 ||
+          sourcemapOffsets.length != 2 ||
+          metadataOffsets.length != 2) {
         globals.printTrace('Invalid manifest byte offsets: $offsets');
         continue;
       }
@@ -453,8 +482,27 @@ class WebAssetServer implements AssetReader {
       final String sourcemapName = '$fileName.map';
       _sourcemaps[sourcemapName] = sourcemapView;
 
+      final int metadataStart = metadataOffsets[0];
+      final int metadataEnd = metadataOffsets[1];
+      if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) {
+        globals.printTrace('Invalid byte index: [$metadataStart, $metadataEnd]');
+        continue;
+      }
+      final Uint8List metadataView = Uint8List.view(
+        metadataBytes.buffer,
+        metadataStart,
+        metadataEnd - metadataStart,
+      );
+      final String metadataName = '$fileName.metadata';
+      _metadataFiles[metadataName] = metadataView;
+
       modules.add(fileName);
     }
+
+    _mergedMetadata = _metadataFiles.values
+      .map((Uint8List encoded) => utf8.decode(encoded))
+      .join('\n');
+
     return modules;
   }
 
@@ -549,7 +597,13 @@ class WebAssetServer implements AssetReader {
   }
 
   @override
-  Future<String> metadataContents(String serverPath) {
+  Future<String> metadataContents(String serverPath) async {
+    if (serverPath == 'main_module.ddc_merged_metadata') {
+      return _mergedMetadata;
+    }
+    if (_metadataFiles.containsKey(serverPath)) {
+      return utf8.decode(_metadataFiles[serverPath]);
+    }
     return null;
   }
 }
@@ -772,13 +826,15 @@ class WebDevFS implements DevFS {
     File codeFile;
     File manifestFile;
     File sourcemapFile;
+    File metadataFile;
     List<String> modules;
     try {
       final Directory parentDirectory = globals.fs.directory(outputDirectoryPath);
       codeFile = parentDirectory.childFile('${compilerOutput.outputFilename}.sources');
       manifestFile = parentDirectory.childFile('${compilerOutput.outputFilename}.json');
       sourcemapFile = parentDirectory.childFile('${compilerOutput.outputFilename}.map');
-      modules = webAssetServer.write(codeFile, manifestFile, sourcemapFile);
+      metadataFile = parentDirectory.childFile('${compilerOutput.outputFilename}.metadata');
+      modules = webAssetServer.write(codeFile, manifestFile, sourcemapFile, metadataFile);
     } on FileSystemException catch (err) {
       throwToolExit('Failed to load recompiled sources:\n$err');
     }

packages/flutter_tools/lib/src/compile.dart

@@ -689,6 +689,10 @@ class DefaultResidentCompiler implements ResidentCompiler {
       // in the frontend_server.
       // https://github.com/flutter/flutter/issues/52693
       '--debugger-module-names',
+      // TODO(annagrin): remove once this becomes the default behavior
+      // in the frontend_server.
+      // https://github.com/flutter/flutter/issues/59902
+      '--experimental-emit-debug-metadata',
       '-Ddart.developer.causal_async_stacks=${buildMode == BuildMode.debug}',
       for (final Object dartDefine in dartDefines)
         '-D$dartDefine',

packages/flutter_tools/pubspec.yaml

@@ -11,7 +11,7 @@ dependencies:
   # To update these, use "flutter update-packages --force-upgrade".
   archive: 2.0.13
   args: 1.6.0
-  dwds: 5.0.0
+  dwds: 5.1.0
   completion: 0.2.2
   coverage: 0.14.0
   crypto: 2.1.5
@@ -107,4 +107,4 @@ dartdoc:
   # Exclude this package from the hosted API docs.
   nodoc: true
 
-# PUBSPEC CHECKSUM: f9c4
+# PUBSPEC CHECKSUM: 85c5

packages/flutter_tools/test/general.shard/web/devfs_web_test.dart

@@ -65,21 +65,25 @@ void main() {
       ..writeAsStringSync('main() {}');
     final File sourcemap = globals.fs.file('sourcemap')
       ..writeAsStringSync('{}');
+    final File metadata = globals.fs.file('metadata')
+      ..writeAsStringSync('{}');
 
     // Missing ending offset.
     final File manifestMissingOffset = globals.fs.file('manifestA')
       ..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
         'code': <int>[0],
         'sourcemap': <int>[0],
+        'metadata': <int>[0],
       }}));
     final File manifestOutOfBounds = globals.fs.file('manifest')
       ..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
         'code': <int>[0, 100],
         'sourcemap': <int>[0],
+        'metadata': <int>[0],
       }}));
 
-    expect(webAssetServer.write(source, manifestMissingOffset, sourcemap), isEmpty);
-    expect(webAssetServer.write(source, manifestOutOfBounds, sourcemap), isEmpty);
+    expect(webAssetServer.write(source, manifestMissingOffset, sourcemap, metadata), isEmpty);
+    expect(webAssetServer.write(source, manifestOutOfBounds, sourcemap, metadata), isEmpty);
   }));
 
   test('serves JavaScript files from in memory cache', () => testbed.run(() async {
@@ -87,12 +91,15 @@ void main() {
       ..writeAsStringSync('main() {}');
     final File sourcemap = globals.fs.file('sourcemap')
       ..writeAsStringSync('{}');
+    final File metadata = globals.fs.file('metadata')
+      ..writeAsStringSync('{}');
     final File manifest = globals.fs.file('manifest')
       ..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
         'code': <int>[0, source.lengthSync()],
         'sourcemap': <int>[0, 2],
+        'metadata':  <int>[0, 2],
       }}));
-    webAssetServer.write(source, manifest, sourcemap);
+    webAssetServer.write(source, manifest, sourcemap, metadata);
 
     final Response response = await webAssetServer
       .handleRequest(Request('GET', Uri.parse('http://foobar/foo.js')));
@@ -107,6 +114,31 @@ void main() {
     Platform: () => linux,
   }));
 
+  test('serves metadata files from in memory cache', () => testbed.run(() async {
+    const String metadataContents = '{"name":"foo"}';
+    final File source = globals.fs.file('source')
+      ..writeAsStringSync('main() {}');
+    final File sourcemap = globals.fs.file('sourcemap')
+      ..writeAsStringSync('{}');
+    final File metadata = globals.fs.file('metadata')
+      ..writeAsStringSync(metadataContents);
+    final File manifest = globals.fs.file('manifest')
+      ..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
+        'code': <int>[0, source.lengthSync()],
+        'sourcemap': <int>[0, sourcemap.lengthSync()],
+        'metadata':  <int>[0, metadata.lengthSync()],
+      }}));
+    webAssetServer.write(source, manifest, sourcemap, metadata);
+
+    final String merged = await webAssetServer.metadataContents('main_module.ddc_merged_metadata');
+    expect(merged, equals(metadataContents));
+
+    final String single = await webAssetServer.metadataContents('foo.js.metadata');
+    expect(single, equals(metadataContents));
+  }, overrides: <Type, Generator>{
+    Platform: () => linux,
+  }));
+
   test('Removes leading slashes for valid requests to avoid requesting outside'
     ' of served directory', () => testbed.run(() async {
     globals.fs.file('foo.png').createSync();
@@ -164,12 +196,15 @@ void main() {
       ..writeAsStringSync('main() {}');
     final File sourcemap = globals.fs.file('sourcemap')
       ..writeAsStringSync('{}');
+    final File metadata = globals.fs.file('metadata')
+      ..writeAsStringSync('{}');
     final File manifest = globals.fs.file('manifest')
       ..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
         'code': <int>[0, source.lengthSync()],
         'sourcemap': <int>[0, 2],
+        'metadata': <int>[0, 2],
       }}));
-    webAssetServer.write(source, manifest, sourcemap);
+    webAssetServer.write(source, manifest, sourcemap, metadata);
 
     final Response response = await webAssetServer
       .handleRequest(Request('GET', Uri.parse('http://foobar/bar.js')));
@@ -191,12 +226,15 @@ void main() {
       ..writeAsStringSync('main() {}');
     final File sourcemap = globals.fs.file('sourcemap')
       ..writeAsStringSync('{}');
+    final File metadata = globals.fs.file('metadata')
+      ..writeAsStringSync('{}');
     final File manifest = globals.fs.file('manifest')
       ..writeAsStringSync(json.encode(<String, Object>{'/foo.dart.lib.js': <String, Object>{
         'code': <int>[0, source.lengthSync()],
         'sourcemap': <int>[0, 2],
+        'metadata': <int>[0, 2],
       }}));
-    webAssetServer.write(source, manifest, sourcemap);
+    webAssetServer.write(source, manifest, sourcemap, metadata);
 
     final Response response = await webAssetServer
       .handleRequest(Request('GET', Uri.parse('http://foobar/foo.dart.js')));
@@ -209,12 +247,15 @@ void main() {
       ..writeAsStringSync('main() {}');
     final File sourcemap = globals.fs.file('sourcemap')
       ..writeAsStringSync('{}');
+    final File metadata = globals.fs.file('metadata')
+      ..writeAsStringSync('{}');
     final File manifest = globals.fs.file('manifest')
       ..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
         'code': <int>[0, source.lengthSync()],
         'sourcemap': <int>[0, 2],
+        'metadata': <int>[0, 2],
       }}));
-    webAssetServer.write(source, manifest, sourcemap);
+    webAssetServer.write(source, manifest, sourcemap, metadata);
     final Response response = await webAssetServer
       .handleRequest(Request('GET', Uri.parse('http://localhost/foo.js')));
 
@@ -401,6 +442,7 @@ void main() {
     outputFile.parent.childFile('a.sources').writeAsStringSync('');
     outputFile.parent.childFile('a.json').writeAsStringSync('{}');
     outputFile.parent.childFile('a.map').writeAsStringSync('{}');
+    outputFile.parent.childFile('a.metadata').writeAsStringSync('{}');
     outputFile.parent.childFile('.packages').writeAsStringSync('\n');
 
     final ResidentCompiler residentCompiler = MockResidentCompiler();
@@ -511,6 +553,7 @@ void main() {
     outputFile.parent.childFile('a.sources').writeAsStringSync('');
     outputFile.parent.childFile('a.json').writeAsStringSync('{}');
     outputFile.parent.childFile('a.map').writeAsStringSync('{}');
+    outputFile.parent.childFile('a.metadata').writeAsStringSync('{}');
     outputFile.parent.childFile('.packages').writeAsStringSync('\n');
 
     final ResidentCompiler residentCompiler = MockResidentCompiler();

packages/flutter_tools/test/general.shard/web/web_expression_compiler_test.dart

@@ -82,7 +82,7 @@ void expectResult(ExpressionCompilationResult result, bool isError, String value
   expect(result,
     const TypeMatcher<ExpressionCompilationResult>()
       .having((ExpressionCompilationResult instance) => instance.isError, 'isError', isError)
-      .having((ExpressionCompilationResult instance) =>instance.result, 'result', value));
+      .having((ExpressionCompilationResult instance) => instance.result, 'result', value));
 }
 
 class MockResidentCompiler extends Mock implements ResidentCompiler {}