extract engine sub-metrics; change reported metrics (#55331)

* extract engine sub-metrics; change reported metrics

- Extract sub-metrics reported by the Web engine: "preroll_frame", "apply_frame".
- Add a concept of unreported metrics: displayed on the benchmark UI, but not on the dashboard.
- Make "sceneBuildDuration" and "windowRenderDuration" unreported, which are too fine-grained. They are included in "drawFrameDuration" already.
- Report outlier ratio instead of outlier average. The ratio is more useful of the two.

dev/benchmarks/macrobenchmarks/lib/src/web/bench_picture_recording.dart

@@ -67,9 +67,9 @@ class BenchPictureRecording extends RawRecorder {
         }
         canvas.restore();
       }
-    });
+    }, reported: true);
     profile.record('estimatePaintBounds', () {
       recorder.endRecording();
-    });
+    }, reported: true);
   }
 }

dev/benchmarks/macrobenchmarks/lib/src/web/bench_text_layout.dart

@@ -58,11 +58,6 @@ void _useCanvasText(bool useCanvasText) {
   );
 }
 
-typedef OnBenchmark = void Function(String name, num value);
-void _onBenchmark(OnBenchmark listener) {
-  js_util.setProperty(html.window, '_flutter_internal_on_benchmark', listener);
-}
-
 /// Repeatedly lays out a paragraph using the DOM measurement approach.
 ///
 /// Creates a different paragraph each time in order to avoid hitting the cache.
@@ -132,21 +127,21 @@ class BenchTextLayout extends RawRecorder {
   }) {
     profile.record('$keyPrefix.layout', () {
       paragraph.layout(ui.ParagraphConstraints(width: maxWidth));
-    });
+    }, reported: true);
     profile.record('$keyPrefix.getBoxesForRange', () {
       for (int start = 0; start < text.length; start += 3) {
         for (int end = start + 1; end < text.length; end *= 2) {
           paragraph.getBoxesForRange(start, end);
         }
       }
-    });
+    }, reported: true);
     profile.record('$keyPrefix.getPositionForOffset', () {
       for (double dx = 0.0; dx < paragraph.width; dx += 10.0) {
         for (double dy = 0.0; dy < paragraph.height; dy += 10.0) {
           paragraph.getPositionForOffset(Offset(dx, dy));
         }
       }
-    });
+    }, reported: true);
   }
 }
 
@@ -179,7 +174,7 @@ class BenchTextCachedLayout extends RawRecorder {
     final ui.Paragraph paragraph = builder.build();
     profile.record('layout', () {
       paragraph.layout(const ui.ParagraphConstraints(width: double.infinity));
-    });
+    }, reported: true);
     _useCanvasText(null);
   }
 }
@@ -242,7 +237,7 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {
     if (mode == _TestMode.useDomTextLayout) {
       _useCanvasText(false);
     }
-    _onBenchmark((String name, num value) {
+    registerEngineBenchmarkValueListener('text_layout', (num value) {
       _textLayoutMicros += value;
     });
   }
@@ -250,7 +245,7 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {
   @override
   Future<void> tearDownAll() async {
     _useCanvasText(null);
-    _onBenchmark(null);
+    stopListeningToEngineBenchmarkValues('text_layout');
   }
 
   @override
@@ -268,6 +263,7 @@ class BenchBuildColorsGrid extends WidgetBuildRecorder {
       profile.addDataPoint(
         'text_layout',
         Duration(microseconds: _textLayoutMicros.toInt()),
+        reported: true,
       );
     }
     super.frameDidDraw();

dev/benchmarks/macrobenchmarks/lib/src/web/recorder.dart

@@ -4,6 +4,7 @@
 
 import 'dart:async';
 import 'dart:html' as html;
+import 'dart:js_util' as js_util;
 import 'dart:math' as math;
 import 'dart:ui';
 
@@ -27,6 +28,16 @@ const int _kMeasuredSampleCount = 100;
 /// The total number of samples collected by a benchmark.
 const int kTotalSampleCount = _kWarmUpSampleCount + _kMeasuredSampleCount;
 
+/// A benchmark metric that includes frame-related computations prior to
+/// submitting layer and picture operations to the underlying renderer, such as
+/// HTML and CanvasKit. During this phase we compute transforms, clips, and
+/// other information needed for rendering.
+const String kProfilePrerollFrame = 'preroll_frame';
+
+/// A benchmark metric that includes submitting layer and picture information
+/// to the renderer.
+const String kProfileApplyFrame = 'apply_frame';
+
 /// Measures the amount of time [action] takes.
 Duration timeAction(VoidCallback action) {
   final Stopwatch stopwatch = Stopwatch()..start();
@@ -221,9 +232,9 @@ abstract class SceneBuilderRecorder extends Recorder {
             final Scene scene = sceneBuilder.build();
             profile.record('windowRenderDuration', () {
               window.render(scene);
-            });
-          });
-        });
+            }, reported: false);
+          }, reported: false);
+        }, reported: true);
         endMeasureFrame();
 
         if (profile.shouldContinue()) {
@@ -331,7 +342,7 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder {
   @mustCallSuper
   void frameDidDraw() {
     endMeasureFrame();
-    profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed);
+    profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, reported: true);
 
     if (profile.shouldContinue()) {
       window.scheduleFrame();
@@ -353,12 +364,30 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder {
     final _RecordingWidgetsBinding binding =
         _RecordingWidgetsBinding.ensureInitialized();
     final Widget widget = createWidget();
+
+    registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) {
+      localProfile.addDataPoint(
+        kProfilePrerollFrame,
+        Duration(microseconds: value.toInt()),
+        reported: false,
+      );
+    });
+    registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) {
+      localProfile.addDataPoint(
+        kProfileApplyFrame,
+        Duration(microseconds: value.toInt()),
+        reported: false,
+      );
+    });
+
     binding._beginRecording(this, widget);
 
     try {
       await _runCompleter.future;
       return localProfile;
     } finally {
+      stopListeningToEngineBenchmarkValues(kProfilePrerollFrame);
+      stopListeningToEngineBenchmarkValues(kProfileApplyFrame);
       _runCompleter = null;
       profile = null;
     }
@@ -421,7 +450,7 @@ abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder {
     // Only record frames that show the widget.
     if (showWidget) {
       endMeasureFrame();
-      profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed);
+      profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, reported: true);
     }
 
     if (profile.shouldContinue()) {
@@ -488,11 +517,21 @@ class _WidgetBuildRecorderHostState extends State<_WidgetBuildRecorderHost> {
 /// calculations will only apply to the latest [_kMeasuredSampleCount] data
 /// points.
 class Timeseries {
-  Timeseries(this.name);
+  Timeseries(this.name, this.isReported);
 
   /// The label of this timeseries used for debugging and result inspection.
   final String name;
 
+  /// Whether this timeseries is reported to the benchmark dashboard.
+  ///
+  /// If `true` a new benchmark card is created for the timeseries and is
+  /// visible on the dashboard.
+  ///
+  /// If `false` the data is stored but it does not show up on the dashboard.
+  /// Use unreported metrics for metrics that are useful for manual inspection
+  /// but that are too fine-grained to be useful for tracking on the dashboard.
+  final bool isReported;
+
   /// List of all the values that have been recorded.
   ///
   /// This list has no limit.
@@ -700,14 +739,20 @@ class Profile {
   final Map<String, dynamic> extraData = <String, dynamic>{};
 
   /// Invokes [callback] and records the duration of its execution under [key].
-  Duration record(String key, VoidCallback callback) {
+  Duration record(String key, VoidCallback callback, { @required bool reported }) {
     final Duration duration = timeAction(callback);
-    addDataPoint(key, duration);
+    addDataPoint(key, duration, reported: reported);
     return duration;
   }
 
-  void addDataPoint(String key, Duration duration) {
-    scoreData.putIfAbsent(key, () => Timeseries(key)).add(duration.inMicroseconds.toDouble());
+  /// Adds a timed sample to the timeseries corresponding to [key].
+  ///
+  /// Set [reported] to `true` to report the timeseries to the dashboard UI.
+  ///
+  /// Set [reported] to `false` to store the data, but not show it on the
+  /// dashboard UI.
+  void addDataPoint(String key, Duration duration, { @required bool reported }) {
+    scoreData.putIfAbsent(key, () => Timeseries(key, reported)).add(duration.inMicroseconds.toDouble());
   }
 
   /// Decides whether the data collected so far is sufficient to stop, or
@@ -740,9 +785,16 @@ class Profile {
     };
 
     for (final String key in scoreData.keys) {
-      scoreKeys.add('$key.average');
-      scoreKeys.add('$key.outlierAverage');
       final Timeseries timeseries = scoreData[key];
+
+      if (timeseries.isReported) {
+        scoreKeys.add('$key.average');
+        // Report `outlierRatio` rather than `outlierAverage`, because
+        // the absolute value of outliers is less interesting than the
+        // ratio.
+        scoreKeys.add('$key.outlierRatio');
+      }
+
       final TimeseriesStats stats = timeseries.computeStats();
       json['$key.average'] = stats.average;
       json['$key.outlierAverage'] = stats.outlierAverage;
@@ -958,3 +1010,55 @@ void endMeasureFrame() {
   );
   _currentFrameNumber += 1;
 }
+
+/// A function that receives a benchmark value from the framework.
+typedef EngineBenchmarkValueListener = void Function(num value);
+
+// Maps from a value label name to a listener.
+final Map<String, EngineBenchmarkValueListener> _engineBenchmarkListeners = <String, EngineBenchmarkValueListener>{};
+
+/// Registers a [listener] for engine benchmark values labeled by [name].
+///
+/// If another listener is already registered, overrides it.
+void registerEngineBenchmarkValueListener(String name, EngineBenchmarkValueListener listener) {
+  if (listener == null) {
+    throw ArgumentError(
+      'Listener must not be null. To stop listening to engine benchmark values '
+      'under label "$name", call stopListeningToEngineBenchmarkValues(\'$name\').',
+    );
+  }
+
+  if (_engineBenchmarkListeners.containsKey(name)) {
+    throw StateError(
+      'A listener for "$name" is already registered.\n'
+      'Call `stopListeningToEngineBenchmarkValues` to unregister the previous '
+      'listener before registering a new one.'
+    );
+  }
+
+  if (_engineBenchmarkListeners.isEmpty) {
+    // The first listener is being registered. Register the global listener.
+    js_util.setProperty(html.window, '_flutter_internal_on_benchmark', _dispatchEngineBenchmarkValue);
+  }
+
+  _engineBenchmarkListeners[name] = listener;
+}
+
+/// Stops listening to engine benchmark values under labeled by [name].
+void stopListeningToEngineBenchmarkValues(String name) {
+  _engineBenchmarkListeners.remove(name);
+  if (_engineBenchmarkListeners.isEmpty) {
+    // The last listener unregistered. Remove the global listener.
+    js_util.setProperty(html.window, '_flutter_internal_on_benchmark', null);
+  }
+}
+
+// Dispatches a benchmark value reported by the engine to the relevant listener.
+//
+// If there are no listeners registered for [name], ignores the value.
+void _dispatchEngineBenchmarkValue(String name, double value) {
+  final EngineBenchmarkValueListener listener = _engineBenchmarkListeners[name];
+  if (listener != null) {
+    listener(value);
+  }
+}