benchmarkLive: a new `LiveTestWidgetsFlutterBindingFramePolicy` for benchmark on device (#61388)

* add benchmarkLive flag and tests

* update handlePointerEventRecord doc

* using e2e 0.6.1

dev/benchmarks/macrobenchmarks/lib/common.dart

@@ -16,5 +16,6 @@ const String kFadingChildAnimationRouteName = '/fading_child_animation';
 const String kImageFilteredTransformAnimationRouteName = '/imagefiltered_transform_animation';
 const String kMultiWidgetConstructionRouteName = '/multi_widget_construction';
 const String kHeavyGridViewRouteName = '/heavy_gridview';
+const String kSimpleScrollRouteName = '/simple_scroll';
 
 const String kScrollableName = '/macrobenchmark_listview';

dev/benchmarks/macrobenchmarks/lib/main.dart

@@ -17,6 +17,7 @@ import 'src/filtered_child_animation.dart';
 import 'src/multi_widget_construction.dart';
 import 'src/post_backdrop_filter.dart';
 import 'src/simple_animation.dart';
+import 'src/simple_scroll.dart';
 import 'src/text.dart';
 
 const String kMacrobenchmarks = 'Macrobenchmarks';
@@ -47,6 +48,7 @@ class MacrobenchmarksApp extends StatelessWidget {
         kImageFilteredTransformAnimationRouteName: (BuildContext context) => const FilteredChildAnimationPage(FilterType.rotateFilter),
         kMultiWidgetConstructionRouteName: (BuildContext context) => const MultiWidgetConstructTable(10, 20),
         kHeavyGridViewRouteName: (BuildContext context) => HeavyGridViewPage(),
+        kSimpleScrollRouteName: (BuildContext context) => SimpleScroll(),
       },
     );
   }

dev/benchmarks/macrobenchmarks/lib/src/simple_scroll.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/material.dart';
+
+class SimpleScroll extends StatelessWidget {
+  @override
+  Widget build(BuildContext context) {
+    return ListView(
+      children: <Widget>[
+        for (int n = 0; n < 200; n += 1)
+          Container(height: 40.0, child: Text('$n')),
+      ],
+    );
+  }
+}

dev/benchmarks/macrobenchmarks/pubspec.yaml

@@ -86,6 +86,7 @@ dev_dependencies:
   flutter_test:
     sdk: flutter
   test: 1.15.3
+  e2e: 0.6.1
 
   _fe_analyzer_shared: 5.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   analyzer: 0.39.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -291,4 +292,4 @@ flutter:
       fonts:
         - asset: packages/flutter_gallery_assets/fonts/GalleryIcons.ttf
 
-# PUBSPEC CHECKSUM: 0d76
+# PUBSPEC CHECKSUM: 36c1

dev/benchmarks/macrobenchmarks/test/frame_policy.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/gestures.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:e2e/e2e.dart';
+
+import 'package:macrobenchmarks/src/simple_scroll.dart';
+
+void main() {
+  final E2EWidgetsFlutterBinding binding =
+      E2EWidgetsFlutterBinding.ensureInitialized() as E2EWidgetsFlutterBinding;
+  testWidgets(
+    'Frame Counter and Input Delay for benchmarkLive',
+    (WidgetTester tester) async {
+      await tester.pumpWidget(MaterialApp(home: Scaffold(body: SimpleScroll())));
+      await tester.pumpAndSettle();
+      final Offset location = tester.getCenter(find.byType(ListView));
+      int frameCount = 0;
+      final FrameCallback frameCounter = (Duration elapsed) {
+        frameCount += 1;
+      };
+      tester.binding.addPersistentFrameCallback(frameCounter);
+
+      const int timeInSecond = 1;
+      const Duration totalTime = Duration(seconds: timeInSecond);
+      const int moveEventNumber = timeInSecond * 120;  // 120Hz
+      const Offset movePerRun = Offset(0.0, -200.0 / moveEventNumber);
+      final List<PointerEventRecord> records = <PointerEventRecord>[
+        PointerEventRecord(Duration.zero, <PointerEvent>[
+          PointerAddedEvent(
+            timeStamp: Duration.zero,
+            position: location,
+          ),
+          PointerDownEvent(
+            timeStamp: Duration.zero,
+            position: location,
+            pointer: 1,
+          ),
+        ]),
+        ...<PointerEventRecord>[
+          for (int t=0; t < moveEventNumber; t++)
+            PointerEventRecord(totalTime * (t / moveEventNumber), <PointerEvent>[
+              PointerMoveEvent(
+                timeStamp: totalTime * (t / moveEventNumber),
+                position: location + movePerRun * t.toDouble(),
+                pointer: 1,
+                delta: movePerRun,
+              )
+            ])
+        ],
+        PointerEventRecord(totalTime, <PointerEvent>[
+          PointerUpEvent(
+            // Deviate a little from integer number of frames to reduce flakiness
+            timeStamp: totalTime - const Duration(milliseconds: 1),
+            position: location + movePerRun * moveEventNumber.toDouble(),
+            pointer: 1,
+          )
+        ])
+      ];
+
+      binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive;
+      List<Duration> delays = await tester.handlePointerEventRecord(records);
+      await tester.pumpAndSettle();
+      binding.reportData = <String, dynamic>{
+        'benchmarkLive': _summarizeResult(frameCount, delays),
+      };
+      await tester.idle();
+      await tester.binding.delayed(const Duration(milliseconds: 250));
+
+      binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
+      frameCount = 0;
+      delays = await tester.handlePointerEventRecord(records);
+      await tester.pumpAndSettle();
+      binding.reportData['fullyLive'] = _summarizeResult(frameCount, delays);
+      await tester.idle();
+    },
+  );
+}
+
+Map<String, dynamic> _summarizeResult(
+  final int frameCount,
+  final List<Duration> delays,
+) {
+  assert(delays.length > 1);
+  final List<int> delayedInMicro = delays.map<int>(
+    (Duration delay) => delay.inMicroseconds,
+  ).toList();
+  final List<int> delayedInMicroSorted = List<int>.from(delayedInMicro)..sort();
+  final int index90th = (delayedInMicroSorted.length * 0.90).round();
+  final int percentile90th = delayedInMicroSorted[index90th];
+  final int sum = delayedInMicroSorted.reduce((int a, int b) => a + b);
+  final double averageDelay = sum.toDouble() / delayedInMicroSorted.length;
+  return <String, dynamic>{
+    'frame_count': frameCount,
+    'average_delay_millis': averageDelay / 1E3,
+    '90th_percentile_delay_millis': percentile90th / 1E3,
+    if (kDebugMode)
+    'delaysInMicro': delayedInMicro,
+  };
+}

dev/benchmarks/macrobenchmarks/test_driver/frame_policy_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:e2e/common.dart' as e2e;
+import 'package:flutter_driver/flutter_driver.dart';
+
+import 'package:path/path.dart' as path;
+
+Future<void> main() async {
+  const Duration timeout = Duration(minutes: 1);
+  const String testName = 'frame_policy';
+
+  final FlutterDriver driver = await FlutterDriver.connect();
+  String jsonResult;
+  jsonResult = await driver.requestData(null, timeout: timeout);
+  final e2e.Response response = e2e.Response.fromJson(jsonResult);
+  await driver.close();
+  final Map<String, dynamic> benchmarkLiveResult =
+      response.data['benchmarkLive'] as Map<String,dynamic>;
+  final Map<String, dynamic> fullyLiveResult =
+      response.data['fullyLive'] as Map<String,dynamic>;
+
+  if (response.allTestsPassed) {
+    if(benchmarkLiveResult['frame_count'] as int < 10
+       || fullyLiveResult['frame_count'] as int < 10) {
+      print('Failure Details:\nNot Enough frames collected:'
+            'benchmarkLive ${benchmarkLiveResult['frameCount']},'
+            '${fullyLiveResult['frameCount']}.');
+      exit(1);
+    }
+    print('All tests passed.');
+    const String destinationDirectory = 'build';
+    await fs.directory(destinationDirectory).create(recursive: true);
+    final File file = fs.file(path.join(
+      destinationDirectory,
+      '${testName}_event_delay.json'
+    ));
+    await file.writeAsString(const JsonEncoder.withIndent('  ').convert(
+      <String, dynamic>{
+        'benchmarkLive': benchmarkLiveResult,
+        'fullyLive': fullyLiveResult,
+      },
+    ));
+    exit(0);
+  } else {
+    print('Failure Details:\n${response.formattedFailureDetails}');
+    exit(1);
+  }
+}

dev/devicelab/bin/tasks/frame_policy_delay_test_android.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(createFramePolicyIntegrationTest());
+}

dev/devicelab/lib/tasks/perf_tests.dart

@@ -262,6 +262,49 @@ TaskFunction createsMultiWidgetConstructPerfTest() {
   ).run;
 }
 
+TaskFunction createFramePolicyIntegrationTest() {
+  final String testDirectory =
+      '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';
+  const String testTarget = 'test/frame_policy.dart';
+  return () {
+    return inDirectory<TaskResult>(testDirectory, () async {
+      final Device device = await devices.workingDevice;
+      await device.unlock();
+      final String deviceId = device.deviceId;
+      await flutter('packages', options: <String>['get']);
+
+      await flutter('drive', options: <String>[
+        '-v',
+        '--verbose-system-logs',
+        '--profile',
+        '-t', testTarget,
+        '-d',
+        deviceId,
+      ]);
+      final Map<String, dynamic> data = json.decode(
+        file('$testDirectory/build/frame_policy_event_delay.json').readAsStringSync(),
+      ) as Map<String, dynamic>;
+      final Map<String, dynamic> fullLiveData = data['fullyLive'] as Map<String, dynamic>;
+      final Map<String, dynamic> benchmarkLiveData = data['benchmarkLive'] as Map<String, dynamic>;
+      final Map<String, dynamic> dataFormated = <String, dynamic>{
+        'average_delay_fullyLive_millis':
+          fullLiveData['average_delay_millis'],
+        'average_delay_benchmarkLive_millis':
+          benchmarkLiveData['average_delay_millis'],
+        '90th_percentile_delay_fullyLive_millis':
+          fullLiveData['90th_percentile_delay_millis'],
+        '90th_percentile_delay_benchmarkLive_millis':
+          benchmarkLiveData['90th_percentile_delay_millis'],
+      };
+
+      return TaskResult.success(
+        dataFormated,
+        benchmarkScoreKeys: dataFormated.keys.toList(),
+      );
+    });
+  };
+}
+
 /// Measure application startup performance.
 class StartupTest {
   const StartupTest(this.testDirectory, { this.reportMetrics = true });

dev/devicelab/manifest.yaml

@@ -167,6 +167,13 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
+  frame_policy_delay_test_android:
+    description: >
+      Tests the effect of LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
   picture_cache_perf__timeline_summary:
     description: >
       Measures the runtime performance of raster caching many pictures on Android.

packages/flutter_test/lib/src/binding.dart

@@ -1264,6 +1264,21 @@ enum LiveTestWidgetsFlutterBindingFramePolicy {
   /// on the [SchedulerBinding.hasScheduledFrame] property to determine when the
   /// application has "settled".
   benchmark,
+
+  /// Ignore any request from pump but respect other requests to schedule a
+  /// frame.
+  ///
+  /// This is used for running the test on a device, where scheduling of new
+  /// frames respects what the engine and the device needed.
+  ///
+  /// Compared to `fullyLive` this policy ignores the frame requests from pump
+  /// of the test code so that the frame scheduling respects the situation of
+  /// that for the real environment, and avoids waiting for the new frame beyond
+  /// the expected time.
+  ///
+  /// Compared to `benchmark` this policy can be used for capturing the
+  /// animation frames requested by the framework.
+  benchmarkLive,
 }
 
 /// A variant of [TestWidgetsFlutterBinding] for executing tests in
@@ -1398,6 +1413,7 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
     assert(_doDrawThisFrame == null);
     if (_expectingFrame ||
         (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.fullyLive) ||
+        (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive) ||
         (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmark) ||
         (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.fadePointers && _viewNeedsPaint)) {
       _doDrawThisFrame = true;
@@ -1489,6 +1505,10 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
     assert(inTest);
     assert(!_expectingFrame);
     assert(_pendingFrame == null);
+    if (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive) {
+      // Ignore all pumps and just wait.
+      return delayed(duration ?? Duration.zero);
+    }
     return TestAsyncUtils.guard<void>(() {
       if (duration != null) {
         Timer(duration, () {

packages/flutter_test/lib/src/controller.dart

@@ -406,9 +406,17 @@ abstract class WidgetController {
   /// The [PointerEventRecord.timeDelay] is used as the time delay of the events
   /// injection relative to the starting point of the method call.
   ///
-  /// Returns a list of the difference between [PointerEventRecord.timeDelay]
-  /// and the real delay time when the [PointerEventRecord.events] are processed.
-  /// The closer these values are to zero the more faithful it is to the
+  /// Returns a list of the difference between the real delay time when the
+  /// [PointerEventRecord.events] are processed and
+  /// [PointerEventRecord.timeDelay].
+  /// - For [AutomatedTestWidgetsFlutterBinding] where the clock is fake, the
+  ///   return value should be exact zeros.
+  /// - For [LiveTestWidgetsFlutterBinding], the values are typically small
+  /// positives, meaning the event happens a little later than the set time,
+  /// but a very small portion may have a tiny negatvie value for about tens of
+  /// microseconds. This is due to the nature of [Future.delayed].
+  ///
+  /// The closer the return values are to zero the more faithful it is to the
   /// `records`.
   ///
   /// See [PointerEventRecord].

packages/flutter_test/lib/src/widget_tester.dart

@@ -480,7 +480,7 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker
         final Duration timeDiff = record.timeDelay - now.difference(startTime);
         if (timeDiff.isNegative) {
           // Flush all past events
-          handleTimeStampDiff.add(timeDiff);
+          handleTimeStampDiff.add(-timeDiff);
           for (final PointerEvent event in record.events) {
             _handlePointerEvent(event, hitTestHistory);
           }
@@ -490,7 +490,7 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker
           await binding.pump();
           await binding.delayed(timeDiff);
           handleTimeStampDiff.add(
-            record.timeDelay - binding.clock.now().difference(startTime),
+            binding.clock.now().difference(startTime) - record.timeDelay,
           );
           for (final PointerEvent event in record.events) {
             _handlePointerEvent(event, hitTestHistory);