A benchmark test case for measuring scroll smoothness (#61998)

dev/benchmarks/complex_layout/android/settings.gradle

@@ -3,3 +3,13 @@
 // found in the LICENSE file.
 
 include ':app'
+
+def localPropertiesFile = new File(rootProject.projectDir, "local.properties")
+def properties = new Properties()
+
+assert localPropertiesFile.exists()
+localPropertiesFile.withReader("UTF-8") { reader -> properties.load(reader) }
+
+def flutterSdkPath = properties.getProperty("flutter.sdk")
+assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
+apply from: "$flutterSdkPath/packages/flutter_tools/gradle/app_plugin_loader.gradle"

dev/benchmarks/complex_layout/lib/main.dart

@@ -109,6 +109,7 @@ class ComplexLayoutState extends State<ComplexLayout> {
           Expanded(
             child: ListView.builder(
               key: const Key('complex-scroll'), // this key is used by the driver test
+              controller: ScrollController(),  // So that the scroll offset can be tracked
               itemBuilder: (BuildContext context, int index) {
                 if (index % 2 == 0)
                   return FancyImageItem(index, key: PageStorageKey<int>(index));

dev/benchmarks/complex_layout/pubspec.yaml

@@ -3,7 +3,7 @@ description: A benchmark of a relatively complex layout.
 
 environment:
   # The pub client defaults to an <2.0.0 sdk constraint which we need to explicitly overwrite.
-  sdk: ">=2.0.0-dev.68.0 <3.0.0"
+  sdk: ">=2.2.2 <3.0.0"
 
 dependencies:
   flutter:
@@ -46,6 +46,7 @@ dev_dependencies:
   flutter_test:
     sdk: flutter
   test: 1.16.0-nullsafety.1
+  e2e: 0.7.0
 
   _fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -90,4 +91,4 @@ flutter:
     - packages/flutter_gallery_assets/people/square/ali.png
     - packages/flutter_gallery_assets/places/india_chettinad_silk_maker.png
 
-# PUBSPEC CHECKSUM: 6832
+# PUBSPEC CHECKSUM: 047d

dev/benchmarks/complex_layout/test/measure_scroll_smoothness.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This test is a use case of flutter/flutter#60796
+// the test should be run as:
+// flutter drive -t test/using_array.dart --driver test_driver/scrolling_test_e2e_test.dart
+
+import 'dart:ui' as ui;
+
+import 'package:flutter/gestures.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:e2e/e2e.dart';
+
+import 'package:complex_layout/main.dart' as app;
+
+class PointerDataTestBinding extends E2EWidgetsFlutterBinding {
+  // PointerData injection would usually be considered device input and therefore
+  // blocked by [TestWidgetsFlutterBinding]. Override this behavior
+  // to help events go into widget tree.
+  @override
+  void dispatchEvent(
+    PointerEvent event,
+    HitTestResult hitTestResult, {
+    TestBindingEventSource source = TestBindingEventSource.device,
+  }) {
+    super.dispatchEvent(event, hitTestResult, source: TestBindingEventSource.test);
+  }
+}
+
+/// A union of [ui.PointerDataPacket] and the time it should be sent.
+class PointerDataRecord {
+  PointerDataRecord(this.timeStamp, List<ui.PointerData> data)
+    : data = ui.PointerDataPacket(data: data);
+  final ui.PointerDataPacket data;
+  final Duration timeStamp;
+}
+
+/// Generates the [PointerDataRecord] to simulate a drag operation from
+/// `center - totalMove/2` to `center + totalMove/2`.
+Iterable<PointerDataRecord> dragInputDatas(
+  final Duration epoch,
+  final Offset center, {
+  final Offset totalMove = const Offset(0, -400),
+  final Duration totalTime = const Duration(milliseconds: 2000),
+  final double frequency = 90,
+}) sync* {
+  final Offset startLocation = (center - totalMove / 2) * ui.window.devicePixelRatio;
+  // The issue is about 120Hz input on 90Hz refresh rate device.
+  // We test 90Hz input on 60Hz device here, which shows similar pattern.
+  final int moveEventCount = totalTime.inMicroseconds * frequency ~/ const Duration(seconds: 1).inMicroseconds;
+  final Offset movePerEvent = totalMove / moveEventCount.toDouble() * ui.window.devicePixelRatio;
+  yield PointerDataRecord(epoch, <ui.PointerData>[
+    ui.PointerData(
+      timeStamp: epoch,
+      change: ui.PointerChange.add,
+      physicalX: startLocation.dx,
+      physicalY: startLocation.dy,
+    ),
+    ui.PointerData(
+      timeStamp: epoch,
+      change: ui.PointerChange.down,
+      physicalX: startLocation.dx,
+      physicalY: startLocation.dy,
+      pointerIdentifier: 1,
+    ),
+  ]);
+  for (int t = 0; t < moveEventCount + 1; t++) {
+    final Offset position = startLocation + movePerEvent * t.toDouble();
+    yield PointerDataRecord(
+      epoch + totalTime * t ~/ moveEventCount,
+      <ui.PointerData>[ui.PointerData(
+        timeStamp: epoch + totalTime * t ~/ moveEventCount,
+        change: ui.PointerChange.move,
+        physicalX: position.dx,
+        physicalY: position.dy,
+        // Scrolling behavior depends on this delta rather
+        // than the position difference.
+        physicalDeltaX: movePerEvent.dx,
+        physicalDeltaY: movePerEvent.dy,
+        pointerIdentifier: 1,
+      )],
+    );
+  }
+  final Offset position = startLocation + totalMove;
+  yield PointerDataRecord(epoch + totalTime, <ui.PointerData>[ui.PointerData(
+    timeStamp: epoch + totalTime,
+    change: ui.PointerChange.up,
+    physicalX: position.dx,
+    physicalY: position.dy,
+    pointerIdentifier: 1,
+  )]);
+}
+
+enum TestScenario {
+  resampleOn90Hz,
+  resampleOn59Hz,
+  resampleOff90Hz,
+  resampleOff59Hz,
+}
+
+class ResampleFlagVariant extends TestVariant<TestScenario> {
+  ResampleFlagVariant(this.binding);
+  final E2EWidgetsFlutterBinding binding;
+
+  @override
+  final Set<TestScenario> values = Set<TestScenario>.from(TestScenario.values);
+
+  TestScenario currentValue;
+  bool get resample {
+    switch(currentValue) {
+      case TestScenario.resampleOn90Hz:
+      case TestScenario.resampleOn59Hz:
+        return true;
+      case TestScenario.resampleOff90Hz:
+      case TestScenario.resampleOff59Hz:
+        return false;
+    }
+    throw ArgumentError;
+  }
+  double get frequency {
+    switch(currentValue) {
+      case TestScenario.resampleOn90Hz:
+      case TestScenario.resampleOff90Hz:
+        return 90.0;
+      case TestScenario.resampleOn59Hz:
+      case TestScenario.resampleOff59Hz:
+        return 59.0;
+    }
+    throw ArgumentError;
+  }
+
+  Map<String, dynamic> result;
+
+  @override
+  String describeValue(TestScenario value) {
+    switch(value) {
+      case TestScenario.resampleOn90Hz:
+        return 'resample on with 90Hz input';
+      case TestScenario.resampleOn59Hz:
+        return 'resample on with 59Hz input';
+      case TestScenario.resampleOff90Hz:
+        return 'resample off with 90Hz input';
+      case TestScenario.resampleOff59Hz:
+        return 'resample off with 59Hz input';
+    }
+    throw ArgumentError;
+  }
+
+  @override
+  Future<bool> setUp(TestScenario value) async {
+    currentValue = value;
+    final bool original = binding.resamplingEnabled;
+    binding.resamplingEnabled = resample;
+    return original;
+  }
+
+  @override
+  Future<void> tearDown(TestScenario value, bool memento) async {
+    binding.resamplingEnabled = memento;
+    binding.reportData[describeValue(value)] = result;
+  }
+}
+
+Future<void> main() async {
+  final PointerDataTestBinding binding = PointerDataTestBinding();
+  assert(WidgetsBinding.instance == binding);
+  binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive;
+  binding.reportData ??= <String, dynamic>{};
+  final ResampleFlagVariant variant = ResampleFlagVariant(binding);
+  testWidgets('Smoothness test', (WidgetTester tester) async {
+    app.main();
+    await tester.pumpAndSettle();
+    final Finder scrollerFinder = find.byKey(const ValueKey<String>('complex-scroll'));
+    final ListView scroller = tester.widget<ListView>(scrollerFinder);
+    final ScrollController controller = scroller.controller;
+    final List<int> frameTimestamp = <int>[];
+    final List<double> scrollOffset = <double>[];
+    final List<Duration> delays = <Duration>[];
+    binding.addPersistentFrameCallback((Duration timeStamp) {
+      if (controller.hasClients) {
+        // This if is necessary because by the end of the test the widget tree
+        // is destroyed.
+        frameTimestamp.add(timeStamp.inMicroseconds);
+        scrollOffset.add(controller.offset);
+      }
+    });
+
+    Duration now() => binding.currentSystemFrameTimeStamp;
+    Future<void> scroll() async {
+      // Extra 50ms to avoid timeouts.
+      final Duration startTime = const Duration(milliseconds: 500) + now();
+      for (final PointerDataRecord record in dragInputDatas(
+        startTime,
+        tester.getCenter(scrollerFinder),
+        frequency: variant.frequency,
+      )) {
+        await tester.binding.delayed(record.timeStamp - now());
+        // This now measures how accurate the above delayed is.
+        final Duration delay = now() - record.timeStamp;
+        if (delays.length < frameTimestamp.length) {
+          while (delays.length < frameTimestamp.length - 1) {
+            delays.add(Duration.zero);
+          }
+          delays.add(delay);
+        } else if (delays.last < delay) {
+          delays.last = delay;
+        }
+        ui.window.onPointerDataPacket(record.data);
+      }
+    }
+
+    for (int n = 0; n < 5; n++) {
+      await scroll();
+    }
+    variant.result = scrollSummary(scrollOffset, delays, frameTimestamp);
+    await tester.pumpAndSettle();
+    scrollOffset.clear();
+    delays.clear();
+    await tester.idle();
+  }, semanticsEnabled: false, variant: variant);
+}
+
+/// Calculates the smoothness measure from `scrollOffset` and `delays` list.
+///
+/// Smoothness (`abs_jerk`) is measured by  the absolute value of the discrete
+/// 2nd derivative of the scroll offset.
+///
+/// It was experimented that jerk (3rd derivative of the position) is a good
+/// measure the smoothness.
+/// Here we are using 2nd derivative instead because the input is completely
+/// linear and the expected acceleration should be strictly zero.
+/// Observed acceleration is jumping from positive to negative within
+/// adjacent frames, meaning mathematically the discrete 3-rd derivative
+/// (`f[3] - 3*f[2] + 3*f[1] - f[0]`) is not a good approximation of jerk
+/// (continuous 3-rd derivative), while discrete 2nd
+/// derivative (`f[2] - 2*f[1] + f[0]`) on the other hand is a better measure
+/// of how the scrolling deviate away from linear, and given the acceleration
+/// should average to zero within two frames, it's also a good approximation
+/// for jerk in terms of physics.
+/// We use abs rather than square because square (2-norm) amplifies the
+/// effect of the data point that's relatively large, but in this metric
+/// we prefer smaller data point to have similar effect.
+/// This is also why we count the number of data that's larger than a
+/// threshold (and the result is tested not sensitive to this threshold),
+/// which is effectively a 0-norm.
+///
+/// Frames that are too slow to build (longer than 40ms) or with input delay
+/// longer than 16ms (1/60Hz) is filtered out to separate the janky due to slow
+/// response.
+///
+/// The returned map has keys:
+/// `average_abs_jerk`: average for the overall smoothness.
+/// `janky_count`: number of frames with `abs_jerk` larger than 0.5.
+/// `dropped_frame_count`: number of frames that are built longer than 40ms and
+///  are not used for smoothness measurement.
+/// `frame_timestamp`: the list of the timestamp for each frame, in the time
+/// order.
+/// `scroll_offset`: the scroll offset for each frame. Its length is the same as
+/// `frame_timestamp`.
+/// `input_delay`: the list of maximum delay time of the input simulation during
+/// a frame. Its length is the same as `frame_timestamp`
+Map<String, dynamic> scrollSummary(
+  List<double> scrollOffset,
+  List<Duration> delays,
+  List<int> frameTimestamp,
+) {
+  double jankyCount = 0;
+  double absJerkAvg = 0;
+  int lostFrame = 0;
+  for (int i = 1; i < scrollOffset.length-1; i += 1) {
+    if (frameTimestamp[i+1] - frameTimestamp[i-1] > 40E3 ||
+        (i >= delays.length || delays[i] > const Duration(milliseconds: 16))) {
+      // filter data points from slow frame building or input simulation artifact
+      lostFrame += 1;
+      continue;
+    }
+    //
+    final double absJerk = (scrollOffset[i-1] + scrollOffset[i+1] - 2*scrollOffset[i]).abs();
+    absJerkAvg += absJerk;
+    if (absJerk > 0.5)
+      jankyCount += 1;
+  }
+  // expect(lostFrame < 0.1 * frameTimestamp.length, true);
+  absJerkAvg /= frameTimestamp.length - lostFrame;
+
+  return <String, dynamic>{
+    'janky_count': jankyCount,
+    'average_abs_jerk': absJerkAvg,
+    'dropped_frame_count': lostFrame,
+    'frame_timestamp': List<int>.from(frameTimestamp),
+    'scroll_offset': List<double>.from(scrollOffset),
+    'input_delay': delays.map<int>((Duration data) => data.inMicroseconds).toList(),
+  };
+}

dev/benchmarks/complex_layout/test_driver/measure_scroll_smoothness_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:e2e/e2e_driver.dart' as driver;
+
+Future<void> main() => driver.e2eDriver(
+  timeout: const Duration(minutes: 5),
+  responseDataCallback: (Map<String, dynamic> data) async {
+    await driver.writeResponseData(
+      data,
+      testOutputFilename: 'scroll_smoothness_test',
+    );
+  }
+);

dev/devicelab/bin/tasks/complex_layout_android__scroll_smoothness.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(createsScrollSmoothnessPerfTest());
+}

dev/devicelab/lib/tasks/perf_tests.dart

@@ -298,6 +298,54 @@ TaskFunction createsMultiWidgetConstructPerfE2ETest() {
   ).run;
 }
 
+TaskFunction createsScrollSmoothnessPerfTest() {
+  final String testDirectory =
+      '${flutterDirectory.path}/dev/benchmarks/complex_layout';
+  const String testTarget = 'test/measure_scroll_smoothness.dart';
+  return () {
+    return inDirectory<TaskResult>(testDirectory, () async {
+      final Device device = await devices.workingDevice;
+      await device.unlock();
+      final String deviceId = device.deviceId;
+      await flutter('packages', options: <String>['get']);
+
+      await flutter('drive', options: <String>[
+        '-v',
+        '--verbose-system-logs',
+        '--profile',
+        '-t', testTarget,
+        '-d',
+        deviceId,
+      ]);
+      final Map<String, dynamic> data = json.decode(
+        file('$testDirectory/build/scroll_smoothness_test.json').readAsStringSync(),
+      ) as Map<String, dynamic>;
+
+      final Map<String, dynamic> result = <String, dynamic>{};
+      void addResult(dynamic data, String suffix) {
+        assert(data is Map<String, dynamic>);
+        const List<String> metricKeys = <String>[
+          'janky_count',
+          'average_abs_jerk',
+          'dropped_frame_count',
+        ];
+        for (final String key in metricKeys) {
+          result[key+suffix] = data[key];
+        }
+      }
+      addResult(data['resample on with 90Hz input'], '_with_resampler_90Hz');
+      addResult(data['resample on with 59Hz input'], '_with_resampler_59Hz');
+      addResult(data['resample off with 90Hz input'], '_without_resampler_90Hz');
+      addResult(data['resample off with 59Hz input'], '_without_resampler_59Hz');
+
+      return TaskResult.success(
+        result,
+        benchmarkScoreKeys: result.keys.toList(),
+      );
+    });
+  };
+}
+
 TaskFunction createFramePolicyIntegrationTest() {
   final String testDirectory =
       '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';

dev/devicelab/manifest.yaml

@@ -114,6 +114,14 @@ tasks:
 
   # Android on-device tests
 
+  complex_layout_android__scroll_smoothness:
+    description: >
+      Measures the smoothness of scrolling of the Complex Layout sample app on
+      Android.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
   complex_layout_scroll_perf__timeline_summary:
     description: >
       Measures the runtime performance of the Complex Layout sample app on

packages/flutter_test/lib/src/binding.dart

@@ -1504,7 +1504,6 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
             renderView._pointers[event.pointer].decay = _kPointerDecay;
           _handleViewNeedsPaint();
         } else if (event.down) {
-          assert(event is PointerDownEvent);
           renderView._pointers[event.pointer] = _LiveTestPointerRecord(
             event.pointer,
             event.position,