Pointer event resampler (#41118) (#60558)

packages/flutter/lib/gestures.dart

@@ -26,6 +26,7 @@ export 'src/gestures/multitap.dart';
 export 'src/gestures/pointer_router.dart';
 export 'src/gestures/pointer_signal_resolver.dart';
 export 'src/gestures/recognizer.dart';
+export 'src/gestures/resampler.dart';
 export 'src/gestures/scale.dart';
 export 'src/gestures/tap.dart';
 export 'src/gestures/team.dart';

packages/flutter/lib/src/gestures/binding.dart

@@ -8,6 +8,7 @@ import 'dart:collection';
 import 'dart:ui' as ui show PointerDataPacket;
 
 import 'package:flutter/foundation.dart';
+import 'package:flutter/scheduler.dart';
 
 import 'arena.dart';
 import 'converter.dart';
@@ -16,6 +17,131 @@ import 'events.dart';
 import 'hit_test.dart';
 import 'pointer_router.dart';
 import 'pointer_signal_resolver.dart';
+import 'resampler.dart';
+
+typedef HandleSampleTimeChangedCallback = void Function();
+
+// Class that handles resampling of touch events for multiple pointer
+// devices.
+//
+// SchedulerBinding's `currentSystemFrameTimeStamp` is used to determine
+// sample time.
+class _Resampler {
+  _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged);
+
+  // Resamplers used to filter incoming pointer events.
+  final Map<int, PointerEventResampler> _resamplers = <int, PointerEventResampler>{};
+
+  // Flag to track if a frame callback has been scheduled.
+  bool _frameCallbackScheduled = false;
+
+  // Current frame time for resampling.
+  Duration _frameTime = Duration.zero;
+
+  // Last sample time and time stamp of last event.
+  //
+  // Only used for debugPrint of resampling margin.
+  Duration _lastSampleTime = Duration.zero;
+  Duration _lastEventTime = Duration.zero;
+
+  // Callback used to handle pointer events.
+  final HandleEventCallback _handlePointerEvent;
+
+  // Callback used to handle sample time changes.
+  final HandleSampleTimeChangedCallback _handleSampleTimeChanged;
+
+  // Enqueue `events` for resampling or dispatch them directly if
+  // not a touch event.
+  void addOrDispatchAll(Queue<PointerEvent> events) {
+    final SchedulerBinding? scheduler = SchedulerBinding.instance;
+    assert(scheduler != null);
+
+    while (events.isNotEmpty) {
+      final PointerEvent event = events.removeFirst();
+
+      // Add touch event to resampler or dispatch pointer event directly.
+      if (event.kind == PointerDeviceKind.touch) {
+        // Save last event time for debugPrint of resampling margin.
+        _lastEventTime = event.timeStamp;
+
+        final PointerEventResampler resampler = _resamplers.putIfAbsent(
+          event.device,
+          () => PointerEventResampler(),
+        );
+        resampler.addEvent(event);
+      } else {
+        _handlePointerEvent(event);
+      }
+    }
+  }
+
+  // Sample and dispatch events.
+  //
+  // `samplingOffset` is relative to the current frame time, which
+  // can be in the past when we're not actively resampling.
+  // `currentSystemFrameTimeStamp` is used to determine the current
+  // frame time.
+  void sample(Duration samplingOffset) {
+    final SchedulerBinding? scheduler = SchedulerBinding.instance;
+    assert(scheduler != null);
+
+    // Determine sample time by adding the offset to the current
+    // frame time. This is expected to be in the past and not
+    // result in any dispatched events unless we're actively
+    // resampling events.
+    final Duration sampleTime = _frameTime + samplingOffset;
+
+    // Iterate over active resamplers and sample pointer events for
+    // current sample time.
+    for (final PointerEventResampler resampler in _resamplers.values) {
+      resampler.sample(sampleTime, _handlePointerEvent);
+    }
+
+    // Remove inactive resamplers.
+    _resamplers.removeWhere((int key, PointerEventResampler resampler) {
+      return !resampler.hasPendingEvents && !resampler.isDown;
+    });
+
+    // Save last sample time for debugPrint of resampling margin.
+    _lastSampleTime = sampleTime;
+
+    // Schedule a frame callback if another call to `sample` is needed.
+    if (!_frameCallbackScheduled && _resamplers.isNotEmpty) {
+      _frameCallbackScheduled = true;
+      scheduler?.scheduleFrameCallback((_) {
+        _frameCallbackScheduled = false;
+        // We use `currentSystemFrameTimeStamp` here as it's critical that
+        // sample time is in the same clock as the event time stamps, and
+        // never adjusted or scaled like `currentFrameTimeStamp`.
+        _frameTime = scheduler.currentSystemFrameTimeStamp;
+        assert(() {
+          if (debugPrintResamplingMargin) {
+            final Duration resamplingMargin = _lastEventTime - _lastSampleTime;
+              debugPrint('$resamplingMargin');
+          }
+          return true;
+        }());
+        _handleSampleTimeChanged();
+      });
+    }
+  }
+
+  // Stop all resampling and dispatched any queued events.
+  void stop() {
+    for (final PointerEventResampler resampler in _resamplers.values) {
+      resampler.stop(_handlePointerEvent);
+    }
+    _resamplers.clear();
+  }
+}
+
+// The default sampling offset.
+//
+// Sampling offset is relative to presentation time. If we produce frames
+// 16.667 ms before presentation and input rate is ~60hz, worst case latency
+// is 33.334 ms. This however assumes zero latency from the input driver.
+// 4.666 ms margin is added for this.
+const Duration _defaultSamplingOffset = Duration(milliseconds: -38);
 
 /// A binding for the gesture subsystem.
 ///
@@ -99,6 +225,17 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
 
   void _flushPointerEventQueue() {
     assert(!locked);
+
+    if (resamplingEnabled) {
+      _resampler.addOrDispatchAll(_pendingPointerEvents);
+      _resampler.sample(samplingOffset);
+      return;
+    }
+
+    // Stop resampler if resampling is not enabled. This is a no-op if
+    // resampling was never enabled.
+    _resampler.stop();
+
     while (_pendingPointerEvents.isNotEmpty)
       _handlePointerEvent(_pendingPointerEvents.removeFirst());
   }
@@ -226,6 +363,38 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
       pointerSignalResolver.resolve(event);
     }
   }
+
+  void _handleSampleTimeChanged() {
+    if (!locked) {
+      _flushPointerEventQueue();
+    }
+  }
+
+  // Resampler used to filter incoming pointer events when resampling
+  // is enabled.
+  late final _Resampler _resampler = _Resampler(
+    _handlePointerEvent,
+    _handleSampleTimeChanged,
+  );
+
+  /// Enable pointer event resampling for touch devices by setting
+  /// this to true.
+  ///
+  /// Resampling results in smoother touch event processing at the
+  /// cost of some added latency. Devices with low frequency sensors
+  /// or when the frequency is not a multiple of the display frequency
+  /// (e.g., 120Hz input and 90Hz display) benefit from this.
+  ///
+  /// This is typically set during application initialization but
+  /// can be adjusted dynamically in case the application only
+  /// wants resampling for some period of time.
+  bool resamplingEnabled = false;
+
+  /// Offset relative to current frame time that should be used for
+  /// resampling. The [samplingOffset] is expected to be negative.
+  /// Non-negative [samplingOffset] is allowed but will effectively
+  /// disable resampling.
+  Duration samplingOffset = _defaultSamplingOffset;
 }
 
 /// Variant of [FlutterErrorDetails] with extra fields for the gesture

packages/flutter/lib/src/gestures/debug.dart

@@ -50,6 +50,20 @@ bool debugPrintGestureArenaDiagnostics = false;
 ///    arenas.
 bool debugPrintRecognizerCallbacksTrace = false;
 
+/// Whether to print the resampling margin to the console.
+///
+/// When this is set, in debug mode, any time resampling is triggered by the
+/// [GestureBinding] the resampling margin is dumped to the console. The
+/// resampling margin is the delta between the time of the last received
+/// touch event and the current sample time. Positive value indicates that
+/// resampling is effective and the resampling offset can potentially be
+/// reduced for improved latency. Negative value indicates that resampling
+/// is failing and resampling offset needs to be increased for smooth
+/// touch event processing.
+///
+/// This has no effect in release builds.
+bool debugPrintResamplingMargin = false;
+
 /// Returns true if none of the gestures library debug variables have been changed.
 ///
 /// This function is used by the test framework to ensure that debug variables
@@ -61,7 +75,8 @@ bool debugAssertAllGesturesVarsUnset(String reason) {
   assert(() {
     if (debugPrintHitTestResults ||
         debugPrintGestureArenaDiagnostics ||
-        debugPrintRecognizerCallbacksTrace)
+        debugPrintRecognizerCallbacksTrace ||
+        debugPrintResamplingMargin)
       throw FlutterError(reason);
     return true;
   }());

packages/flutter/test/gestures/gesture_binding_test.dart

@@ -8,13 +8,16 @@ import 'dart:ui' as ui;
 
 import 'package:flutter/foundation.dart';
 import 'package:flutter/gestures.dart';
+import 'package:flutter/scheduler.dart';
 
 import '../flutter_test_alternative.dart';
 
 typedef HandleEventCallback = void Function(PointerEvent event);
 
-class TestGestureFlutterBinding extends BindingBase with GestureBinding {
+class TestGestureFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding {
   HandleEventCallback callback;
+  FrameCallback frameCallback;
+  Duration frameTime;
 
   @override
   void handleEvent(PointerEvent event, HitTestEntry entry) {
@@ -22,6 +25,18 @@ class TestGestureFlutterBinding extends BindingBase with GestureBinding {
     if (callback != null)
       callback(event);
   }
+
+  @override
+  Duration get currentSystemFrameTimeStamp {
+    assert(frameTime != null);
+    return frameTime;
+  }
+
+  @override
+  int scheduleFrameCallback(FrameCallback callback, {bool rescheduling = false}) {
+    frameCallback = callback;
+    return 0;
+  }
 }
 
 TestGestureFlutterBinding _binding = TestGestureFlutterBinding();
@@ -29,6 +44,7 @@ TestGestureFlutterBinding _binding = TestGestureFlutterBinding();
 void ensureTestGestureBinding() {
   _binding ??= TestGestureFlutterBinding();
   assert(GestureBinding.instance != null);
+  assert(SchedulerBinding.instance != null);
 }
 
 void main() {
@@ -299,4 +315,107 @@ void main() {
       expect(events[4].buttons, equals(0));
     }
   });
+
+  test('Pointer event resampling', () {
+    const ui.PointerDataPacket packet = ui.PointerDataPacket(
+      data: <ui.PointerData>[
+        ui.PointerData(
+            change: ui.PointerChange.add,
+            physicalX: 0.0,
+            timeStamp: Duration(milliseconds: 0),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.down,
+            physicalX: 0.0,
+            timeStamp: Duration(milliseconds: 1),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 10.0,
+            timeStamp: Duration(milliseconds: 2),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 20.0,
+            timeStamp: Duration(milliseconds: 3),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 30.0,
+            timeStamp: Duration(milliseconds: 4),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.up,
+            physicalX: 40.0,
+            timeStamp: Duration(milliseconds: 5),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.remove,
+            physicalX: 40.0,
+            timeStamp: Duration(milliseconds: 6),
+        ),
+      ],
+    );
+
+    final List<PointerEvent> pointerRouterEvents = <PointerEvent>[];
+    GestureBinding.instance.pointerRouter.addGlobalRoute(pointerRouterEvents.add);
+
+    final List<PointerEvent> events = <PointerEvent>[];
+    _binding.callback = events.add;
+
+    GestureBinding.instance.resamplingEnabled = true;
+    GestureBinding.instance.samplingOffset = const Duration(microseconds: -5500);
+
+    ui.window.onPointerDataPacket(packet);
+
+    // No pointer events should have been dispatched yet.
+    expect(events.length, 0);
+
+    // Frame callback should have been requested.
+    FrameCallback callback = _binding.frameCallback;
+    _binding.frameCallback = null;
+    expect(callback, isNotNull);
+
+    _binding.frameTime = const Duration(milliseconds: 7);
+    callback(Duration.zero);
+
+    // One pointer event should have been dispatched.
+    expect(events.length, 1);
+    expect(events[0].runtimeType, equals(PointerDownEvent));
+    expect(events[0].timeStamp, _binding.frameTime + GestureBinding.instance.samplingOffset);
+    expect(events[0].position, Offset(5.0 / ui.window.devicePixelRatio, 0.0));
+
+    // Second frame callback should have been requested.
+    callback = _binding.frameCallback;
+    _binding.frameCallback = null;
+    expect(callback, isNotNull);
+
+    _binding.frameTime = const Duration(milliseconds: 9);
+    callback(Duration.zero);
+
+    // Second pointer event should have been dispatched.
+    expect(events.length, 2);
+    expect(events[1].timeStamp, _binding.frameTime + GestureBinding.instance.samplingOffset);
+    expect(events[1].runtimeType, equals(PointerMoveEvent));
+    expect(events[1].position, Offset(25.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].delta, Offset(20.0 / ui.window.devicePixelRatio, 0.0));
+
+    // Third frame callback should have been requested.
+    callback = _binding.frameCallback;
+    _binding.frameCallback = null;
+    expect(callback, isNotNull);
+
+    _binding.frameTime = const Duration(milliseconds: 11);
+    callback(Duration.zero);
+
+    // Third pointer event should have been dispatched.
+    expect(events.length, 3);
+    expect(events[2].timeStamp, _binding.frameTime + GestureBinding.instance.samplingOffset);
+    expect(events[2].runtimeType, equals(PointerUpEvent));
+    expect(events[2].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0));
+
+    // No frame callback should have been requested.
+    callback = _binding.frameCallback;
+    expect(callback, isNull);
+  });
 }