Timer based pointer event resampling (#73042)

Timer based pointer event resampling

This provides two improvements to pointer event resampling:

1. PostFrameCallbacks are used instead of scheduling frames. This
avoids unnecessary rendering work when resampling is used.

2. Resampling continues when frames are not produced. I.e. input
events keep being delivered at a fixed frequency even if frame
production is taking a long time.

This fixes #72924
Co-authored-by: David Reveman <reveman@google.com>
Co-authored-by: Tong Mu <dkwingsmt@users.noreply.github.com>

packages/flutter/lib/src/gestures/binding.dart

@@ -21,13 +21,24 @@ import 'resampler.dart';
 
 typedef _HandleSampleTimeChangedCallback = void Function();
 
+/// Class that implements clock used for sampling.
+class SamplingClock {
+  /// Returns current time.
+  DateTime now() => DateTime.now();
+
+  /// Returns a new stopwatch that uses the current time as reported by `this`.
+  Stopwatch stopwatch() => Stopwatch();
+}
+
 // Class that handles resampling of touch events for multiple pointer
 // devices.
 //
+// The `samplingInterval` is used to determine the approximate next
+// time for resampling.
 // SchedulerBinding's `currentSystemFrameTimeStamp` is used to determine
 // sample time.
 class _Resampler {
-  _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged);
+  _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged, this._samplingInterval);
 
   // Resamplers used to filter incoming pointer events.
   final Map<int, PointerEventResampler> _resamplers = <int, PointerEventResampler>{};
@@ -35,9 +46,12 @@ class _Resampler {
   // Flag to track if a frame callback has been scheduled.
   bool _frameCallbackScheduled = false;
 
-  // Current frame time for resampling.
+  // Last frame time for resampling.
   Duration _frameTime = Duration.zero;
 
+  // Time since `_frameTime` was updated.
+  Stopwatch _frameTimeAge = Stopwatch();
+
   // Last sample time and time stamp of last event.
   //
   // Only used for debugPrint of resampling margin.
@@ -50,12 +64,18 @@ class _Resampler {
   // Callback used to handle sample time changes.
   final _HandleSampleTimeChangedCallback _handleSampleTimeChanged;
 
+  // Interval used for sampling.
+  final Duration _samplingInterval;
+
+  // Timer used to schedule resampling.
+  Timer? _timer;
+
   // Add `event` for resampling or dispatch it directly if
   // not a touch event.
   void addOrDispatch(PointerEvent event) {
     final SchedulerBinding? scheduler = SchedulerBinding.instance;
     assert(scheduler != null);
-      // Add touch event to resampler or dispatch pointer event directly.
+    // Add touch event to resampler or dispatch pointer event directly.
     if (event.kind == PointerDeviceKind.touch) {
       // Save last event time for debugPrint of resampling margin.
       _lastEventTime = event.timeStamp;
@@ -72,25 +92,43 @@ class _Resampler {
 
   // Sample and dispatch events.
   //
-  // `samplingOffset` is relative to the current frame time, which
+  // The `samplingOffset` is relative to the current frame time, which
   // can be in the past when we're not actively resampling.
-  // `samplingInterval` is used to determine the approximate next
-  // time for resampling.
-  // `currentSystemFrameTimeStamp` is used to determine the current
-  // frame time.
-  void sample(Duration samplingOffset, Duration samplingInterval) {
+  // The `samplingClock` is the clock used to determine frame time age.
+  void sample(Duration samplingOffset, SamplingClock clock) {
     final SchedulerBinding? scheduler = SchedulerBinding.instance;
     assert(scheduler != null);
 
+    // Initialize `_frameTime` if needed. This will be used for periodic
+    // sampling when frame callbacks are not received.
+    if (_frameTime == Duration.zero) {
+      _frameTime = Duration(milliseconds: clock.now().millisecondsSinceEpoch);
+      _frameTimeAge = clock.stopwatch()..start();
+    }
+
+    // Schedule periodic resampling if `_timer` is not already active.
+    if (_timer?.isActive == false) {
+       _timer = Timer.periodic(_samplingInterval, (_) => _onSampleTimeChanged());
+    }
+
+    // Calculate the effective frame time by taking the number
+    // of sampling intervals since last time `_frameTime` was
+    // updated into account. This allows us to advance sample
+    // time without having to receive frame callbacks.
+    final int samplingIntervalUs = _samplingInterval.inMicroseconds;
+    final int elapsedIntervals = _frameTimeAge.elapsedMicroseconds ~/ samplingIntervalUs;
+    final int elapsedUs = elapsedIntervals * samplingIntervalUs;
+    final Duration frameTime = _frameTime + Duration(microseconds: elapsedUs);
+
     // Determine sample time by adding the offset to the current
     // frame time. This is expected to be in the past and not
     // result in any dispatched events unless we're actively
     // resampling events.
-    final Duration sampleTime = _frameTime + samplingOffset;
+    final Duration sampleTime = frameTime + samplingOffset;
 
     // Determine next sample time by adding the sampling interval
     // to the current sample time.
-    final Duration nextSampleTime = sampleTime + samplingInterval;
+    final Duration nextSampleTime = sampleTime + _samplingInterval;
 
     // Iterate over active resamplers and sample pointer events for
     // current sample time.
@@ -106,23 +144,30 @@ class _Resampler {
     // Save last sample time for debugPrint of resampling margin.
     _lastSampleTime = sampleTime;
 
+    // Early out if another call to `sample` isn't needed.
+    if (_resamplers.isEmpty) {
+      _timer!.cancel();
+      return;
+    }
+
     // Schedule a frame callback if another call to `sample` is needed.
-    if (!_frameCallbackScheduled && _resamplers.isNotEmpty) {
+    if (!_frameCallbackScheduled) {
       _frameCallbackScheduled = true;
-      scheduler?.scheduleFrameCallback((_) {
+      // Add a post frame callback as this avoids producing unnecessary
+      // frames but ensures that sampling phase is adjusted to frame
+      // time when frames are produced.
+      scheduler?.addPostFrameCallback((_) {
         _frameCallbackScheduled = false;
         // We use `currentSystemFrameTimeStamp` here as it's critical that
         // sample time is in the same clock as the event time stamps, and
         // never adjusted or scaled like `currentFrameTimeStamp`.
         _frameTime = scheduler.currentSystemFrameTimeStamp;
-        assert(() {
-          if (debugPrintResamplingMargin) {
-            final Duration resamplingMargin = _lastEventTime - _lastSampleTime;
-              debugPrint('$resamplingMargin');
-          }
-          return true;
-        }());
-        _handleSampleTimeChanged();
+        _frameTimeAge.reset();
+        // Reset timer to match phase of latest frame callback.
+        _timer?.cancel();
+        _timer = Timer.periodic(_samplingInterval, (_) => _onSampleTimeChanged());
+        // Trigger an immediate sample time change.
+        _onSampleTimeChanged();
       });
     }
   }
@@ -133,6 +178,18 @@ class _Resampler {
       resampler.stop(_handlePointerEvent);
     }
     _resamplers.clear();
+    _frameTime = Duration.zero;
+  }
+
+  void _onSampleTimeChanged() {
+    assert(() {
+      if (debugPrintResamplingMargin) {
+        final Duration resamplingMargin = _lastEventTime - _lastSampleTime;
+          debugPrint('$resamplingMargin');
+      }
+      return true;
+    }());
+    _handleSampleTimeChanged();
   }
 }
 
@@ -147,7 +204,8 @@ const Duration _defaultSamplingOffset = Duration(milliseconds: -38);
 // The sampling interval.
 //
 // Sampling interval is used to determine the approximate time for subsequent
-// sampling. This is used to decide if early processing of up and removed events
+// sampling. This is used to sample events when frame callbacks are not
+// being received and decide if early processing of up and removed events
 // is appropriate. 16667 us for 60hz sampling interval.
 const Duration _samplingInterval = Duration(microseconds: 16667);
 
@@ -270,7 +328,7 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
 
     if (resamplingEnabled) {
       _resampler.addOrDispatch(event);
-      _resampler.sample(samplingOffset, _samplingInterval);
+      _resampler.sample(samplingOffset, _samplingClock);
       return;
     }
 
@@ -398,10 +456,16 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
     _hitTests.clear();
   }
 
+  /// Overrides the sampling clock for debugging and testing.
+  ///
+  /// This value is ignored in non-debug builds.
+  @protected
+  SamplingClock? get debugSamplingClock => null;
+
   void _handleSampleTimeChanged() {
     if (!locked) {
       if (resamplingEnabled) {
-        _resampler.sample(samplingOffset, _samplingInterval);
+        _resampler.sample(samplingOffset, _samplingClock);
       }
       else {
         _resampler.stop();
@@ -409,11 +473,23 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
     }
   }
 
+  SamplingClock get _samplingClock {
+    SamplingClock value = SamplingClock();
+    assert(() {
+      final SamplingClock? debugValue = debugSamplingClock;
+      if (debugValue != null)
+        value = debugValue;
+      return true;
+    }());
+    return value;
+  }
+
   // Resampler used to filter incoming pointer events when resampling
   // is enabled.
   late final _Resampler _resampler = _Resampler(
     _handlePointerEventImmediately,
     _handleSampleTimeChanged,
+    _samplingInterval,
   );
 
   /// Enable pointer event resampling for touch devices by setting

packages/flutter/test/gestures/gesture_binding_resample_event_on_widget_test.dart

@@ -10,15 +10,35 @@
 
 import 'dart:ui' as ui;
 
+import 'package:clock/clock.dart';
 import 'package:flutter/gestures.dart';
+import 'package:flutter/scheduler.dart';
 import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';
 
+class TestResampleEventFlutterBinding extends AutomatedTestWidgetsFlutterBinding {
+  @override
+  SamplingClock? get debugSamplingClock => TestSamplingClock(this.clock);
+}
+
+class TestSamplingClock implements SamplingClock {
+  TestSamplingClock(this._clock);
+
+  @override
+  DateTime now() => _clock.now();
+
+  @override
+  Stopwatch stopwatch() => _clock.stopwatch();
+
+  final Clock _clock;
+}
+
 void main() {
-  final TestWidgetsFlutterBinding binding = AutomatedTestWidgetsFlutterBinding();
+  final TestWidgetsFlutterBinding binding = TestResampleEventFlutterBinding();
   testWidgets('PointerEvent resampling on a widget', (WidgetTester tester) async {
     assert(WidgetsBinding.instance == binding);
     Duration currentTestFrameTime() => Duration(milliseconds: binding.clock.now().millisecondsSinceEpoch);
+    void requestFrame() => SchedulerBinding.instance!.scheduleFrameCallback((_) {});
     final Duration epoch = currentTestFrameTime();
     final ui.PointerDataPacket packet = ui.PointerDataPacket(
       data: <ui.PointerData>[
@@ -30,37 +50,37 @@ void main() {
         ui.PointerData(
             change: ui.PointerChange.down,
             physicalX: 0.0,
+            timeStamp: epoch + const Duration(milliseconds: 0),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 15.0,
             timeStamp: epoch + const Duration(milliseconds: 10),
         ),
         ui.PointerData(
             change: ui.PointerChange.move,
-            physicalX: 10.0,
+            physicalX: 30.0,
             timeStamp: epoch + const Duration(milliseconds: 20),
         ),
         ui.PointerData(
             change: ui.PointerChange.move,
-            physicalX: 20.0,
+            physicalX: 45.0,
             timeStamp: epoch + const Duration(milliseconds: 30),
         ),
         ui.PointerData(
             change: ui.PointerChange.move,
-            physicalX: 30.0,
+            physicalX: 50.0,
             timeStamp: epoch + const Duration(milliseconds: 40),
         ),
-        ui.PointerData(
-            change: ui.PointerChange.move,
-            physicalX: 40.0,
-            timeStamp: epoch + const Duration(milliseconds: 50),
-        ),
         ui.PointerData(
             change: ui.PointerChange.up,
-            physicalX: 40.0,
-            timeStamp: epoch + const Duration(milliseconds: 60),
+            physicalX: 60.0,
+            timeStamp: epoch + const Duration(milliseconds: 40),
         ),
         ui.PointerData(
             change: ui.PointerChange.remove,
-            physicalX: 40.0,
-            timeStamp: epoch + const Duration(milliseconds: 70),
+            physicalX: 60.0,
+            timeStamp: epoch + const Duration(milliseconds: 40),
         ),
       ],
     );
@@ -84,29 +104,31 @@ void main() {
     ui.window.onPointerDataPacket!(packet);
     expect(events.length, 0);
 
-    await tester.pump(const Duration(milliseconds: 20));
+    requestFrame();
+    await tester.pump(const Duration(milliseconds: 10));
     expect(events.length, 1);
     expect(events[0], isA<PointerDownEvent>());
     expect(events[0].timeStamp, currentTestFrameTime() + kSamplingOffset);
-    expect(events[0].position, Offset(5.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[0].position, Offset(7.5 / ui.window.devicePixelRatio, 0.0));
 
-    // Now the system time is epoch + 40ms
-    await tester.pump(const Duration(milliseconds: 20));
+    // Now the system time is epoch + 20ms
+    requestFrame();
+    await tester.pump(const Duration(milliseconds: 10));
     expect(events.length, 2);
     expect(events[1].timeStamp, currentTestFrameTime() + kSamplingOffset);
     expect(events[1], isA<PointerMoveEvent>());
-    expect(events[1].position, Offset(25.0 / ui.window.devicePixelRatio, 0.0));
-    expect(events[1].delta, Offset(20.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].position, Offset(22.5 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].delta, Offset(15.0 / ui.window.devicePixelRatio, 0.0));
 
-    // Now the system time is epoch + 60ms
-    await tester.pump(const Duration(milliseconds: 20));
+    // Now the system time is epoch + 30ms
+    requestFrame();
+    await tester.pump(const Duration(milliseconds: 10));
     expect(events.length, 4);
     expect(events[2].timeStamp, currentTestFrameTime() + kSamplingOffset);
     expect(events[2], isA<PointerMoveEvent>());
-    expect(events[2].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[2].position, Offset(37.5 / ui.window.devicePixelRatio, 0.0));
     expect(events[2].delta, Offset(15.0 / ui.window.devicePixelRatio, 0.0));
     expect(events[3].timeStamp, currentTestFrameTime() + kSamplingOffset);
     expect(events[3], isA<PointerUpEvent>());
-    expect(events[3].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0));
   });
 }

packages/flutter/test/gestures/gesture_binding_resample_event_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:ui' as ui;
+
+import 'package:clock/clock.dart';
+import 'package:fake_async/fake_async.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/gestures.dart';
+import 'package:flutter/scheduler.dart';
+
+import '../flutter_test_alternative.dart';
+
+typedef HandleEventCallback = void Function(PointerEvent event);
+
+class TestResampleEventFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding {
+  HandleEventCallback? callback;
+  FrameCallback? postFrameCallback;
+  Duration? frameTime;
+
+  @override
+  void handleEvent(PointerEvent event, HitTestEntry entry) {
+    super.handleEvent(event, entry);
+    if (callback != null)
+      callback?.call(event);
+  }
+
+  @override
+  Duration get currentSystemFrameTimeStamp {
+    assert(frameTime != null);
+    return frameTime!;
+  }
+
+  @override
+  int addPostFrameCallback(FrameCallback callback) {
+    postFrameCallback = callback;
+    return 0;
+  }
+
+  @override
+  SamplingClock? get debugSamplingClock => TestSamplingClock();
+}
+
+class TestSamplingClock implements SamplingClock {
+  @override
+  DateTime now() => clock.now();
+
+  @override
+  Stopwatch stopwatch() => clock.stopwatch();
+}
+
+typedef ResampleEventTest = void Function(FakeAsync async);
+
+void testResampleEvent(String description, ResampleEventTest callback) {
+  test(description, () {
+    fakeAsync((FakeAsync async) {
+      callback(async);
+    }, initialTime: DateTime.utc(2015, 1, 1));
+  }, skip: isBrowser); // Fake clock is not working with the web platform.
+}
+
+void main() {
+  final TestResampleEventFlutterBinding binding = TestResampleEventFlutterBinding();
+  testResampleEvent('Pointer event resampling', (FakeAsync async) {
+    Duration currentTime() => Duration(milliseconds: clock.now().millisecondsSinceEpoch);
+    final Duration epoch = currentTime();
+    final ui.PointerDataPacket packet = ui.PointerDataPacket(
+      data: <ui.PointerData>[
+        ui.PointerData(
+            change: ui.PointerChange.add,
+            physicalX: 0.0,
+            timeStamp: epoch + const Duration(milliseconds: 0),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.down,
+            physicalX: 0.0,
+            timeStamp: epoch + const Duration(milliseconds: 10),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 10.0,
+            timeStamp: epoch + const Duration(milliseconds: 20),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 20.0,
+            timeStamp: epoch + const Duration(milliseconds: 30),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 30.0,
+            timeStamp: epoch + const Duration(milliseconds: 40),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 40.0,
+            timeStamp: epoch + const Duration(milliseconds: 50),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 50.0,
+            timeStamp: epoch + const Duration(milliseconds: 60),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.up,
+            physicalX: 50.0,
+            timeStamp: epoch + const Duration(milliseconds: 70),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.remove,
+            physicalX: 50.0,
+            timeStamp: epoch + const Duration(milliseconds: 70),
+        ),
+      ],
+    );
+
+    const Duration samplingOffset = Duration(milliseconds: -5);
+    const Duration frameInterval = Duration(microseconds: 16667);
+
+    GestureBinding.instance!.resamplingEnabled = true;
+    GestureBinding.instance!.samplingOffset = samplingOffset;
+
+    final List<PointerEvent> events = <PointerEvent>[];
+    binding.callback = events.add;
+
+    ui.window.onPointerDataPacket?.call(packet);
+
+    // No pointer events should have been dispatched yet.
+    expect(events.length, 0);
+
+    // Frame callback should have been requested.
+    FrameCallback? callback = binding.postFrameCallback;
+    binding.postFrameCallback = null;
+    expect(callback, isNotNull);
+
+    binding.frameTime = epoch + const Duration(milliseconds: 15);
+    callback!(Duration.zero);
+
+    // One pointer event should have been dispatched.
+    expect(events.length, 1);
+    expect(events[0], isA<PointerDownEvent>());
+    expect(events[0].timeStamp, binding.frameTime! + samplingOffset);
+    expect(events[0].position, Offset(0.0 / ui.window.devicePixelRatio, 0.0));
+
+    // Second frame callback should have been requested.
+    callback = binding.postFrameCallback;
+    binding.postFrameCallback = null;
+    expect(callback, isNotNull);
+
+    final Duration frameTime = epoch + const Duration(milliseconds: 25);
+    binding.frameTime = frameTime;
+    callback!(Duration.zero);
+
+    // Second pointer event should have been dispatched.
+    expect(events.length, 2);
+    expect(events[1], isA<PointerMoveEvent>());
+    expect(events[1].timeStamp, binding.frameTime! + samplingOffset);
+    expect(events[1].position, Offset(10.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].delta, Offset(10.0 / ui.window.devicePixelRatio, 0.0));
+
+    // Verify that resampling continues without a frame callback.
+    async.elapse(frameInterval * 1.5);
+
+    // Third pointer event should have been dispatched.
+    expect(events.length, 3);
+    expect(events[2], isA<PointerMoveEvent>());
+    expect(events[2].timeStamp, frameTime + frameInterval + samplingOffset);
+
+    async.elapse(frameInterval);
+
+    // Remaining pointer events should have been dispatched.
+    expect(events.length, 5);
+    expect(events[3], isA<PointerMoveEvent>());
+    expect(events[3].timeStamp, frameTime + frameInterval * 2 + samplingOffset);
+    expect(events[4], isA<PointerUpEvent>());
+    expect(events[4].timeStamp, frameTime + frameInterval * 2 + samplingOffset);
+
+    async.elapse(frameInterval);
+
+    // No more pointer events should have been dispatched.
+    expect(events.length, 5);
+
+    GestureBinding.instance!.resamplingEnabled = false;
+  });
+}

packages/flutter/test/gestures/gesture_binding_test.dart

@@ -14,8 +14,6 @@ typedef HandleEventCallback = void Function(PointerEvent event);
 
 class TestGestureFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding {
   HandleEventCallback? callback;
-  FrameCallback? frameCallback;
-  Duration? frameTime;
 
   @override
   void handleEvent(PointerEvent event, HitTestEntry entry) {
@@ -23,18 +21,6 @@ class TestGestureFlutterBinding extends BindingBase with GestureBinding, Schedul
     if (callback != null)
       callback?.call(event);
   }
-
-  @override
-  Duration get currentSystemFrameTimeStamp {
-    assert(frameTime != null);
-    return frameTime!;
-  }
-
-  @override
-  int scheduleFrameCallback(FrameCallback callback, {bool rescheduling = false}) {
-    frameCallback = callback;
-    return 0;
-  }
 }
 
 TestGestureFlutterBinding? _binding;