Reland: Timer based pointer event resampling (#76195)

This reverts #76179 and relands #73042 with an active timer check fix.
Co-authored-by: David Reveman <reveman@google.com>

packages/flutter/lib/src/gestures/binding.dart

@@ -21,13 +21,24 @@ import 'resampler.dart';
 
 typedef _HandleSampleTimeChangedCallback = void Function();
 
+/// Class that implements clock used for sampling.
+class SamplingClock {
+  /// Returns current time.
+  DateTime now() => DateTime.now();
+
+  /// Returns a new stopwatch that uses the current time as reported by `this`.
+  Stopwatch stopwatch() => Stopwatch();
+}
+
 // Class that handles resampling of touch events for multiple pointer
 // devices.
 //
+// The `samplingInterval` is used to determine the approximate next
+// time for resampling.
 // SchedulerBinding's `currentSystemFrameTimeStamp` is used to determine
 // sample time.
 class _Resampler {
-  _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged);
+  _Resampler(this._handlePointerEvent, this._handleSampleTimeChanged, this._samplingInterval);
 
   // Resamplers used to filter incoming pointer events.
   final Map<int, PointerEventResampler> _resamplers = <int, PointerEventResampler>{};
@@ -35,9 +46,12 @@ class _Resampler {
   // Flag to track if a frame callback has been scheduled.
   bool _frameCallbackScheduled = false;
 
-  // Current frame time for resampling.
+  // Last frame time for resampling.
   Duration _frameTime = Duration.zero;
 
+  // Time since `_frameTime` was updated.
+  Stopwatch _frameTimeAge = Stopwatch();
+
   // Last sample time and time stamp of last event.
   //
   // Only used for debugPrint of resampling margin.
@@ -50,12 +64,18 @@ class _Resampler {
   // Callback used to handle sample time changes.
   final _HandleSampleTimeChangedCallback _handleSampleTimeChanged;
 
+  // Interval used for sampling.
+  final Duration _samplingInterval;
+
+  // Timer used to schedule resampling.
+  Timer? _timer;
+
   // Add `event` for resampling or dispatch it directly if
   // not a touch event.
   void addOrDispatch(PointerEvent event) {
     final SchedulerBinding? scheduler = SchedulerBinding.instance;
     assert(scheduler != null);
-      // Add touch event to resampler or dispatch pointer event directly.
+    // Add touch event to resampler or dispatch pointer event directly.
     if (event.kind == PointerDeviceKind.touch) {
       // Save last event time for debugPrint of resampling margin.
       _lastEventTime = event.timeStamp;
@@ -72,25 +92,43 @@ class _Resampler {
 
   // Sample and dispatch events.
   //
-  // `samplingOffset` is relative to the current frame time, which
+  // The `samplingOffset` is relative to the current frame time, which
   // can be in the past when we're not actively resampling.
-  // `samplingInterval` is used to determine the approximate next
-  // time for resampling.
-  // `currentSystemFrameTimeStamp` is used to determine the current
-  // frame time.
-  void sample(Duration samplingOffset, Duration samplingInterval) {
+  // The `samplingClock` is the clock used to determine frame time age.
+  void sample(Duration samplingOffset, SamplingClock clock) {
     final SchedulerBinding? scheduler = SchedulerBinding.instance;
     assert(scheduler != null);
 
+    // Initialize `_frameTime` if needed. This will be used for periodic
+    // sampling when frame callbacks are not received.
+    if (_frameTime == Duration.zero) {
+      _frameTime = Duration(milliseconds: clock.now().millisecondsSinceEpoch);
+      _frameTimeAge = clock.stopwatch()..start();
+    }
+
+    // Schedule periodic resampling if `_timer` is not already active.
+    if (_timer?.isActive != true) {
+       _timer = Timer.periodic(_samplingInterval, (_) => _onSampleTimeChanged());
+    }
+
+    // Calculate the effective frame time by taking the number
+    // of sampling intervals since last time `_frameTime` was
+    // updated into account. This allows us to advance sample
+    // time without having to receive frame callbacks.
+    final int samplingIntervalUs = _samplingInterval.inMicroseconds;
+    final int elapsedIntervals = _frameTimeAge.elapsedMicroseconds ~/ samplingIntervalUs;
+    final int elapsedUs = elapsedIntervals * samplingIntervalUs;
+    final Duration frameTime = _frameTime + Duration(microseconds: elapsedUs);
+
     // Determine sample time by adding the offset to the current
     // frame time. This is expected to be in the past and not
     // result in any dispatched events unless we're actively
     // resampling events.
-    final Duration sampleTime = _frameTime + samplingOffset;
+    final Duration sampleTime = frameTime + samplingOffset;
 
     // Determine next sample time by adding the sampling interval
     // to the current sample time.
-    final Duration nextSampleTime = sampleTime + samplingInterval;
+    final Duration nextSampleTime = sampleTime + _samplingInterval;
 
     // Iterate over active resamplers and sample pointer events for
     // current sample time.
@@ -106,23 +144,30 @@ class _Resampler {
     // Save last sample time for debugPrint of resampling margin.
     _lastSampleTime = sampleTime;
 
+    // Early out if another call to `sample` isn't needed.
+    if (_resamplers.isEmpty) {
+      _timer!.cancel();
+      return;
+    }
+
     // Schedule a frame callback if another call to `sample` is needed.
-    if (!_frameCallbackScheduled && _resamplers.isNotEmpty) {
+    if (!_frameCallbackScheduled) {
       _frameCallbackScheduled = true;
-      scheduler?.scheduleFrameCallback((_) {
+      // Add a post frame callback as this avoids producing unnecessary
+      // frames but ensures that sampling phase is adjusted to frame
+      // time when frames are produced.
+      scheduler?.addPostFrameCallback((_) {
         _frameCallbackScheduled = false;
         // We use `currentSystemFrameTimeStamp` here as it's critical that
         // sample time is in the same clock as the event time stamps, and
         // never adjusted or scaled like `currentFrameTimeStamp`.
         _frameTime = scheduler.currentSystemFrameTimeStamp;
-        assert(() {
-          if (debugPrintResamplingMargin) {
-            final Duration resamplingMargin = _lastEventTime - _lastSampleTime;
-              debugPrint('$resamplingMargin');
-          }
-          return true;
-        }());
-        _handleSampleTimeChanged();
+        _frameTimeAge.reset();
+        // Reset timer to match phase of latest frame callback.
+        _timer?.cancel();
+        _timer = Timer.periodic(_samplingInterval, (_) => _onSampleTimeChanged());
+        // Trigger an immediate sample time change.
+        _onSampleTimeChanged();
       });
     }
   }
@@ -133,6 +178,18 @@ class _Resampler {
       resampler.stop(_handlePointerEvent);
     }
     _resamplers.clear();
+    _frameTime = Duration.zero;
+  }
+
+  void _onSampleTimeChanged() {
+    assert(() {
+      if (debugPrintResamplingMargin) {
+        final Duration resamplingMargin = _lastEventTime - _lastSampleTime;
+          debugPrint('$resamplingMargin');
+      }
+      return true;
+    }());
+    _handleSampleTimeChanged();
   }
 }
 
@@ -147,7 +204,8 @@ const Duration _defaultSamplingOffset = Duration(milliseconds: -38);
 // The sampling interval.
 //
 // Sampling interval is used to determine the approximate time for subsequent
-// sampling. This is used to decide if early processing of up and removed events
+// sampling. This is used to sample events when frame callbacks are not
+// being received and decide if early processing of up and removed events
 // is appropriate. 16667 us for 60hz sampling interval.
 const Duration _samplingInterval = Duration(microseconds: 16667);
 
@@ -270,7 +328,7 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
 
     if (resamplingEnabled) {
       _resampler.addOrDispatch(event);
-      _resampler.sample(samplingOffset, _samplingInterval);
+      _resampler.sample(samplingOffset, _samplingClock);
       return;
     }
 
@@ -398,10 +456,16 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
     _hitTests.clear();
   }
 
+  /// Overrides the sampling clock for debugging and testing.
+  ///
+  /// This value is ignored in non-debug builds.
+  @protected
+  SamplingClock? get debugSamplingClock => null;
+
   void _handleSampleTimeChanged() {
     if (!locked) {
       if (resamplingEnabled) {
-        _resampler.sample(samplingOffset, _samplingInterval);
+        _resampler.sample(samplingOffset, _samplingClock);
       }
       else {
         _resampler.stop();
@@ -409,11 +473,23 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
     }
   }
 
+  SamplingClock get _samplingClock {
+    SamplingClock value = SamplingClock();
+    assert(() {
+      final SamplingClock? debugValue = debugSamplingClock;
+      if (debugValue != null)
+        value = debugValue;
+      return true;
+    }());
+    return value;
+  }
+
   // Resampler used to filter incoming pointer events when resampling
   // is enabled.
   late final _Resampler _resampler = _Resampler(
     _handlePointerEventImmediately,
     _handleSampleTimeChanged,
+    _samplingInterval,
   );
 
   /// Enable pointer event resampling for touch devices by setting

packages/flutter/test/gestures/gesture_binding_resample_event_on_widget_test.dart

@@ -10,15 +10,35 @@
 
 import 'dart:ui' as ui;
 
+import 'package:clock/clock.dart';
 import 'package:flutter/gestures.dart';
+import 'package:flutter/scheduler.dart';
 import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';
 
+class TestResampleEventFlutterBinding extends AutomatedTestWidgetsFlutterBinding {
+  @override
+  SamplingClock? get debugSamplingClock => TestSamplingClock(this.clock);
+}
+
+class TestSamplingClock implements SamplingClock {
+  TestSamplingClock(this._clock);
+
+  @override
+  DateTime now() => _clock.now();
+
+  @override
+  Stopwatch stopwatch() => _clock.stopwatch();
+
+  final Clock _clock;
+}
+
 void main() {
-  final TestWidgetsFlutterBinding binding = AutomatedTestWidgetsFlutterBinding();
+  final TestWidgetsFlutterBinding binding = TestResampleEventFlutterBinding();
   testWidgets('PointerEvent resampling on a widget', (WidgetTester tester) async {
     assert(WidgetsBinding.instance == binding);
     Duration currentTestFrameTime() => Duration(milliseconds: binding.clock.now().millisecondsSinceEpoch);
+    void requestFrame() => SchedulerBinding.instance!.scheduleFrameCallback((_) {});
     final Duration epoch = currentTestFrameTime();
     final ui.PointerDataPacket packet = ui.PointerDataPacket(
       data: <ui.PointerData>[
@@ -30,37 +50,37 @@ void main() {
         ui.PointerData(
             change: ui.PointerChange.down,
             physicalX: 0.0,
+            timeStamp: epoch + const Duration(milliseconds: 0),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 15.0,
             timeStamp: epoch + const Duration(milliseconds: 10),
         ),
         ui.PointerData(
             change: ui.PointerChange.move,
-            physicalX: 10.0,
+            physicalX: 30.0,
             timeStamp: epoch + const Duration(milliseconds: 20),
         ),
         ui.PointerData(
             change: ui.PointerChange.move,
-            physicalX: 20.0,
+            physicalX: 45.0,
             timeStamp: epoch + const Duration(milliseconds: 30),
         ),
         ui.PointerData(
             change: ui.PointerChange.move,
-            physicalX: 30.0,
+            physicalX: 50.0,
             timeStamp: epoch + const Duration(milliseconds: 40),
         ),
-        ui.PointerData(
-            change: ui.PointerChange.move,
-            physicalX: 40.0,
-            timeStamp: epoch + const Duration(milliseconds: 50),
-        ),
         ui.PointerData(
             change: ui.PointerChange.up,
-            physicalX: 40.0,
-            timeStamp: epoch + const Duration(milliseconds: 60),
+            physicalX: 60.0,
+            timeStamp: epoch + const Duration(milliseconds: 40),
         ),
         ui.PointerData(
             change: ui.PointerChange.remove,
-            physicalX: 40.0,
-            timeStamp: epoch + const Duration(milliseconds: 70),
+            physicalX: 60.0,
+            timeStamp: epoch + const Duration(milliseconds: 40),
         ),
       ],
     );
@@ -84,29 +104,31 @@ void main() {
     ui.window.onPointerDataPacket!(packet);
     expect(events.length, 0);
 
-    await tester.pump(const Duration(milliseconds: 20));
+    requestFrame();
+    await tester.pump(const Duration(milliseconds: 10));
     expect(events.length, 1);
     expect(events[0], isA<PointerDownEvent>());
     expect(events[0].timeStamp, currentTestFrameTime() + kSamplingOffset);
-    expect(events[0].position, Offset(5.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[0].position, Offset(7.5 / ui.window.devicePixelRatio, 0.0));
 
-    // Now the system time is epoch + 40ms
-    await tester.pump(const Duration(milliseconds: 20));
+    // Now the system time is epoch + 20ms
+    requestFrame();
+    await tester.pump(const Duration(milliseconds: 10));
     expect(events.length, 2);
     expect(events[1].timeStamp, currentTestFrameTime() + kSamplingOffset);
     expect(events[1], isA<PointerMoveEvent>());
-    expect(events[1].position, Offset(25.0 / ui.window.devicePixelRatio, 0.0));
-    expect(events[1].delta, Offset(20.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].position, Offset(22.5 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].delta, Offset(15.0 / ui.window.devicePixelRatio, 0.0));
 
-    // Now the system time is epoch + 60ms
-    await tester.pump(const Duration(milliseconds: 20));
+    // Now the system time is epoch + 30ms
+    requestFrame();
+    await tester.pump(const Duration(milliseconds: 10));
     expect(events.length, 4);
     expect(events[2].timeStamp, currentTestFrameTime() + kSamplingOffset);
     expect(events[2], isA<PointerMoveEvent>());
-    expect(events[2].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[2].position, Offset(37.5 / ui.window.devicePixelRatio, 0.0));
     expect(events[2].delta, Offset(15.0 / ui.window.devicePixelRatio, 0.0));
     expect(events[3].timeStamp, currentTestFrameTime() + kSamplingOffset);
     expect(events[3], isA<PointerUpEvent>());
-    expect(events[3].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0));
   });
 }

packages/flutter/test/gestures/gesture_binding_resample_event_test.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:ui' as ui;
+
+import 'package:clock/clock.dart';
+import 'package:fake_async/fake_async.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/gestures.dart';
+import 'package:flutter/scheduler.dart';
+
+import '../flutter_test_alternative.dart';
+
+typedef HandleEventCallback = void Function(PointerEvent event);
+
+class TestResampleEventFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding {
+  HandleEventCallback? callback;
+  FrameCallback? postFrameCallback;
+  Duration? frameTime;
+
+  @override
+  void handleEvent(PointerEvent event, HitTestEntry entry) {
+    super.handleEvent(event, entry);
+    if (callback != null)
+      callback?.call(event);
+  }
+
+  @override
+  Duration get currentSystemFrameTimeStamp {
+    assert(frameTime != null);
+    return frameTime!;
+  }
+
+  @override
+  int addPostFrameCallback(FrameCallback callback) {
+    postFrameCallback = callback;
+    return 0;
+  }
+
+  @override
+  SamplingClock? get debugSamplingClock => TestSamplingClock();
+}
+
+class TestSamplingClock implements SamplingClock {
+  @override
+  DateTime now() => clock.now();
+
+  @override
+  Stopwatch stopwatch() => clock.stopwatch();
+}
+
+typedef ResampleEventTest = void Function(FakeAsync async);
+
+void testResampleEvent(String description, ResampleEventTest callback) {
+  test(description, () {
+    fakeAsync((FakeAsync async) {
+      callback(async);
+    }, initialTime: DateTime.utc(2015, 1, 1));
+  }, skip: isBrowser); // Fake clock is not working with the web platform.
+}
+
+void main() {
+  final TestResampleEventFlutterBinding binding = TestResampleEventFlutterBinding();
+  testResampleEvent('Pointer event resampling', (FakeAsync async) {
+    Duration currentTime() => Duration(milliseconds: clock.now().millisecondsSinceEpoch);
+    final Duration epoch = currentTime();
+    final ui.PointerDataPacket packet = ui.PointerDataPacket(
+      data: <ui.PointerData>[
+        ui.PointerData(
+            change: ui.PointerChange.add,
+            physicalX: 0.0,
+            timeStamp: epoch + const Duration(milliseconds: 0),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.down,
+            physicalX: 0.0,
+            timeStamp: epoch + const Duration(milliseconds: 10),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 10.0,
+            timeStamp: epoch + const Duration(milliseconds: 20),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 20.0,
+            timeStamp: epoch + const Duration(milliseconds: 30),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 30.0,
+            timeStamp: epoch + const Duration(milliseconds: 40),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 40.0,
+            timeStamp: epoch + const Duration(milliseconds: 50),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.move,
+            physicalX: 50.0,
+            timeStamp: epoch + const Duration(milliseconds: 60),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.up,
+            physicalX: 50.0,
+            timeStamp: epoch + const Duration(milliseconds: 70),
+        ),
+        ui.PointerData(
+            change: ui.PointerChange.remove,
+            physicalX: 50.0,
+            timeStamp: epoch + const Duration(milliseconds: 70),
+        ),
+      ],
+    );
+
+    const Duration samplingOffset = Duration(milliseconds: -5);
+    const Duration frameInterval = Duration(microseconds: 16667);
+
+    GestureBinding.instance!.resamplingEnabled = true;
+    GestureBinding.instance!.samplingOffset = samplingOffset;
+
+    final List<PointerEvent> events = <PointerEvent>[];
+    binding.callback = events.add;
+
+    ui.window.onPointerDataPacket?.call(packet);
+
+    // No pointer events should have been dispatched yet.
+    expect(events.length, 0);
+
+    // Frame callback should have been requested.
+    FrameCallback? callback = binding.postFrameCallback;
+    binding.postFrameCallback = null;
+    expect(callback, isNotNull);
+
+    binding.frameTime = epoch + const Duration(milliseconds: 15);
+    callback!(Duration.zero);
+
+    // One pointer event should have been dispatched.
+    expect(events.length, 1);
+    expect(events[0], isA<PointerDownEvent>());
+    expect(events[0].timeStamp, binding.frameTime! + samplingOffset);
+    expect(events[0].position, Offset(0.0 / ui.window.devicePixelRatio, 0.0));
+
+    // Second frame callback should have been requested.
+    callback = binding.postFrameCallback;
+    binding.postFrameCallback = null;
+    expect(callback, isNotNull);
+
+    final Duration frameTime = epoch + const Duration(milliseconds: 25);
+    binding.frameTime = frameTime;
+    callback!(Duration.zero);
+
+    // Second pointer event should have been dispatched.
+    expect(events.length, 2);
+    expect(events[1], isA<PointerMoveEvent>());
+    expect(events[1].timeStamp, binding.frameTime! + samplingOffset);
+    expect(events[1].position, Offset(10.0 / ui.window.devicePixelRatio, 0.0));
+    expect(events[1].delta, Offset(10.0 / ui.window.devicePixelRatio, 0.0));
+
+    // Verify that resampling continues without a frame callback.
+    async.elapse(frameInterval * 1.5);
+
+    // Third pointer event should have been dispatched.
+    expect(events.length, 3);
+    expect(events[2], isA<PointerMoveEvent>());
+    expect(events[2].timeStamp, frameTime + frameInterval + samplingOffset);
+
+    async.elapse(frameInterval);
+
+    // Remaining pointer events should have been dispatched.
+    expect(events.length, 5);
+    expect(events[3], isA<PointerMoveEvent>());
+    expect(events[3].timeStamp, frameTime + frameInterval * 2 + samplingOffset);
+    expect(events[4], isA<PointerUpEvent>());
+    expect(events[4].timeStamp, frameTime + frameInterval * 2 + samplingOffset);
+
+    async.elapse(frameInterval);
+
+    // No more pointer events should have been dispatched.
+    expect(events.length, 5);
+
+    GestureBinding.instance!.resamplingEnabled = false;
+  });
+}

packages/flutter/test/gestures/gesture_binding_test.dart

@@ -14,8 +14,6 @@ typedef HandleEventCallback = void Function(PointerEvent event);
 
 class TestGestureFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding {
   HandleEventCallback? callback;
-  FrameCallback? frameCallback;
-  Duration? frameTime;
 
   @override
   void handleEvent(PointerEvent event, HitTestEntry entry) {
@@ -23,18 +21,6 @@ class TestGestureFlutterBinding extends BindingBase with GestureBinding, Schedul
     if (callback != null)
       callback?.call(event);
   }
-
-  @override
-  Duration get currentSystemFrameTimeStamp {
-    assert(frameTime != null);
-    return frameTime!;
-  }
-
-  @override
-  int scheduleFrameCallback(FrameCallback callback, {bool rescheduling = false}) {
-    frameCallback = callback;
-    return 0;
-  }
 }
 
 TestGestureFlutterBinding? _binding;