Unverified Commit 02612bfe authored by David Reveman's avatar David Reveman Committed by GitHub

Pointer event resampler (#41118) (#60558)

parent 2f937703
...@@ -26,6 +26,7 @@ export 'src/gestures/multitap.dart'; ...@@ -26,6 +26,7 @@ export 'src/gestures/multitap.dart';
export 'src/gestures/pointer_router.dart'; export 'src/gestures/pointer_router.dart';
export 'src/gestures/pointer_signal_resolver.dart'; export 'src/gestures/pointer_signal_resolver.dart';
export 'src/gestures/recognizer.dart'; export 'src/gestures/recognizer.dart';
export 'src/gestures/resampler.dart';
export 'src/gestures/scale.dart'; export 'src/gestures/scale.dart';
export 'src/gestures/tap.dart'; export 'src/gestures/tap.dart';
export 'src/gestures/team.dart'; export 'src/gestures/team.dart';
......
...@@ -8,6 +8,7 @@ import 'dart:collection'; ...@@ -8,6 +8,7 @@ import 'dart:collection';
import 'dart:ui' as ui show PointerDataPacket; import 'dart:ui' as ui show PointerDataPacket;
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:flutter/scheduler.dart';
import 'arena.dart'; import 'arena.dart';
import 'converter.dart'; import 'converter.dart';
...@@ -16,6 +17,131 @@ import 'events.dart'; ...@@ -16,6 +17,131 @@ import 'events.dart';
import 'hit_test.dart'; import 'hit_test.dart';
import 'pointer_router.dart'; import 'pointer_router.dart';
import 'pointer_signal_resolver.dart'; import 'pointer_signal_resolver.dart';
import 'resampler.dart';
typedef HandleSampleTimeChangedCallback = void Function();
// Class that handles resampling of touch events for multiple pointer
// devices.
//
// SchedulerBinding's `currentSystemFrameTimeStamp` is used to determine
// sample time.
class _Resampler {
_Resampler(this._handlePointerEvent, this._handleSampleTimeChanged);
// Resamplers used to filter incoming pointer events.
final Map<int, PointerEventResampler> _resamplers = <int, PointerEventResampler>{};
// Flag to track if a frame callback has been scheduled.
bool _frameCallbackScheduled = false;
// Current frame time for resampling.
Duration _frameTime = Duration.zero;
// Last sample time and time stamp of last event.
//
// Only used for debugPrint of resampling margin.
Duration _lastSampleTime = Duration.zero;
Duration _lastEventTime = Duration.zero;
// Callback used to handle pointer events.
final HandleEventCallback _handlePointerEvent;
// Callback used to handle sample time changes.
final HandleSampleTimeChangedCallback _handleSampleTimeChanged;
// Enqueue `events` for resampling or dispatch them directly if
// not a touch event.
void addOrDispatchAll(Queue<PointerEvent> events) {
final SchedulerBinding? scheduler = SchedulerBinding.instance;
assert(scheduler != null);
while (events.isNotEmpty) {
final PointerEvent event = events.removeFirst();
// Add touch event to resampler or dispatch pointer event directly.
if (event.kind == PointerDeviceKind.touch) {
// Save last event time for debugPrint of resampling margin.
_lastEventTime = event.timeStamp;
final PointerEventResampler resampler = _resamplers.putIfAbsent(
event.device,
() => PointerEventResampler(),
);
resampler.addEvent(event);
} else {
_handlePointerEvent(event);
}
}
}
// Sample and dispatch events.
//
// `samplingOffset` is relative to the current frame time, which
// can be in the past when we're not actively resampling.
// `currentSystemFrameTimeStamp` is used to determine the current
// frame time.
void sample(Duration samplingOffset) {
final SchedulerBinding? scheduler = SchedulerBinding.instance;
assert(scheduler != null);
// Determine sample time by adding the offset to the current
// frame time. This is expected to be in the past and not
// result in any dispatched events unless we're actively
// resampling events.
final Duration sampleTime = _frameTime + samplingOffset;
// Iterate over active resamplers and sample pointer events for
// current sample time.
for (final PointerEventResampler resampler in _resamplers.values) {
resampler.sample(sampleTime, _handlePointerEvent);
}
// Remove inactive resamplers.
_resamplers.removeWhere((int key, PointerEventResampler resampler) {
return !resampler.hasPendingEvents && !resampler.isDown;
});
// Save last sample time for debugPrint of resampling margin.
_lastSampleTime = sampleTime;
// Schedule a frame callback if another call to `sample` is needed.
if (!_frameCallbackScheduled && _resamplers.isNotEmpty) {
_frameCallbackScheduled = true;
scheduler?.scheduleFrameCallback((_) {
_frameCallbackScheduled = false;
// We use `currentSystemFrameTimeStamp` here as it's critical that
// sample time is in the same clock as the event time stamps, and
// never adjusted or scaled like `currentFrameTimeStamp`.
_frameTime = scheduler.currentSystemFrameTimeStamp;
assert(() {
if (debugPrintResamplingMargin) {
final Duration resamplingMargin = _lastEventTime - _lastSampleTime;
debugPrint('$resamplingMargin');
}
return true;
}());
_handleSampleTimeChanged();
});
}
}
// Stop all resampling and dispatched any queued events.
void stop() {
for (final PointerEventResampler resampler in _resamplers.values) {
resampler.stop(_handlePointerEvent);
}
_resamplers.clear();
}
}
// The default sampling offset.
//
// Sampling offset is relative to presentation time. If we produce frames
// 16.667 ms before presentation and input rate is ~60hz, worst case latency
// is 33.334 ms. This however assumes zero latency from the input driver.
// 4.666 ms margin is added for this.
const Duration _defaultSamplingOffset = Duration(milliseconds: -38);
/// A binding for the gesture subsystem. /// A binding for the gesture subsystem.
/// ///
...@@ -99,6 +225,17 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H ...@@ -99,6 +225,17 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
void _flushPointerEventQueue() { void _flushPointerEventQueue() {
assert(!locked); assert(!locked);
if (resamplingEnabled) {
_resampler.addOrDispatchAll(_pendingPointerEvents);
_resampler.sample(samplingOffset);
return;
}
// Stop resampler if resampling is not enabled. This is a no-op if
// resampling was never enabled.
_resampler.stop();
while (_pendingPointerEvents.isNotEmpty) while (_pendingPointerEvents.isNotEmpty)
_handlePointerEvent(_pendingPointerEvents.removeFirst()); _handlePointerEvent(_pendingPointerEvents.removeFirst());
} }
...@@ -226,6 +363,38 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H ...@@ -226,6 +363,38 @@ mixin GestureBinding on BindingBase implements HitTestable, HitTestDispatcher, H
pointerSignalResolver.resolve(event); pointerSignalResolver.resolve(event);
} }
} }
void _handleSampleTimeChanged() {
if (!locked) {
_flushPointerEventQueue();
}
}
// Resampler used to filter incoming pointer events when resampling
// is enabled.
late final _Resampler _resampler = _Resampler(
_handlePointerEvent,
_handleSampleTimeChanged,
);
/// Enable pointer event resampling for touch devices by setting
/// this to true.
///
/// Resampling results in smoother touch event processing at the
/// cost of some added latency. Devices with low frequency sensors
/// or when the frequency is not a multiple of the display frequency
/// (e.g., 120Hz input and 90Hz display) benefit from this.
///
/// This is typically set during application initialization but
/// can be adjusted dynamically in case the application only
/// wants resampling for some period of time.
bool resamplingEnabled = false;
/// Offset relative to current frame time that should be used for
/// resampling. The [samplingOffset] is expected to be negative.
/// Non-negative [samplingOffset] is allowed but will effectively
/// disable resampling.
Duration samplingOffset = _defaultSamplingOffset;
} }
/// Variant of [FlutterErrorDetails] with extra fields for the gesture /// Variant of [FlutterErrorDetails] with extra fields for the gesture
......
...@@ -50,6 +50,20 @@ bool debugPrintGestureArenaDiagnostics = false; ...@@ -50,6 +50,20 @@ bool debugPrintGestureArenaDiagnostics = false;
/// arenas. /// arenas.
bool debugPrintRecognizerCallbacksTrace = false; bool debugPrintRecognizerCallbacksTrace = false;
/// Whether to print the resampling margin to the console.
///
/// When this is set, in debug mode, any time resampling is triggered by the
/// [GestureBinding] the resampling margin is dumped to the console. The
/// resampling margin is the delta between the time of the last received
/// touch event and the current sample time. Positive value indicates that
/// resampling is effective and the resampling offset can potentially be
/// reduced for improved latency. Negative value indicates that resampling
/// is failing and resampling offset needs to be increased for smooth
/// touch event processing.
///
/// This has no effect in release builds.
bool debugPrintResamplingMargin = false;
/// Returns true if none of the gestures library debug variables have been changed. /// Returns true if none of the gestures library debug variables have been changed.
/// ///
/// This function is used by the test framework to ensure that debug variables /// This function is used by the test framework to ensure that debug variables
...@@ -61,7 +75,8 @@ bool debugAssertAllGesturesVarsUnset(String reason) { ...@@ -61,7 +75,8 @@ bool debugAssertAllGesturesVarsUnset(String reason) {
assert(() { assert(() {
if (debugPrintHitTestResults || if (debugPrintHitTestResults ||
debugPrintGestureArenaDiagnostics || debugPrintGestureArenaDiagnostics ||
debugPrintRecognizerCallbacksTrace) debugPrintRecognizerCallbacksTrace ||
debugPrintResamplingMargin)
throw FlutterError(reason); throw FlutterError(reason);
return true; return true;
}()); }());
......
This diff is collapsed.
...@@ -8,13 +8,16 @@ import 'dart:ui' as ui; ...@@ -8,13 +8,16 @@ import 'dart:ui' as ui;
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:flutter/gestures.dart'; import 'package:flutter/gestures.dart';
import 'package:flutter/scheduler.dart';
import '../flutter_test_alternative.dart'; import '../flutter_test_alternative.dart';
typedef HandleEventCallback = void Function(PointerEvent event); typedef HandleEventCallback = void Function(PointerEvent event);
class TestGestureFlutterBinding extends BindingBase with GestureBinding { class TestGestureFlutterBinding extends BindingBase with GestureBinding, SchedulerBinding {
HandleEventCallback callback; HandleEventCallback callback;
FrameCallback frameCallback;
Duration frameTime;
@override @override
void handleEvent(PointerEvent event, HitTestEntry entry) { void handleEvent(PointerEvent event, HitTestEntry entry) {
...@@ -22,6 +25,18 @@ class TestGestureFlutterBinding extends BindingBase with GestureBinding { ...@@ -22,6 +25,18 @@ class TestGestureFlutterBinding extends BindingBase with GestureBinding {
if (callback != null) if (callback != null)
callback(event); callback(event);
} }
@override
Duration get currentSystemFrameTimeStamp {
assert(frameTime != null);
return frameTime;
}
@override
int scheduleFrameCallback(FrameCallback callback, {bool rescheduling = false}) {
frameCallback = callback;
return 0;
}
} }
TestGestureFlutterBinding _binding = TestGestureFlutterBinding(); TestGestureFlutterBinding _binding = TestGestureFlutterBinding();
...@@ -29,6 +44,7 @@ TestGestureFlutterBinding _binding = TestGestureFlutterBinding(); ...@@ -29,6 +44,7 @@ TestGestureFlutterBinding _binding = TestGestureFlutterBinding();
void ensureTestGestureBinding() { void ensureTestGestureBinding() {
_binding ??= TestGestureFlutterBinding(); _binding ??= TestGestureFlutterBinding();
assert(GestureBinding.instance != null); assert(GestureBinding.instance != null);
assert(SchedulerBinding.instance != null);
} }
void main() { void main() {
...@@ -299,4 +315,107 @@ void main() { ...@@ -299,4 +315,107 @@ void main() {
expect(events[4].buttons, equals(0)); expect(events[4].buttons, equals(0));
} }
}); });
test('Pointer event resampling', () {
const ui.PointerDataPacket packet = ui.PointerDataPacket(
data: <ui.PointerData>[
ui.PointerData(
change: ui.PointerChange.add,
physicalX: 0.0,
timeStamp: Duration(milliseconds: 0),
),
ui.PointerData(
change: ui.PointerChange.down,
physicalX: 0.0,
timeStamp: Duration(milliseconds: 1),
),
ui.PointerData(
change: ui.PointerChange.move,
physicalX: 10.0,
timeStamp: Duration(milliseconds: 2),
),
ui.PointerData(
change: ui.PointerChange.move,
physicalX: 20.0,
timeStamp: Duration(milliseconds: 3),
),
ui.PointerData(
change: ui.PointerChange.move,
physicalX: 30.0,
timeStamp: Duration(milliseconds: 4),
),
ui.PointerData(
change: ui.PointerChange.up,
physicalX: 40.0,
timeStamp: Duration(milliseconds: 5),
),
ui.PointerData(
change: ui.PointerChange.remove,
physicalX: 40.0,
timeStamp: Duration(milliseconds: 6),
),
],
);
final List<PointerEvent> pointerRouterEvents = <PointerEvent>[];
GestureBinding.instance.pointerRouter.addGlobalRoute(pointerRouterEvents.add);
final List<PointerEvent> events = <PointerEvent>[];
_binding.callback = events.add;
GestureBinding.instance.resamplingEnabled = true;
GestureBinding.instance.samplingOffset = const Duration(microseconds: -5500);
ui.window.onPointerDataPacket(packet);
// No pointer events should have been dispatched yet.
expect(events.length, 0);
// Frame callback should have been requested.
FrameCallback callback = _binding.frameCallback;
_binding.frameCallback = null;
expect(callback, isNotNull);
_binding.frameTime = const Duration(milliseconds: 7);
callback(Duration.zero);
// One pointer event should have been dispatched.
expect(events.length, 1);
expect(events[0].runtimeType, equals(PointerDownEvent));
expect(events[0].timeStamp, _binding.frameTime + GestureBinding.instance.samplingOffset);
expect(events[0].position, Offset(5.0 / ui.window.devicePixelRatio, 0.0));
// Second frame callback should have been requested.
callback = _binding.frameCallback;
_binding.frameCallback = null;
expect(callback, isNotNull);
_binding.frameTime = const Duration(milliseconds: 9);
callback(Duration.zero);
// Second pointer event should have been dispatched.
expect(events.length, 2);
expect(events[1].timeStamp, _binding.frameTime + GestureBinding.instance.samplingOffset);
expect(events[1].runtimeType, equals(PointerMoveEvent));
expect(events[1].position, Offset(25.0 / ui.window.devicePixelRatio, 0.0));
expect(events[1].delta, Offset(20.0 / ui.window.devicePixelRatio, 0.0));
// Third frame callback should have been requested.
callback = _binding.frameCallback;
_binding.frameCallback = null;
expect(callback, isNotNull);
_binding.frameTime = const Duration(milliseconds: 11);
callback(Duration.zero);
// Third pointer event should have been dispatched.
expect(events.length, 3);
expect(events[2].timeStamp, _binding.frameTime + GestureBinding.instance.samplingOffset);
expect(events[2].runtimeType, equals(PointerUpEvent));
expect(events[2].position, Offset(40.0 / ui.window.devicePixelRatio, 0.0));
// No frame callback should have been requested.
callback = _binding.frameCallback;
expect(callback, isNull);
});
} }
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment