Reland 3: Make LiveTestWidgetsFlutterBinding work with setSurfaceSize and live tests (#87239)

* Revert "Revert "Reland 2: Make LiveTestWidgetsFlutterBinding work with setSurfaceSize and live tests (#86912)" (#87233)"

This reverts commit 5b80d32a.

* Record tap positions

packages/flutter/test/animation/live_binding_test.dart

@@ -11,11 +11,12 @@ void main() {
    * because [matchesGoldenFile] does not use Skia Gold in its native package.
    */
 
-  LiveTestWidgetsFlutterBinding();
+  LiveTestWidgetsFlutterBinding().framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.onlyPumps;
 
   testWidgets('Should show event indicator for pointer events', (WidgetTester tester) async {
     final AnimationSheetBuilder animationSheet = AnimationSheetBuilder(frameSize: const Size(200, 200), allLayers: true);
-    final Widget target = Container(
+    final List<Offset> taps = <Offset>[];
+    Widget target({bool recording = true}) => Container(
       padding: const EdgeInsets.fromLTRB(20, 10, 25, 20),
       child: animationSheet.record(
         MaterialApp(
@@ -25,37 +26,105 @@ void main() {
               border: Border.all(color: const Color.fromARGB(255, 0, 0, 0)),
             ),
             child: Center(
-              child: GestureDetector(
-                onTap: () {},
-                child: const Text('Test'),
+              child: Container(
+                width: 40,
+                height: 40,
+                color: Colors.black,
+                child: GestureDetector(
+                  onTapDown: (TapDownDetails details) {
+                    taps.add(details.globalPosition);
+                  },
+                ),
               ),
             ),
           ),
         ),
+        recording: recording,
       ),
     );
 
-    await tester.pumpWidget(target);
+    await tester.pumpWidget(target(recording: false));
 
-    await tester.pumpFrames(target, const Duration(milliseconds: 50));
+    await tester.pumpFrames(target(), const Duration(milliseconds: 50));
 
-    final TestGesture gesture1 = await tester.createGesture();
-    await gesture1.down(tester.getCenter(find.byType(Text)) + const Offset(10, 10));
+    final TestGesture gesture1 = await tester.createGesture(pointer: 1);
+    await gesture1.down(tester.getCenter(find.byType(GestureDetector)) + const Offset(10, 10));
+    expect(taps, equals(const <Offset>[Offset(130, 120)]));
+    taps.clear();
 
-    await tester.pumpFrames(target, const Duration(milliseconds: 100));
+    await tester.pumpFrames(target(), const Duration(milliseconds: 100));
 
-    final TestGesture gesture2 = await tester.createGesture();
-    await gesture2.down(tester.getTopLeft(find.byType(Text)) + const Offset(30, -10));
+    final TestGesture gesture2 = await tester.createGesture(pointer: 2);
+    await gesture2.down(tester.getTopLeft(find.byType(GestureDetector)) + const Offset(30, -10));
     await gesture1.moveBy(const Offset(50, 50));
 
-    await tester.pumpFrames(target, const Duration(milliseconds: 100));
+    await tester.pumpFrames(target(), const Duration(milliseconds: 100));
     await gesture1.up();
     await gesture2.up();
-    await tester.pumpFrames(target, const Duration(milliseconds: 50));
+    await tester.pumpFrames(target(), const Duration(milliseconds: 50));
+    expect(taps, isEmpty);
 
     await expectLater(
       animationSheet.collate(6),
       matchesGoldenFile('LiveBinding.press.animation.png'),
     );
+  }, skip: isBrowser); // https://github.com/flutter/flutter/issues/42767
+
+  testWidgets('Should show event indicator for pointer events with setSurfaceSize', (WidgetTester tester) async {
+    final AnimationSheetBuilder animationSheet = AnimationSheetBuilder(frameSize: const Size(200, 200), allLayers: true);
+    final List<Offset> taps = <Offset>[];
+    Widget target({bool recording = true}) => Container(
+      padding: const EdgeInsets.fromLTRB(20, 10, 25, 20),
+      child: animationSheet.record(
+        MaterialApp(
+          home: Container(
+            decoration: BoxDecoration(
+              color: const Color.fromARGB(255, 128, 128, 128),
+              border: Border.all(color: const Color.fromARGB(255, 0, 0, 0)),
+            ),
+            child: Center(
+              child: Container(
+                width: 40,
+                height: 40,
+                color: Colors.black,
+                child: GestureDetector(
+                  onTapDown: (TapDownDetails details) {
+                    taps.add(details.globalPosition);
+                  },
+                ),
+              ),
+            ),
+          ),
+        ),
+        recording: recording,
+      ),
+    );
+
+    await tester.binding.setSurfaceSize(const Size(300, 300));
+    await tester.pumpWidget(target(recording: false));
+
+    await tester.pumpFrames(target(), const Duration(milliseconds: 50));
+
+    final TestGesture gesture1 = await tester.createGesture(pointer: 1);
+    await gesture1.down(tester.getCenter(find.byType(GestureDetector)) + const Offset(10, 10));
+    expect(taps, equals(const <Offset>[Offset(130, 120)]));
+    taps.clear();
+
+    await tester.pumpFrames(target(), const Duration(milliseconds: 100));
+
+    final TestGesture gesture2 = await tester.createGesture(pointer: 2);
+    await gesture2.down(tester.getTopLeft(find.byType(GestureDetector)) + const Offset(30, -10));
+    await gesture1.moveBy(const Offset(50, 50));
+
+    await tester.pumpFrames(target(), const Duration(milliseconds: 100));
+    await gesture1.up();
+    await gesture2.up();
+    await tester.pumpFrames(target(), const Duration(milliseconds: 50));
+    expect(taps, isEmpty);
+
+    await expectLater(
+      animationSheet.collate(6),
+      matchesGoldenFile('LiveBinding.press.animation.2.png'),
+    );
   }, skip: isBrowser); // https://github.com/flutter/flutter/issues/56001
 }

packages/flutter_test/lib/src/animation_sheet.dart

@@ -134,11 +134,11 @@ class AnimationSheetBuilder {
   /// The returned widget wraps `child` in a box with a fixed size specified by
   /// [frameSize]. The `key` is also applied to the returned widget.
   ///
-  /// The `recording` defaults to true, which means the painted result of each
-  /// frame will be stored and later available for [display]. If `recording` is
-  /// false, then frames are not recorded. This is useful during the setup phase
-  /// that shouldn't be recorded; if the target widget isn't wrapped in [record]
-  /// during the setup phase, the states will be lost when it starts recording.
+  /// The frame is only recorded if the `recording` argument is true, or during
+  /// a procedure that is wrapped within [recording]. In either case, the
+  /// painted result of each frame will be stored and later available for
+  /// [display]. If neither condition is met, the frames are not recorded, which
+  /// is useful during setup phases.
   ///
   /// The `child` must not be null.
   ///
@@ -274,7 +274,10 @@ class AnimationSheetBuilder {
   ///
   /// An example of using this method can be found at [AnimationSheetBuilder].
   Future<ui.Image> collate(int cellsPerRow) async {
-    return _collateFrames(await _frames, frameSize, cellsPerRow);
+    final List<ui.Image> frames = await _frames;
+    assert(frames.isNotEmpty,
+      'No frames are collected. Have you forgot to set `recording` to true?');
+    return _collateFrames(frames, frameSize, cellsPerRow);
   }
 
   /// Returns the smallest size that can contain all recorded frames.

packages/flutter_test/lib/src/binding.dart

@@ -119,6 +119,20 @@ mixin TestDefaultBinaryMessengerBinding on BindingBase, ServicesBinding {
 /// that actually needs to make a network call should provide its own
 /// `HttpClient` to the code making the call, so that it can appropriately mock
 /// or fake responses.
+///
+/// ### Coordinate spaces
+///
+/// [TestWidgetsFlutterBinding] might be run on devices of different screen
+/// sizes, while the testing widget is still told the same size to ensure
+/// consistent results. Consequently, code that deals with positions (such as
+/// pointer events or painting) must distinguish between two coordinate spaces:
+///
+///  * The _local coordinate space_ is the one used by the testing widget
+///    (typically an 800 by 600 window, but can be altered by [setSurfaceSize]).
+///  * The _global coordinate space_ is the one used by the device.
+///
+/// Positions can be transformed between coordinate spaces with [localToGlobal]
+/// and [globalToLocal].
 abstract class TestWidgetsFlutterBinding extends BindingBase
   with SchedulerBinding,
        ServicesBinding,
@@ -447,14 +461,16 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
     });
   }
 
-  /// Convert the given point from the global coordinate system (as used by
-  /// pointer events from the device) to the coordinate system used by the
-  /// tests (an 800 by 600 window).
+  /// Convert the given point from the global coordinate space to the local
+  /// one.
+  ///
+  /// For definitions for coordinate spaces, see [TestWidgetsFlutterBinding].
   Offset globalToLocal(Offset point) => point;
 
-  /// Convert the given point from the coordinate system used by the tests (an
-  /// 800 by 600 window) to the global coordinate system (as used by pointer
-  /// events from the device).
+  /// Convert the given point from the local coordinate space to the global
+  /// one.
+  ///
+  /// For definitions for coordinate spaces, see [TestWidgetsFlutterBinding].
   Offset localToGlobal(Offset point) => point;
 
   /// The source of the current pointer event.
@@ -462,15 +478,34 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
   /// The [pointerEventSource] is set as the `source` parameter of
   /// [handlePointerEventForSource] and can be used in the immediate enclosing
   /// [dispatchEvent].
+  ///
+  /// When [handlePointerEvent] is called directly, [pointerEventSource]
+  /// is [TestBindingEventSource.device].
   TestBindingEventSource get pointerEventSource => _pointerEventSource;
   TestBindingEventSource _pointerEventSource = TestBindingEventSource.device;
 
   /// Dispatch an event to the targets found by a hit test on its position,
   /// and remember its source as [pointerEventSource].
   ///
-  /// This method sets [pointerEventSource] to `source`, runs
+  /// This method sets [pointerEventSource] to `source`, forwards the call to
   /// [handlePointerEvent], then resets [pointerEventSource] to the previous
   /// value.
+  ///
+  /// If `source` is [TestBindingEventSource.device], then the `event` is based
+  /// in the global coordinate space (for definitions for coordinate spaces,
+  /// see [TestWidgetsFlutterBinding]) and the event is likely triggered by the
+  /// user physically interacting with the screen during a live test on a real
+  /// device (see [LiveTestWidgetsFlutterBinding]).
+  ///
+  /// If `source` is [TestBindingEventSource.test], then the `event` is based
+  /// in the local coordinate space and the event is likely triggered by
+  /// programatically simulated pointer events, such as:
+  ///
+  ///  * [WidgetController.tap] and alike methods, as well as directly using
+  ///    [TestGesture]. They are usually used in
+  ///    [AutomatedTestWidgetsFlutterBinding] but sometimes in live tests too.
+  ///  * [WidgetController.timedDrag] and alike methods. They are usually used
+  ///    in macrobenchmarks.
   void handlePointerEventForSource(
     PointerEvent event, {
     TestBindingEventSource source = TestBindingEventSource.device,
@@ -482,7 +517,7 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
   /// to the previous value.
   @protected
   void withPointerEventSource(TestBindingEventSource source, VoidCallback task) {
-    final TestBindingEventSource previousSource = source;
+    final TestBindingEventSource previousSource = _pointerEventSource;
     _pointerEventSource = source;
     try {
       task();
@@ -1504,11 +1539,15 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
   /// Events dispatched by [TestGesture] are not affected by this.
   HitTestDispatcher? deviceEventDispatcher;
 
-
   /// Dispatch an event to the targets found by a hit test on its position.
   ///
-  /// Apart from forwarding the event to [GestureBinding.dispatchEvent],
-  /// This also paint all events that's down on the screen.
+  /// If the [pointerEventSource] is [TestBindingEventSource.test], then
+  /// the event is forwarded to [GestureBinding.dispatchEvent] as usual;
+  /// additionally, down pointers are painted on the screen.
+  ///
+  /// If the [pointerEventSource] is [TestBindingEventSource.device], then
+  /// the event, after being transformed to the local coordinate system, is
+  /// forwarded to [deviceEventDispatcher].
   @override
   void handlePointerEvent(PointerEvent event) {
     switch (pointerEventSource) {
@@ -1530,8 +1569,12 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
         break;
       case TestBindingEventSource.device:
         if (deviceEventDispatcher != null) {
+          // The pointer events received with this source has a global position
+          // (see [handlePointerEventForSource]). Transform it to the local
+          // coordinate space used by the testing widgets.
+          final PointerEvent localEvent = event.copyWith(position: globalToLocal(event.position));
           withPointerEventSource(TestBindingEventSource.device,
-            () => super.handlePointerEvent(event)
+            () => super.handlePointerEvent(localEvent)
           );
         }
         break;
@@ -1545,9 +1588,10 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding {
         super.dispatchEvent(event, hitTestResult);
         break;
       case TestBindingEventSource.device:
-        assert(hitTestResult != null);
+        assert(hitTestResult != null || event is PointerAddedEvent || event is PointerRemovedEvent);
         assert(deviceEventDispatcher != null);
-        deviceEventDispatcher!.dispatchEvent(event, hitTestResult!);
+        if (hitTestResult != null)
+          deviceEventDispatcher!.dispatchEvent(event, hitTestResult);
         break;
     }
   }
@@ -1782,15 +1826,6 @@ class _LiveTestRenderView extends RenderView {
     onNeedPaint();
   }
 
-  @override
-  bool hitTest(HitTestResult result, { required Offset position }) {
-    final Matrix4 transform = configuration.toHitTestMatrix();
-    final double det = transform.invert();
-    assert(det != 0.0);
-    position = MatrixUtils.transformPoint(transform, position);
-    return super.hitTest(result, position: position);
-  }
-
   @override
   void paint(PaintingContext context, Offset offset) {
     assert(offset == Offset.zero);

packages/flutter_test/lib/src/widget_tester.dart

@@ -59,6 +59,21 @@ export 'package:test_api/test_api.dart' hide
 /// Signature for callback to [testWidgets] and [benchmarkWidgets].
 typedef WidgetTesterCallback = Future<void> Function(WidgetTester widgetTester);
 
+// Return the last element that satisifes `test`, or return null if not found.
+E? _lastWhereOrNull<E>(Iterable<E> list, bool Function(E) test) {
+  late E result;
+  bool foundMatching = false;
+  for (final E element in list) {
+    if (test(element)) {
+      result = element;
+      foundMatching = true;
+    }
+  }
+  if (foundMatching)
+    return result;
+  return null;
+}
+
 /// Runs the [callback] inside the Flutter test environment.
 ///
 /// Use this function for testing custom [StatelessWidget]s and
@@ -829,15 +844,12 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker
         .map((HitTestEntry candidate) => candidate.target)
         .whereType<RenderObject>()
         .first;
-      final Element? innerTargetElement = collectAllElementsFrom(
-        binding.renderViewElement!,
-        skipOffstage: true,
-      ).cast<Element?>().lastWhere(
-        (Element? element) => element!.renderObject == innerTarget,
-        orElse: () => null,
+      final Element? innerTargetElement = _lastWhereOrNull(
+        collectAllElementsFrom(binding.renderViewElement!, skipOffstage: true),
+        (Element element) => element.renderObject == innerTarget,
       );
       if (innerTargetElement == null) {
-        printToConsole('No widgets found at ${binding.globalToLocal(event.position)}.');
+        printToConsole('No widgets found at ${event.position}.');
         return;
       }
       final List<Element> candidates = <Element>[];
@@ -850,7 +862,7 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker
       int numberOfWithTexts = 0;
       int numberOfTypes = 0;
       int totalNumber = 0;
-      printToConsole('Some possible finders for the widgets at ${binding.globalToLocal(event.position)}:');
+      printToConsole('Some possible finders for the widgets at ${event.position}:');
       for (final Element element in candidates) {
         if (totalNumber > 13) // an arbitrary number of finders that feels useful without being overwhelming
           break;

packages/flutter_test/test/widget_tester_live_device_test.dart

@@ -6,6 +6,13 @@ import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
 
+// Only check the initial lines of the message, since the message walks the
+// entire widget tree back, and any changes to the widget tree break these
+// tests if we check the entire message.
+void _expectStartsWith(List<String?> actual, List<String?> matcher) {
+  expect(actual.sublist(0, matcher.length), equals(matcher));
+}
+
 void main() {
   final _MockLiveTestWidgetsFlutterBinding binding = _MockLiveTestWidgetsFlutterBinding();
 
@@ -14,8 +21,9 @@ void main() {
 
     int invocations = 0;
     await tester.pumpWidget(
-      MaterialApp(
-        home: Center(
+      Directionality(
+        textDirection: TextDirection.ltr,
+        child: Center(
           child: GestureDetector(
             onTap: () {
               invocations++;
@@ -42,39 +50,87 @@ void main() {
     await tester.pump();
     expect(invocations, 0);
 
-    expect(printedMessages, equals('''
+    _expectStartsWith(printedMessages, '''
 Some possible finders for the widgets at Offset(400.0, 300.0):
   find.text('Test')
-  find.widgetWithText(RawGestureDetector, 'Test')
-  find.byType(GestureDetector)
-  find.byType(Center)
-  find.widgetWithText(IgnorePointer, 'Test')
-  find.byType(FadeTransition)
-  find.byType(FractionalTranslation)
-  find.byType(SlideTransition)
-  find.widgetWithText(FocusTrap, 'Test')
-  find.widgetWithText(PrimaryScrollController, 'Test')
-  find.widgetWithText(PageStorage, 'Test')
-'''.trim().split('\n')));
+'''.trim().split('\n'));
     printedMessages.clear();
 
     await binding.collectDebugPrints(printedMessages, () async {
       await tester.tapAt(const Offset(1, 1));
     });
     expect(printedMessages, equals('''
-Some possible finders for the widgets at Offset(1.0, 1.0):
-  find.byType(MouseRegion)
-  find.byType(ExcludeSemantics)
-  find.byType(BlockSemantics)
-  find.byType(ModalBarrier)
-  find.byType(Overlay)
+No widgets found at Offset(1.0, 1.0).
+'''.trim().split('\n')));
+  });
+
+  testWidgets('Should print message on pointer events with setSurfaceSize', (WidgetTester tester) async {
+    final List<String?> printedMessages = <String?>[];
+
+    int invocations = 0;
+    await tester.pumpWidget(
+      Directionality(
+        textDirection: TextDirection.ltr,
+        child: Center(
+          child:GestureDetector(
+            onTap: () {
+              invocations++;
+            },
+            child: const Text('Test'),
+          ),
+        ),
+      ),
+    );
+
+    final Size originalSize = tester.binding.createViewConfiguration().size;
+    await tester.binding.setSurfaceSize(const Size(2000, 1800));
+    try {
+      await tester.pump();
+
+      final Offset widgetCenter = tester.getRect(find.byType(Text)).center;
+      expect(widgetCenter.dx, 1000);
+      expect(widgetCenter.dy, 900);
+
+      await binding.collectDebugPrints(printedMessages, () async {
+        await tester.tap(find.byType(Text));
+      });
+      await tester.pump();
+      expect(invocations, 0);
+
+      _expectStartsWith(printedMessages, '''
+Some possible finders for the widgets at Offset(1000.0, 900.0):
+  find.text('Test')
+'''.trim().split('\n'));
+      printedMessages.clear();
+
+      await binding.collectDebugPrints(printedMessages, () async {
+        await tester.tapAt(const Offset(1, 1));
+      });
+      expect(printedMessages, equals('''
+No widgets found at Offset(1.0, 1.0).
 '''.trim().split('\n')));
+    } finally {
+      await tester.binding.setSurfaceSize(originalSize);
+    }
   });
 }
 
 class _MockLiveTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
   @override
-  TestBindingEventSource get pointerEventSource => TestBindingEventSource.device;
+  void handlePointerEventForSource(
+    PointerEvent event, {
+    TestBindingEventSource source = TestBindingEventSource.device,
+  }) {
+    // In this test we use `WidgetTester.tap` to simulate real device touches.
+    // `WidgetTester.tap` sends events in the local coordinate system, while
+    // real devices touches sends event in the global coordinate system.
+    // See the documentation of [handlePointerEventForSource] for details.
+    if (source == TestBindingEventSource.test) {
+      final PointerEvent globalEvent = event.copyWith(position: localToGlobal(event.position));
+      return super.handlePointerEventForSource(globalEvent, source: TestBindingEventSource.device);
+    }
+    return super.handlePointerEventForSource(event, source: source);
+  }
 
   List<String?>? _storeDebugPrints;