Positioning IME bars on iOS (#61981)

packages/flutter/lib/src/rendering/editable.dart

@@ -1587,6 +1587,30 @@ class RenderEditable extends RenderBox with RelayoutWhenSystemFontsChangeMixin {
     }
   }
 
+  /// Returns the smallest [Rect], in the local coordinate system, that covers
+  /// the text within the [TextRange] specified.
+  ///
+  /// This method is used to calculate the approximate position of the IME bar
+  /// on iOS.
+  ///
+  /// Returns null if [TextRange.isValid] is false for the given `range`, or the
+  /// given `range` is collapsed.
+  Rect? getRectForComposingRange(TextRange range) {
+    assert(constraints != null);
+    if (!range.isValid || range.isCollapsed)
+      return null;
+    _layoutText(minWidth: constraints.minWidth, maxWidth: constraints.maxWidth);
+
+    final List<ui.TextBox> boxes = _textPainter.getBoxesForSelection(
+      TextSelection(baseOffset: range.start, extentOffset: range.end),
+    );
+
+    return boxes.fold(
+      null,
+      (Rect? accum, TextBox incoming) => accum?.expandToInclude(incoming.toRect()) ?? incoming.toRect(),
+    )?.shift(_paintOffset);
+  }
+
   /// Returns the position in the text for the given global coordinate.
   ///
   /// See also:

packages/flutter/lib/src/services/text_input.dart

@@ -9,6 +9,7 @@ import 'dart:ui' show
   FontWeight,
   Offset,
   Size,
+  Rect,
   TextAffinity,
   TextAlign,
   TextDirection,
@@ -839,6 +840,7 @@ class TextInputConnection {
 
   Size? _cachedSize;
   Matrix4? _cachedTransform;
+  Rect? _cachedRect;
 
   static int _nextId = 1;
   final int _id;
@@ -916,6 +918,29 @@ class TextInputConnection {
     }
   }
 
+  /// Send the smallest rect that covers the text in the client that's currently
+  /// being composed.
+  ///
+  /// The given `rect` can not be null. If any of the 4 coordinates of the given
+  /// [Rect] is not finite, a [Rect] of size (-1, -1) will be sent instead.
+  ///
+  /// The information is currently only used on iOS, for positioning the IME bar.
+  void setComposingRect(Rect rect) {
+    assert(rect != null);
+    if (rect == _cachedRect)
+      return;
+    _cachedRect = rect;
+    final Rect validRect = rect.isFinite ? rect : Offset.zero & const Size(-1, -1);
+    TextInput._instance._setComposingTextRect(
+      <String, dynamic>{
+        'width': validRect.width,
+        'height': validRect.height,
+        'x': validRect.left,
+        'y': validRect.top,
+      },
+    );
+  }
+
   /// Send text styling information.
   ///
   /// This information is used by the Flutter Web Engine to change the style
@@ -1249,6 +1274,13 @@ class TextInput {
     );
   }
 
+  void _setComposingTextRect(Map<String, dynamic> args) {
+    _channel.invokeMethod<void>(
+      'TextInput.setMarkedTextRect',
+      args,
+    );
+  }
+
   void _setStyle(Map<String, dynamic> args) {
     _channel.invokeMethod<void>(
       'TextInput.setStyle',

packages/flutter/lib/src/widgets/editable_text.dart

@@ -1662,7 +1662,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
       return;
     } else if (value.text == _value.text && value.composing == _value.composing && value.selection != _value.selection) {
       // `selection` is the only change.
-      _handleSelectionChanged(value.selection, renderEditable!, SelectionChangedCause.keyboard);
+      _handleSelectionChanged(value.selection, renderEditable, SelectionChangedCause.keyboard);
     } else {
       _formatAndSetValue(value);
     }
@@ -1728,7 +1728,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
   // Because the center of the cursor is preferredLineHeight / 2 below the touch
   // origin, but the touch origin is used to determine which line the cursor is
   // on, we need this offset to correctly render and move the cursor.
-  Offset get _floatingCursorOffset => Offset(0, renderEditable!.preferredLineHeight / 2);
+  Offset get _floatingCursorOffset => Offset(0, renderEditable.preferredLineHeight / 2);
 
   @override
   void updateFloatingCursor(RawFloatingCursorPoint point) {
@@ -1742,20 +1742,20 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
         // we cache the position.
         _pointOffsetOrigin = point.offset;
 
-        final TextPosition currentTextPosition = TextPosition(offset: renderEditable!.selection!.baseOffset);
-        _startCaretRect = renderEditable!.getLocalRectForCaret(currentTextPosition);
+        final TextPosition currentTextPosition = TextPosition(offset: renderEditable.selection!.baseOffset);
+        _startCaretRect = renderEditable.getLocalRectForCaret(currentTextPosition);
 
         _lastBoundedOffset = _startCaretRect!.center - _floatingCursorOffset;
         _lastTextPosition = currentTextPosition;
-        renderEditable!.setFloatingCursor(point.state, _lastBoundedOffset!, _lastTextPosition!);
+        renderEditable.setFloatingCursor(point.state, _lastBoundedOffset!, _lastTextPosition!);
         break;
       case FloatingCursorDragState.Update:
         final Offset centeredPoint = point.offset! - _pointOffsetOrigin!;
         final Offset rawCursorOffset = _startCaretRect!.center + centeredPoint - _floatingCursorOffset;
 
-        _lastBoundedOffset = renderEditable!.calculateBoundedFloatingCursorOffset(rawCursorOffset);
-        _lastTextPosition = renderEditable!.getPositionForPoint(renderEditable!.localToGlobal(_lastBoundedOffset! + _floatingCursorOffset));
-        renderEditable!.setFloatingCursor(point.state, _lastBoundedOffset!, _lastTextPosition!);
+        _lastBoundedOffset = renderEditable.calculateBoundedFloatingCursorOffset(rawCursorOffset);
+        _lastTextPosition = renderEditable.getPositionForPoint(renderEditable.localToGlobal(_lastBoundedOffset! + _floatingCursorOffset));
+        renderEditable.setFloatingCursor(point.state, _lastBoundedOffset!, _lastTextPosition!);
         break;
       case FloatingCursorDragState.End:
         // We skip animation if no update has happened.
@@ -1768,12 +1768,12 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
   }
 
   void _onFloatingCursorResetTick() {
-    final Offset finalPosition = renderEditable!.getLocalRectForCaret(_lastTextPosition!).centerLeft - _floatingCursorOffset;
+    final Offset finalPosition = renderEditable.getLocalRectForCaret(_lastTextPosition!).centerLeft - _floatingCursorOffset;
     if (_floatingCursorResetController.isCompleted) {
-      renderEditable!.setFloatingCursor(FloatingCursorDragState.End, finalPosition, _lastTextPosition!);
-      if (_lastTextPosition!.offset != renderEditable!.selection!.baseOffset)
+      renderEditable.setFloatingCursor(FloatingCursorDragState.End, finalPosition, _lastTextPosition!);
+      if (_lastTextPosition!.offset != renderEditable.selection!.baseOffset)
         // The cause is technically the force cursor, but the cause is listed as tap as the desired functionality is the same.
-        _handleSelectionChanged(TextSelection.collapsed(offset: _lastTextPosition!.offset), renderEditable!, SelectionChangedCause.forcePress);
+        _handleSelectionChanged(TextSelection.collapsed(offset: _lastTextPosition!.offset), renderEditable, SelectionChangedCause.forcePress);
       _startCaretRect = null;
       _lastTextPosition = null;
       _pointOffsetOrigin = null;
@@ -1783,7 +1783,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
       final double lerpX = ui.lerpDouble(_lastBoundedOffset!.dx, finalPosition.dx, lerpValue)!;
       final double lerpY = ui.lerpDouble(_lastBoundedOffset!.dy, finalPosition.dy, lerpValue)!;
 
-      renderEditable!.setFloatingCursor(FloatingCursorDragState.Update, Offset(lerpX, lerpY), _lastTextPosition!, resetLerpValue: lerpValue);
+      renderEditable.setFloatingCursor(FloatingCursorDragState.Update, Offset(lerpX, lerpY), _lastTextPosition!, resetLerpValue: lerpValue);
     }
   }
 
@@ -1862,7 +1862,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
     if (!_scrollController!.position.allowImplicitScrolling)
       return RevealedOffset(offset: _scrollController!.offset, rect: rect);
 
-    final Size editableSize = renderEditable!.size;
+    final Size editableSize = renderEditable.size;
     double additionalOffset;
     Offset unitOffset;
 
@@ -1881,7 +1881,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
       final Rect expandedRect = Rect.fromCenter(
         center: rect.center,
         width: rect.width,
-        height: math.max(rect.height, renderEditable!.preferredLineHeight),
+        height: math.max(rect.height, renderEditable.preferredLineHeight),
       );
 
       additionalOffset = expandedRect.height >= editableSize.height
@@ -1927,6 +1927,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
         : TextInput.attach(this, _createTextInputConfiguration(_isInAutofillContext || _needsAutofill));
       _textInputConnection!.show();
       _updateSizeAndTransform();
+      _updateComposingRectIfNeeded();
       if (_needsAutofill) {
         // Request autofill AFTER the size and the transform have been sent to
         // the platform text input plugin.
@@ -2071,7 +2072,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
         return;
       }
 
-      final double lineHeight = renderEditable!.preferredLineHeight;
+      final double lineHeight = renderEditable.preferredLineHeight;
 
       // Enlarge the target rect by scrollPadding to ensure that caret is not
       // positioned directly at the edge after scrolling.
@@ -2106,7 +2107,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
         curve: _caretAnimationCurve,
       );
 
-      renderEditable!.showOnScreen(
+      renderEditable.showOnScreen(
         rect: caretPadding.inflateRect(targetOffset.rect),
         duration: _caretAnimationDuration,
         curve: _caretAnimationCurve,
@@ -2167,7 +2168,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
   }
 
   void _onCursorColorTick() {
-    renderEditable!.cursorColor = widget.cursorColor.withOpacity(_cursorBlinkOpacityController.value);
+    renderEditable.cursorColor = widget.cursorColor.withOpacity(_cursorBlinkOpacityController.value);
     _cursorVisibilityNotifier.value = widget.showCursor && _cursorBlinkOpacityController.value > 0;
   }
 
@@ -2273,7 +2274,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
       _showCaretOnScreen();
       if (!_value.selection.isValid) {
         // Place cursor at the end if the selection is invalid when we receive focus.
-        _handleSelectionChanged(TextSelection.collapsed(offset: _value.text.length), renderEditable!, null);
+        _handleSelectionChanged(TextSelection.collapsed(offset: _value.text.length), renderEditable, null);
       }
     } else {
       WidgetsBinding.instance!.removeObserver(this);
@@ -2286,14 +2287,37 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
 
   void _updateSizeAndTransform() {
     if (_hasInputConnection) {
-      final Size size = renderEditable!.size;
-      final Matrix4 transform = renderEditable!.getTransformTo(null);
+      final Size size = renderEditable.size;
+      final Matrix4 transform = renderEditable.getTransformTo(null);
       _textInputConnection!.setEditableSizeAndTransform(size, transform);
       SchedulerBinding.instance!
           .addPostFrameCallback((Duration _) => _updateSizeAndTransform());
     }
   }
 
+  // Sends the current composing rect to the iOS text input plugin via the text
+  // input channel. We need to keep sending the information even if no text is
+  // currently marked, as the information usually lags behind. The text input
+  // plugin needs to estimate the composing rect based on the latest caret rect,
+  // when the composing rect info didn't arrive in time.
+  void _updateComposingRectIfNeeded() {
+    final TextRange composingRange = _value.composing;
+    if (_hasInputConnection) {
+      assert(mounted);
+      Rect? composingRect = renderEditable.getRectForComposingRange(composingRange);
+      // Send the caret location instead if there's no marked text yet.
+      if (composingRect == null) {
+        assert(!composingRange.isValid || composingRange.isCollapsed);
+        final int offset = composingRange.isValid ? composingRange.start : 0;
+        composingRect = renderEditable.getLocalRectForCaret(TextPosition(offset: offset));
+      }
+      assert(composingRect != null);
+      _textInputConnection!.setComposingRect(composingRect);
+      SchedulerBinding.instance!
+          .addPostFrameCallback((Duration _) => _updateComposingRectIfNeeded());
+    }
+  }
+
   TextDirection get _textDirection {
     final TextDirection? result = widget.textDirection ?? Directionality.of(context);
     assert(result != null, '$runtimeType created without a textDirection and with no ambient Directionality.');
@@ -2304,7 +2328,7 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
   ///
   /// This property is typically used to notify the renderer of input gestures
   /// when [RenderEditable.ignorePointer] is true.
-  RenderEditable? get renderEditable => _editableKey.currentContext!.findRenderObject() as RenderEditable?;
+  RenderEditable get renderEditable => _editableKey.currentContext!.findRenderObject()! as RenderEditable;
 
   @override
   TextEditingValue get textEditingValue => _value;
@@ -2319,11 +2343,11 @@ class EditableTextState extends State<EditableText> with AutomaticKeepAliveClien
 
   @override
   void bringIntoView(TextPosition position) {
-    final Rect localRect = renderEditable!.getLocalRectForCaret(position);
+    final Rect localRect = renderEditable.getLocalRectForCaret(position);
     final RevealedOffset targetOffset = _getOffsetToRevealCaret(localRect);
 
     _scrollController!.jumpTo(targetOffset.offset);
-    renderEditable!.showOnScreen(rect: targetOffset.rect);
+    renderEditable.showOnScreen(rect: targetOffset.rect);
   }
 
   /// Shows the selection toolbar at the location of the current cursor.

packages/flutter/lib/src/widgets/text_selection.dart

@@ -903,7 +903,7 @@ class TextSelectionGestureDetectorBuilder {
   /// The [RenderObject] of the [EditableText] for which the builder will
   /// provide a [TextSelectionGestureDetector].
   @protected
-  RenderEditable get renderEditable => editableText!.renderEditable!;
+  RenderEditable get renderEditable => editableText!.renderEditable;
 
   /// Handler for [TextSelectionGestureDetector.onTapDown].
   ///

packages/flutter/test/rendering/editable_test.dart

@@ -1003,6 +1003,59 @@ void main() {
     });
   });
 
+  group('getRectForComposingRange', () {
+    const TextSpan emptyTextSpan = TextSpan(text: '\u200e');
+    final TextSelectionDelegate delegate = FakeEditableTextState();
+    final RenderEditable editable = RenderEditable(
+      maxLines: null,
+      textAlign: TextAlign.start,
+      textDirection: TextDirection.ltr,
+      offset: ViewportOffset.zero(),
+      textSelectionDelegate: delegate,
+      startHandleLayerLink: LayerLink(),
+      endHandleLayerLink: LayerLink(),
+    );
+
+    test('returns null when no composing range', () {
+      editable.text = const TextSpan(text: '123');
+      editable.layout(const BoxConstraints.tightFor(width: 200));
+
+      // Invalid range.
+      expect(editable.getRectForComposingRange(const TextRange(start: -1, end: 2)), isNull);
+      // Collapsed range.
+      expect(editable.getRectForComposingRange(const TextRange.collapsed(2)), isNull);
+
+      // Empty Editable.
+      editable.text = emptyTextSpan;
+      editable.layout(const BoxConstraints.tightFor(width: 200));
+
+      expect(
+        editable.getRectForComposingRange(const TextRange(start: 0, end: 1)),
+        // On web this evaluates to a zero-width Rect.
+        anyOf(isNull, (Rect rect) => rect.width == 0));
+    });
+
+    test('more than 1 run on the same line', () {
+      const TextStyle tinyText = TextStyle(fontSize: 1, fontFamily: 'Ahem');
+      const TextStyle normalText = TextStyle(fontSize: 10, fontFamily: 'Ahem');
+      editable.text = TextSpan(
+        children: <TextSpan>[
+          const TextSpan(text: 'A', style: tinyText),
+          TextSpan(text: 'A' * 20, style: normalText),
+          const TextSpan(text: 'A', style: tinyText)
+        ],
+      );
+      // Give it a width that forces the editable to wrap.
+      editable.layout(const BoxConstraints.tightFor(width: 200));
+
+      final Rect composingRect = editable.getRectForComposingRange(const TextRange(start: 0, end: 20 + 2));
+
+      // Since the range covers an entire line, the Rect should also be almost
+      // as wide as the entire paragraph (give or take 1 character).
+      expect(composingRect?.width, greaterThan(200 - 10));
+    }, skip: isBrowser); // https://github.com/flutter/flutter/issues/66089
+  });
+
   group('previousCharacter', () {
     test('handles normal strings correctly', () {
       expect(RenderEditable.previousCharacter(8, '01234567'), 7);

packages/flutter/test/widgets/editable_text_test.dart

@@ -18,6 +18,30 @@ import '../rendering/mock_canvas.dart';
 import 'editable_text_utils.dart';
 import 'semantics_tester.dart';
 
+Matcher matchesMethodCall(String method, { dynamic args }) => _MatchesMethodCall(method, arguments: args == null ? null : wrapMatcher(args));
+
+class _MatchesMethodCall extends Matcher {
+  const _MatchesMethodCall(this.name, {this.arguments});
+
+  final String name;
+  final Matcher arguments;
+
+  @override
+  bool matches(dynamic item, Map<dynamic, dynamic> matchState) {
+    if (item is MethodCall && item.method == name)
+      return arguments?.matches(item.arguments, matchState) ?? true;
+    return false;
+  }
+
+  @override
+  Description describe(Description description) {
+    final Description newDescription = description.add('has method name: ').addDescriptionOf(name);
+    if (arguments != null)
+        newDescription.add(' with arguments: ').addDescriptionOf(arguments);
+    return newDescription;
+  }
+}
+
 TextEditingController controller;
 final FocusNode focusNode = FocusNode(debugLabel: 'EditableText Node');
 final FocusScopeNode focusScopeNode = FocusScopeNode(debugLabel: 'EditableText Scope Node');
@@ -3326,6 +3350,98 @@ void main() {
     );
   });
 
+  group('setMarkedTextRect', () {
+    Widget builder() {
+      return MaterialApp(
+        home: MediaQuery(
+          data: const MediaQueryData(devicePixelRatio: 1.0),
+          child: Directionality(
+            textDirection: TextDirection.ltr,
+            child: Center(
+              child: Material(
+                child: EditableText(
+                  backgroundCursorColor: Colors.grey,
+                  controller: controller,
+                  focusNode: FocusNode(),
+                  style: textStyle,
+                  cursorColor: Colors.blue,
+                  selectionControls: materialTextSelectionControls,
+                  keyboardType: TextInputType.text,
+                  onChanged: (String value) {},
+                ),
+              ),
+            ),
+          ),
+        ),
+      );
+    }
+
+    testWidgets(
+      'called when the composing range changes',
+      (WidgetTester tester) async {
+        controller.value = TextEditingValue(text: 'a' * 100);
+        await tester.pumpWidget(builder());
+        await tester.showKeyboard(find.byType(EditableText));
+
+        expect(tester.testTextInput.log, contains(
+          matchesMethodCall(
+            'TextInput.setMarkedTextRect',
+            args: allOf(
+              // No composing text so the width should not be too wide because
+              // it's empty.
+              containsPair('width', lessThanOrEqualTo(5)),
+              containsPair('x', lessThanOrEqualTo(1)),
+            ),
+          ),
+        ));
+
+        tester.testTextInput.log.clear();
+
+        controller.value = TextEditingValue(text: 'a' * 100, composing: const TextRange(start: 0, end: 10));
+        await tester.pump();
+
+        expect(tester.testTextInput.log, contains(
+          matchesMethodCall(
+            'TextInput.setMarkedTextRect',
+            // Now the composing range is not empty.
+            args: containsPair('width', greaterThanOrEqualTo(10)),
+          ),
+        ));
+    }, skip: isBrowser); // Related to https://github.com/flutter/flutter/issues/66089
+
+    testWidgets(
+      'only send updates when necessary',
+      (WidgetTester tester) async {
+        controller.value = TextEditingValue(text: 'a' * 100, composing: const TextRange(start: 0, end: 10));
+        await tester.pumpWidget(builder());
+        await tester.showKeyboard(find.byType(EditableText));
+
+        expect(tester.testTextInput.log, contains(matchesMethodCall('TextInput.setMarkedTextRect')));
+
+        tester.testTextInput.log.clear();
+
+        // Should not send updates every frame.
+        await tester.pump();
+
+        expect(tester.testTextInput.log, isNot(contains(matchesMethodCall('TextInput.setMarkedTextRect'))));
+    });
+
+    testWidgets(
+      'does not throw when sending infinite Rect',
+      (WidgetTester tester) async {
+        controller.value = TextEditingValue(text: 'a' * 100, composing: const TextRange(start: 0, end: 10));
+        await tester.pumpWidget(FittedBox(child: SizedBox.fromSize(size: Size.zero, child: builder())));
+        await tester.showKeyboard(find.byType(EditableText));
+        expect(tester.testTextInput.log, contains(matchesMethodCall('TextInput.setMarkedTextRect', args: <String, dynamic> {
+          'width': -1,
+          'height': -1,
+          'x': 0,
+          'y': 0,
+        })));
+    });
+  });
+
+
   testWidgets('custom keyboardAppearance is respected', (WidgetTester tester) async {
     // Regression test for https://github.com/flutter/flutter/issues/22212.
 
@@ -4645,17 +4761,16 @@ void main() {
       'TextInput.setClient',
       'TextInput.show',
       'TextInput.setEditableSizeAndTransform',
+      'TextInput.setMarkedTextRect',
       'TextInput.setStyle',
       'TextInput.setEditingState',
       'TextInput.setEditingState',
       'TextInput.show',
     ];
-    expect(tester.testTextInput.log.length, 7);
-    int index = 0;
-    for (final MethodCall m in tester.testTextInput.log) {
-      expect(m.method, logOrder[index]);
-      index++;
-    }
+    expect(
+      tester.testTextInput.log.map((MethodCall m) => m.method),
+      logOrder,
+    );
   });
 
   testWidgets('setEditingState is not called when text changes', (WidgetTester tester) async {
@@ -4689,6 +4804,7 @@ void main() {
       'TextInput.setClient',
       'TextInput.show',
       'TextInput.setEditableSizeAndTransform',
+      'TextInput.setMarkedTextRect',
       'TextInput.setStyle',
       'TextInput.setEditingState',
       'TextInput.setEditingState',
@@ -4736,18 +4852,18 @@ void main() {
       'TextInput.setClient',
       'TextInput.show',
       'TextInput.setEditableSizeAndTransform',
+      'TextInput.setMarkedTextRect',
       'TextInput.setStyle',
       'TextInput.setEditingState',
       'TextInput.setEditingState',
       'TextInput.show',
       'TextInput.setEditingState',
     ];
-    expect(tester.testTextInput.log.length, logOrder.length);
-    int index = 0;
-    for (final MethodCall m in tester.testTextInput.log) {
-      expect(m.method, logOrder[index]);
-      index++;
-    }
+
+    expect(
+      tester.testTextInput.log.map((MethodCall m) => m.method),
+      logOrder,
+    );
     expect(tester.testTextInput.editingState['text'], 'flutter is the best!...');
   });