Reland Characters Usage (#59778)

Use Dart's characters package to fix user-facing grapheme cluster bugs.

packages/flutter/lib/src/material/text_field.dart

@@ -6,6 +6,7 @@
 
 import 'dart:ui' as ui show BoxHeightStyle, BoxWidthStyle;
 
+import 'package:characters/characters.dart';
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/rendering.dart';
 import 'package:flutter/services.dart';
@@ -818,9 +819,9 @@ class _TextFieldState extends State<TextField> implements TextSelectionGestureDe
 
   bool get _isEnabled =>  widget.enabled ?? widget.decoration?.enabled ?? true;
 
-  int get _currentLength => _effectiveController.value.text.runes.length;
+  int get _currentLength => _effectiveController.value.text.characters.length;
 
-  bool get _hasIntrinsicError => widget.maxLength != null && widget.maxLength > 0 && _effectiveController.value.text.runes.length > widget.maxLength;
+  bool get _hasIntrinsicError => widget.maxLength != null && widget.maxLength > 0 && _effectiveController.value.text.characters.length > widget.maxLength;
 
   bool get _hasError => widget.decoration?.errorText != null || _hasIntrinsicError;
 

packages/flutter/lib/src/painting/text_painter.dart

@@ -603,7 +603,7 @@ class TextPainter {
 
   // Complex glyphs can be represented by two or more UTF16 codepoints. This
   // checks if the value represents a UTF16 glyph by itself or is a 'surrogate'.
-  bool _isUtf16Surrogate(int value) {
+  static bool _isUtf16Surrogate(int value) {
     return value & 0xF800 == 0xD800;
   }
 
@@ -611,7 +611,7 @@ class TextPainter {
   // up zero space and do not have valid bounding boxes around them.
   //
   // We do not directly use the [Unicode] constants since they are strings.
-  bool _isUnicodeDirectionality(int value) {
+  static bool _isUnicodeDirectionality(int value) {
     return value == 0x200F || value == 0x200E;
   }
 
@@ -640,15 +640,13 @@ class TextPainter {
 
   // Get the Rect of the cursor (in logical pixels) based off the near edge
   // of the character upstream from the given string offset.
-  // TODO(garyq): Use actual extended grapheme cluster length instead of
-  // an increasing cluster length amount to achieve deterministic performance.
   Rect _getRectFromUpstream(int offset, Rect caretPrototype) {
     final String flattenedText = _text.toPlainText(includePlaceholders: false);
     final int prevCodeUnit = _text.codeUnitAt(max(0, offset - 1));
     if (prevCodeUnit == null)
       return null;
 
-    // Check for multi-code-unit glyphs such as emojis or zero width joiner
+    // Check for multi-code-unit glyphs such as emojis or zero width joiner.
     final bool needsSearch = _isUtf16Surrogate(prevCodeUnit) || _text.codeUnitAt(offset) == _zwjUtf16 || _isUnicodeDirectionality(prevCodeUnit);
     int graphemeClusterLength = needsSearch ? 2 : 1;
     List<TextBox> boxes = <TextBox>[];
@@ -691,8 +689,6 @@ class TextPainter {
 
   // Get the Rect of the cursor (in logical pixels) based off the near edge
   // of the character downstream from the given string offset.
-  // TODO(garyq): Use actual extended grapheme cluster length instead of
-  // an increasing cluster length amount to achieve deterministic performance.
   Rect _getRectFromDownstream(int offset, Rect caretPrototype) {
     final String flattenedText = _text.toPlainText(includePlaceholders: false);
     // We cap the offset at the final index of the _text.

packages/flutter/lib/src/services/text_formatter.dart

@@ -6,6 +6,7 @@
 
 import 'dart:math' as math;
 
+import 'package:characters/characters.dart';
 import 'package:flutter/foundation.dart';
 
 import 'text_editing.dart';
@@ -344,24 +345,24 @@ class LengthLimitingTextInputFormatter extends TextInputFormatter {
   /// characters.
   final int maxLength;
 
-  // TODO(justinmc): This should be updated to use characters instead of runes,
-  // see the comment in formatEditUpdate.
-  /// Truncate the given TextEditingValue to maxLength runes.
+  /// Truncate the given TextEditingValue to maxLength characters.
+  ///
+  /// See also:
+  ///  * [Dart's characters package](https://pub.dev/packages/characters).
+  ///  * [Dart's documenetation on runes and grapheme clusters](https://dart.dev/guides/language/language-tour#runes-and-grapheme-clusters).
   @visibleForTesting
   static TextEditingValue truncate(TextEditingValue value, int maxLength) {
-    final TextSelection newSelection = value.selection.copyWith(
-        baseOffset: math.min(value.selection.start, maxLength),
-        extentOffset: math.min(value.selection.end, maxLength),
-    );
-    final RuneIterator iterator = RuneIterator(value.text);
-    if (iterator.moveNext())
-      for (int count = 0; count < maxLength; ++count)
-        if (!iterator.moveNext())
-          break;
-    final String truncated = value.text.substring(0, iterator.rawIndex);
+    final CharacterRange iterator = CharacterRange(value.text);
+    if (value.text.characters.length > maxLength) {
+      iterator.expandNext(maxLength);
+    }
+    final String truncated = iterator.current;
     return TextEditingValue(
       text: truncated,
-      selection: newSelection,
+      selection: value.selection.copyWith(
+        baseOffset: math.min(value.selection.start, truncated.length),
+        extentOffset: math.min(value.selection.end, truncated.length),
+      ),
       composing: TextRange.empty,
     );
   }
@@ -371,18 +372,10 @@ class LengthLimitingTextInputFormatter extends TextInputFormatter {
     TextEditingValue oldValue, // unused.
     TextEditingValue newValue,
   ) {
-    // This does not count grapheme clusters (i.e. characters visible to the user),
-    // it counts Unicode runes, which leaves out a number of useful possible
-    // characters (like many emoji), so this will be inaccurate in the
-    // presence of those characters. The Dart lang bug
-    // https://github.com/dart-lang/sdk/issues/28404 has been filed to
-    // address this in Dart.
-    // TODO(justinmc): convert this to count actual characters using Dart's
-    // characters package (https://pub.dev/packages/characters).
-    if (maxLength != null && maxLength > 0 && newValue.text.runes.length > maxLength) {
+    if (maxLength != null && maxLength > 0 && newValue.text.characters.length > maxLength) {
       // If already at the maximum and tried to enter even more, keep the old
       // value.
-      if (oldValue.text.runes.length == maxLength) {
+      if (oldValue.text.characters.length == maxLength) {
         return oldValue;
       }
       return truncate(newValue, maxLength);

packages/flutter/test/material/text_field_test.dart

@@ -3518,6 +3518,36 @@ void main() {
     expect(textController.text, '0123456789');
   });
 
+  testWidgets('maxLength limits input with surrogate pairs.', (WidgetTester tester) async {
+    final TextEditingController textController = TextEditingController();
+
+    await tester.pumpWidget(boilerplate(
+      child: TextField(
+        controller: textController,
+        maxLength: 10,
+      ),
+    ));
+
+    const String surrogatePair = '😆';
+    await tester.enterText(find.byType(TextField), surrogatePair + '0123456789101112');
+    expect(textController.text, surrogatePair + '012345678');
+  });
+
+  testWidgets('maxLength limits input with grapheme clusters.', (WidgetTester tester) async {
+    final TextEditingController textController = TextEditingController();
+
+    await tester.pumpWidget(boilerplate(
+      child: TextField(
+        controller: textController,
+        maxLength: 10,
+      ),
+    ));
+
+    const String graphemeCluster = '👨‍👩‍👦';
+    await tester.enterText(find.byType(TextField), graphemeCluster + '0123456789101112');
+    expect(textController.text, graphemeCluster + '012345678');
+  });
+
   testWidgets('maxLength limits input in the center of a maxed-out field.', (WidgetTester tester) async {
     // Regression test for https://github.com/flutter/flutter/issues/37420.
     final TextEditingController textController = TextEditingController();
@@ -3645,6 +3675,96 @@ void main() {
     expect(counterTextWidget.style.color, isNot(equals(Colors.deepPurpleAccent)));
   });
 
+  testWidgets('maxLength shows warning when maxLengthEnforced is false with surrogate pairs.', (WidgetTester tester) async {
+    final TextEditingController textController = TextEditingController();
+    const TextStyle testStyle = TextStyle(color: Colors.deepPurpleAccent);
+
+    await tester.pumpWidget(boilerplate(
+      child: TextField(
+        decoration: const InputDecoration(errorStyle: testStyle),
+        controller: textController,
+        maxLength: 10,
+        maxLengthEnforced: false,
+      ),
+    ));
+
+    await tester.enterText(find.byType(TextField), '😆012345678910111');
+    await tester.pump();
+
+    expect(textController.text, '😆012345678910111');
+    expect(find.text('16/10'), findsOneWidget);
+    Text counterTextWidget = tester.widget(find.text('16/10'));
+    expect(counterTextWidget.style.color, equals(Colors.deepPurpleAccent));
+
+    await tester.enterText(find.byType(TextField), '😆012345678');
+    await tester.pump();
+
+    expect(textController.text, '😆012345678');
+    expect(find.text('10/10'), findsOneWidget);
+    counterTextWidget = tester.widget(find.text('10/10'));
+    expect(counterTextWidget.style.color, isNot(equals(Colors.deepPurpleAccent)));
+  });
+
+  testWidgets('maxLength shows warning when maxLengthEnforced is false with grapheme clusters.', (WidgetTester tester) async {
+    final TextEditingController textController = TextEditingController();
+    const TextStyle testStyle = TextStyle(color: Colors.deepPurpleAccent);
+
+    await tester.pumpWidget(boilerplate(
+      child: TextField(
+        decoration: const InputDecoration(errorStyle: testStyle),
+        controller: textController,
+        maxLength: 10,
+        maxLengthEnforced: false,
+      ),
+    ));
+
+    await tester.enterText(find.byType(TextField), '👨‍👩‍👦012345678910111');
+    await tester.pump();
+
+    expect(textController.text, '👨‍👩‍👦012345678910111');
+    expect(find.text('16/10'), findsOneWidget);
+    Text counterTextWidget = tester.widget(find.text('16/10'));
+    expect(counterTextWidget.style.color, equals(Colors.deepPurpleAccent));
+
+    await tester.enterText(find.byType(TextField), '👨‍👩‍👦012345678');
+    await tester.pump();
+
+    expect(textController.text, '👨‍👩‍👦012345678');
+    expect(find.text('10/10'), findsOneWidget);
+    counterTextWidget = tester.widget(find.text('10/10'));
+    expect(counterTextWidget.style.color, isNot(equals(Colors.deepPurpleAccent)));
+  });
+
+  testWidgets('maxLength limits input with surrogate pairs.', (WidgetTester tester) async {
+    final TextEditingController textController = TextEditingController();
+
+    await tester.pumpWidget(boilerplate(
+      child: TextField(
+        controller: textController,
+        maxLength: 10,
+      ),
+    ));
+
+    const String surrogatePair = '😆';
+    await tester.enterText(find.byType(TextField), surrogatePair + '0123456789101112');
+    expect(textController.text, surrogatePair + '012345678');
+  });
+
+  testWidgets('maxLength limits input with grapheme clusters.', (WidgetTester tester) async {
+    final TextEditingController textController = TextEditingController();
+
+    await tester.pumpWidget(boilerplate(
+      child: TextField(
+        controller: textController,
+        maxLength: 10,
+      ),
+    ));
+
+    const String graphemeCluster = '👨‍👩‍👦';
+    await tester.enterText(find.byType(TextField), graphemeCluster + '0123456789101112');
+    expect(textController.text, graphemeCluster + '012345678');
+  });
+
   testWidgets('setting maxLength shows counter', (WidgetTester tester) async {
     await tester.pumpWidget(const MaterialApp(
       home: Material(
@@ -3665,6 +3785,48 @@ void main() {
     expect(find.text('5/10'), findsOneWidget);
   });
 
+  testWidgets('maxLength counter measures surrogate pairs as one character', (WidgetTester tester) async {
+    await tester.pumpWidget(const MaterialApp(
+      home: Material(
+        child: Center(
+            child: TextField(
+              maxLength: 10,
+            ),
+          ),
+        ),
+      ),
+    );
+
+    expect(find.text('0/10'), findsOneWidget);
+
+    const String surrogatePair = '😆';
+    await tester.enterText(find.byType(TextField), surrogatePair);
+    await tester.pump();
+
+    expect(find.text('1/10'), findsOneWidget);
+  });
+
+  testWidgets('maxLength counter measures grapheme clusters as one character', (WidgetTester tester) async {
+    await tester.pumpWidget(const MaterialApp(
+      home: Material(
+        child: Center(
+            child: TextField(
+              maxLength: 10,
+            ),
+          ),
+        ),
+      ),
+    );
+
+    expect(find.text('0/10'), findsOneWidget);
+
+    const String familyEmoji = '👨‍👩‍👦';
+    await tester.enterText(find.byType(TextField), familyEmoji);
+    await tester.pump();
+
+    expect(find.text('1/10'), findsOneWidget);
+  });
+
   testWidgets('setting maxLength to TextField.noMaxLength shows only entered length', (WidgetTester tester) async {
     await tester.pumpWidget(const MaterialApp(
       home: Material(

packages/flutter/test/painting/text_painter_test.dart

@@ -27,7 +27,8 @@ void main() {
     caretOffset = painter.getOffsetForCaret(ui.TextPosition(offset: text.length), ui.Rect.zero);
     expect(caretOffset.dx, painter.width);
 
-    // Check that getOffsetForCaret handles a character that is encoded as a surrogate pair.
+    // Check that getOffsetForCaret handles a character that is encoded as a
+    // surrogate pair.
     text = 'A\u{1F600}';
     painter.text = TextSpan(text: text);
     painter.layout();

packages/flutter/test/rendering/editable_test.dart

@@ -788,6 +788,106 @@ void main() {
     expect(delegate.textEditingValue.text, 'est');
   }, skip: kIsWeb);
 
+  test('arrow keys and delete handle surrogate pairs correctly', () async {
+    final TextSelectionDelegate delegate = FakeEditableTextState();
+    final ViewportOffset viewportOffset = ViewportOffset.zero();
+    TextSelection currentSelection;
+    final RenderEditable editable = RenderEditable(
+      backgroundCursorColor: Colors.grey,
+      selectionColor: Colors.black,
+      textDirection: TextDirection.ltr,
+      cursorColor: Colors.red,
+      offset: viewportOffset,
+      textSelectionDelegate: delegate,
+      onSelectionChanged: (TextSelection selection, RenderEditable renderObject, SelectionChangedCause cause) {
+        currentSelection = selection;
+      },
+      startHandleLayerLink: LayerLink(),
+      endHandleLayerLink: LayerLink(),
+      text: const TextSpan(
+        text: '0123😆6789',
+        style: TextStyle(
+          height: 1.0, fontSize: 10.0, fontFamily: 'Ahem',
+        ),
+      ),
+      selection: const TextSelection.collapsed(
+        offset: 0,
+      ),
+    );
+
+    layout(editable);
+    editable.hasFocus = true;
+
+    editable.selection = const TextSelection.collapsed(offset: 4);
+    pumpFrame();
+
+    await simulateKeyDownEvent(LogicalKeyboardKey.arrowRight, platform: 'android');
+    await simulateKeyUpEvent(LogicalKeyboardKey.arrowRight, platform: 'android');
+    expect(currentSelection.isCollapsed, true);
+    expect(currentSelection.baseOffset, 6);
+    editable.selection = currentSelection;
+
+    await simulateKeyDownEvent(LogicalKeyboardKey.arrowLeft, platform: 'android');
+    await simulateKeyUpEvent(LogicalKeyboardKey.arrowLeft, platform: 'android');
+    expect(currentSelection.isCollapsed, true);
+    expect(currentSelection.baseOffset, 4);
+    editable.selection = currentSelection;
+
+    await simulateKeyDownEvent(LogicalKeyboardKey.delete, platform: 'android');
+    await simulateKeyUpEvent(LogicalKeyboardKey.delete, platform: 'android');
+    expect(delegate.textEditingValue.text, '01236789');
+  }, skip: kIsWeb);
+
+  test('arrow keys and delete handle grapheme clusters correctly', () async {
+    final TextSelectionDelegate delegate = FakeEditableTextState();
+    final ViewportOffset viewportOffset = ViewportOffset.zero();
+    TextSelection currentSelection;
+    final RenderEditable editable = RenderEditable(
+      backgroundCursorColor: Colors.grey,
+      selectionColor: Colors.black,
+      textDirection: TextDirection.ltr,
+      cursorColor: Colors.red,
+      offset: viewportOffset,
+      textSelectionDelegate: delegate,
+      onSelectionChanged: (TextSelection selection, RenderEditable renderObject, SelectionChangedCause cause) {
+        currentSelection = selection;
+      },
+      startHandleLayerLink: LayerLink(),
+      endHandleLayerLink: LayerLink(),
+      text: const TextSpan(
+        text: '0123👨‍👩‍👦2345',
+        style: TextStyle(
+          height: 1.0, fontSize: 10.0, fontFamily: 'Ahem',
+        ),
+      ),
+      selection: const TextSelection.collapsed(
+        offset: 0,
+      ),
+    );
+
+    layout(editable);
+    editable.hasFocus = true;
+
+    editable.selection = const TextSelection.collapsed(offset: 4);
+    pumpFrame();
+
+    await simulateKeyDownEvent(LogicalKeyboardKey.arrowRight, platform: 'android');
+    await simulateKeyUpEvent(LogicalKeyboardKey.arrowRight, platform: 'android');
+    expect(currentSelection.isCollapsed, true);
+    expect(currentSelection.baseOffset, 12);
+    editable.selection = currentSelection;
+
+    await simulateKeyDownEvent(LogicalKeyboardKey.arrowLeft, platform: 'android');
+    await simulateKeyUpEvent(LogicalKeyboardKey.arrowLeft, platform: 'android');
+    expect(currentSelection.isCollapsed, true);
+    expect(currentSelection.baseOffset, 4);
+    editable.selection = currentSelection;
+
+    await simulateKeyDownEvent(LogicalKeyboardKey.delete, platform: 'android');
+    await simulateKeyUpEvent(LogicalKeyboardKey.delete, platform: 'android');
+    expect(delegate.textEditingValue.text, '01232345');
+  }, skip: kIsWeb);
+
   test('arrow keys and delete handle surrogate pairs correctly', () async {
     final TextSelectionDelegate delegate = FakeEditableTextState();
     final ViewportOffset viewportOffset = ViewportOffset.zero();
@@ -855,4 +955,98 @@ void main() {
       const TextSelection(baseOffset: 0, extentOffset: 1));
     expect(endpoints[0].point.dx, 0);
   });
+
+  group('nextCharacter', () {
+    test('handles normal strings correctly', () {
+      expect(RenderEditable.nextCharacter(0, '01234567'), 1);
+      expect(RenderEditable.nextCharacter(3, '01234567'), 4);
+      expect(RenderEditable.nextCharacter(7, '01234567'), 8);
+      expect(RenderEditable.nextCharacter(8, '01234567'), 8);
+    });
+
+    test('throws for invalid indices', () {
+      expect(() => RenderEditable.nextCharacter(-1, '01234567'), throwsAssertionError);
+      expect(() => RenderEditable.nextCharacter(9, '01234567'), throwsAssertionError);
+    });
+
+    test('skips spaces in normal strings when includeWhitespace is false', () {
+      expect(RenderEditable.nextCharacter(3, '0123 5678', false), 5);
+      expect(RenderEditable.nextCharacter(4, '0123 5678', false), 5);
+      expect(RenderEditable.nextCharacter(3, '0123      0123', false), 10);
+      expect(RenderEditable.nextCharacter(2, '0123      0123', false), 3);
+      expect(RenderEditable.nextCharacter(4, '0123      0123', false), 10);
+      expect(RenderEditable.nextCharacter(9, '0123      0123', false), 10);
+      expect(RenderEditable.nextCharacter(10, '0123      0123', false), 11);
+      // If the subsequent characters are all whitespace, it returns the length
+      // of the string.
+      expect(RenderEditable.nextCharacter(5, '0123      ', false), 10);
+    });
+
+    test('handles surrogate pairs correctly', () {
+      expect(RenderEditable.nextCharacter(3, '0123👨👩👦0123'), 4);
+      expect(RenderEditable.nextCharacter(4, '0123👨👩👦0123'), 6);
+      expect(RenderEditable.nextCharacter(5, '0123👨👩👦0123'), 6);
+      expect(RenderEditable.nextCharacter(6, '0123👨👩👦0123'), 8);
+      expect(RenderEditable.nextCharacter(7, '0123👨👩👦0123'), 8);
+      expect(RenderEditable.nextCharacter(8, '0123👨👩👦0123'), 10);
+      expect(RenderEditable.nextCharacter(9, '0123👨👩👦0123'), 10);
+      expect(RenderEditable.nextCharacter(10, '0123👨👩👦0123'), 11);
+    });
+
+    test('handles extended grapheme clusters correctly', () {
+      expect(RenderEditable.nextCharacter(3, '0123👨‍👩‍👦2345'), 4);
+      expect(RenderEditable.nextCharacter(4, '0123👨‍👩‍👦2345'), 12);
+      // Even when extent falls within an extended grapheme cluster, it still
+      // identifies the whole grapheme cluster.
+      expect(RenderEditable.nextCharacter(5, '0123👨‍👩‍👦2345'), 12);
+      expect(RenderEditable.nextCharacter(12, '0123👨‍👩‍👦2345'), 13);
+    });
+  });
+
+  group('previousCharacter', () {
+    test('handles normal strings correctly', () {
+      expect(RenderEditable.previousCharacter(8, '01234567'), 7);
+      expect(RenderEditable.previousCharacter(0, '01234567'), 0);
+      expect(RenderEditable.previousCharacter(1, '01234567'), 0);
+      expect(RenderEditable.previousCharacter(5, '01234567'), 4);
+      expect(RenderEditable.previousCharacter(8, '01234567'), 7);
+    });
+
+    test('throws for invalid indices', () {
+      expect(() => RenderEditable.previousCharacter(-1, '01234567'), throwsAssertionError);
+      expect(() => RenderEditable.previousCharacter(9, '01234567'), throwsAssertionError);
+    });
+
+    test('skips spaces in normal strings when includeWhitespace is false', () {
+      expect(RenderEditable.previousCharacter(10, '0123      0123', false), 3);
+      expect(RenderEditable.previousCharacter(11, '0123      0123', false), 10);
+      expect(RenderEditable.previousCharacter(9, '0123      0123', false), 3);
+      expect(RenderEditable.previousCharacter(4, '0123      0123', false), 3);
+      expect(RenderEditable.previousCharacter(3, '0123      0123', false), 2);
+      // If the previous characters are all whitespace, it returns zero.
+      expect(RenderEditable.previousCharacter(3, '          0123', false), 0);
+    });
+
+    test('handles surrogate pairs correctly', () {
+      expect(RenderEditable.previousCharacter(11, '0123👨👩👦0123'), 10);
+      expect(RenderEditable.previousCharacter(10, '0123👨👩👦0123'), 8);
+      expect(RenderEditable.previousCharacter(9, '0123👨👩👦0123'), 8);
+      expect(RenderEditable.previousCharacter(8, '0123👨👩👦0123'), 6);
+      expect(RenderEditable.previousCharacter(7, '0123👨👩👦0123'), 6);
+      expect(RenderEditable.previousCharacter(6, '0123👨👩👦0123'), 4);
+      expect(RenderEditable.previousCharacter(5, '0123👨👩👦0123'), 4);
+      expect(RenderEditable.previousCharacter(4, '0123👨👩👦0123'), 3);
+      expect(RenderEditable.previousCharacter(3, '0123👨👩👦0123'), 2);
+    });
+
+    test('handles extended grapheme clusters correctly', () {
+      expect(RenderEditable.previousCharacter(13, '0123👨‍👩‍👦2345'), 12);
+      // Even when extent falls within an extended grapheme cluster, it still
+      // identifies the whole grapheme cluster.
+      expect(RenderEditable.previousCharacter(12, '0123👨‍👩‍👦2345'), 4);
+      expect(RenderEditable.previousCharacter(11, '0123👨‍👩‍👦2345'), 4);
+      expect(RenderEditable.previousCharacter(5, '0123👨‍👩‍👦2345'), 4);
+      expect(RenderEditable.previousCharacter(4, '0123👨‍👩‍👦2345'), 3);
+    });
+  });
 }