Synchronize modifier keys in RawKeyboard.keysPressed with modifier flags on events. (#43948)

Currently, we listen to keyboard events to find out which keys should be represented in RawKeyboard.instance.keysPressed, but that's not sufficient to represent the physical state of the keys, since modifier keys could have been pressed when the overall app did not have keyboard focus (especially on desktop platforms).

This PR synchronizes the list of modifier keys in keysPressed with the modifier key flags that are present in the raw key event so that they can be relied upon to represent the current state of the keyboard. When synchronizing these states, we don't send any new key events, since they didn't happen when the app had keyboard focus, but if you ask "is this key down", we'll give the right answer

packages/flutter/lib/src/services/raw_keyboard.dart

@@ -3,6 +3,8 @@
 // found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:io';
+import 'dart:ui';
 
 import 'package:flutter/foundation.dart';
 
@@ -295,8 +297,8 @@ abstract class RawKeyEvent extends Diagnosticable {
         );
         break;
       default:
-        // We don't yet implement raw key events on iOS or other platforms, but
-        // we don't hit this exception because the engine never sends us these
+        // Raw key events are not yet implemented  on iOS or other platforms,
+        // but this exception isn't hit, because the engine never sends these
         // messages.
         throw FlutterError('Unknown keymap for key events: $keymap');
     }
@@ -506,6 +508,9 @@ class RawKeyboard {
     if (event is RawKeyUpEvent) {
       _keysPressed.remove(event.logicalKey);
     }
+    // Make sure that the modifiers reflect reality, in case a modifier key was
+    // pressed/released while the app didn't have focus.
+    _synchronizeModifiers(event);
     if (_listeners.isEmpty) {
       return;
     }
@@ -516,6 +521,68 @@ class RawKeyboard {
     }
   }
 
+  static final Map<_ModifierSidePair, Set<LogicalKeyboardKey>> _modifierKeyMap = <_ModifierSidePair, Set<LogicalKeyboardKey>>{
+    const _ModifierSidePair(ModifierKey.altModifier, KeyboardSide.left): <LogicalKeyboardKey>{LogicalKeyboardKey.altLeft},
+    const _ModifierSidePair(ModifierKey.altModifier, KeyboardSide.right): <LogicalKeyboardKey>{LogicalKeyboardKey.altRight},
+    const _ModifierSidePair(ModifierKey.altModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.altLeft, LogicalKeyboardKey.altRight},
+    const _ModifierSidePair(ModifierKey.altModifier, KeyboardSide.any): <LogicalKeyboardKey>{LogicalKeyboardKey.altLeft},
+    const _ModifierSidePair(ModifierKey.shiftModifier, KeyboardSide.left): <LogicalKeyboardKey>{LogicalKeyboardKey.shiftLeft},
+    const _ModifierSidePair(ModifierKey.shiftModifier, KeyboardSide.right): <LogicalKeyboardKey>{LogicalKeyboardKey.shiftRight},
+    const _ModifierSidePair(ModifierKey.shiftModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.shiftLeft, LogicalKeyboardKey.shiftRight},
+    const _ModifierSidePair(ModifierKey.shiftModifier, KeyboardSide.any): <LogicalKeyboardKey>{LogicalKeyboardKey.shiftLeft},
+    const _ModifierSidePair(ModifierKey.controlModifier, KeyboardSide.left): <LogicalKeyboardKey>{LogicalKeyboardKey.controlLeft},
+    const _ModifierSidePair(ModifierKey.controlModifier, KeyboardSide.right): <LogicalKeyboardKey>{LogicalKeyboardKey.controlRight},
+    const _ModifierSidePair(ModifierKey.controlModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.controlLeft, LogicalKeyboardKey.controlRight},
+    const _ModifierSidePair(ModifierKey.controlModifier, KeyboardSide.any): <LogicalKeyboardKey>{LogicalKeyboardKey.controlLeft},
+    const _ModifierSidePair(ModifierKey.metaModifier, KeyboardSide.left): <LogicalKeyboardKey>{LogicalKeyboardKey.metaLeft},
+    const _ModifierSidePair(ModifierKey.metaModifier, KeyboardSide.right): <LogicalKeyboardKey>{LogicalKeyboardKey.metaRight},
+    const _ModifierSidePair(ModifierKey.metaModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.metaLeft, LogicalKeyboardKey.metaRight},
+    const _ModifierSidePair(ModifierKey.metaModifier, KeyboardSide.any): <LogicalKeyboardKey>{LogicalKeyboardKey.metaLeft},
+    const _ModifierSidePair(ModifierKey.capsLockModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.capsLock},
+    const _ModifierSidePair(ModifierKey.numLockModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.numLock},
+    const _ModifierSidePair(ModifierKey.scrollLockModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.scrollLock},
+    const _ModifierSidePair(ModifierKey.functionModifier, KeyboardSide.all): <LogicalKeyboardKey>{LogicalKeyboardKey.fn},
+    // The symbolModifier doesn't have a key representation on any of the
+    // platforms, so don't map it here.
+  };
+
+  // The list of all modifier keys that are represented in modifier key bit
+  // masks on all platforms, so that they can be cleared out of pressedKeys when
+  // synchronizing.
+  static final Set<LogicalKeyboardKey> _allModifiers = <LogicalKeyboardKey>{
+    LogicalKeyboardKey.altLeft,
+    LogicalKeyboardKey.altRight,
+    LogicalKeyboardKey.shiftLeft,
+    LogicalKeyboardKey.shiftRight,
+    LogicalKeyboardKey.controlLeft,
+    LogicalKeyboardKey.controlRight,
+    LogicalKeyboardKey.metaLeft,
+    LogicalKeyboardKey.metaRight,
+    LogicalKeyboardKey.capsLock,
+    LogicalKeyboardKey.numLock,
+    LogicalKeyboardKey.scrollLock,
+    LogicalKeyboardKey.fn,
+  };
+
+  void _synchronizeModifiers(RawKeyEvent event) {
+    final Map<ModifierKey, KeyboardSide> modifiersPressed = event.data.modifiersPressed;
+    final Set<LogicalKeyboardKey> modifierKeys = <LogicalKeyboardKey>{};
+    for (ModifierKey key in modifiersPressed.keys) {
+      final Set<LogicalKeyboardKey> mappedKeys = _modifierKeyMap[_ModifierSidePair(key, modifiersPressed[key])];
+      assert(mappedKeys != null,
+        'Platform key support for ${Platform.operatingSystem} is '
+        'producing unsupported modifier combinations.');
+      modifierKeys.addAll(mappedKeys);
+    }
+    // Don't send any key events for these changes, since there *should* be
+    // separate events for each modifier key down/up that occurs while the app
+    // has focus. This is just to synchronize the modifier keys when they are
+    // pressed/released while the app doesn't have focus, to make sure that
+    // _keysPressed reflects reality at all times.
+    _keysPressed.removeAll(_allModifiers);
+    _keysPressed.addAll(modifierKeys);
+  }
+
   final Set<LogicalKeyboardKey> _keysPressed = <LogicalKeyboardKey>{};
 
   /// Returns the set of keys currently pressed.
@@ -531,3 +598,20 @@ class RawKeyboard {
     _keysPressed.clear();
   }
 }
+
+class _ModifierSidePair extends Object {
+  const _ModifierSidePair(this.modifier, this.side);
+
+  final ModifierKey modifier;
+  final KeyboardSide side;
+
+  @override
+  bool operator ==(dynamic other) {
+    return runtimeType == other.runtimeType
+        && modifier == other.modifier
+        && side == other.side;
+  }
+
+  @override
+  int get hashCode => hashValues(modifier, side);
+}

packages/flutter/lib/src/services/raw_keyboard_macos.dart

@@ -113,8 +113,8 @@ class RawKeyEventDataMacOs extends RawKeyEventData {
       );
     }
 
-    // This is a non-printable key that we don't know about, so we mint a new
-    // code with the autogenerated bit set.
+    // This is a non-printable key that is unrecognized, so a new code is minted
+    // with the autogenerated bit set.
     const int macOsKeyIdPlane = 0x00500000000;
 
     return LogicalKeyboardKey(
@@ -154,13 +154,15 @@ class RawKeyEventDataMacOs extends RawKeyEventData {
         return _isLeftRightModifierPressed(side, independentModifier & modifierCommand, modifierLeftCommand, modifierRightCommand);
       case ModifierKey.capsLockModifier:
         return independentModifier & modifierCapsLock != 0;
-      case ModifierKey.numLockModifier:
-        return independentModifier & modifierNumericPad != 0;
+    // On macOS, the function modifier bit is set for any function key, like F1,
+    // F2, etc., but the meaning of ModifierKey.modifierFunction in Flutter is
+    // that of the Fn modifier key, so there's no good way to emulate that on
+    // macOS.
       case ModifierKey.functionModifier:
-        return independentModifier & modifierFunction != 0;
+      case ModifierKey.numLockModifier:
       case ModifierKey.symbolModifier:
       case ModifierKey.scrollLockModifier:
-        // These are not used in macOS keyboards.
+        // These modifier masks are not used in macOS keyboards.
         return false;
     }
     return false;

packages/flutter/test/widgets/focus_traversal_test.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 import 'dart:ui';
 
+import 'package:flutter/foundation.dart';
 import 'package:flutter/painting.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter_test/flutter_test.dart';
@@ -751,7 +752,7 @@ void main() {
         }
       }
 
-      // Test to make sure that we follow the same path backwards and forwards.
+      // Test to make sure that the same path is followed backwards and forwards.
       await tester.pump();
       expectState(<bool>[null, null, null, null, true, null]);
       clear();
@@ -1004,7 +1005,7 @@ void main() {
       expect(Focus.of(lowerLeftKey.currentContext).hasPrimaryFocus, isTrue);
       await tester.sendKeyEvent(LogicalKeyboardKey.arrowUp);
       expect(Focus.of(upperLeftKey.currentContext).hasPrimaryFocus, isTrue);
-    });
+    }, skip: kIsWeb);
     testWidgets('Arrow focus traversal actions can be re-enabled for text fields.', (WidgetTester tester) async {
       final GlobalKey upperLeftKey = GlobalKey(debugLabel: 'upperLeftKey');
       final GlobalKey upperRightKey = GlobalKey(debugLabel: 'upperRightKey');

packages/flutter_test/lib/src/event_simulation.dart

@@ -8,8 +8,11 @@ import 'dart:io';
 import 'package:flutter/services.dart';
 import 'test_async_utils.dart';
 
-// For the synonym keys, just return the left version of it.
-LogicalKeyboardKey _getSynonym(LogicalKeyboardKey origKey) {
+/// A class that serves as a namespace for a bunch of keyboard-key generation
+/// utilities.
+class KeyEventSimulator {
+  // Look up a synonym key, and just return the left version of it.
+  static LogicalKeyboardKey _getKeySynonym(LogicalKeyboardKey origKey) {
     if (origKey == LogicalKeyboardKey.shift) {
       return LogicalKeyboardKey.shiftLeft;
     }
@@ -23,9 +26,9 @@ LogicalKeyboardKey _getSynonym(LogicalKeyboardKey origKey) {
       return LogicalKeyboardKey.controlLeft;
     }
     return origKey;
-}
+  }
 
-bool _osIsSupported(String platform) {
+  static bool _osIsSupported(String platform) {
     switch (platform) {
       case 'android':
       case 'fuchsia':
@@ -34,9 +37,9 @@ bool _osIsSupported(String platform) {
         return true;
     }
     return false;
-}
+  }
 
-int _getScanCode(LogicalKeyboardKey key, String platform) {
+  static int _getScanCode(LogicalKeyboardKey key, String platform) {
     assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
     int scanCode;
     Map<int, PhysicalKeyboardKey> map;
@@ -61,9 +64,9 @@ int _getScanCode(LogicalKeyboardKey key, String platform) {
       }
     }
     return scanCode;
-}
+  }
 
-int _getKeyCode(LogicalKeyboardKey key, String platform) {
+  static int _getKeyCode(LogicalKeyboardKey key, String platform) {
     assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
     int keyCode;
     Map<int, LogicalKeyboardKey> map;
@@ -88,12 +91,13 @@ int _getKeyCode(LogicalKeyboardKey key, String platform) {
       }
     }
     return keyCode;
-}
+  }
 
-Map<String, dynamic> _getKeyData(LogicalKeyboardKey key, {String platform, bool isDown = true}) {
+  /// Get a raw key data map given a [LogicalKeyboardKey] and a platform.
+  static Map<String, dynamic> getKeyData(LogicalKeyboardKey key, {String platform, bool isDown = true}) {
     assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
 
-  key = _getSynonym(key);
+    key = _getKeySynonym(key);
 
     assert(key.debugName != null);
     final int keyCode = platform == 'macos' ? -1 : _getKeyCode(key, platform);
@@ -133,9 +137,9 @@ Map<String, dynamic> _getKeyData(LogicalKeyboardKey key, {String platform, bool
         break;
     }
     return result;
-}
+  }
 
-int _getAndroidModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
+  static int _getAndroidModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
     int result = 0;
     final Set<LogicalKeyboardKey> pressed = RawKeyboard.instance.keysPressed;
     if (isDown) {
@@ -180,9 +184,9 @@ int _getAndroidModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
       result |= RawKeyEventDataAndroid.modifierCapsLock;
     }
     return result;
-}
+  }
 
-int _getGlfwModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
+  static int _getGlfwModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
     int result = 0;
     final Set<LogicalKeyboardKey> pressed = RawKeyboard.instance.keysPressed;
     if (isDown) {
@@ -206,9 +210,9 @@ int _getGlfwModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
       result |= GLFWKeyHelper.modifierCapsLock;
     }
     return result;
-}
+  }
 
-int _getFuchsiaModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
+  static int _getFuchsiaModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
     int result = 0;
     final Set<LogicalKeyboardKey> pressed = RawKeyboard.instance.keysPressed;
     if (isDown) {
@@ -244,9 +248,9 @@ int _getFuchsiaModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
       result |= RawKeyEventDataFuchsia.modifierCapsLock;
     }
     return result;
-}
+  }
 
-int _getMacOsModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
+  static int _getMacOsModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
     int result = 0;
     final Set<LogicalKeyboardKey> pressed = RawKeyboard.instance.keysPressed;
     if (isDown) {
@@ -311,6 +315,63 @@ int _getMacOsModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
       result |= RawKeyEventDataMacOs.modifierCapsLock;
     }
     return result;
+  }
+
+  /// Simulates sending a hardware key down event through the system channel.
+  ///
+  /// This only simulates key presses coming from a physical keyboard, not from a
+  /// soft keyboard.
+  ///
+  /// Specify `platform` as one of the platforms allowed in
+  /// [Platform.operatingSystem] to make the event appear to be from that type of
+  /// system. Defaults to the operating system that the test is running on. Some
+  /// platforms (e.g. Windows, iOS) are not yet supported.
+  ///
+  /// Keys that are down when the test completes are cleared after each test.
+  ///
+  /// See also:
+  ///
+  ///  - [simulateKeyUpEvent] to simulate the corresponding key up event.
+  static Future<void> simulateKeyDownEvent(LogicalKeyboardKey key, {String platform}) async {
+    return TestAsyncUtils.guard<void>(() async {
+      platform ??= Platform.operatingSystem;
+      assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
+
+      final Map<String, dynamic> data = getKeyData(key, platform: platform, isDown: true);
+      await ServicesBinding.instance.defaultBinaryMessenger.handlePlatformMessage(
+        SystemChannels.keyEvent.name,
+        SystemChannels.keyEvent.codec.encodeMessage(data),
+            (ByteData data) {},
+      );
+    });
+  }
+
+  /// Simulates sending a hardware key up event through the system channel.
+  ///
+  /// This only simulates key presses coming from a physical keyboard, not from a
+  /// soft keyboard.
+  ///
+  /// Specify `platform` as one of the platforms allowed in
+  /// [Platform.operatingSystem] to make the event appear to be from that type of
+  /// system. Defaults to the operating system that the test is running on. Some
+  /// platforms (e.g. Windows, iOS) are not yet supported.
+  ///
+  /// See also:
+  ///
+  ///  - [simulateKeyDownEvent] to simulate the corresponding key down event.
+  static Future<void> simulateKeyUpEvent(LogicalKeyboardKey key, {String platform}) async {
+    return TestAsyncUtils.guard<void>(() async {
+      platform ??= Platform.operatingSystem;
+      assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
+
+      final Map<String, dynamic> data = getKeyData(key, platform: platform, isDown: false);
+      await ServicesBinding.instance.defaultBinaryMessenger.handlePlatformMessage(
+        SystemChannels.keyEvent.name,
+        SystemChannels.keyEvent.codec.encodeMessage(data),
+            (ByteData data) {},
+      );
+    });
+  }
 }
 
 /// Simulates sending a hardware key down event through the system channel.
@@ -328,18 +389,8 @@ int _getMacOsModifierFlags(LogicalKeyboardKey newKey, bool isDown) {
 /// See also:
 ///
 ///  - [simulateKeyUpEvent] to simulate the corresponding key up event.
-Future<void> simulateKeyDownEvent(LogicalKeyboardKey key, {String platform}) async {
-  return TestAsyncUtils.guard<void>(() async {
-    platform ??= Platform.operatingSystem;
-    assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
-
-    final Map<String, dynamic> data = _getKeyData(key, platform: platform, isDown: true);
-    await ServicesBinding.instance.defaultBinaryMessenger.handlePlatformMessage(
-      SystemChannels.keyEvent.name,
-      SystemChannels.keyEvent.codec.encodeMessage(data),
-      (ByteData data) {},
-    );
-  });
+Future<void> simulateKeyDownEvent(LogicalKeyboardKey key, {String platform}) {
+  return KeyEventSimulator.simulateKeyDownEvent(key, platform: platform);
 }
 
 /// Simulates sending a hardware key up event through the system channel.
@@ -355,16 +406,6 @@ Future<void> simulateKeyDownEvent(LogicalKeyboardKey key, {String platform}) asy
 /// See also:
 ///
 ///  - [simulateKeyDownEvent] to simulate the corresponding key down event.
-Future<void> simulateKeyUpEvent(LogicalKeyboardKey key, {String platform}) async {
-  return TestAsyncUtils.guard<void>(() async {
-    platform ??= Platform.operatingSystem;
-    assert(_osIsSupported(platform), 'Platform $platform not supported for key simulation');
-
-    final Map<String, dynamic> data = _getKeyData(key, platform: platform, isDown: false);
-    await ServicesBinding.instance.defaultBinaryMessenger.handlePlatformMessage(
-      SystemChannels.keyEvent.name,
-      SystemChannels.keyEvent.codec.encodeMessage(data),
-      (ByteData data) {},
-    );
-  });
+Future<void> simulateKeyUpEvent(LogicalKeyboardKey key, {String platform}) {
+  return KeyEventSimulator.simulateKeyUpEvent(key, platform: platform);
 }