Migrates android semanitcs integration to integration test (#127128)

I think the flake is due to setclipboard or semantics update race condition. I migrated the test to use integration test package which relies less on timing

fixes https://github.com/flutter/flutter/issues/124636

dev/devicelab/lib/framework/talkback.dart

+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+
+import 'package:path/path.dart' as path;
+import 'package:pub_semver/pub_semver.dart';
+
+String adbPath() {
+  final String? androidHome = io.Platform.environment['ANDROID_HOME'] ?? io.Platform.environment['ANDROID_SDK_ROOT'];
+  if (androidHome == null) {
+    return 'adb';
+  } else {
+    return path.join(androidHome, 'platform-tools', 'adb');
+  }
+}
+
+Future<Version> getTalkbackVersion() async {
+  final io.ProcessResult result = await io.Process.run(adbPath(), const <String>[
+    'shell',
+    'dumpsys',
+    'package',
+    'com.google.android.marvin.talkback',
+  ]);
+  if (result.exitCode != 0) {
+    throw Exception('Failed to get TalkBack version: ${result.stdout as String}\n${result.stderr as String}');
+  }
+  final List<String> lines = (result.stdout as String).split('\n');
+  String? version;
+  for (final String line in lines) {
+    if (line.contains('versionName')) {
+      version = line.replaceAll(RegExp(r'\s*versionName='), '');
+      break;
+    }
+  }
+  if (version == null) {
+    throw Exception('Unable to determine TalkBack version.');
+  }
+
+  // Android doesn't quite use semver, so convert the version string to semver form.
+  final RegExp startVersion = RegExp(r'(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)(\.(?<build>\d+))?');
+  final RegExpMatch? match = startVersion.firstMatch(version);
+  if (match == null) {
+    return Version(0, 0, 0);
+  }
+  return Version(
+    int.parse(match.namedGroup('major')!),
+    int.parse(match.namedGroup('minor')!),
+    int.parse(match.namedGroup('patch')!),
+    build: match.namedGroup('build'),
+  );
+}
+
+Future<void> enableTalkBack() async {
+  final io.Process run = await io.Process.start(adbPath(), const <String>[
+    'shell',
+    'settings',
+    'put',
+    'secure',
+    'enabled_accessibility_services',
+    'com.google.android.marvin.talkback/com.google.android.marvin.talkback.TalkBackService',
+  ]);
+  await run.exitCode;
+
+  print('TalkBack version is ${await getTalkbackVersion()}');
+}
+
+Future<void> disableTalkBack() async {
+  final io.Process run = await io.Process.start(adbPath(), const <String>[
+    'shell',
+    'settings',
+    'put',
+    'secure',
+    'enabled_accessibility_services',
+    'null',
+  ]);
+  await run.exitCode;
+}

dev/devicelab/lib/tasks/integration_tests.dart

@@ -4,6 +4,7 @@
 
 import '../framework/devices.dart';
 import '../framework/framework.dart';
+import '../framework/talkback.dart';
 import '../framework/task_result.dart';
 import '../framework/utils.dart';
 
@@ -74,9 +75,10 @@ TaskFunction createHybridAndroidViewsIntegrationTest() {
 }
 
 TaskFunction createAndroidSemanticsIntegrationTest() {
-  return DriverTest(
+  return IntegrationTest(
     '${flutterDirectory.path}/dev/integration_tests/android_semantics_testing',
-    'lib/main.dart',
+    'integration_test/main_test.dart',
+    withTalkBack: true,
   ).call;
 }
 
@@ -213,6 +215,7 @@ class IntegrationTest {
     this.testTarget, {
       this.extraOptions = const <String>[],
       this.createPlatforms = const <String>[],
+      this.withTalkBack = false,
     }
   );
 
@@ -220,6 +223,7 @@ class IntegrationTest {
   final String testTarget;
   final List<String> extraOptions;
   final List<String> createPlatforms;
+  final bool withTalkBack;
 
   Future<TaskResult> call() {
     return inDirectory<TaskResult>(testDirectory, () async {
@@ -237,6 +241,13 @@ class IntegrationTest {
         ]);
       }
 
+      if (withTalkBack) {
+        if (device is! AndroidDevice) {
+          return TaskResult.failure('A test that enables TalkBack can only be run on Android devices');
+        }
+        await enableTalkBack();
+      }
+
       final List<String> options = <String>[
         '-v',
         '-d',
@@ -246,6 +257,10 @@ class IntegrationTest {
       ];
       await flutter('test', options: options);
 
+      if (withTalkBack) {
+        await disableTalkBack();
+      }
+
       return TaskResult.success(null);
     });
   }

dev/integration_tests/android_semantics_testing/android/app/src/main/java/com/yourcompany/platforminteraction/MainActivity.java

@@ -20,7 +20,6 @@ import android.content.ClipData;
 import android.content.Context;
 import androidx.annotation.NonNull;
 
-import io.flutter.embedding.android.FlutterActivity;
 import io.flutter.embedding.engine.FlutterEngine;
 import io.flutter.embedding.android.FlutterActivity;
 import io.flutter.embedding.android.FlutterView;
@@ -123,11 +122,60 @@ public class MainActivity extends FlutterActivity {
         if (actionList.size() > 0) {
             ArrayList<Integer> actions = new ArrayList<>();
             for (AccessibilityNodeInfo.AccessibilityAction action : actionList) {
-                actions.add(action.getId());
+                if (kIdByAction.containsKey(action)) {
+                    actions.add(kIdByAction.get(action).intValue());
+                }
             }
             result.put("actions", actions);
         }
         return result;
     }
   }
+
+  // These indices need to be in sync with android_semantics_testing/lib/src/constants.dart
+  static final int kFocusIndex = 1 << 0;
+  static final int kClearFocusIndex = 1 << 1;
+  static final int kSelectIndex = 1 << 2;
+  static final int kClearSelectionIndex = 1 << 3;
+  static final int kClickIndex = 1 << 4;
+  static final int kLongClickIndex = 1 << 5;
+  static final int kAccessibilityFocusIndex = 1 << 6;
+  static final int kClearAccessibilityFocusIndex = 1 << 7;
+  static final int kNextAtMovementGranularityIndex = 1 << 8;
+  static final int kPreviousAtMovementGranularityIndex = 1 << 9;
+  static final int kNextHtmlElementIndex = 1 << 10;
+  static final int kPreviousHtmlElementIndex = 1 << 11;
+  static final int kScrollForwardIndex = 1 << 12;
+  static final int kScrollBackwardIndex = 1 << 13;
+  static final int kCutIndex = 1 << 14;
+  static final int kCopyIndex = 1 << 15;
+  static final int kPasteIndex = 1 << 16;
+  static final int kSetSelectionIndex = 1 << 17;
+  static final int kExpandIndex = 1 << 18;
+  static final int kCollapseIndex = 1 << 19;
+  static final int kSetText = 1 << 21;
+
+  static final Map<AccessibilityNodeInfo.AccessibilityAction, Integer> kIdByAction = new HashMap<AccessibilityNodeInfo.AccessibilityAction, Integer>() {{
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_FOCUS, kFocusIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_FOCUS, kClearFocusIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_SELECT, kSelectIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_SELECTION, kClearSelectionIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK, kClickIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_LONG_CLICK, kLongClickIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS, kAccessibilityFocusIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS, kClearAccessibilityFocusIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_NEXT_AT_MOVEMENT_GRANULARITY, kNextAtMovementGranularityIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY, kPreviousAtMovementGranularityIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_NEXT_HTML_ELEMENT, kNextHtmlElementIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_PREVIOUS_HTML_ELEMENT, kPreviousHtmlElementIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_FORWARD, kScrollForwardIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_BACKWARD, kScrollBackwardIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_CUT, kCutIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_COPY, kCopyIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_PASTE, kPasteIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_SELECTION, kSetSelectionIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_EXPAND, kExpandIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_COLLAPSE, kCollapseIndex);
+    put(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_TEXT, kSetText);
+  }};
 }

dev/integration_tests/android_semantics_testing/lib/main.dart

@@ -2,13 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:async';
-import 'dart:convert';
-
 import 'package:flutter/material.dart';
-import 'package:flutter/scheduler.dart';
-import 'package:flutter/services.dart';
-import 'package:flutter_driver/driver_extension.dart';
 
 import 'src/tests/controls_page.dart';
 import 'src/tests/headings_page.dart';
@@ -16,49 +10,9 @@ import 'src/tests/popup_page.dart';
 import 'src/tests/text_field_page.dart';
 
 void main() {
-  timeDilation = 0.05; // remove animations.
-  enableFlutterDriverExtension(handler: dataHandler);
   runApp(const TestApp());
 }
 
-const MethodChannel kSemanticsChannel = MethodChannel('semantics');
-
-Future<String> dataHandler(String? message) async {
-  if (message != null && message.contains('getSemanticsNode')) {
-    final Completer<String> completer = Completer<String>();
-    final int id = int.tryParse(message.split('#')[1]) ?? 0;
-    Future<void> completeSemantics([Object? _]) async {
-      final dynamic result = await kSemanticsChannel.invokeMethod<dynamic>('getSemanticsNode', <String, dynamic>{
-        'id': id,
-      });
-      completer.complete(json.encode(result));
-    }
-    if (SchedulerBinding.instance.hasScheduledFrame) {
-      SchedulerBinding.instance.addPostFrameCallback(completeSemantics);
-    } else {
-      completeSemantics();
-    }
-    return completer.future;
-  }
-  if (message != null && message.contains('setClipboard')) {
-    final Completer<String> completer = Completer<String>();
-    final String str = message.split('#')[1];
-    Future<void> completeSetClipboard([Object? _]) async {
-      await kSemanticsChannel.invokeMethod<dynamic>('setClipboard', <String, dynamic>{
-        'message': str,
-      });
-      completer.complete('');
-    }
-    if (SchedulerBinding.instance.hasScheduledFrame) {
-      SchedulerBinding.instance.addPostFrameCallback(completeSetClipboard);
-    } else {
-      completeSetClipboard();
-    }
-    return completer.future;
-  }
-  throw UnimplementedError();
-}
-
 Map<String, WidgetBuilder> routes = <String, WidgetBuilder>{
   selectionControlsRoute : (BuildContext context) => const SelectionControlsPage(),
   popupControlsRoute : (BuildContext context) => const PopupControlsPage(),

dev/integration_tests/android_semantics_testing/lib/src/constants.dart

@@ -98,6 +98,7 @@ enum AndroidSemanticsAction {
   /// The Android id of the action.
   final int id;
 
+  // These indices need to be in sync with android_semantics_testing/android/app/src/main/java/com/yourcompany/platforminteraction/MainActivity.java
   static const int _kFocusIndex = 1 << 0;
   static const int _kClearFocusIndex = 1 << 1;
   static const int _kSelectIndex = 1 << 2;

dev/integration_tests/android_semantics_testing/lib/src/matcher.dart

@@ -77,7 +77,8 @@ class _AndroidSemanticsMatcher extends Matcher {
     this.isHeading,
     this.isPassword,
     this.isLongClickable,
-  });
+  }) : assert(ignoredActions == null || actions != null, 'actions must not be null if ignoredActions is not null'),
+       assert(ignoredActions == null || !actions!.any(ignoredActions.contains));
 
   final String? text;
   final String? className;
@@ -115,6 +116,9 @@ class _AndroidSemanticsMatcher extends Matcher {
     if (actions != null) {
       description.add(' with actions: $actions');
     }
+    if (ignoredActions != null) {
+      description.add(' with ignoredActions: $ignoredActions');
+    }
     if (rect != null) {
       description.add(' with rect: $rect');
     }
@@ -170,13 +174,15 @@ class _AndroidSemanticsMatcher extends Matcher {
     }
     if (actions != null) {
       final List<AndroidSemanticsAction> itemActions = item.getActions();
+      if (ignoredActions != null) {
+        itemActions.removeWhere(ignoredActions!.contains);
+      }
       if (!unorderedEquals(actions!).matches(itemActions, matchState)) {
         final List<String> actionsString = actions!.map<String>((AndroidSemanticsAction action) => action.toString()).toList()..sort();
         final List<String> itemActionsString = itemActions.map<String>((AndroidSemanticsAction action) => action.toString()).toList()..sort();
-        final Set<AndroidSemanticsAction> unexpected = itemActions.toSet().difference(actions!.toSet());
         final Set<String> unexpectedInString = itemActionsString.toSet().difference(actionsString.toSet());
         final Set<String> missingInString = actionsString.toSet().difference(itemActionsString.toSet());
-        if (missingInString.isEmpty && ignoredActions != null && unexpected.every(ignoredActions!.contains)) {
+        if (missingInString.isEmpty && unexpectedInString.isEmpty) {
           return true;
         }
         return _failWithMessage('Expected actions: $actionsString\nActual actions: $itemActionsString\nUnexpected: $unexpectedInString\nMissing: $missingInString', matchState);

dev/integration_tests/android_semantics_testing/pubspec.yaml

@@ -6,7 +6,7 @@ environment:
 dependencies:
   flutter:
     sdk: flutter
-  flutter_driver:
+  integration_test:
     sdk: flutter
   flutter_test:
     sdk: flutter