1 /* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- 2 * This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this 4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 package org.mozilla.geckoview; 7 8 import org.mozilla.gecko.GeckoThread; 9 import org.mozilla.gecko.annotation.WrapForJNI; 10 import org.mozilla.gecko.GeckoAppShell; 11 import org.mozilla.gecko.util.GeckoBundle; 12 import org.mozilla.gecko.util.ThreadUtils; 13 import org.mozilla.gecko.mozglue.JNIObject; 14 15 import android.content.Context; 16 import android.graphics.Matrix; 17 import android.graphics.Rect; 18 import android.os.Build; 19 import android.os.Bundle; 20 import androidx.annotation.NonNull; 21 import androidx.annotation.Nullable; 22 import androidx.annotation.UiThread; 23 import android.text.TextUtils; 24 import android.util.Log; 25 import android.util.SparseArray; 26 import android.view.InputDevice; 27 import android.view.MotionEvent; 28 import android.view.View; 29 import android.view.ViewParent; 30 import android.view.accessibility.AccessibilityEvent; 31 import android.view.accessibility.AccessibilityManager; 32 import android.view.accessibility.AccessibilityNodeInfo; 33 import android.view.accessibility.AccessibilityNodeInfo.RangeInfo; 34 import android.view.accessibility.AccessibilityNodeInfo.CollectionItemInfo; 35 import android.view.accessibility.AccessibilityNodeInfo.CollectionInfo; 36 import android.view.accessibility.AccessibilityNodeProvider; 37 38 import java.util.Iterator; 39 import java.util.LinkedList; 40 41 @UiThread 42 public class SessionAccessibility { 43 private static final String LOGTAG = "GeckoAccessibility"; 44 45 // This is the number BrailleBack uses to start indexing routing keys. 46 private static final int BRAILLE_CLICK_BASE_INDEX = -275000000; 47 private static final String ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE = 48 "ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE"; 49 50 @WrapForJNI static final int FLAG_ACCESSIBILITY_FOCUSED = 0; 51 @WrapForJNI static final int FLAG_CHECKABLE = 1 << 1; 52 @WrapForJNI static final int FLAG_CHECKED = 1 << 2; 53 @WrapForJNI static final int FLAG_CLICKABLE = 1 << 3; 54 @WrapForJNI static final int FLAG_CONTENT_INVALID = 1 << 4; 55 @WrapForJNI static final int FLAG_CONTEXT_CLICKABLE = 1 << 5; 56 @WrapForJNI static final int FLAG_EDITABLE = 1 << 6; 57 @WrapForJNI static final int FLAG_ENABLED = 1 << 7; 58 @WrapForJNI static final int FLAG_FOCUSABLE = 1 << 8; 59 @WrapForJNI static final int FLAG_FOCUSED = 1 << 9; 60 @WrapForJNI static final int FLAG_LONG_CLICKABLE = 1 << 10; 61 @WrapForJNI static final int FLAG_MULTI_LINE = 1 << 11; 62 @WrapForJNI static final int FLAG_PASSWORD = 1 << 12; 63 @WrapForJNI static final int FLAG_SCROLLABLE = 1 << 13; 64 @WrapForJNI static final int FLAG_SELECTED = 1 << 14; 65 @WrapForJNI static final int FLAG_VISIBLE_TO_USER = 1 << 15; 66 @WrapForJNI static final int FLAG_SELECTABLE = 1 << 16; 67 @WrapForJNI static final int FLAG_EXPANDABLE = 1 << 17; 68 @WrapForJNI static final int FLAG_EXPANDED = 1 << 18; 69 70 static final int CLASSNAME_UNKNOWN = -1; 71 @WrapForJNI static final int CLASSNAME_VIEW = 0; 72 @WrapForJNI static final int CLASSNAME_BUTTON = 1; 73 @WrapForJNI static final int CLASSNAME_CHECKBOX = 2; 74 @WrapForJNI static final int CLASSNAME_DIALOG = 3; 75 @WrapForJNI static final int CLASSNAME_EDITTEXT = 4; 76 @WrapForJNI static final int CLASSNAME_GRIDVIEW = 5; 77 @WrapForJNI static final int CLASSNAME_IMAGE = 6; 78 @WrapForJNI static final int CLASSNAME_LISTVIEW = 7; 79 @WrapForJNI static final int CLASSNAME_MENUITEM = 8; 80 @WrapForJNI static final int CLASSNAME_PROGRESSBAR = 9; 81 @WrapForJNI static final int CLASSNAME_RADIOBUTTON = 10; 82 @WrapForJNI static final int CLASSNAME_SEEKBAR = 11; 83 @WrapForJNI static final int CLASSNAME_SPINNER = 12; 84 @WrapForJNI static final int CLASSNAME_TABWIDGET = 13; 85 @WrapForJNI static final int CLASSNAME_TOGGLEBUTTON = 14; 86 @WrapForJNI static final int CLASSNAME_WEBVIEW = 15; 87 88 private static final String[] CLASSNAMES = { 89 "android.view.View", 90 "android.widget.Button", 91 "android.widget.CheckBox", 92 "android.app.Dialog", 93 "android.widget.EditText", 94 "android.widget.GridView", 95 "android.widget.Image", 96 "android.widget.ListView", 97 "android.view.MenuItem", 98 "android.widget.ProgressBar", 99 "android.widget.RadioButton", 100 "android.widget.SeekBar", 101 "android.widget.Spinner", 102 "android.widget.TabWidget", 103 "android.widget.ToggleButton", 104 "android.webkit.WebView" 105 }; 106 107 @WrapForJNI static final int HTML_GRANULARITY_DEFAULT = -1; 108 @WrapForJNI static final int HTML_GRANULARITY_ARTICLE = 0; 109 @WrapForJNI static final int HTML_GRANULARITY_BUTTON = 1; 110 @WrapForJNI static final int HTML_GRANULARITY_CHECKBOX = 2; 111 @WrapForJNI static final int HTML_GRANULARITY_COMBOBOX = 3; 112 @WrapForJNI static final int HTML_GRANULARITY_CONTROL = 4; 113 @WrapForJNI static final int HTML_GRANULARITY_FOCUSABLE = 5; 114 @WrapForJNI static final int HTML_GRANULARITY_FRAME = 6; 115 @WrapForJNI static final int HTML_GRANULARITY_GRAPHIC = 7; 116 @WrapForJNI static final int HTML_GRANULARITY_H1 = 8; 117 @WrapForJNI static final int HTML_GRANULARITY_H2 = 9; 118 @WrapForJNI static final int HTML_GRANULARITY_H3 = 10; 119 @WrapForJNI static final int HTML_GRANULARITY_H4 = 11; 120 @WrapForJNI static final int HTML_GRANULARITY_H5 = 12; 121 @WrapForJNI static final int HTML_GRANULARITY_H6 = 13; 122 @WrapForJNI static final int HTML_GRANULARITY_HEADING = 14; 123 @WrapForJNI static final int HTML_GRANULARITY_LANDMARK = 15; 124 @WrapForJNI static final int HTML_GRANULARITY_LINK = 16; 125 @WrapForJNI static final int HTML_GRANULARITY_LIST = 17; 126 @WrapForJNI static final int HTML_GRANULARITY_LIST_ITEM = 18; 127 @WrapForJNI static final int HTML_GRANULARITY_MAIN = 19; 128 @WrapForJNI static final int HTML_GRANULARITY_MEDIA = 20; 129 @WrapForJNI static final int HTML_GRANULARITY_RADIO = 21; 130 @WrapForJNI static final int HTML_GRANULARITY_SECTION = 22; 131 @WrapForJNI static final int HTML_GRANULARITY_TABLE = 23; 132 @WrapForJNI static final int HTML_GRANULARITY_TEXT_FIELD = 24; 133 @WrapForJNI static final int HTML_GRANULARITY_UNVISITED_LINK = 25; 134 @WrapForJNI static final int HTML_GRANULARITY_VISITED_LINK = 26; 135 136 private static String[] sHtmlGranularities = { 137 "ARTICLE", 138 "BUTTON", 139 "CHECKBOX", 140 "COMBOBOX", 141 "CONTROL", 142 "FOCUSABLE", 143 "FRAME", 144 "GRAPHIC", 145 "H1", 146 "H2", 147 "H3", 148 "H4", 149 "H5", 150 "H6", 151 "HEADING", 152 "LANDMARK", 153 "LINK", 154 "LIST", 155 "LIST_ITEM", 156 "MAIN", 157 "MEDIA", 158 "RADIO", 159 "SECTION", 160 "TABLE", 161 "TEXT_FIELD", 162 "UNVISITED_LINK", 163 "VISITED_LINK" }; 164 getClassName(final int index)165 static private String getClassName(final int index) { 166 if (index >= 0 && index < CLASSNAMES.length) { 167 return CLASSNAMES[index]; 168 } 169 170 Log.e(LOGTAG, "Index " + index + " our of CLASSNAME bounds."); 171 return "android.view.View"; // Fallback class is View 172 } 173 174 /* package */ final class NodeProvider extends AccessibilityNodeProvider { 175 @Override createAccessibilityNodeInfo(final int virtualDescendantId)176 public AccessibilityNodeInfo createAccessibilityNodeInfo(final int virtualDescendantId) { 177 AccessibilityNodeInfo node = null; 178 if (mAttached) { 179 node = mSession.getSettings().getFullAccessibilityTree() ? 180 getNodeFromGecko(virtualDescendantId) : getNodeFromCache(virtualDescendantId); 181 } 182 183 if (node == null) { 184 Log.w(LOGTAG, "Failed to retrieve accessible node virtualDescendantId=" + 185 virtualDescendantId + " mAttached=" + mAttached); 186 node = AccessibilityNodeInfo.obtain(mView, View.NO_ID); 187 if (Build.VERSION.SDK_INT < 17 || mView.getDisplay() != null) { 188 // When running junit tests we don't have a display 189 mView.onInitializeAccessibilityNodeInfo(node); 190 } 191 node.setClassName("android.webkit.WebView"); 192 } 193 194 return node; 195 } 196 197 @Override performAction(final int virtualViewId, final int action, final Bundle arguments)198 public boolean performAction(final int virtualViewId, final int action, 199 final Bundle arguments) { 200 final GeckoBundle data; 201 202 switch (action) { 203 case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: 204 sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED, virtualViewId, CLASSNAME_UNKNOWN, null); 205 return true; 206 case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: 207 sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, virtualViewId, 208 virtualViewId == View.NO_ID ? CLASSNAME_WEBVIEW : CLASSNAME_UNKNOWN, null); 209 return true; 210 case AccessibilityNodeInfo.ACTION_CLICK: 211 case AccessibilityNodeInfo.ACTION_EXPAND: 212 case AccessibilityNodeInfo.ACTION_COLLAPSE: 213 nativeProvider.click(virtualViewId); 214 final GeckoBundle nodeInfo = getMostRecentBundle(virtualViewId); 215 if (nodeInfo != null) { 216 if ((nodeInfo.getInt("flags") & (FLAG_SELECTABLE | FLAG_CHECKABLE | FLAG_EXPANDABLE)) == 0) { 217 sendEvent(AccessibilityEvent.TYPE_VIEW_CLICKED, virtualViewId, nodeInfo.getInt("className"), null); 218 } 219 } 220 return true; 221 case AccessibilityNodeInfo.ACTION_LONG_CLICK: 222 // XXX: Implement long press. 223 return true; 224 case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD: 225 if (virtualViewId == View.NO_ID) { 226 // Scroll the viewport forwards by approximately 80%. 227 mSession.getPanZoomController().scrollBy( 228 ScreenLength.zero(), ScreenLength.fromVisualViewportHeight(0.8), 229 PanZoomController.SCROLL_BEHAVIOR_AUTO); 230 } else { 231 // XXX: It looks like we never call scroll on virtual views. 232 // If we did, we should synthesize a wheel event on it's center coordinate. 233 } 234 return true; 235 case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD: 236 if (virtualViewId == View.NO_ID) { 237 // Scroll the viewport backwards by approximately 80%. 238 mSession.getPanZoomController().scrollBy( 239 ScreenLength.zero(), ScreenLength.fromVisualViewportHeight(-0.8), 240 PanZoomController.SCROLL_BEHAVIOR_AUTO); 241 } else { 242 // XXX: It looks like we never call scroll on virtual views. 243 // If we did, we should synthesize a wheel event on it's center coordinate. 244 } 245 return true; 246 case AccessibilityNodeInfo.ACTION_SELECT: 247 nativeProvider.click(virtualViewId); 248 return true; 249 case AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT: 250 requestViewFocus(); 251 return pivot(virtualViewId, arguments != null ? 252 arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING) : "", 253 true, false); 254 case AccessibilityNodeInfo.ACTION_PREVIOUS_HTML_ELEMENT: 255 requestViewFocus(); 256 return pivot(virtualViewId, arguments != null ? 257 arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING) : "", 258 false, false); 259 case AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY: 260 case AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY: 261 // XXX: Self brailling gives this action with a bogus argument instead of an actual click action; 262 // the argument value is the BRAILLE_CLICK_BASE_INDEX - the index of the routing key that was hit. 263 // Other negative values are used by ChromeVox, but we don't support them. 264 // FAKE_GRANULARITY_READ_CURRENT = -1 265 // FAKE_GRANULARITY_READ_TITLE = -2 266 // FAKE_GRANULARITY_STOP_SPEECH = -3 267 // FAKE_GRANULARITY_CHANGE_SHIFTER = -4 268 final int granularity = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT); 269 if (granularity <= BRAILLE_CLICK_BASE_INDEX) { 270 // XXX: Use click offset to update caret position in editables (BRAILLE_CLICK_BASE_INDEX - granularity). 271 nativeProvider.click(virtualViewId); 272 } else if (granularity > 0) { 273 final boolean extendSelection = arguments.getBoolean(AccessibilityNodeInfo.ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN); 274 final boolean next = action == AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY; 275 // We must return false if we're already at the edge. 276 if (next) { 277 if (mAtEndOfText) { 278 return false; 279 } 280 if (granularity == AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD && mAtLastWord) { 281 return false; 282 } 283 } else if (mAtStartOfText) { 284 return false; 285 } 286 nativeProvider.navigateText(virtualViewId, granularity, mStartOffset, mEndOffset, next, extendSelection); 287 } 288 return true; 289 case AccessibilityNodeInfo.ACTION_SET_SELECTION: 290 if (arguments == null) { 291 return false; 292 } 293 final int selectionStart = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT); 294 final int selectionEnd = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT); 295 nativeProvider.setSelection(virtualViewId, selectionStart, selectionEnd); 296 return true; 297 case AccessibilityNodeInfo.ACTION_CUT: 298 nativeProvider.cut(virtualViewId); 299 return true; 300 case AccessibilityNodeInfo.ACTION_COPY: 301 nativeProvider.copy(virtualViewId); 302 return true; 303 case AccessibilityNodeInfo.ACTION_PASTE: 304 nativeProvider.paste(virtualViewId); 305 return true; 306 case AccessibilityNodeInfo.ACTION_SET_TEXT: 307 final String value = arguments.getString(Build.VERSION.SDK_INT >= 21 308 ? AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE 309 : ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE); 310 if (mAttached) { 311 nativeProvider.setText(virtualViewId, value); 312 } 313 return true; 314 } 315 316 return mView.performAccessibilityAction(action, arguments); 317 } 318 319 @Override findFocus(final int focus)320 public AccessibilityNodeInfo findFocus(final int focus) { 321 switch (focus) { 322 case AccessibilityNodeInfo.FOCUS_ACCESSIBILITY: 323 if (mAccessibilityFocusedNode != 0) { 324 return createAccessibilityNodeInfo(mAccessibilityFocusedNode); 325 } 326 break; 327 case AccessibilityNodeInfo.FOCUS_INPUT: 328 if (mFocusedNode != 0) { 329 return createAccessibilityNodeInfo(mFocusedNode); 330 } 331 break; 332 } 333 334 return super.findFocus(focus); 335 } 336 getNodeFromGecko(final int virtualViewId)337 private AccessibilityNodeInfo getNodeFromGecko(final int virtualViewId) { 338 final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(mView, virtualViewId); 339 populateNodeFromBundle(node, nativeProvider.getNodeInfo(virtualViewId), false); 340 return node; 341 } 342 getNodeFromCache(final int virtualViewId)343 private AccessibilityNodeInfo getNodeFromCache(final int virtualViewId) { 344 synchronized (SessionAccessibility.this) { 345 AccessibilityNodeInfo node = null; 346 for (final SparseArray<GeckoBundle> cache : mCaches) { 347 final GeckoBundle bundle = cache.get(virtualViewId); 348 if (bundle == null) { 349 continue; 350 } 351 352 if (node == null) { 353 node = AccessibilityNodeInfo.obtain(mView, virtualViewId); 354 } 355 populateNodeFromBundle(node, bundle, true); 356 } 357 358 if (node == null) { 359 Log.e(LOGTAG, "No cached node for " + virtualViewId); 360 } 361 362 return node; 363 } 364 } 365 populateNodeFromBundle(final AccessibilityNodeInfo node, final GeckoBundle nodeInfo, final boolean fromCache)366 private void populateNodeFromBundle(final AccessibilityNodeInfo node, final GeckoBundle nodeInfo, final boolean fromCache) { 367 if (mView == null || nodeInfo == null) { 368 return; 369 } 370 371 final int id = nodeInfo.getInt("id"); 372 final boolean isRoot = id == View.NO_ID; 373 if (isRoot) { 374 if (Build.VERSION.SDK_INT < 17 || mView.getDisplay() != null) { 375 // When running junit tests we don't have a display 376 mView.onInitializeAccessibilityNodeInfo(node); 377 } 378 node.addAction(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD); 379 node.addAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD); 380 } else { 381 node.setParent(mView, nodeInfo.getInt("parentId", View.NO_ID)); 382 } 383 384 final int flags = nodeInfo.getInt("flags"); 385 386 // The basics 387 node.setPackageName(GeckoAppShell.getApplicationContext().getPackageName()); 388 node.setClassName(getClassName(nodeInfo.getInt("className"))); 389 390 if (nodeInfo.containsKey("text")) { 391 node.setText(nodeInfo.getString("text")); 392 } 393 394 if (nodeInfo.containsKey("description")) { 395 node.setContentDescription(nodeInfo.getString("description")); 396 } 397 398 // Add actions 399 node.addAction(AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT); 400 node.addAction(AccessibilityNodeInfo.ACTION_PREVIOUS_HTML_ELEMENT); 401 node.addAction(AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY); 402 node.addAction(AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY); 403 node.setMovementGranularities(AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER | 404 AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD | 405 AccessibilityNodeInfo.MOVEMENT_GRANULARITY_LINE | 406 AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH); 407 if ((flags & FLAG_CLICKABLE) != 0) { 408 node.addAction(AccessibilityNodeInfo.ACTION_CLICK); 409 } 410 411 412 // Set boolean properties 413 node.setCheckable((flags & FLAG_CHECKABLE) != 0); 414 node.setChecked((flags & FLAG_CHECKED) != 0); 415 node.setClickable((flags & FLAG_CLICKABLE) != 0); 416 node.setEnabled((flags & FLAG_ENABLED) != 0); 417 node.setFocusable((flags & FLAG_FOCUSABLE) != 0); 418 node.setLongClickable((flags & FLAG_LONG_CLICKABLE) != 0); 419 node.setPassword((flags & FLAG_PASSWORD) != 0); 420 node.setScrollable((flags & FLAG_SCROLLABLE) != 0); 421 node.setSelected((flags & FLAG_SELECTED) != 0); 422 node.setVisibleToUser((flags & FLAG_VISIBLE_TO_USER) != 0); 423 // Other boolean properties to consider later: 424 // setHeading, setImportantForAccessibility, setScreenReaderFocusable, setShowingHintText, setDismissable 425 426 if (mAccessibilityFocusedNode == id) { 427 node.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS); 428 node.setAccessibilityFocused(true); 429 } else { 430 node.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS); 431 } 432 node.setFocused(mFocusedNode == id); 433 434 // Bounds 435 final int[] b = nodeInfo.getIntArray("bounds"); 436 if (b != null) { 437 final Rect screenBounds = new Rect(b[0], b[1], b[2], b[3]); 438 node.setBoundsInScreen(screenBounds); 439 440 final Matrix matrix = new Matrix(); 441 mSession.getClientToScreenMatrix(matrix); 442 final float[] origin = new float[2]; 443 matrix.mapPoints(origin); 444 final Rect parentBounds = new Rect(b[0] - (int)origin[0], b[1] - (int)origin[1], b[2], b[3]); 445 node.setBoundsInParent(parentBounds); 446 } 447 448 // Children 449 final int[] children = nodeInfo.getIntArray("children"); 450 if (node.getChildCount() == 0 && children != null) { 451 for (final int childId : children) { 452 final GeckoBundle childBundle = getMostRecentBundle(childId); 453 if (!fromCache || (childBundle != null && childBundle.getInt("parentId") == id)) { 454 // If this node is from cache, only populate with children that are cached as well. 455 node.addChild(mView, childId); 456 } 457 } 458 } 459 460 // SDK 18 and above 461 if (Build.VERSION.SDK_INT >= 18) { 462 node.setViewIdResourceName(nodeInfo.getString("viewIdResourceName")); 463 464 if ((flags & FLAG_EDITABLE) != 0) { 465 node.addAction(AccessibilityNodeInfo.ACTION_SET_SELECTION); 466 node.addAction(AccessibilityNodeInfo.ACTION_CUT); 467 node.addAction(AccessibilityNodeInfo.ACTION_COPY); 468 node.addAction(AccessibilityNodeInfo.ACTION_PASTE); 469 node.setEditable(true); 470 } 471 } 472 473 // SDK 19 and above 474 if (Build.VERSION.SDK_INT >= 19) { 475 node.setMultiLine((flags & FLAG_MULTI_LINE) != 0); 476 node.setContentInvalid((flags & FLAG_CONTENT_INVALID) != 0); 477 478 // Set bundle keys like role and hint 479 final Bundle bundle = node.getExtras(); 480 if (nodeInfo.containsKey("hint")) { 481 final String hint = nodeInfo.getString("hint"); 482 bundle.putCharSequence("AccessibilityNodeInfo.hint", hint); 483 if (Build.VERSION.SDK_INT >= 26) { 484 node.setHintText(hint); 485 } 486 } 487 if (nodeInfo.containsKey("geckoRole")) { 488 bundle.putCharSequence("AccessibilityNodeInfo.geckoRole", nodeInfo.getString("geckoRole")); 489 } 490 if (nodeInfo.containsKey("roleDescription")) { 491 bundle.putCharSequence("AccessibilityNodeInfo.roleDescription", nodeInfo.getString("roleDescription")); 492 } 493 if (isRoot) { 494 // Argument values for ACTION_NEXT_HTML_ELEMENT/ACTION_PREVIOUS_HTML_ELEMENT. 495 // This is mostly here to let TalkBack know we are a legit "WebView". 496 bundle.putCharSequence( 497 "ACTION_ARGUMENT_HTML_ELEMENT_STRING_VALUES", 498 TextUtils.join(",", sHtmlGranularities)); 499 } 500 501 502 // Set RangeInfo 503 final GeckoBundle rangeBundle = nodeInfo.getBundle("rangeInfo"); 504 if (rangeBundle != null) { 505 final RangeInfo rangeInfo = RangeInfo.obtain( 506 rangeBundle.getInt("type"), 507 (float)rangeBundle.getDouble("min", Float.NEGATIVE_INFINITY), 508 (float)rangeBundle.getDouble("max", Float.POSITIVE_INFINITY), 509 (float)rangeBundle.getDouble("current", 0)); 510 node.setRangeInfo(rangeInfo); 511 } 512 513 // Set CollectionItemInfo 514 final GeckoBundle collectionItemBundle = nodeInfo.getBundle("collectionItemInfo"); 515 if (collectionItemBundle != null) { 516 final CollectionItemInfo collectionItemInfo = CollectionItemInfo.obtain( 517 collectionItemBundle.getInt("rowIndex"), 518 collectionItemBundle.getInt("rowSpan"), 519 collectionItemBundle.getInt("columnIndex"), 520 collectionItemBundle.getInt("columnSpan"), false); 521 node.setCollectionItemInfo(collectionItemInfo); 522 } 523 524 // Set CollectionInfo 525 final GeckoBundle collectionBundle = nodeInfo.getBundle("collectionInfo"); 526 if (collectionBundle != null) { 527 // selectionMode is only supported in SDK >= 21. 528 final CollectionInfo collectionInfo = Build.VERSION.SDK_INT >= 21 529 ? CollectionInfo.obtain( 530 collectionBundle.getInt("rowCount"), 531 collectionBundle.getInt("columnCount"), 532 collectionBundle.getBoolean("isHierarchical", false), 533 collectionBundle.getInt("selectionMode", 0)) 534 : CollectionInfo.obtain( 535 collectionBundle.getInt("rowCount"), 536 collectionBundle.getInt("columnCount"), 537 collectionBundle.getBoolean("isHierarchical", false)); 538 node.setCollectionInfo(collectionInfo); 539 } 540 541 node.setInputType(nodeInfo.getInt("inputType")); 542 } 543 544 // SDK 21 and above 545 if (Build.VERSION.SDK_INT >= 21) { 546 if ((flags & FLAG_EXPANDABLE) != 0) { 547 if ((flags & FLAG_EXPANDED) != 0) { 548 node.removeAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_EXPAND); 549 node.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_COLLAPSE); 550 } else { 551 node.removeAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_COLLAPSE); 552 node.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_EXPAND); 553 } 554 } 555 } 556 557 // SDK 23 and above 558 if (Build.VERSION.SDK_INT >= 23) { 559 node.setContextClickable((flags & FLAG_CONTEXT_CLICKABLE) != 0); 560 } 561 } 562 } 563 564 // Gecko session we are proxying 565 /* package */ final GeckoSession mSession; 566 // This is the view that delegates accessibility to us. We also sends event through it. 567 private View mView; 568 // The native portion of the node provider. 569 /* package */ final NativeProvider nativeProvider = new NativeProvider(); 570 private boolean mAttached = false; 571 // The current node with accessibility focus 572 private int mAccessibilityFocusedNode = 0; 573 // The first accessibility focusable node 574 private int mFirstAccessibilityFocusable = 0; 575 // The last accessibility focusable node 576 private int mLastAccessibilityFocusable = 0; 577 // The current node with focus 578 private int mFocusedNode = 0; 579 private int mStartOffset = -1; 580 private int mEndOffset = -1; 581 private boolean mAtStartOfText = false; 582 private boolean mAtEndOfText = false; 583 private boolean mAtLastWord = false; 584 // Viewport cache 585 final SparseArray<GeckoBundle> mViewportCache = new SparseArray<>(); 586 // Focus cache 587 final SparseArray<GeckoBundle> mFocusPathCache = new SparseArray<>(); 588 // List of caches in descending order from last updated. 589 LinkedList<SparseArray<GeckoBundle>> mCaches = new LinkedList<>(); 590 private boolean mViewFocusRequested = false; 591 SessionAccessibility(final GeckoSession session)592 /* package */ SessionAccessibility(final GeckoSession session) { 593 mSession = session; 594 Settings.updateAccessibilitySettings(); 595 } 596 setForceEnabled(final boolean forceEnabled)597 /* package */ static void setForceEnabled(final boolean forceEnabled) { 598 Settings.setForceEnabled(forceEnabled); 599 } 600 601 /** 602 * Get the View instance that delegates accessibility to this session. 603 * 604 * @return View instance. 605 */ getView()606 public @Nullable View getView() { 607 ThreadUtils.assertOnUiThread(); 608 609 return mView; 610 } 611 612 /** 613 * Set the View instance that should delegate accessibility to this session. 614 * 615 * @param view View instance. 616 */ 617 @UiThread setView(final @Nullable View view)618 public void setView(final @Nullable View view) { 619 ThreadUtils.assertOnUiThread(); 620 621 if (mView != null) { 622 mView.setAccessibilityDelegate(null); 623 } 624 625 mView = view; 626 627 if (mView == null) { 628 return; 629 } 630 631 mView.setAccessibilityDelegate(new View.AccessibilityDelegate() { 632 private NodeProvider mProvider; 633 634 @Override 635 public AccessibilityNodeProvider getAccessibilityNodeProvider(final View hostView) { 636 if (hostView != mView) { 637 return null; 638 } 639 if (mProvider == null) { 640 mProvider = new NodeProvider(); 641 } 642 return mProvider; 643 } 644 645 @Override 646 public void sendAccessibilityEvent(final View host, final int eventType) { 647 if (eventType == AccessibilityEvent.TYPE_VIEW_FOCUSED) { 648 // We rely on the focus events sent from Gecko. 649 return; 650 } 651 652 super.sendAccessibilityEvent(host, eventType); 653 } 654 }); 655 } 656 isInTest()657 private boolean isInTest() { 658 return Build.VERSION.SDK_INT >= 17 && mView != null && mView.getDisplay() == null; 659 } 660 requestViewFocus()661 private void requestViewFocus() { 662 if (!mView.isFocused() && !isInTest()) { 663 mViewFocusRequested = true; 664 mView.requestFocus(); 665 } 666 } 667 668 private static class Settings { 669 private static volatile boolean sEnabled; 670 private static volatile boolean sTouchExplorationEnabled; 671 private static volatile boolean sForceEnabled; 672 setForceEnabled(final boolean forceEnabled)673 public static void setForceEnabled(final boolean forceEnabled) { 674 sForceEnabled = forceEnabled; 675 dispatch(); 676 } 677 678 static { 679 final Context context = GeckoAppShell.getApplicationContext(); 680 final AccessibilityManager accessibilityManager = 681 (AccessibilityManager) context.getSystemService(Context.ACCESSIBILITY_SERVICE); 682 683 accessibilityManager.addAccessibilityStateChangeListener(enabled -> 684 updateAccessibilitySettings()); 685 686 if (Build.VERSION.SDK_INT >= 19) { 687 accessibilityManager.addTouchExplorationStateChangeListener(enabled -> 688 updateAccessibilitySettings()); 689 } 690 } 691 isEnabled()692 public static boolean isEnabled() { 693 return sEnabled || sForceEnabled; 694 } 695 isTouchExplorationEnabled()696 public static boolean isTouchExplorationEnabled() { 697 return sTouchExplorationEnabled || sForceEnabled; 698 } 699 updateAccessibilitySettings()700 public static void updateAccessibilitySettings() { 701 final AccessibilityManager accessibilityManager = (AccessibilityManager) 702 GeckoAppShell.getApplicationContext().getSystemService(Context.ACCESSIBILITY_SERVICE); 703 sEnabled = accessibilityManager.isEnabled(); 704 sTouchExplorationEnabled = sEnabled && accessibilityManager.isTouchExplorationEnabled(); 705 dispatch(); 706 } 707 dispatch()708 /* package */ static void dispatch() { 709 if (GeckoThread.isStateAtLeast(GeckoThread.State.PROFILE_READY)) { 710 toggleNativeAccessibility(isEnabled()); 711 } else { 712 GeckoThread.queueNativeCallUntil( 713 GeckoThread.State.PROFILE_READY, 714 Settings.class, "toggleNativeAccessibility", isEnabled()); 715 } 716 } 717 718 @WrapForJNI(dispatchTo = "gecko") toggleNativeAccessibility(boolean enable)719 private static native void toggleNativeAccessibility(boolean enable); 720 } 721 722 @SuppressWarnings("checkstyle:javadocmethod") onMotionEvent(final @NonNull MotionEvent event)723 public boolean onMotionEvent(final @NonNull MotionEvent event) { 724 ThreadUtils.assertOnUiThread(); 725 726 if (!Settings.isTouchExplorationEnabled()) { 727 return false; 728 } 729 730 if (event.getSource() != InputDevice.SOURCE_TOUCHSCREEN) { 731 return false; 732 } 733 734 final int action = event.getActionMasked(); 735 if ((action != MotionEvent.ACTION_HOVER_MOVE) && 736 (action != MotionEvent.ACTION_HOVER_ENTER) && 737 (action != MotionEvent.ACTION_HOVER_EXIT)) { 738 return false; 739 } 740 741 requestViewFocus(); 742 743 nativeProvider.exploreByTouch( 744 mAccessibilityFocusedNode != 0 ? mAccessibilityFocusedNode : View.NO_ID, 745 event.getRawX(), event.getRawY()); 746 747 return true; 748 } 749 sendEvent(final int eventType, final int sourceId, final int className, final GeckoBundle eventData)750 /* package */ void sendEvent(final int eventType, final int sourceId, final int className, final GeckoBundle eventData) { 751 ThreadUtils.assertOnUiThread(); 752 if (mView == null) { 753 return; 754 } 755 756 if (mViewFocusRequested && className == CLASSNAME_WEBVIEW) { 757 // If the view was focused from an accessiblity action or 758 // explore-by-touch, we supress this focus event to avoid noise. 759 mViewFocusRequested = false; 760 return; 761 } 762 763 final GeckoBundle cachedBundle = getMostRecentBundle(sourceId); 764 if (cachedBundle == null && sourceId != View.NO_ID) { 765 // Suppress events from non cached nodes. 766 return; 767 } 768 769 final AccessibilityEvent event = AccessibilityEvent.obtain(eventType); 770 event.setPackageName(GeckoAppShell.getApplicationContext().getPackageName()); 771 event.setSource(mView, sourceId); 772 event.setEnabled(true); 773 if (className == CLASSNAME_UNKNOWN && cachedBundle != null) { 774 event.setClassName(getClassName(cachedBundle.getInt("className"))); 775 } else { 776 event.setClassName(getClassName(className)); 777 } 778 779 if (eventData != null) { 780 if (eventData.containsKey("text")) { 781 event.getText().add(eventData.getString("text")); 782 } 783 event.setContentDescription(eventData.getString("description", "")); 784 event.setAddedCount(eventData.getInt("addedCount", -1)); 785 event.setRemovedCount(eventData.getInt("removedCount", -1)); 786 event.setFromIndex(eventData.getInt("fromIndex", -1)); 787 event.setItemCount(eventData.getInt("itemCount", -1)); 788 event.setCurrentItemIndex(eventData.getInt("currentItemIndex", -1)); 789 event.setBeforeText(eventData.getString("beforeText", "")); 790 event.setToIndex(eventData.getInt("toIndex", -1)); 791 event.setScrollX(eventData.getInt("scrollX", -1)); 792 event.setScrollY(eventData.getInt("scrollY", -1)); 793 event.setMaxScrollX(eventData.getInt("maxScrollX", -1)); 794 event.setMaxScrollY(eventData.getInt("maxScrollY", -1)); 795 event.setChecked((eventData.getInt("flags") & FLAG_CHECKED) != 0); 796 } 797 798 // Update cache and stored state from this event. 799 switch (eventType) { 800 case AccessibilityEvent.TYPE_VIEW_CLICKED: 801 if (cachedBundle != null && eventData != null && eventData.containsKey("flags")) { 802 final int flags = eventData.getInt("flags"); 803 if ((flags & FLAG_CHECKABLE) != 0) { 804 if ((flags & FLAG_CHECKED) != 0) { 805 cachedBundle.putInt("flags", cachedBundle.getInt("flags") | FLAG_CHECKED); 806 } else { 807 cachedBundle.putInt("flags", cachedBundle.getInt("flags") & ~FLAG_CHECKED); 808 } 809 } 810 811 if ((flags & FLAG_EXPANDABLE) != 0) { 812 if ((flags & FLAG_EXPANDED) != 0) { 813 cachedBundle.putInt("flags", cachedBundle.getInt("flags") | FLAG_EXPANDED); 814 } else { 815 cachedBundle.putInt("flags", cachedBundle.getInt("flags") & ~FLAG_EXPANDED); 816 } 817 } 818 } 819 break; 820 case AccessibilityEvent.TYPE_VIEW_SELECTED: 821 if (cachedBundle != null && eventData != null && eventData.containsKey("selected")) { 822 if (eventData.getInt("selected") != 0) { 823 cachedBundle.putInt("flags", cachedBundle.getInt("flags") | FLAG_SELECTED); 824 } else { 825 cachedBundle.putInt("flags", cachedBundle.getInt("flags") & ~FLAG_SELECTED); 826 } 827 } 828 break; 829 case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED: 830 if (mAccessibilityFocusedNode == sourceId) { 831 mAccessibilityFocusedNode = 0; 832 } 833 break; 834 case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: 835 mStartOffset = -1; 836 mEndOffset = -1; 837 mAtStartOfText = false; 838 mAtEndOfText = false; 839 mAtLastWord = false; 840 mAccessibilityFocusedNode = sourceId; 841 break; 842 case AccessibilityEvent.TYPE_VIEW_FOCUSED: 843 mFocusedNode = sourceId; 844 if (!mView.isFocused() && !isInTest()) { 845 // Don't dispatch a focus event if the parent view is not focused 846 return; 847 } 848 break; 849 case AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY: 850 mStartOffset = event.getFromIndex(); 851 mEndOffset = event.getToIndex(); 852 // We must synchronously return false for text navigation 853 // actions if the user attempts to navigate past the edge. 854 // Because we do navigation async, we can't query this 855 // on demand when the action is performed. Therefore, we cache 856 // whether we're at either edge here. 857 mAtStartOfText = mStartOffset == 0; 858 final CharSequence text = event.getText().get(0); 859 mAtEndOfText = mEndOffset >= text.length(); 860 mAtLastWord = mAtEndOfText; 861 if (!mAtLastWord) { 862 // Words exclude trailing spaces. To figure out whether 863 // we're at the last word, we need to get the text after 864 // our end offset and check if it's just spaces. 865 final CharSequence afterText = text.subSequence(mEndOffset, text.length()); 866 if (TextUtils.getTrimmedLength(afterText) == 0) { 867 mAtLastWord = true; 868 } 869 } 870 break; 871 } 872 873 try { 874 ((ViewParent) mView).requestSendAccessibilityEvent(mView, event); 875 } catch (final IllegalStateException ex) { 876 // Accessibility could be activated in Gecko via xpcom, for example when using a11y 877 // devtools. Events that are forwarded to the platform will throw an exception. 878 } 879 } 880 getMostRecentBundle(final int virtualViewId)881 private synchronized GeckoBundle getMostRecentBundle(final int virtualViewId) { 882 final Iterator<SparseArray<GeckoBundle>> iter = mCaches.descendingIterator(); 883 while (iter.hasNext()) { 884 final GeckoBundle bundle = iter.next().get(virtualViewId); 885 if (bundle != null) { 886 return bundle; 887 } 888 } 889 890 return null; 891 } 892 pivot(final int id, final String granularity, final boolean forward, final boolean inclusive)893 private boolean pivot(final int id, final String granularity, final boolean forward, final boolean inclusive) { 894 final int gran = java.util.Arrays.asList(sHtmlGranularities).indexOf(granularity); 895 if (forward && id == mLastAccessibilityFocusable) { 896 return false; 897 } 898 899 if (!forward) { 900 if (id == View.NO_ID) { 901 return false; 902 } 903 904 if (id == mFirstAccessibilityFocusable) { 905 sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, View.NO_ID, CLASSNAME_WEBVIEW, null); 906 return true; 907 } 908 909 } 910 911 nativeProvider.pivotNative(id, gran, forward, inclusive); 912 return true; 913 } 914 915 /* package */ final class NativeProvider extends JNIObject { 916 @WrapForJNI(calledFrom = "ui") setAttached(final boolean attached)917 private void setAttached(final boolean attached) { 918 mAttached = attached; 919 } 920 921 @Override // JNIObject disposeNative()922 protected void disposeNative() { 923 // Disposal happens in native code. 924 throw new UnsupportedOperationException(); 925 } 926 927 @WrapForJNI(dispatchTo = "current") getNodeInfo(int id)928 public native GeckoBundle getNodeInfo(int id); 929 930 @WrapForJNI(dispatchTo = "gecko") setText(int id, String text)931 public native void setText(int id, String text); 932 933 @WrapForJNI(dispatchTo = "gecko") click(int id)934 public native void click(int id); 935 936 @WrapForJNI(dispatchTo = "gecko", stubName = "Pivot") pivotNative(int id, int granularity, boolean forward, boolean inclusive)937 public native void pivotNative(int id, int granularity, boolean forward, boolean inclusive); 938 939 @WrapForJNI(dispatchTo = "gecko") exploreByTouch(int id, float x, float y)940 public native void exploreByTouch(int id, float x, float y); 941 942 @WrapForJNI(dispatchTo = "gecko") navigateText(int id, int granularity, int startOffset, int endOffset, boolean forward, boolean select)943 public native void navigateText(int id, int granularity, int startOffset, int endOffset, boolean forward, boolean select); 944 945 @WrapForJNI(dispatchTo = "gecko") setSelection(int id, int start, int end)946 public native void setSelection(int id, int start, int end); 947 948 @WrapForJNI(dispatchTo = "gecko") cut(int id)949 public native void cut(int id); 950 951 @WrapForJNI(dispatchTo = "gecko") copy(int id)952 public native void copy(int id); 953 954 @WrapForJNI(dispatchTo = "gecko") paste(int id)955 public native void paste(int id); 956 957 @WrapForJNI(calledFrom = "gecko", stubName = "SendEvent") sendEventNative(final int eventType, final int sourceId, final int className, final GeckoBundle eventData)958 private void sendEventNative(final int eventType, final int sourceId, final int className, final GeckoBundle eventData) { 959 ThreadUtils.runOnUiThread(new Runnable() { 960 @Override 961 public void run() { 962 sendEvent(eventType, sourceId, className, eventData); 963 } 964 }); 965 } 966 967 @WrapForJNI(calledFrom = "gecko") replaceViewportCache(final GeckoBundle[] bundles)968 private void replaceViewportCache(final GeckoBundle[] bundles) { 969 synchronized (SessionAccessibility.this) { 970 mViewportCache.clear(); 971 for (final GeckoBundle bundle : bundles) { 972 if (bundle == null) { 973 continue; 974 } 975 mViewportCache.append(bundle.getInt("id"), bundle); 976 } 977 mCaches.remove(mViewportCache); 978 mCaches.add(mViewportCache); 979 } 980 } 981 982 @WrapForJNI(calledFrom = "gecko") replaceFocusPathCache(final GeckoBundle[] bundles)983 private void replaceFocusPathCache(final GeckoBundle[] bundles) { 984 synchronized (SessionAccessibility.this) { 985 mFocusPathCache.clear(); 986 for (final GeckoBundle bundle : bundles) { 987 if (bundle == null) { 988 continue; 989 } 990 mFocusPathCache.append(bundle.getInt("id"), bundle); 991 } 992 mCaches.remove(mFocusPathCache); 993 mCaches.add(mFocusPathCache); 994 } 995 } 996 997 @WrapForJNI(calledFrom = "gecko") updateCachedBounds(final GeckoBundle[] bundles)998 private void updateCachedBounds(final GeckoBundle[] bundles) { 999 synchronized (SessionAccessibility.this) { 1000 for (final GeckoBundle bundle : bundles) { 1001 final GeckoBundle cachedBundle = getMostRecentBundle(bundle.getInt("id")); 1002 if (cachedBundle == null) { 1003 Log.e(LOGTAG, "Can't update bounds of uncached node " + bundle.getInt("id")); 1004 continue; 1005 } 1006 cachedBundle.putIntArray("bounds", bundle.getIntArray("bounds")); 1007 } 1008 } 1009 } 1010 1011 @WrapForJNI(calledFrom = "gecko") updateAccessibleFocusBoundaries(final int firstNode, final int lastNode)1012 private void updateAccessibleFocusBoundaries(final int firstNode, final int lastNode) { 1013 synchronized (SessionAccessibility.this) { 1014 mFirstAccessibilityFocusable = firstNode; 1015 mLastAccessibilityFocusable = lastNode; 1016 } 1017 } 1018 } 1019 } 1020