1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "ui/accessibility/ax_assistant_structure.h"
6
7 #include <string>
8
9 #include "base/logging.h"
10 #include "base/optional.h"
11 #include "base/strings/stringprintf.h"
12 #include "base/strings/utf_string_conversions.h"
13 #include "ui/accessibility/ax_enums.mojom.h"
14 #include "ui/accessibility/ax_node.h"
15 #include "ui/accessibility/ax_role_properties.h"
16 #include "ui/accessibility/ax_serializable_tree.h"
17 #include "ui/accessibility/platform/ax_android_constants.h"
18 #include "ui/gfx/geometry/rect_conversions.h"
19 #include "ui/gfx/range/range.h"
20 #include "ui/gfx/transform.h"
21
22 namespace ui {
23
24 namespace {
25
HasFocusableChild(const AXNode * node)26 bool HasFocusableChild(const AXNode* node) {
27 for (size_t i = 0; i < node->GetUnignoredChildCount(); ++i) {
28 AXNode* child = node->GetUnignoredChildAtIndex(i);
29 if (child->data().HasState(ax::mojom::State::kFocusable) ||
30 HasFocusableChild(child)) {
31 return true;
32 }
33 }
34 return false;
35 }
36
HasOnlyTextChildren(const AXNode * node)37 bool HasOnlyTextChildren(const AXNode* node) {
38 for (size_t i = 0; i < node->GetUnignoredChildCount(); ++i) {
39 AXNode* child = node->GetUnignoredChildAtIndex(i);
40 if (!child->IsText())
41 return false;
42 }
43 return true;
44 }
45
46 // TODO(muyuanli): share with BrowserAccessibility.
IsSimpleTextControl(const AXNode * node,uint32_t state)47 bool IsSimpleTextControl(const AXNode* node, uint32_t state) {
48 return (node->data().role == ax::mojom::Role::kTextField ||
49 node->data().role == ax::mojom::Role::kTextFieldWithComboBox ||
50 node->data().role == ax::mojom::Role::kSearchBox ||
51 node->data().HasBoolAttribute(
52 ax::mojom::BoolAttribute::kEditableRoot)) &&
53 !node->data().HasState(ax::mojom::State::kRichlyEditable);
54 }
55
IsRichTextEditable(const AXNode * node)56 bool IsRichTextEditable(const AXNode* node) {
57 const AXNode* parent = node->GetUnignoredParent();
58 return node->data().HasState(ax::mojom::State::kRichlyEditable) &&
59 (!parent ||
60 !parent->data().HasState(ax::mojom::State::kRichlyEditable));
61 }
62
IsNativeTextControl(const AXNode * node)63 bool IsNativeTextControl(const AXNode* node) {
64 const std::string& html_tag =
65 node->data().GetStringAttribute(ax::mojom::StringAttribute::kHtmlTag);
66 if (html_tag == "input") {
67 std::string input_type;
68 if (!node->data().GetHtmlAttribute("type", &input_type))
69 return true;
70 return input_type.empty() || input_type == "email" ||
71 input_type == "password" || input_type == "search" ||
72 input_type == "tel" || input_type == "text" || input_type == "url" ||
73 input_type == "number";
74 }
75 return html_tag == "textarea";
76 }
77
IsLeaf(const AXNode * node)78 bool IsLeaf(const AXNode* node) {
79 if (node->children().empty())
80 return true;
81
82 if (IsNativeTextControl(node) || node->IsText()) {
83 return true;
84 }
85
86 switch (node->data().role) {
87 case ax::mojom::Role::kImage:
88 case ax::mojom::Role::kMeter:
89 case ax::mojom::Role::kScrollBar:
90 case ax::mojom::Role::kSlider:
91 case ax::mojom::Role::kSplitter:
92 case ax::mojom::Role::kProgressIndicator:
93 case ax::mojom::Role::kDate:
94 case ax::mojom::Role::kDateTime:
95 case ax::mojom::Role::kInputTime:
96 return true;
97 default:
98 return false;
99 }
100 }
101
GetInnerText(const AXNode * node)102 base::string16 GetInnerText(const AXNode* node) {
103 if (node->IsText()) {
104 return node->data().GetString16Attribute(ax::mojom::StringAttribute::kName);
105 }
106 base::string16 text;
107 for (size_t i = 0; i < node->GetUnignoredChildCount(); ++i) {
108 AXNode* child = node->GetUnignoredChildAtIndex(i);
109 text += GetInnerText(child);
110 }
111 return text;
112 }
113
GetValue(const AXNode * node,bool show_password)114 base::string16 GetValue(const AXNode* node, bool show_password) {
115 base::string16 value =
116 node->data().GetString16Attribute(ax::mojom::StringAttribute::kValue);
117
118 if (value.empty() &&
119 (IsSimpleTextControl(node, node->data().state) ||
120 IsRichTextEditable(node)) &&
121 !IsNativeTextControl(node)) {
122 value = GetInnerText(node);
123 }
124
125 if (node->data().HasState(ax::mojom::State::kProtected)) {
126 if (!show_password) {
127 value = base::string16(value.size(), kSecurePasswordBullet);
128 }
129 }
130
131 return value;
132 }
133
HasOnlyTextAndImageChildren(const AXNode * node)134 bool HasOnlyTextAndImageChildren(const AXNode* node) {
135 for (size_t i = 0; i < node->GetUnignoredChildCount(); ++i) {
136 AXNode* child = node->GetUnignoredChildAtIndex(i);
137 if (!child->IsText() && !ui::IsImage(child->data().role)) {
138 return false;
139 }
140 }
141 return true;
142 }
143
IsFocusable(const AXNode * node)144 bool IsFocusable(const AXNode* node) {
145 if (node->data().role == ax::mojom::Role::kIframe ||
146 node->data().role == ax::mojom::Role::kIframePresentational ||
147 (node->data().role == ax::mojom::Role::kRootWebArea &&
148 node->GetUnignoredParent())) {
149 return node->data().HasStringAttribute(ax::mojom::StringAttribute::kName);
150 }
151 return node->data().HasState(ax::mojom::State::kFocusable);
152 }
153
GetText(const AXNode * node,bool show_password)154 base::string16 GetText(const AXNode* node, bool show_password) {
155 if (node->data().role == ax::mojom::Role::kWebArea ||
156 node->data().role == ax::mojom::Role::kIframe ||
157 node->data().role == ax::mojom::Role::kIframePresentational) {
158 return base::string16();
159 }
160
161 ax::mojom::NameFrom name_from = static_cast<ax::mojom::NameFrom>(
162 node->data().GetIntAttribute(ax::mojom::IntAttribute::kNameFrom));
163 if (ui::IsListItem(node->data().role) &&
164 name_from == ax::mojom::NameFrom::kContents) {
165 if (!node->children().empty() && !HasOnlyTextChildren(node))
166 return base::string16();
167 }
168
169 base::string16 value = GetValue(node, show_password);
170
171 if (!value.empty()) {
172 if (node->data().HasState(ax::mojom::State::kEditable))
173 return value;
174
175 switch (node->data().role) {
176 case ax::mojom::Role::kComboBoxMenuButton:
177 case ax::mojom::Role::kTextFieldWithComboBox:
178 case ax::mojom::Role::kPopUpButton:
179 case ax::mojom::Role::kTextField:
180 return value;
181 default:
182 break;
183 }
184 }
185
186 if (node->data().role == ax::mojom::Role::kColorWell) {
187 unsigned int color = static_cast<unsigned int>(
188 node->data().GetIntAttribute(ax::mojom::IntAttribute::kColorValue));
189 unsigned int red = color >> 16 & 0xFF;
190 unsigned int green = color >> 8 & 0xFF;
191 unsigned int blue = color >> 0 & 0xFF;
192 return base::UTF8ToUTF16(
193 base::StringPrintf("#%02X%02X%02X", red, green, blue));
194 }
195
196 base::string16 text =
197 node->data().GetString16Attribute(ax::mojom::StringAttribute::kName);
198 base::string16 description = node->data().GetString16Attribute(
199 ax::mojom::StringAttribute::kDescription);
200 if (!description.empty()) {
201 if (!text.empty())
202 text += base::ASCIIToUTF16(" ");
203 text += description;
204 }
205
206 if (text.empty())
207 text = value;
208
209 if (node->data().role == ax::mojom::Role::kRootWebArea)
210 return text;
211
212 if (text.empty() &&
213 (HasOnlyTextChildren(node) ||
214 (IsFocusable(node) && HasOnlyTextAndImageChildren(node)))) {
215 for (size_t i = 0; i < node->GetUnignoredChildCount(); ++i) {
216 AXNode* child = node->GetUnignoredChildAtIndex(i);
217 text += GetText(child, show_password);
218 }
219 }
220
221 if (text.empty() && (ui::IsLink(node->data().role) ||
222 node->data().role == ax::mojom::Role::kImage)) {
223 base::string16 url =
224 node->data().GetString16Attribute(ax::mojom::StringAttribute::kUrl);
225 text = AXUrlBaseText(url);
226 }
227 return text;
228 }
229
230 // Get string representation of ax::mojom::Role. We are not using ToString() in
231 // ax_enums.h since the names are subject to change in the future and
232 // we are only interested in a subset of the roles.
AXRoleToString(ax::mojom::Role role)233 base::Optional<std::string> AXRoleToString(ax::mojom::Role role) {
234 switch (role) {
235 case ax::mojom::Role::kArticle:
236 return base::Optional<std::string>("article");
237 case ax::mojom::Role::kBanner:
238 return base::Optional<std::string>("banner");
239 case ax::mojom::Role::kCaption:
240 return base::Optional<std::string>("caption");
241 case ax::mojom::Role::kComplementary:
242 return base::Optional<std::string>("complementary");
243 case ax::mojom::Role::kDate:
244 return base::Optional<std::string>("date");
245 case ax::mojom::Role::kDateTime:
246 return base::Optional<std::string>("date_time");
247 case ax::mojom::Role::kDefinition:
248 return base::Optional<std::string>("definition");
249 case ax::mojom::Role::kDetails:
250 return base::Optional<std::string>("details");
251 case ax::mojom::Role::kDocument:
252 return base::Optional<std::string>("document");
253 case ax::mojom::Role::kFeed:
254 return base::Optional<std::string>("feed");
255 case ax::mojom::Role::kHeading:
256 return base::Optional<std::string>("heading");
257 case ax::mojom::Role::kIframe:
258 return base::Optional<std::string>("iframe");
259 case ax::mojom::Role::kIframePresentational:
260 return base::Optional<std::string>("iframe_presentational");
261 case ax::mojom::Role::kList:
262 return base::Optional<std::string>("list");
263 case ax::mojom::Role::kListItem:
264 return base::Optional<std::string>("list_item");
265 case ax::mojom::Role::kMain:
266 return base::Optional<std::string>("main");
267 case ax::mojom::Role::kParagraph:
268 return base::Optional<std::string>("paragraph");
269 default:
270 return base::Optional<std::string>();
271 }
272 }
273
AddChild(AssistantTree * tree)274 AssistantNode* AddChild(AssistantTree* tree) {
275 auto node = std::make_unique<AssistantNode>();
276 tree->nodes.push_back(std::move(node));
277 return tree->nodes.back().get();
278 }
279
280 struct WalkAXTreeConfig {
281 bool should_select_leaf;
282 const bool show_password;
283 };
284
WalkAXTreeDepthFirst(const AXNode * node,const gfx::Rect & rect,const AXTreeUpdate & update,const AXTree * tree,WalkAXTreeConfig * config,AssistantTree * assistant_tree,AssistantNode * result)285 void WalkAXTreeDepthFirst(const AXNode* node,
286 const gfx::Rect& rect,
287 const AXTreeUpdate& update,
288 const AXTree* tree,
289 WalkAXTreeConfig* config,
290 AssistantTree* assistant_tree,
291 AssistantNode* result) {
292 result->text = GetText(node, config->show_password);
293 result->class_name =
294 AXRoleToAndroidClassName(node->data().role, node->GetUnignoredParent());
295 result->role = AXRoleToString(node->data().role);
296
297 result->text_size = -1.0;
298 result->bgcolor = 0;
299 result->color = 0;
300 result->bold = 0;
301 result->italic = 0;
302 result->line_through = 0;
303 result->underline = 0;
304
305 if (node->data().HasFloatAttribute(ax::mojom::FloatAttribute::kFontSize)) {
306 gfx::RectF text_size_rect(
307 0, 0, 1,
308 node->data().GetFloatAttribute(ax::mojom::FloatAttribute::kFontSize));
309 gfx::Rect scaled_text_size_rect =
310 gfx::ToEnclosingRect(tree->RelativeToTreeBounds(node, text_size_rect));
311 result->text_size = scaled_text_size_rect.height();
312
313 result->color =
314 node->data().GetIntAttribute(ax::mojom::IntAttribute::kColor);
315 result->bgcolor =
316 node->data().GetIntAttribute(ax::mojom::IntAttribute::kBackgroundColor);
317 result->bold = node->data().HasTextStyle(ax::mojom::TextStyle::kBold);
318 result->italic = node->data().HasTextStyle(ax::mojom::TextStyle::kItalic);
319 result->line_through =
320 node->data().HasTextStyle(ax::mojom::TextStyle::kLineThrough);
321 result->underline =
322 node->data().HasTextStyle(ax::mojom::TextStyle::kUnderline);
323 }
324
325 const gfx::Rect& absolute_rect =
326 gfx::ToEnclosingRect(tree->GetTreeBounds(node));
327 gfx::Rect parent_relative_rect = absolute_rect;
328 bool is_root = !node->GetUnignoredParent();
329 if (!is_root) {
330 parent_relative_rect.Offset(-rect.OffsetFromOrigin());
331 }
332 result->rect = gfx::Rect(parent_relative_rect.x(), parent_relative_rect.y(),
333 absolute_rect.width(), absolute_rect.height());
334
335 if (IsLeaf(node) && update.has_tree_data) {
336 int start_selection = 0;
337 int end_selection = 0;
338 AXTree::Selection unignored_selection = tree->GetUnignoredSelection();
339 if (unignored_selection.anchor_object_id == node->id()) {
340 start_selection = unignored_selection.anchor_offset;
341 config->should_select_leaf = true;
342 }
343
344 if (config->should_select_leaf) {
345 end_selection =
346 static_cast<int32_t>(GetText(node, config->show_password).length());
347 }
348
349 if (unignored_selection.focus_object_id == node->id()) {
350 end_selection = unignored_selection.focus_offset;
351 config->should_select_leaf = false;
352 }
353 if (end_selection > 0)
354 result->selection =
355 base::make_optional<gfx::Range>(start_selection, end_selection);
356 }
357
358 for (size_t i = 0; i < node->GetUnignoredChildCount(); ++i) {
359 AXNode* child = node->GetUnignoredChildAtIndex(i);
360 auto* n = AddChild(assistant_tree);
361 result->children_indices.push_back(assistant_tree->nodes.size() - 1);
362 WalkAXTreeDepthFirst(child, absolute_rect, update, tree, config,
363 assistant_tree, n);
364 }
365 }
366
367 } // namespace
368
369 AssistantNode::AssistantNode() = default;
370 AssistantNode::AssistantNode(const AssistantNode& other) = default;
371 AssistantNode::~AssistantNode() = default;
372
373 AssistantTree::AssistantTree() = default;
374 AssistantTree::~AssistantTree() = default;
375
AssistantTree(const AssistantTree & other)376 AssistantTree::AssistantTree(const AssistantTree& other) {
377 for (const auto& node : other.nodes)
378 nodes.emplace_back(std::make_unique<AssistantNode>(*node));
379 }
380
CreateAssistantTree(const AXTreeUpdate & update,bool show_password)381 std::unique_ptr<AssistantTree> CreateAssistantTree(const AXTreeUpdate& update,
382 bool show_password) {
383 auto tree = std::make_unique<AXSerializableTree>();
384 auto assistant_tree = std::make_unique<AssistantTree>();
385 auto* root = AddChild(assistant_tree.get());
386 if (!tree->Unserialize(update))
387 LOG(FATAL) << tree->error();
388 WalkAXTreeConfig config{
389 false, // should_select_leaf
390 show_password // show_password
391 };
392 WalkAXTreeDepthFirst(tree->root(), gfx::Rect(), update, tree.get(), &config,
393 assistant_tree.get(), root);
394 return assistant_tree;
395 }
396
AXUrlBaseText(base::string16 url)397 base::string16 AXUrlBaseText(base::string16 url) {
398 // Given a url like http://foo.com/bar/baz.png, just return the
399 // base text, e.g., "baz".
400 int trailing_slashes = 0;
401 while (url.size() - trailing_slashes > 0 &&
402 url[url.size() - trailing_slashes - 1] == '/') {
403 trailing_slashes++;
404 }
405 if (trailing_slashes)
406 url = url.substr(0, url.size() - trailing_slashes);
407 size_t slash_index = url.rfind('/');
408 if (slash_index != std::string::npos)
409 url = url.substr(slash_index + 1);
410 size_t dot_index = url.rfind('.');
411 if (dot_index != std::string::npos)
412 url = url.substr(0, dot_index);
413 return url;
414 }
415
AXRoleToAndroidClassName(ax::mojom::Role role,bool has_parent)416 const char* AXRoleToAndroidClassName(ax::mojom::Role role, bool has_parent) {
417 switch (role) {
418 case ax::mojom::Role::kSearchBox:
419 case ax::mojom::Role::kSpinButton:
420 case ax::mojom::Role::kTextField:
421 case ax::mojom::Role::kTextFieldWithComboBox:
422 return kAXEditTextClassname;
423 case ax::mojom::Role::kSlider:
424 return kAXSeekBarClassname;
425 case ax::mojom::Role::kColorWell:
426 case ax::mojom::Role::kComboBoxMenuButton:
427 case ax::mojom::Role::kDate:
428 case ax::mojom::Role::kPopUpButton:
429 case ax::mojom::Role::kInputTime:
430 return kAXSpinnerClassname;
431 case ax::mojom::Role::kButton:
432 case ax::mojom::Role::kPdfActionableHighlight:
433 return kAXButtonClassname;
434 case ax::mojom::Role::kCheckBox:
435 case ax::mojom::Role::kSwitch:
436 return kAXCheckBoxClassname;
437 case ax::mojom::Role::kRadioButton:
438 return kAXRadioButtonClassname;
439 case ax::mojom::Role::kToggleButton:
440 return kAXToggleButtonClassname;
441 case ax::mojom::Role::kCanvas:
442 case ax::mojom::Role::kImage:
443 case ax::mojom::Role::kSvgRoot:
444 return kAXImageClassname;
445 case ax::mojom::Role::kMeter:
446 case ax::mojom::Role::kProgressIndicator:
447 return kAXProgressBarClassname;
448 case ax::mojom::Role::kTabList:
449 return kAXTabWidgetClassname;
450 case ax::mojom::Role::kGrid:
451 case ax::mojom::Role::kTreeGrid:
452 case ax::mojom::Role::kTable:
453 return kAXGridViewClassname;
454 case ax::mojom::Role::kList:
455 case ax::mojom::Role::kListBox:
456 case ax::mojom::Role::kDescriptionList:
457 return kAXListViewClassname;
458 case ax::mojom::Role::kDialog:
459 return kAXDialogClassname;
460 case ax::mojom::Role::kRootWebArea:
461 return has_parent ? kAXViewClassname : kAXWebViewClassname;
462 case ax::mojom::Role::kMenuItem:
463 case ax::mojom::Role::kMenuItemCheckBox:
464 case ax::mojom::Role::kMenuItemRadio:
465 return kAXMenuItemClassname;
466 case ax::mojom::Role::kStaticText:
467 return kAXTextViewClassname;
468 default:
469 return kAXViewClassname;
470 }
471 }
472
473 } // namespace ui
474