1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef CONTENT_RENDERER_ACCESSIBILITY_RENDER_ACCESSIBILITY_IMPL_H_ 6 #define CONTENT_RENDERER_ACCESSIBILITY_RENDER_ACCESSIBILITY_IMPL_H_ 7 8 #include <memory> 9 #include <unordered_map> 10 #include <vector> 11 12 #include "base/macros.h" 13 #include "base/memory/weak_ptr.h" 14 #include "content/common/content_export.h" 15 #include "content/common/render_accessibility.mojom.h" 16 #include "content/public/renderer/plugin_ax_tree_source.h" 17 #include "content/public/renderer/render_accessibility.h" 18 #include "content/public/renderer/render_frame.h" 19 #include "content/public/renderer/render_frame_observer.h" 20 #include "content/renderer/accessibility/blink_ax_tree_source.h" 21 #include "third_party/blink/public/web/web_ax_context.h" 22 #include "third_party/blink/public/web/web_ax_object.h" 23 #include "ui/accessibility/ax_event.h" 24 #include "ui/accessibility/ax_node_data.h" 25 #include "ui/accessibility/ax_relative_bounds.h" 26 #include "ui/accessibility/ax_tree.h" 27 #include "ui/accessibility/ax_tree_data.h" 28 #include "ui/accessibility/ax_tree_serializer.h" 29 #include "ui/accessibility/ax_tree_update.h" 30 #include "ui/gfx/geometry/rect_f.h" 31 32 namespace base { 33 class ElapsedTimer; 34 } // namespace base 35 36 namespace blink { 37 class WebDocument; 38 } // namespace blink 39 40 namespace ui { 41 struct AXActionData; 42 class AXActionTarget; 43 struct AXEvent; 44 } 45 46 namespace ukm { 47 class MojoUkmRecorder; 48 } 49 50 namespace content { 51 52 class AXImageAnnotator; 53 class RenderFrameImpl; 54 class RenderAccessibilityManager; 55 56 using BlinkAXTreeSerializer = 57 ui::AXTreeSerializer<blink::WebAXObject, ui::AXNodeData, ui::AXTreeData>; 58 59 class AXTreeSnapshotterImpl : public AXTreeSnapshotter { 60 public: 61 explicit AXTreeSnapshotterImpl(RenderFrameImpl* render_frame); 62 ~AXTreeSnapshotterImpl() override; 63 64 // AXTreeSnapshotter implementation. 65 void Snapshot(ui::AXMode ax_mode, 66 size_t max_node_count, 67 ui::AXTreeUpdate* accessibility_tree) override; 68 69 // Same as above, but returns in |accessibility_tree| a ui::AXTreeUpdate 70 // with content-specific metadata, instead of an AXTreeUpdate. 71 void SnapshotContentTree(ui::AXMode ax_mode, 72 size_t max_node_count, 73 ui::AXTreeUpdate* accessibility_tree); 74 75 private: 76 RenderFrameImpl* render_frame_; 77 std::unique_ptr<blink::WebAXContext> context_; 78 79 AXTreeSnapshotterImpl(const AXTreeSnapshotterImpl&) = delete; 80 AXTreeSnapshotterImpl& operator=(const AXTreeSnapshotterImpl&) = delete; 81 }; 82 83 // The browser process implements native accessibility APIs, allowing assistive 84 // technology (e.g., screen readers, magnifiers) to access and control the web 85 // contents with high-level APIs. These APIs are also used by automation tools, 86 // and Windows 8 uses them to determine when the on-screen keyboard should be 87 // shown. 88 // 89 // An instance of this class belongs to the RenderAccessibilityManager object. 90 // Accessibility is initialized based on the ui::AXMode passed from the browser 91 // process to the manager object; it lazily starts as Off or EditableTextOnly 92 // depending on the operating system, and switches to Complete if assistive 93 // technology is detected or a flag is set. 94 // 95 // A tree of accessible objects is built here and sent to the browser process; 96 // the browser process maintains this as a tree of platform-native accessible 97 // objects that can be used to respond to accessibility requests from other 98 // processes. 99 // 100 // This class implements complete accessibility support for assistive 101 // technology. It turns on Blink's accessibility code and sends a serialized 102 // representation of that tree whenever it changes. It also handles requests 103 // from the browser to perform accessibility actions on nodes in the tree (e.g., 104 // change focus, or click on a button). 105 class CONTENT_EXPORT RenderAccessibilityImpl : public RenderAccessibility, 106 public RenderFrameObserver { 107 public: 108 // Request a one-time snapshot of the accessibility tree without 109 // enabling accessibility if it wasn't already enabled. 110 static void SnapshotAccessibilityTree(RenderFrameImpl* render_frame, 111 ui::AXTreeUpdate* response, 112 ui::AXMode ax_mode); 113 114 RenderAccessibilityImpl( 115 RenderAccessibilityManager* const render_accessibility_manager, 116 RenderFrameImpl* const render_frame, 117 ui::AXMode mode); 118 ~RenderAccessibilityImpl() override; 119 GetAccessibilityMode()120 ui::AXMode GetAccessibilityMode() { 121 return tree_source_->accessibility_mode(); 122 } 123 124 // RenderAccessibility implementation. 125 int GenerateAXID() override; 126 void SetPluginTreeSource(PluginAXTreeSource* source) override; 127 void OnPluginRootNodeUpdated() override; 128 void ShowPluginContextMenu() override; 129 130 // RenderFrameObserver implementation. 131 void DidCreateNewDocument() override; 132 void DidCommitProvisionalLoad(ui::PageTransition transition) override; 133 void AccessibilityModeChanged(const ui::AXMode& mode) override; 134 135 void HitTest(const gfx::Point& point, 136 ax::mojom::Event event_to_fire, 137 int request_id, 138 mojom::RenderAccessibility::HitTestCallback callback); 139 void PerformAction(const ui::AXActionData& data); 140 void Reset(int32_t reset_token); 141 142 // Called when an accessibility notification occurs in Blink. 143 void HandleWebAccessibilityEvent(const ui::AXEvent& event); 144 void MarkWebAXObjectDirty(const blink::WebAXObject& obj, bool subtree); 145 146 void HandleAXEvent(const ui::AXEvent& event); 147 148 // Returns the main top-level document for this page, or NULL if there's 149 // no view or frame. 150 blink::WebDocument GetMainDocument(); 151 152 // Returns the page language. 153 std::string GetLanguage(); 154 155 // Access the UKM recorder. ukm_recorder()156 ukm::MojoUkmRecorder* ukm_recorder() const { return ukm_recorder_.get(); } 157 158 protected: 159 // Send queued events from the renderer to the browser. 160 void SendPendingAccessibilityEvents(); 161 162 // Check the entire accessibility tree to see if any nodes have 163 // changed location, by comparing their locations to the cached 164 // versions. If any have moved, send an IPC with the new locations. 165 void SendLocationChanges(); 166 167 // Return true if the event indicates that the current batch of changes 168 // should be processed immediately in order for the user to get fast 169 // feedback, e.g. for navigation or data entry activities. 170 bool IsImmediateProcessingRequiredForEvent(const ui::AXEvent&) const; 171 172 // Get the amount of time, in ms, that event processing should be deferred 173 // in order to more efficiently batch changes. 174 int GetDeferredEventsDelay(); 175 176 private: 177 struct DirtyObject { 178 DirtyObject(); 179 DirtyObject(const DirtyObject& other); 180 ~DirtyObject(); 181 blink::WebAXObject obj; 182 ax::mojom::EventFrom event_from; 183 std::vector<ui::AXEventIntent> event_intents; 184 }; 185 186 enum class EventScheduleMode { kDeferEvents, kProcessEventsImmediately }; 187 188 enum class EventScheduleStatus { 189 // Events have been scheduled with a delay, but have not been sent. 190 kScheduledDeferred, 191 // Events have been scheduled without a delay, but have not been sent. 192 kScheduledImmediate, 193 // Events have been sent, waiting for callback. 194 kWaitingForAck, 195 // Events are not scheduled and we are not waiting for an ack. 196 kNotWaiting 197 }; 198 199 // Callback that will be called from the browser upon handling the message 200 // previously sent to it via SendPendingAccessibilityEvents(). 201 void OnAccessibilityEventsHandled(); 202 203 // RenderFrameObserver implementation. 204 void OnDestruct() override; 205 206 // Handlers for messages from the browser to the renderer. 207 void OnLoadInlineTextBoxes(const ui::AXActionTarget* target); 208 void OnGetImageData(const ui::AXActionTarget* target, 209 const gfx::Size& max_size); 210 void AddPluginTreeToUpdate(ui::AXTreeUpdate* update, 211 bool invalidate_plugin_subtree); 212 213 // Creates and takes ownership of an instance of the class that automatically 214 // labels images for accessibility. 215 void CreateAXImageAnnotator(); 216 217 // Automatically labels images for accessibility if the accessibility mode for 218 // this feature is turned on, otherwise stops automatic labeling and removes 219 // any automatic annotations that might have been added before. 220 void StartOrStopLabelingImages(ui::AXMode old_mode, ui::AXMode new_mode); 221 222 // Marks all AXObjects with the given role in the current tree dirty. 223 void MarkAllAXObjectsDirty(ax::mojom::Role role); 224 225 void Scroll(const ui::AXActionTarget* target, 226 ax::mojom::Action scroll_action); 227 228 // Whether an event should mark its associated object dirty. 229 bool ShouldSerializeNodeForEvent(const blink::WebAXObject& obj, 230 const ui::AXEvent& event) const; 231 232 // If we are calling this from a task, scheduling is allowed even if there is 233 // a running task 234 void ScheduleSendPendingAccessibilityEvents( 235 bool scheduling_from_task = false); 236 void AddImageAnnotationDebuggingAttributes( 237 const std::vector<ui::AXTreeUpdate>& updates); 238 239 // Returns the document for the active popup if any. 240 blink::WebDocument GetPopupDocument(); 241 242 // Searches the accessibility tree for plugin's root object and returns it. 243 // Returns an empty WebAXObject if no root object is present. 244 blink::WebAXObject GetPluginRoot(); 245 246 // Cancels scheduled events that are not yet in flight 247 void CancelScheduledEvents(); 248 249 // Sends the URL-keyed metrics for the maximum amount of time spent in 250 // SendPendingAccessibilityEvents if they meet the minimum criteria for 251 // sending. 252 void MaybeSendUKM(); 253 254 // Reset all of the UKM data. This can be called after sending UKM data, 255 // or after navigating to a new page when any previous data will no 256 // longer be valid. 257 void ResetUKMData(); 258 259 // The initial accessibility tree root still needs to be created. Like other 260 // accessible objects, it must be created when layout is clean. 261 bool needs_initial_ax_tree_root_ = true; 262 263 // The RenderAccessibilityManager that owns us. 264 RenderAccessibilityManager* render_accessibility_manager_; 265 266 // The associated RenderFrameImpl by means of the RenderAccessibilityManager. 267 RenderFrameImpl* render_frame_; 268 269 // This keeps accessibility enabled as long as it lives. 270 std::unique_ptr<blink::WebAXContext> ax_context_; 271 272 // Manages the automatic image annotations, if enabled. 273 std::unique_ptr<AXImageAnnotator> ax_image_annotator_; 274 275 // Events from Blink are collected until they are ready to be 276 // sent to the browser. 277 std::vector<ui::AXEvent> pending_events_; 278 279 // Objects that need to be re-serialized, the next time 280 // we send an event bundle to the browser - but don't specifically need 281 // an event fired. 282 std::vector<DirtyObject> dirty_objects_; 283 284 // The adapter that exposes Blink's accessibility tree to AXTreeSerializer. 285 std::unique_ptr<BlinkAXTreeSource> tree_source_; 286 287 // The serializer that sends accessibility messages to the browser process. 288 std::unique_ptr<BlinkAXTreeSerializer> serializer_; 289 290 using PluginAXTreeSerializer = ui::AXTreeSerializer<const ui::AXNode*, 291 ui::AXNodeData, 292 ui::AXTreeData>; 293 std::unique_ptr<PluginAXTreeSerializer> plugin_serializer_; 294 PluginAXTreeSource* plugin_tree_source_; 295 blink::WebAXObject plugin_host_node_; 296 297 // The most recently observed scroll offset of the root document element. 298 // TODO(dmazzoni): remove once https://bugs.webkit.org/show_bug.cgi?id=73460 299 // is fixed. 300 gfx::Size last_scroll_offset_; 301 302 // Current event scheduling status 303 EventScheduleStatus event_schedule_status_; 304 305 // Nonzero if the browser requested we reset the accessibility state. 306 // We need to return this token in the next IPC. 307 int reset_token_; 308 309 // Whether or not we've injected a stylesheet in this document 310 // (only when debugging flags are enabled, never under normal circumstances). 311 bool has_injected_stylesheet_ = false; 312 313 // We defer events to improve performance during the initial page load. 314 EventScheduleMode event_schedule_mode_; 315 316 // Whether we should highlight annotation results visually on the page 317 // for debugging. 318 bool image_annotation_debugging_ = false; 319 320 // The specified page language, or empty if unknown. 321 std::string page_language_; 322 323 // The URL-keyed metrics recorder interface. 324 std::unique_ptr<ukm::MojoUkmRecorder> ukm_recorder_; 325 326 // The longest amount of time spent serializing the accessibility tree 327 // in SendPendingAccessibilityEvents. This is periodically uploaded as 328 // a UKM and then reset. 329 int slowest_serialization_ms_ = 0; 330 331 // The amount of time since the last UKM upload. 332 std::unique_ptr<base::ElapsedTimer> ukm_timer_; 333 334 // The UKM Source ID that corresponds to the web page represented by 335 // slowest_serialization_ms_. We report UKM before the user navigates 336 // away, or every few minutes. 337 ukm::SourceId last_ukm_source_id_; 338 std::string last_ukm_url_; 339 340 // So we can queue up tasks to be executed later. 341 base::WeakPtrFactory<RenderAccessibilityImpl> 342 weak_factory_for_pending_events_{this}; 343 344 friend class AXImageAnnotatorTest; 345 friend class PluginActionHandlingTest; 346 friend class RenderAccessibilityImplTest; 347 friend class RenderAccessibilityImplUKMTest; 348 349 DISALLOW_COPY_AND_ASSIGN(RenderAccessibilityImpl); 350 }; 351 352 } // namespace content 353 354 #endif // CONTENT_RENDERER_ACCESSIBILITY_RENDER_ACCESSIBILITY_IMPL_H_ 355