1 // Copyright 2019 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef CONTENT_RENDERER_ACCESSIBILITY_AX_IMAGE_ANNOTATOR_H_ 6 #define CONTENT_RENDERER_ACCESSIBILITY_AX_IMAGE_ANNOTATOR_H_ 7 8 #include <string> 9 #include <unordered_map> 10 11 #include "base/bind.h" 12 #include "base/macros.h" 13 #include "base/memory/weak_ptr.h" 14 #include "base/observer_list_types.h" 15 #include "base/optional.h" 16 #include "content/common/content_export.h" 17 #include "content/renderer/accessibility/render_accessibility_impl.h" 18 #include "mojo/public/cpp/bindings/pending_remote.h" 19 #include "mojo/public/cpp/bindings/remote.h" 20 #include "services/image_annotation/public/cpp/image_processor.h" 21 #include "services/image_annotation/public/mojom/image_annotation.mojom-forward.h" 22 #include "third_party/skia/include/core/SkBitmap.h" 23 #include "ui/accessibility/ax_enums.mojom.h" 24 25 namespace blink { 26 27 class WebAXObject; 28 29 } // namespace blink 30 31 namespace content { 32 33 class ContentClient; 34 35 // This class gets notified that certain images have been added, removed or 36 // updated on a page. This class is then responsible for retrieving the 37 // automatic label for all images and notifying the RenderAccessibility that 38 // owns it to update the relevant image annotations. 39 class CONTENT_EXPORT AXImageAnnotator : public base::CheckedObserver { 40 public: 41 AXImageAnnotator( 42 RenderAccessibilityImpl* const render_accessibility, 43 const std::string& preferred_language, 44 mojo::PendingRemote<image_annotation::mojom::Annotator> annotator); 45 ~AXImageAnnotator() override; 46 47 void Destroy(); 48 49 std::string GetImageAnnotation(blink::WebAXObject& image) const; 50 ax::mojom::ImageAnnotationStatus GetImageAnnotationStatus( 51 blink::WebAXObject& image) const; 52 bool HasAnnotationInCache(blink::WebAXObject& image) const; 53 bool HasImageInCache(const blink::WebAXObject& image) const; 54 55 void OnImageAdded(blink::WebAXObject& image); 56 void OnImageUpdated(blink::WebAXObject& image); 57 void OnImageRemoved(blink::WebAXObject& image); 58 set_preferred_language(const std::string & language)59 void set_preferred_language(const std::string& language) { 60 preferred_language_ = language; 61 } 62 63 private: 64 // Keeps track of the image data and the automatic annotation for each image. 65 class ImageInfo final { 66 public: 67 ImageInfo() = default; 68 ImageInfo(const blink::WebAXObject& image); 69 virtual ~ImageInfo(); 70 ImageInfo(ImageInfo&&) = default; 71 ImageInfo& operator=(ImageInfo&&) = default; 72 73 mojo::PendingRemote<image_annotation::mojom::ImageProcessor> 74 GetImageProcessor(); 75 bool HasAnnotation() const; 76 status()77 ax::mojom::ImageAnnotationStatus status() const { return status_; } 78 set_status(ax::mojom::ImageAnnotationStatus status)79 void set_status(ax::mojom::ImageAnnotationStatus status) { 80 DCHECK_NE(status, ax::mojom::ImageAnnotationStatus::kNone); 81 status_ = status; 82 } 83 annotation()84 std::string annotation() const { 85 return annotation_.value_or(""); 86 } 87 set_annotation(std::string annotation)88 void set_annotation(std::string annotation) { annotation_ = annotation; } 89 90 private: 91 image_annotation::ImageProcessor image_processor_; 92 ax::mojom::ImageAnnotationStatus status_; 93 base::Optional<std::string> annotation_; 94 }; 95 96 // Retrieves the image data from the renderer. 97 static SkBitmap GetImageData(const blink::WebAXObject& image); 98 99 // Used by tests to override the content client. 100 virtual ContentClient* GetContentClient() const; 101 102 // Given a WebImage, it uses the URL of the main document and the src 103 // attribute of the image, to generate a unique identifier for the image that 104 // could be provided to the image annotation service. 105 // 106 // This method is virtual to allow overriding it from tests. 107 virtual std::string GenerateImageSourceId( 108 const blink::WebAXObject& image) const; 109 110 // Removes the automatic image annotations from all images. 111 void MarkAllImagesDirty(); 112 113 // Marks a node in the accessibility tree dirty when an image annotation 114 // changes. Also marks dirty a link or document that immediately contains 115 // an image. 116 void MarkDirty(const blink::WebAXObject& image) const; 117 118 // Gets called when an image gets annotated by the image annotation service. 119 void OnImageAnnotated(const blink::WebAXObject& image, 120 image_annotation::mojom::AnnotateImageResultPtr result); 121 122 // Only for local logging when running with --v=1. 123 std::string GetDocumentUrl() const; 124 125 // Weak, owns us. 126 RenderAccessibilityImpl* const render_accessibility_; 127 128 // The language in which to request image descriptions. 129 std::string preferred_language_; 130 131 // A pointer to the automatic image annotation service. 132 mojo::Remote<image_annotation::mojom::Annotator> annotator_; 133 134 // Keeps track of the image data and the automatic annotations for each image. 135 // 136 // The key is retrieved using WebAXObject::AxID(). 137 std::unordered_map<int, ImageInfo> image_annotations_; 138 139 // This member needs to be last because it should destructed first. 140 base::WeakPtrFactory<AXImageAnnotator> weak_factory_{this}; 141 142 DISALLOW_COPY_AND_ASSIGN(AXImageAnnotator); 143 }; 144 145 } // namespace content 146 147 #endif // CONTENT_RENDERER_ACCESSIBILITY_AX_IMAGE_ANNOTATOR_H_ 148