1 // Copyright 2018 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_VIDEO_CAPTURE_OVERLAY_H_ 6 #define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_VIDEO_CAPTURE_OVERLAY_H_ 7 8 #include <stdint.h> 9 10 #include <memory> 11 #include <vector> 12 13 #include "base/callback.h" 14 #include "base/macros.h" 15 #include "base/memory/ref_counted.h" 16 #include "base/sequence_checker.h" 17 #include "components/viz/service/viz_service_export.h" 18 #include "media/base/video_types.h" 19 #include "mojo/public/cpp/bindings/pending_receiver.h" 20 #include "mojo/public/cpp/bindings/receiver.h" 21 #include "services/viz/privileged/mojom/compositing/frame_sink_video_capture.mojom.h" 22 #include "third_party/skia/include/core/SkBitmap.h" 23 #include "ui/gfx/color_space.h" 24 #include "ui/gfx/color_transform.h" 25 #include "ui/gfx/geometry/rect.h" 26 #include "ui/gfx/geometry/rect_f.h" 27 #include "ui/gfx/geometry/size.h" 28 29 namespace media { 30 class VideoFrame; 31 } 32 33 namespace viz { 34 35 // An overlay image to be blitted onto video frames. A mojo client sets the 36 // image, position, and size of the overlay in the video frame; and then this 37 // VideoCaptureOverlay scales the image and maps its color space to match that 38 // of the video frame before the blitting. 39 // 40 // As an optimization, the client's bitmap image is transformed (scaled, color 41 // space converted, and pre-multiplied by alpha), and then this cached Sprite is 42 // re-used for blitting to all successive video frames until some change 43 // requires a different transformation. MakeRenderer() produces a Renderer 44 // callback that holds a reference to an existing Sprite, or will create a new 45 // one if necessary. The Renderer callback can then be run at any point in the 46 // future, unaffected by later image, size, or color space settings changes. 47 // 48 // The blit algorithm uses naive linear blending. Thus, the use of non-linear 49 // color spaces will cause loses in color accuracy. 50 class VIZ_SERVICE_EXPORT VideoCaptureOverlay 51 : public mojom::FrameSinkVideoCaptureOverlay { 52 public: 53 // Interface for notifying the frame source when changes to the overlay's 54 // state occur. 55 class VIZ_SERVICE_EXPORT FrameSource { 56 public: 57 // Returns the current size of the source, or empty if unknown. 58 virtual gfx::Size GetSourceSize() = 0; 59 60 // Notifies the FrameSource that the given source |rect| needs to be 61 // re-captured soon. One or more calls to this method will be followed-up 62 // with a call to RequestRefreshFrame(). 63 virtual void InvalidateRect(const gfx::Rect& rect) = 0; 64 65 // Notifies the FrameSource that another frame should be captured and have 66 // its VideoCaptureOverlay re-rendered soon to reflect an updated overlay 67 // image and/or position. 68 virtual void RequestRefreshFrame() = 0; 69 70 // Notifies the FrameSource that the VideoCaptureOverlay has lost its mojo 71 // binding. 72 virtual void OnOverlayConnectionLost(VideoCaptureOverlay* overlay) = 0; 73 74 protected: 75 virtual ~FrameSource(); 76 }; 77 78 // A OnceCallback that, when run, renders the overlay on a VideoFrame. 79 using OnceRenderer = base::OnceCallback<void(media::VideoFrame*)>; 80 81 // |frame_source| must outlive this instance. 82 VideoCaptureOverlay( 83 FrameSource* frame_source, 84 mojo::PendingReceiver<mojom::FrameSinkVideoCaptureOverlay> receiver); 85 86 ~VideoCaptureOverlay() final; 87 88 // mojom::FrameSinkVideoCaptureOverlay implementation: 89 void SetImageAndBounds(const SkBitmap& image, const gfx::RectF& bounds) final; 90 void SetBounds(const gfx::RectF& bounds) final; 91 92 // Returns a OnceCallback that, when run, renders this VideoCaptureOverlay on 93 // a VideoFrame. The overlay's position and size are computed based on the 94 // given content |region_in_frame|. Returns a null OnceCallback if there is 95 // nothing to render at this time. 96 OnceRenderer MakeRenderer(const gfx::Rect& region_in_frame, 97 const media::VideoPixelFormat frame_format); 98 99 // Returns a OnceCallback that renders all of the given |overlays| in 100 // order. The remaining arguments are the same as in MakeRenderer(). This is a 101 // convenience that produces a single callback, so that client code need not 102 // deal with collections of callbacks. Returns a null OnceCallback if there is 103 // nothing to render at this time. 104 static OnceRenderer MakeCombinedRenderer( 105 const std::vector<VideoCaptureOverlay*>& overlays, 106 const gfx::Rect& region_in_frame, 107 const media::VideoPixelFormat frame_format); 108 109 private: 110 // Transforms the overlay SkBitmap image by scaling and converting its color 111 // space, and then blitting it onto a VideoFrame. The transformation is lazy: 112 // Meaning, the transformation is executed upon the first call to Blit(), and 113 // the result is cached for re-use for later Blit() calls. The transformation 114 // is re-executed if the color space of the VideoFrame changes (rarely). 115 class Sprite : public base::RefCounted<Sprite> { 116 public: 117 Sprite(const SkBitmap& image, 118 const gfx::Size& size, 119 const media::VideoPixelFormat format); 120 size()121 const gfx::Size& size() const { return size_; } format()122 media::VideoPixelFormat format() const { return format_; } 123 124 void Blit(const gfx::Point& position, 125 const gfx::Rect& blit_rect, 126 media::VideoFrame* frame); 127 128 private: 129 friend class base::RefCounted<Sprite>; 130 ~Sprite(); 131 132 void TransformImage(); 133 134 // As Sprites can be long-lived and hidden from external code within 135 // callbacks, ensure that all Blit() calls are in-sequence. 136 SEQUENCE_CHECKER(sequence_checker_); 137 138 // Starts-out as the original, unscaled overlay image. The first call to 139 // TransformImage() replaces it with a scaled one. 140 SkBitmap image_; 141 142 // The size, format, and color space of the cached transformed image. 143 const gfx::Size size_; 144 const media::VideoPixelFormat format_; 145 gfx::ColorSpace color_space_; 146 147 // The transformed source image data. For blitting to ARGB format video 148 // frames, the source image data will consist of 4 elements per pixel pixel 149 // (A, R, G, B). For blitting to the I420 format, the source image data is 150 // not interlaved: Instead, there are 5 planes of data (one minus alpha, Y, 151 // subsampled one minus alpha, U, V). For both formats, the color components 152 // are premultiplied for more-efficient Blit()'s. 153 std::unique_ptr<float[]> transformed_image_; 154 155 DISALLOW_COPY_AND_ASSIGN(Sprite); 156 }; 157 158 // Computes the region of the source that, if changed, would require 159 // re-rendering the overlay. 160 gfx::Rect ComputeSourceMutationRect() const; 161 162 FrameSource* const frame_source_; 163 164 mojo::Receiver<mojom::FrameSinkVideoCaptureOverlay> receiver_; 165 166 // The currently-set overlay image. 167 SkBitmap image_; 168 169 // If empty, the overlay is currently hidden. Otherwise, this consists of 170 // coordinates where the range [0.0,1.0) indicates the relative position+size 171 // within the bounds of the video frame's content region (e.g., 0.0 refers to 172 // the top or left edge; 1.0 to just after the bottom or right edge). 173 gfx::RectF bounds_; 174 175 // The current Sprite. This is set to null whenever a settings change requires 176 // a new Sprite to be generated from the |image_|. 177 scoped_refptr<Sprite> sprite_; 178 179 DISALLOW_COPY_AND_ASSIGN(VideoCaptureOverlay); 180 }; 181 182 } // namespace viz 183 184 #endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_VIDEO_CAPTURE_OVERLAY_H_ 185