1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_ 6 #define MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_ 7 8 #include <stddef.h> 9 #include <stdint.h> 10 11 #include "base/macros.h" 12 #include "base/memory/ref_counted.h" 13 #include "base/optional.h" 14 #include "base/threading/thread_checker.h" 15 #include "base/time/time.h" 16 #include "base/timer/timer.h" 17 #include "cc/paint/paint_canvas.h" 18 #include "cc/paint/paint_flags.h" 19 #include "cc/paint/paint_image.h" 20 #include "gpu/command_buffer/common/mailbox.h" 21 #include "media/base/media_export.h" 22 #include "media/base/timestamp_constants.h" 23 #include "media/base/video_frame.h" 24 #include "media/base/video_transformation.h" 25 #include "media/renderers/video_frame_yuv_converter.h" 26 27 namespace gfx { 28 class RectF; 29 } 30 31 namespace gpu { 32 struct Capabilities; 33 34 namespace gles2 { 35 class GLES2Interface; 36 } 37 } // namespace gpu 38 39 namespace viz { 40 class RasterContextProvider; 41 } 42 43 namespace media { 44 45 // Handles rendering of VideoFrames to PaintCanvases. 46 class MEDIA_EXPORT PaintCanvasVideoRenderer { 47 public: 48 PaintCanvasVideoRenderer(); 49 ~PaintCanvasVideoRenderer(); 50 51 // Paints |video_frame| translated and scaled to |dest_rect| on |canvas|. 52 // 53 // If the format of |video_frame| is PIXEL_FORMAT_NATIVE_TEXTURE, |context_3d| 54 // and |context_support| must be provided. 55 // 56 // If |video_frame| is nullptr or an unsupported format, |dest_rect| will be 57 // painted black. 58 void Paint(scoped_refptr<VideoFrame> video_frame, 59 cc::PaintCanvas* canvas, 60 const gfx::RectF& dest_rect, 61 cc::PaintFlags& flags, 62 VideoTransformation video_transformation, 63 viz::RasterContextProvider* raster_context_provider); 64 65 // Paints |video_frame|, scaled to its |video_frame->visible_rect().size()| 66 // on |canvas|. Note that the origin of |video_frame->visible_rect()| is 67 // ignored -- the copy is done to the origin of |canvas|. 68 // 69 // If the format of |video_frame| is PIXEL_FORMAT_NATIVE_TEXTURE, |context_3d| 70 // and |context_support| must be provided. 71 void Copy(scoped_refptr<VideoFrame> video_frame, 72 cc::PaintCanvas* canvas, 73 viz::RasterContextProvider* raster_context_provider); 74 75 // Convert the contents of |video_frame| to raw RGB pixels. |rgb_pixels| 76 // should point into a buffer large enough to hold as many 32 bit RGBA pixels 77 // as are in the visible_rect() area of the frame. |premultiply_alpha| 78 // indicates whether the R, G, B samples in |rgb_pixels| should be multiplied 79 // by alpha. 80 // 81 // NOTE: If |video_frame| doesn't have an alpha plane, all the A samples in 82 // |rgb_pixels| will be 255 (equivalent to an alpha of 1.0) and therefore the 83 // value of |premultiply_alpha| has no effect on the R, G, B samples in 84 // |rgb_pixels|. 85 static void ConvertVideoFrameToRGBPixels(const media::VideoFrame* video_frame, 86 void* rgb_pixels, 87 size_t row_bytes, 88 bool premultiply_alpha = true); 89 90 // Copy the visible rect size contents of texture of |video_frame| to 91 // texture |texture|. |level|, |internal_format|, |type| specify target 92 // texture |texture|. The format of |video_frame| must be 93 // VideoFrame::NATIVE_TEXTURE. 94 static void CopyVideoFrameSingleTextureToGLTexture( 95 gpu::gles2::GLES2Interface* gl, 96 VideoFrame* video_frame, 97 unsigned int target, 98 unsigned int texture, 99 unsigned int internal_format, 100 unsigned int format, 101 unsigned int type, 102 int level, 103 bool premultiply_alpha, 104 bool flip_y); 105 106 // Copy the contents of |video_frame| to |texture| of |destination_gl|. 107 // 108 // The format of |video_frame| must be VideoFrame::NATIVE_TEXTURE. 109 bool CopyVideoFrameTexturesToGLTexture( 110 viz::RasterContextProvider* raster_context_provider, 111 gpu::gles2::GLES2Interface* destination_gl, 112 scoped_refptr<VideoFrame> video_frame, 113 unsigned int target, 114 unsigned int texture, 115 unsigned int internal_format, 116 unsigned int format, 117 unsigned int type, 118 int level, 119 bool premultiply_alpha, 120 bool flip_y); 121 122 bool PrepareVideoFrameForWebGL( 123 viz::RasterContextProvider* raster_context_provider, 124 gpu::gles2::GLES2Interface* gl, 125 scoped_refptr<VideoFrame> video_frame, 126 unsigned int target, 127 unsigned int texture); 128 129 // Copy the CPU-side YUV contents of |video_frame| to texture |texture| in 130 // context |destination_gl|. 131 // |level|, |internal_format|, |type| specify target texture |texture|. 132 // The format of |video_frame| must be mappable. 133 // |context_3d| has a GrContext that may be used during the copy. 134 // CorrectLastImageDimensions() ensures that the source texture will be 135 // cropped to |visible_rect|. Returns true on success. 136 bool CopyVideoFrameYUVDataToGLTexture( 137 viz::RasterContextProvider* raster_context_provider, 138 gpu::gles2::GLES2Interface* destination_gl, 139 const VideoFrame& video_frame, 140 unsigned int target, 141 unsigned int texture, 142 unsigned int internal_format, 143 unsigned int format, 144 unsigned int type, 145 int level, 146 bool premultiply_alpha, 147 bool flip_y); 148 149 // Calls texImage2D where the texture image data source is the contents of 150 // |video_frame|. Texture |texture| needs to be created and bound to |target| 151 // before this call and the binding is active upon return. 152 // This is an optimization of WebGL |video_frame| TexImage2D implementation 153 // for specific combinations of |video_frame| and |texture| formats; e.g. if 154 // |frame format| is Y16, optimizes conversion of normalized 16-bit content 155 // and calls texImage2D to |texture|. |level|, |internal_format|, |format| and 156 // |type| are WebGL texImage2D parameters. 157 // Returns false if there is no implementation for given parameters. 158 static bool TexImage2D(unsigned target, 159 unsigned texture, 160 gpu::gles2::GLES2Interface* gl, 161 const gpu::Capabilities& gpu_capabilities, 162 VideoFrame* video_frame, 163 int level, 164 int internalformat, 165 unsigned format, 166 unsigned type, 167 bool flip_y, 168 bool premultiply_alpha); 169 170 // Calls texSubImage2D where the texture image data source is the contents of 171 // |video_frame|. 172 // This is an optimization of WebGL |video_frame| TexSubImage2D implementation 173 // for specific combinations of |video_frame| and texture |format| and |type|; 174 // e.g. if |frame format| is Y16, converts unsigned 16-bit value to target 175 // |format| and calls WebGL texSubImage2D. |level|, |format|, |type|, 176 // |xoffset| and |yoffset| are texSubImage2D parameters. 177 // Returns false if there is no implementation for given parameters. 178 static bool TexSubImage2D(unsigned target, 179 gpu::gles2::GLES2Interface* gl, 180 VideoFrame* video_frame, 181 int level, 182 unsigned format, 183 unsigned type, 184 int xoffset, 185 int yoffset, 186 bool flip_y, 187 bool premultiply_alpha); 188 189 // In general, We hold the most recently painted frame to increase the 190 // performance for the case that the same frame needs to be painted 191 // repeatedly. Call this function if you are sure the most recent frame will 192 // never be painted again, so we can release the resource. 193 void ResetCache(); 194 195 // Used for unit test. 196 gfx::Size LastImageDimensionsForTesting(); 197 198 private: 199 // This structure wraps information extracted out of a VideoFrame and/or 200 // constructed out of it. The various calls in PaintCanvasVideoRenderer must 201 // not keep a reference to the VideoFrame so necessary data is extracted out 202 // of it. 203 struct Cache { 204 explicit Cache(int frame_id); 205 ~Cache(); 206 207 // VideoFrame::unique_id() of the videoframe used to generate the cache. 208 int frame_id; 209 210 // A PaintImage that can be used to draw into a PaintCanvas. This is sized 211 // to the visible size of the VideoFrame. Its contents are generated lazily. 212 cc::PaintImage paint_image; 213 214 // The context provider used to generate |source_mailbox| and 215 // |source_texture|. This is only set if the VideoFrame was texture-backed. 216 scoped_refptr<viz::RasterContextProvider> raster_context_provider; 217 218 // The mailbox for the source texture. This can be either the source 219 // VideoFrame's texture (if |wraps_video_frame_texture| is true) or a newly 220 // allocated shared image (if |wraps_video_frame_texture| is false) if a 221 // copy or conversion was necessary. 222 // This is only set if the VideoFrame was texture-backed. 223 gpu::Mailbox source_mailbox; 224 225 // The texture ID created when importing |source_mailbox|. 226 // This is only set if the VideoFrame was texture-backed. 227 uint32_t source_texture = 0; 228 229 // The allocated size of |source_mailbox|. 230 // This is only set if the VideoFrame was texture-backed. 231 gfx::Size coded_size; 232 233 // The visible subrect of |coded_size| that represents the logical contents 234 // of the frame after cropping. 235 // This is only set if the VideoFrame was texture-backed. 236 gfx::Rect visible_rect; 237 238 // Whether |source_mailbox| directly points to a texture of the VideoFrame 239 // (if true), or to an allocated shared image (if false). 240 bool wraps_video_frame_texture = false; 241 242 // Whether the texture pointed by |paint_image| is owned by skia or not. 243 bool texture_ownership_in_skia = false; 244 245 // Used to allow recycling of the previous shared image. This requires that 246 // no external users have access to this resource via SkImage. Returns true 247 // if the existing resource can be recycled. 248 bool Recycle(); 249 }; 250 251 // Update the cache holding the most-recently-painted frame. Returns false 252 // if the image couldn't be updated. 253 bool UpdateLastImage(scoped_refptr<VideoFrame> video_frame, 254 viz::RasterContextProvider* raster_context_provider, 255 bool allow_wrap_texture); 256 257 bool PrepareVideoFrame(scoped_refptr<VideoFrame> video_frame, 258 viz::RasterContextProvider* raster_context_provider, 259 const gpu::MailboxHolder& dest_holder); 260 261 bool UploadVideoFrameToGLTexture( 262 viz::RasterContextProvider* raster_context_provider, 263 gpu::gles2::GLES2Interface* destination_gl, 264 scoped_refptr<VideoFrame> video_frame, 265 unsigned int target, 266 unsigned int texture, 267 unsigned int internal_format, 268 unsigned int format, 269 unsigned int type, 270 bool flip_y); 271 272 base::Optional<Cache> cache_; 273 274 // If |cache_| is not used for a while, it's deleted to save memory. 275 base::DelayTimer cache_deleting_timer_; 276 // Stable paint image id to provide to draw image calls. 277 cc::PaintImage::Id renderer_stable_id_; 278 279 // Used for DCHECKs to ensure method calls executed in the correct thread. 280 base::ThreadChecker thread_checker_; 281 282 struct YUVTextureCache { 283 YUVTextureCache(); 284 ~YUVTextureCache(); 285 void Reset(); 286 287 // The ContextProvider that holds the texture. 288 scoped_refptr<viz::RasterContextProvider> raster_context_provider; 289 290 // The size of the texture. 291 gfx::Size size; 292 293 // The shared image backing the texture. 294 gpu::Mailbox mailbox; 295 296 // Used to perform YUV->RGB conversion on video frames. Internally caches 297 // shared images that are created to upload CPU video frame data to the GPU. 298 VideoFrameYUVConverter yuv_converter; 299 300 // A SyncToken after last usage, used for reusing or destroying texture and 301 // shared image. 302 gpu::SyncToken sync_token; 303 }; 304 YUVTextureCache yuv_cache_; 305 306 DISALLOW_COPY_AND_ASSIGN(PaintCanvasVideoRenderer); 307 }; 308 309 } // namespace media 310 311 #endif // MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_ 312