1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/renderers/video_resource_updater.h"
6
7 #include <stddef.h>
8 #include <stdint.h>
9
10 #include <algorithm>
11 #include <string>
12
13 #include "base/atomic_sequence_num.h"
14 #include "base/bind.h"
15 #include "base/bit_cast.h"
16 #include "base/memory/shared_memory_mapping.h"
17 #include "base/memory/unsafe_shared_memory_region.h"
18 #include "base/strings/stringprintf.h"
19 #include "base/threading/thread_task_runner_handle.h"
20 #include "base/trace_event/memory_dump_manager.h"
21 #include "base/trace_event/process_memory_dump.h"
22 #include "base/trace_event/trace_event.h"
23 #include "build/build_config.h"
24 #include "cc/base/math_util.h"
25 #include "cc/paint/skia_paint_canvas.h"
26 #include "components/viz/client/client_resource_provider.h"
27 #include "components/viz/client/shared_bitmap_reporter.h"
28 #include "components/viz/common/gpu/context_provider.h"
29 #include "components/viz/common/gpu/raster_context_provider.h"
30 #include "components/viz/common/quads/render_pass.h"
31 #include "components/viz/common/quads/stream_video_draw_quad.h"
32 #include "components/viz/common/quads/texture_draw_quad.h"
33 #include "components/viz/common/quads/video_hole_draw_quad.h"
34 #include "components/viz/common/quads/yuv_video_draw_quad.h"
35 #include "components/viz/common/resources/bitmap_allocation.h"
36 #include "components/viz/common/resources/resource_sizes.h"
37 #include "gpu/GLES2/gl2extchromium.h"
38 #include "gpu/command_buffer/client/context_support.h"
39 #include "gpu/command_buffer/client/gles2_interface.h"
40 #include "gpu/command_buffer/client/shared_image_interface.h"
41 #include "gpu/command_buffer/common/shared_image_trace_utils.h"
42 #include "gpu/command_buffer/common/shared_image_usage.h"
43 #include "media/base/video_frame.h"
44 #include "media/renderers/paint_canvas_video_renderer.h"
45 #include "media/video/half_float_maker.h"
46 #include "third_party/khronos/GLES2/gl2.h"
47 #include "third_party/khronos/GLES2/gl2ext.h"
48 #include "third_party/libyuv/include/libyuv.h"
49 #include "third_party/skia/include/core/SkCanvas.h"
50 #include "ui/gfx/geometry/size_conversions.h"
51 #include "ui/gfx/skia_util.h"
52 #include "ui/gl/gl_enums.h"
53 #include "ui/gl/trace_util.h"
54
55 namespace media {
56 namespace {
57
58 // Generates process-unique IDs to use for tracing video resources.
59 base::AtomicSequenceNumber g_next_video_resource_updater_id;
60
ExternalResourceTypeForHardwarePlanes(VideoPixelFormat format,GLuint target,int num_textures,gfx::BufferFormat buffer_formats[VideoFrame::kMaxPlanes],bool use_stream_video_draw_quad)61 VideoFrameResourceType ExternalResourceTypeForHardwarePlanes(
62 VideoPixelFormat format,
63 GLuint target,
64 int num_textures,
65 gfx::BufferFormat buffer_formats[VideoFrame::kMaxPlanes],
66 bool use_stream_video_draw_quad) {
67 switch (format) {
68 case PIXEL_FORMAT_ARGB:
69 case PIXEL_FORMAT_XRGB:
70 case PIXEL_FORMAT_ABGR:
71 case PIXEL_FORMAT_BGRA:
72 DCHECK_EQ(num_textures, 1);
73 // This maps VideoPixelFormat back to GMB BufferFormat
74 // NOTE: ABGR == RGBA and ARGB == BGRA, they differ only byte order
75 // See: VideoFormat function in gpu_memory_buffer_video_frame_pool
76 // https://cs.chromium.org/chromium/src/media/video/gpu_memory_buffer_video_frame_pool.cc?type=cs&g=0&l=281
77 buffer_formats[0] = (format == PIXEL_FORMAT_ABGR)
78 ? gfx::BufferFormat::RGBA_8888
79 : gfx::BufferFormat::BGRA_8888;
80 switch (target) {
81 case GL_TEXTURE_EXTERNAL_OES:
82 if (use_stream_video_draw_quad)
83 return VideoFrameResourceType::STREAM_TEXTURE;
84 FALLTHROUGH;
85 case GL_TEXTURE_2D:
86 case GL_TEXTURE_RECTANGLE_ARB:
87 return (format == PIXEL_FORMAT_XRGB)
88 ? VideoFrameResourceType::RGB
89 : VideoFrameResourceType::RGBA_PREMULTIPLIED;
90 default:
91 NOTREACHED();
92 break;
93 }
94 break;
95 case PIXEL_FORMAT_XR30:
96 case PIXEL_FORMAT_XB30:
97 buffer_formats[0] = (format == PIXEL_FORMAT_XR30)
98 ? gfx::BufferFormat::BGRA_1010102
99 : gfx::BufferFormat::RGBA_1010102;
100 return VideoFrameResourceType::RGB;
101 case PIXEL_FORMAT_I420:
102 DCHECK_EQ(num_textures, 3);
103 buffer_formats[0] = gfx::BufferFormat::R_8;
104 buffer_formats[1] = gfx::BufferFormat::R_8;
105 buffer_formats[2] = gfx::BufferFormat::R_8;
106 return VideoFrameResourceType::YUV;
107
108 case PIXEL_FORMAT_NV12:
109 // |target| is set to 0 for Vulkan textures.
110 DCHECK(target == 0 || target == GL_TEXTURE_EXTERNAL_OES ||
111 target == GL_TEXTURE_2D || target == GL_TEXTURE_RECTANGLE_ARB)
112 << "Unsupported target " << gl::GLEnums::GetStringEnum(target);
113
114 if (num_textures == 1) {
115 // Single-texture multi-planar frames can be sampled as RGB.
116 buffer_formats[0] = gfx::BufferFormat::YUV_420_BIPLANAR;
117 return VideoFrameResourceType::RGB;
118 }
119
120 buffer_formats[0] = gfx::BufferFormat::R_8;
121 buffer_formats[1] = gfx::BufferFormat::RG_88;
122 return VideoFrameResourceType::YUV;
123
124 case PIXEL_FORMAT_UYVY:
125 NOTREACHED();
126 FALLTHROUGH;
127 case PIXEL_FORMAT_YV12:
128 case PIXEL_FORMAT_I422:
129 case PIXEL_FORMAT_I444:
130 case PIXEL_FORMAT_I420A:
131 case PIXEL_FORMAT_NV21:
132 case PIXEL_FORMAT_YUY2:
133 case PIXEL_FORMAT_RGB24:
134 case PIXEL_FORMAT_MJPEG:
135 case PIXEL_FORMAT_YUV420P9:
136 case PIXEL_FORMAT_YUV422P9:
137 case PIXEL_FORMAT_YUV444P9:
138 case PIXEL_FORMAT_YUV420P10:
139 case PIXEL_FORMAT_YUV422P10:
140 case PIXEL_FORMAT_YUV444P10:
141 case PIXEL_FORMAT_YUV420P12:
142 case PIXEL_FORMAT_YUV422P12:
143 case PIXEL_FORMAT_YUV444P12:
144 case PIXEL_FORMAT_Y16:
145 case PIXEL_FORMAT_XBGR:
146 case PIXEL_FORMAT_P016LE:
147 case PIXEL_FORMAT_UNKNOWN:
148 break;
149 }
150 return VideoFrameResourceType::NONE;
151 }
152
153 class SyncTokenClientImpl : public VideoFrame::SyncTokenClient {
154 public:
SyncTokenClientImpl(gpu::gles2::GLES2Interface * gl,gpu::SyncToken sync_token)155 SyncTokenClientImpl(gpu::gles2::GLES2Interface* gl, gpu::SyncToken sync_token)
156 : gl_(gl), sync_token_(sync_token) {}
157 ~SyncTokenClientImpl() override = default;
158
GenerateSyncToken(gpu::SyncToken * sync_token)159 void GenerateSyncToken(gpu::SyncToken* sync_token) override {
160 if (sync_token_.HasData()) {
161 *sync_token = sync_token_;
162 } else {
163 gl_->GenSyncTokenCHROMIUM(sync_token->GetData());
164 }
165 }
166
WaitSyncToken(const gpu::SyncToken & sync_token)167 void WaitSyncToken(const gpu::SyncToken& sync_token) override {
168 if (sync_token.HasData()) {
169 gl_->WaitSyncTokenCHROMIUM(sync_token.GetConstData());
170 if (sync_token_.HasData() && sync_token_ != sync_token) {
171 gl_->WaitSyncTokenCHROMIUM(sync_token_.GetConstData());
172 sync_token_.Clear();
173 }
174 }
175 }
176
177 private:
178 gpu::gles2::GLES2Interface* gl_;
179 gpu::SyncToken sync_token_;
180 DISALLOW_COPY_AND_ASSIGN(SyncTokenClientImpl);
181 };
182
183 // Sync tokens passed downstream to the compositor can be unverified.
GenerateCompositorSyncToken(gpu::gles2::GLES2Interface * gl,gpu::SyncToken * sync_token)184 void GenerateCompositorSyncToken(gpu::gles2::GLES2Interface* gl,
185 gpu::SyncToken* sync_token) {
186 gl->GenUnverifiedSyncTokenCHROMIUM(sync_token->GetData());
187 }
188
189 // For frames that we receive in software format, determine the dimensions of
190 // each plane in the frame.
SoftwarePlaneDimension(VideoFrame * input_frame,bool software_compositor,size_t plane_index)191 gfx::Size SoftwarePlaneDimension(VideoFrame* input_frame,
192 bool software_compositor,
193 size_t plane_index) {
194 gfx::Size coded_size = input_frame->coded_size();
195 if (software_compositor)
196 return coded_size;
197
198 int plane_width = VideoFrame::Columns(plane_index, input_frame->format(),
199 coded_size.width());
200 int plane_height =
201 VideoFrame::Rows(plane_index, input_frame->format(), coded_size.height());
202 return gfx::Size(plane_width, plane_height);
203 }
204
205 } // namespace
206
207 VideoFrameExternalResources::VideoFrameExternalResources() = default;
208 VideoFrameExternalResources::~VideoFrameExternalResources() = default;
209
210 VideoFrameExternalResources::VideoFrameExternalResources(
211 VideoFrameExternalResources&& other) = default;
212 VideoFrameExternalResources& VideoFrameExternalResources::operator=(
213 VideoFrameExternalResources&& other) = default;
214
215 // Resource for a video plane allocated and owned by VideoResourceUpdater. There
216 // can be multiple plane resources for each video frame, depending on the
217 // format. These will be reused when possible.
218 class VideoResourceUpdater::PlaneResource {
219 public:
PlaneResource(uint32_t plane_resource_id,const gfx::Size & resource_size,viz::ResourceFormat resource_format,bool is_software)220 PlaneResource(uint32_t plane_resource_id,
221 const gfx::Size& resource_size,
222 viz::ResourceFormat resource_format,
223 bool is_software)
224 : plane_resource_id_(plane_resource_id),
225 resource_size_(resource_size),
226 resource_format_(resource_format),
227 is_software_(is_software) {}
228 virtual ~PlaneResource() = default;
229
230 // Casts |this| to SoftwarePlaneResource for software compositing.
231 SoftwarePlaneResource* AsSoftware();
232
233 // Casts |this| to HardwarePlaneResource for GPU compositing.
234 HardwarePlaneResource* AsHardware();
235
236 // Returns true if this resource matches the unique identifiers of another
237 // VideoFrame resource.
Matches(int unique_frame_id,size_t plane_index)238 bool Matches(int unique_frame_id, size_t plane_index) {
239 return has_unique_frame_id_and_plane_index_ &&
240 unique_frame_id_ == unique_frame_id && plane_index_ == plane_index;
241 }
242
243 // Sets the unique identifiers for this resource, may only be called when
244 // there is a single reference to the resource (i.e. |ref_count_| == 1).
SetUniqueId(int unique_frame_id,size_t plane_index)245 void SetUniqueId(int unique_frame_id, size_t plane_index) {
246 DCHECK_EQ(ref_count_, 1);
247 plane_index_ = plane_index;
248 unique_frame_id_ = unique_frame_id;
249 has_unique_frame_id_and_plane_index_ = true;
250 }
251
252 // Accessors for resource identifiers provided at construction time.
plane_resource_id() const253 uint32_t plane_resource_id() const { return plane_resource_id_; }
resource_size() const254 const gfx::Size& resource_size() const { return resource_size_; }
resource_format() const255 viz::ResourceFormat resource_format() const { return resource_format_; }
256
257 // Various methods for managing references. See |ref_count_| for details.
add_ref()258 void add_ref() { ++ref_count_; }
remove_ref()259 void remove_ref() { --ref_count_; }
clear_refs()260 void clear_refs() { ref_count_ = 0; }
has_refs() const261 bool has_refs() const { return ref_count_ != 0; }
262
263 private:
264 const uint32_t plane_resource_id_;
265 const gfx::Size resource_size_;
266 const viz::ResourceFormat resource_format_;
267 const bool is_software_;
268
269 // The number of times this resource has been imported vs number of times this
270 // resource has returned.
271 int ref_count_ = 0;
272
273 // These two members are used for identifying the data stored in this
274 // resource; they uniquely identify a VideoFrame plane.
275 int unique_frame_id_ = 0;
276 size_t plane_index_ = 0u;
277 // Indicates if the above two members have been set or not.
278 bool has_unique_frame_id_and_plane_index_ = false;
279
280 DISALLOW_COPY_AND_ASSIGN(PlaneResource);
281 };
282
283 class VideoResourceUpdater::SoftwarePlaneResource
284 : public VideoResourceUpdater::PlaneResource {
285 public:
SoftwarePlaneResource(uint32_t plane_resource_id,const gfx::Size & size,viz::SharedBitmapReporter * shared_bitmap_reporter)286 SoftwarePlaneResource(uint32_t plane_resource_id,
287 const gfx::Size& size,
288 viz::SharedBitmapReporter* shared_bitmap_reporter)
289 : PlaneResource(plane_resource_id,
290 size,
291 viz::ResourceFormat::RGBA_8888,
292 /*is_software=*/true),
293 shared_bitmap_reporter_(shared_bitmap_reporter),
294 shared_bitmap_id_(viz::SharedBitmap::GenerateId()) {
295 DCHECK(shared_bitmap_reporter_);
296
297 // Allocate SharedMemory and notify display compositor of the allocation.
298 base::MappedReadOnlyRegion shm =
299 viz::bitmap_allocation::AllocateSharedBitmap(
300 resource_size(), viz::ResourceFormat::RGBA_8888);
301 shared_mapping_ = std::move(shm.mapping);
302 shared_bitmap_reporter_->DidAllocateSharedBitmap(std::move(shm.region),
303 shared_bitmap_id_);
304 }
~SoftwarePlaneResource()305 ~SoftwarePlaneResource() override {
306 shared_bitmap_reporter_->DidDeleteSharedBitmap(shared_bitmap_id_);
307 }
308
shared_bitmap_id() const309 const viz::SharedBitmapId& shared_bitmap_id() const {
310 return shared_bitmap_id_;
311 }
pixels()312 void* pixels() { return shared_mapping_.memory(); }
313
314 // Returns a memory dump GUID consistent across processes.
GetSharedMemoryGuid() const315 base::UnguessableToken GetSharedMemoryGuid() const {
316 return shared_mapping_.guid();
317 }
318
319 private:
320 viz::SharedBitmapReporter* const shared_bitmap_reporter_;
321 const viz::SharedBitmapId shared_bitmap_id_;
322 base::WritableSharedMemoryMapping shared_mapping_;
323
324 DISALLOW_COPY_AND_ASSIGN(SoftwarePlaneResource);
325 };
326
327 class VideoResourceUpdater::HardwarePlaneResource
328 : public VideoResourceUpdater::PlaneResource {
329 public:
330 // Provides a RAII scope to access the HardwarePlaneResource as a texture on a
331 // GL context. This will wait on the sync token and provide the shared image
332 // access scope.
333 class ScopedTexture {
334 public:
ScopedTexture(gpu::gles2::GLES2Interface * gl,HardwarePlaneResource * resource)335 ScopedTexture(gpu::gles2::GLES2Interface* gl,
336 HardwarePlaneResource* resource)
337 : gl_(gl) {
338 texture_id_ = gl_->CreateAndTexStorage2DSharedImageCHROMIUM(
339 resource->mailbox().name);
340 gl_->BeginSharedImageAccessDirectCHROMIUM(
341 texture_id_, GL_SHARED_IMAGE_ACCESS_MODE_READWRITE_CHROMIUM);
342 }
343
~ScopedTexture()344 ~ScopedTexture() {
345 gl_->EndSharedImageAccessDirectCHROMIUM(texture_id_);
346 gl_->DeleteTextures(1, &texture_id_);
347 }
348
texture_id() const349 GLuint texture_id() const { return texture_id_; }
350
351 private:
352 gpu::gles2::GLES2Interface* gl_;
353 GLuint texture_id_;
354 };
355
HardwarePlaneResource(uint32_t plane_resource_id,const gfx::Size & size,viz::ResourceFormat format,const gfx::ColorSpace & color_space,bool use_gpu_memory_buffer_resources,viz::ContextProvider * context_provider,viz::RasterContextProvider * raster_context_provider)356 HardwarePlaneResource(uint32_t plane_resource_id,
357 const gfx::Size& size,
358 viz::ResourceFormat format,
359 const gfx::ColorSpace& color_space,
360 bool use_gpu_memory_buffer_resources,
361 viz::ContextProvider* context_provider,
362 viz::RasterContextProvider* raster_context_provider)
363 : PlaneResource(plane_resource_id, size, format, /*is_software=*/false),
364 context_provider_(context_provider),
365 raster_context_provider_(raster_context_provider) {
366 DCHECK(context_provider_ || raster_context_provider_);
367 const gpu::Capabilities& caps =
368 raster_context_provider_
369 ? raster_context_provider_->ContextCapabilities()
370 : context_provider_->ContextCapabilities();
371 overlay_candidate_ = use_gpu_memory_buffer_resources &&
372 caps.texture_storage_image &&
373 IsGpuMemoryBufferFormatSupported(format);
374 uint32_t shared_image_usage =
375 gpu::SHARED_IMAGE_USAGE_GLES2 | gpu::SHARED_IMAGE_USAGE_DISPLAY;
376 if (overlay_candidate_) {
377 shared_image_usage |= gpu::SHARED_IMAGE_USAGE_SCANOUT;
378 texture_target_ = gpu::GetBufferTextureTarget(gfx::BufferUsage::SCANOUT,
379 BufferFormat(format), caps);
380 }
381 auto* sii = SharedImageInterface();
382 mailbox_ =
383 sii->CreateSharedImage(format, size, color_space, shared_image_usage);
384 ContextGL()->WaitSyncTokenCHROMIUM(
385 sii->GenUnverifiedSyncToken().GetConstData());
386 }
387
~HardwarePlaneResource()388 ~HardwarePlaneResource() override {
389 gpu::SyncToken sync_token;
390 ContextGL()->GenUnverifiedSyncTokenCHROMIUM(sync_token.GetData());
391 SharedImageInterface()->DestroySharedImage(sync_token, mailbox_);
392 }
393
mailbox() const394 const gpu::Mailbox& mailbox() const { return mailbox_; }
395
texture_target() const396 GLenum texture_target() const { return texture_target_; }
overlay_candidate() const397 bool overlay_candidate() const { return overlay_candidate_; }
398
399 private:
SharedImageInterface()400 gpu::SharedImageInterface* SharedImageInterface() {
401 auto* sii = raster_context_provider_
402 ? raster_context_provider_->SharedImageInterface()
403 : context_provider_->SharedImageInterface();
404 DCHECK(sii);
405 return sii;
406 }
407
ContextGL()408 gpu::gles2::GLES2Interface* ContextGL() {
409 auto* gl = raster_context_provider_ ? raster_context_provider_->ContextGL()
410 : context_provider_->ContextGL();
411 DCHECK(gl);
412 return gl;
413 }
414
415 viz::ContextProvider* const context_provider_;
416 viz::RasterContextProvider* const raster_context_provider_;
417 gpu::Mailbox mailbox_;
418 GLenum texture_target_ = GL_TEXTURE_2D;
419 bool overlay_candidate_ = false;
420
421 DISALLOW_COPY_AND_ASSIGN(HardwarePlaneResource);
422 };
423
424 VideoResourceUpdater::SoftwarePlaneResource*
AsSoftware()425 VideoResourceUpdater::PlaneResource::AsSoftware() {
426 DCHECK(is_software_);
427 return static_cast<SoftwarePlaneResource*>(this);
428 }
429
430 VideoResourceUpdater::HardwarePlaneResource*
AsHardware()431 VideoResourceUpdater::PlaneResource::AsHardware() {
432 DCHECK(!is_software_);
433 return static_cast<HardwarePlaneResource*>(this);
434 }
435
VideoResourceUpdater(viz::ContextProvider * context_provider,viz::RasterContextProvider * raster_context_provider,viz::SharedBitmapReporter * shared_bitmap_reporter,viz::ClientResourceProvider * resource_provider,bool use_stream_video_draw_quad,bool use_gpu_memory_buffer_resources,bool use_r16_texture,int max_resource_size)436 VideoResourceUpdater::VideoResourceUpdater(
437 viz::ContextProvider* context_provider,
438 viz::RasterContextProvider* raster_context_provider,
439 viz::SharedBitmapReporter* shared_bitmap_reporter,
440 viz::ClientResourceProvider* resource_provider,
441 bool use_stream_video_draw_quad,
442 bool use_gpu_memory_buffer_resources,
443 bool use_r16_texture,
444 int max_resource_size)
445 : context_provider_(context_provider),
446 raster_context_provider_(raster_context_provider),
447 shared_bitmap_reporter_(shared_bitmap_reporter),
448 resource_provider_(resource_provider),
449 use_stream_video_draw_quad_(use_stream_video_draw_quad),
450 use_gpu_memory_buffer_resources_(use_gpu_memory_buffer_resources),
451 use_r16_texture_(use_r16_texture),
452 max_resource_size_(max_resource_size),
453 tracing_id_(g_next_video_resource_updater_id.GetNext()) {
454 DCHECK(context_provider_ || raster_context_provider_ ||
455 shared_bitmap_reporter_);
456
457 base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
458 this, "media::VideoResourceUpdater", base::ThreadTaskRunnerHandle::Get());
459 }
460
~VideoResourceUpdater()461 VideoResourceUpdater::~VideoResourceUpdater() {
462 base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
463 this);
464 }
465
ObtainFrameResources(scoped_refptr<VideoFrame> video_frame)466 void VideoResourceUpdater::ObtainFrameResources(
467 scoped_refptr<VideoFrame> video_frame) {
468 if (video_frame->metadata()->GetUnguessableToken(
469 VideoFrameMetadata::OVERLAY_PLANE_ID, &overlay_plane_id_)) {
470 // This is a hole punching VideoFrame, there is nothing to display.
471 frame_resource_type_ = VideoFrameResourceType::VIDEO_HOLE;
472 return;
473 }
474
475 VideoFrameExternalResources external_resources =
476 CreateExternalResourcesFromVideoFrame(video_frame);
477 frame_resource_type_ = external_resources.type;
478
479 if (external_resources.type == VideoFrameResourceType::YUV) {
480 frame_resource_offset_ = external_resources.offset;
481 frame_resource_multiplier_ = external_resources.multiplier;
482 frame_bits_per_channel_ = external_resources.bits_per_channel;
483 }
484
485 DCHECK_EQ(external_resources.resources.size(),
486 external_resources.release_callbacks.size());
487 for (size_t i = 0; i < external_resources.resources.size(); ++i) {
488 viz::ResourceId resource_id = resource_provider_->ImportResource(
489 external_resources.resources[i],
490 viz::SingleReleaseCallback::Create(
491 std::move(external_resources.release_callbacks[i])));
492 frame_resources_.push_back(
493 {resource_id, external_resources.resources[i].size});
494 }
495 TRACE_EVENT_INSTANT1("media", "VideoResourceUpdater::ObtainFrameResources",
496 TRACE_EVENT_SCOPE_THREAD, "Timestamp",
497 video_frame->timestamp().InMicroseconds());
498 }
499
ReleaseFrameResources()500 void VideoResourceUpdater::ReleaseFrameResources() {
501 for (auto& frame_resource : frame_resources_)
502 resource_provider_->RemoveImportedResource(frame_resource.id);
503 frame_resources_.clear();
504 }
505
AppendQuads(viz::RenderPass * render_pass,scoped_refptr<VideoFrame> frame,gfx::Transform transform,gfx::Rect quad_rect,gfx::Rect visible_quad_rect,const gfx::RRectF & rounded_corner_bounds,gfx::Rect clip_rect,bool is_clipped,bool contents_opaque,float draw_opacity,int sorting_context_id)506 void VideoResourceUpdater::AppendQuads(viz::RenderPass* render_pass,
507 scoped_refptr<VideoFrame> frame,
508 gfx::Transform transform,
509 gfx::Rect quad_rect,
510 gfx::Rect visible_quad_rect,
511 const gfx::RRectF& rounded_corner_bounds,
512 gfx::Rect clip_rect,
513 bool is_clipped,
514 bool contents_opaque,
515 float draw_opacity,
516 int sorting_context_id) {
517 DCHECK(frame.get());
518
519 viz::SharedQuadState* shared_quad_state =
520 render_pass->CreateAndAppendSharedQuadState();
521 shared_quad_state->SetAll(transform, quad_rect, visible_quad_rect,
522 rounded_corner_bounds, clip_rect, is_clipped,
523 contents_opaque, draw_opacity,
524 SkBlendMode::kSrcOver, sorting_context_id);
525
526 bool needs_blending = !contents_opaque;
527
528 gfx::Rect visible_rect = frame->visible_rect();
529 gfx::Size coded_size = frame->coded_size();
530
531 const float tex_width_scale =
532 static_cast<float>(visible_rect.width()) / coded_size.width();
533 const float tex_height_scale =
534 static_cast<float>(visible_rect.height()) / coded_size.height();
535
536 const gfx::PointF uv_top_left(0.f, 0.f);
537 const gfx::PointF uv_bottom_right(tex_width_scale, tex_height_scale);
538
539 switch (frame_resource_type_) {
540 case VideoFrameResourceType::VIDEO_HOLE: {
541 auto* video_hole_quad =
542 render_pass->CreateAndAppendDrawQuad<viz::VideoHoleDrawQuad>();
543 video_hole_quad->SetNew(shared_quad_state, quad_rect, visible_quad_rect,
544 overlay_plane_id_);
545 break;
546 }
547 case VideoFrameResourceType::YUV: {
548 const gfx::Size ya_tex_size = coded_size;
549
550 int u_width = VideoFrame::Columns(VideoFrame::kUPlane, frame->format(),
551 coded_size.width());
552 int u_height = VideoFrame::Rows(VideoFrame::kUPlane, frame->format(),
553 coded_size.height());
554 gfx::Size uv_tex_size(u_width, u_height);
555
556 DCHECK_EQ(frame_resources_.size(),
557 VideoFrame::NumPlanes(frame->format()));
558 if (frame->HasTextures()) {
559 DCHECK(frame->format() == PIXEL_FORMAT_NV12 ||
560 frame->format() == PIXEL_FORMAT_I420);
561 }
562
563 // Compute the UV sub-sampling factor based on the ratio between
564 // |ya_tex_size| and |uv_tex_size|.
565 float uv_subsampling_factor_x =
566 static_cast<float>(ya_tex_size.width()) / uv_tex_size.width();
567 float uv_subsampling_factor_y =
568 static_cast<float>(ya_tex_size.height()) / uv_tex_size.height();
569 gfx::RectF ya_tex_coord_rect(visible_rect);
570 gfx::RectF uv_tex_coord_rect(
571 visible_rect.x() / uv_subsampling_factor_x,
572 visible_rect.y() / uv_subsampling_factor_y,
573 visible_rect.width() / uv_subsampling_factor_x,
574 visible_rect.height() / uv_subsampling_factor_y);
575
576 auto* yuv_video_quad =
577 render_pass->CreateAndAppendDrawQuad<viz::YUVVideoDrawQuad>();
578 yuv_video_quad->SetNew(
579 shared_quad_state, quad_rect, visible_quad_rect, needs_blending,
580 ya_tex_coord_rect, uv_tex_coord_rect, ya_tex_size, uv_tex_size,
581 frame_resources_[0].id, frame_resources_[1].id,
582 frame_resources_.size() > 2 ? frame_resources_[2].id
583 : frame_resources_[1].id,
584 frame_resources_.size() > 3 ? frame_resources_[3].id : 0,
585 frame->ColorSpace(), frame_resource_offset_,
586 frame_resource_multiplier_, frame_bits_per_channel_);
587 if (frame->metadata()->IsTrue(VideoFrameMetadata::PROTECTED_VIDEO)) {
588 if (frame->metadata()->IsTrue(VideoFrameMetadata::HW_PROTECTED)) {
589 yuv_video_quad->protected_video_type =
590 gfx::ProtectedVideoType::kHardwareProtected;
591 } else {
592 yuv_video_quad->protected_video_type =
593 gfx::ProtectedVideoType::kSoftwareProtected;
594 }
595 }
596
597 for (viz::ResourceId resource_id : yuv_video_quad->resources) {
598 resource_provider_->ValidateResource(resource_id);
599 }
600 break;
601 }
602 case VideoFrameResourceType::RGBA:
603 case VideoFrameResourceType::RGBA_PREMULTIPLIED:
604 case VideoFrameResourceType::RGB: {
605 DCHECK_EQ(frame_resources_.size(), 1u);
606 if (frame_resources_.size() < 1u)
607 break;
608 bool premultiplied_alpha =
609 frame_resource_type_ == VideoFrameResourceType::RGBA_PREMULTIPLIED;
610
611 float opacity[] = {1.0f, 1.0f, 1.0f, 1.0f};
612 bool flipped = false;
613 bool nearest_neighbor = false;
614 gfx::ProtectedVideoType protected_video_type =
615 gfx::ProtectedVideoType::kClear;
616 if (frame->metadata()->IsTrue(VideoFrameMetadata::PROTECTED_VIDEO)) {
617 if (frame->metadata()->IsTrue(VideoFrameMetadata::HW_PROTECTED))
618 protected_video_type = gfx::ProtectedVideoType::kHardwareProtected;
619 else
620 protected_video_type = gfx::ProtectedVideoType::kSoftwareProtected;
621 }
622
623 const gfx::Vector2dF offset(
624 static_cast<float>(visible_rect.x()) / coded_size.width(),
625 static_cast<float>(visible_rect.y()) / coded_size.height());
626
627 auto* texture_quad =
628 render_pass->CreateAndAppendDrawQuad<viz::TextureDrawQuad>();
629 texture_quad->SetNew(
630 shared_quad_state, quad_rect, visible_quad_rect, needs_blending,
631 frame_resources_[0].id, premultiplied_alpha, uv_top_left + offset,
632 uv_bottom_right + offset, SK_ColorTRANSPARENT, opacity, flipped,
633 nearest_neighbor, false, protected_video_type);
634 texture_quad->set_resource_size_in_pixels(coded_size);
635 for (viz::ResourceId resource_id : texture_quad->resources) {
636 resource_provider_->ValidateResource(resource_id);
637 }
638
639 break;
640 }
641 case VideoFrameResourceType::STREAM_TEXTURE: {
642 DCHECK_EQ(frame_resources_.size(), 1u);
643 if (frame_resources_.size() < 1u)
644 break;
645 auto* stream_video_quad =
646 render_pass->CreateAndAppendDrawQuad<viz::StreamVideoDrawQuad>();
647 stream_video_quad->SetNew(shared_quad_state, quad_rect, visible_quad_rect,
648 needs_blending, frame_resources_[0].id,
649 frame_resources_[0].size_in_pixels, uv_top_left,
650 uv_bottom_right);
651 for (viz::ResourceId resource_id : stream_video_quad->resources) {
652 resource_provider_->ValidateResource(resource_id);
653 }
654 break;
655 }
656 case VideoFrameResourceType::NONE:
657 NOTIMPLEMENTED();
658 break;
659 }
660 }
661
662 VideoFrameExternalResources
CreateExternalResourcesFromVideoFrame(scoped_refptr<VideoFrame> video_frame)663 VideoResourceUpdater::CreateExternalResourcesFromVideoFrame(
664 scoped_refptr<VideoFrame> video_frame) {
665 if (video_frame->format() == PIXEL_FORMAT_UNKNOWN)
666 return VideoFrameExternalResources();
667 DCHECK(video_frame->HasTextures() || video_frame->IsMappable());
668 if (video_frame->HasTextures())
669 return CreateForHardwarePlanes(std::move(video_frame));
670 else
671 return CreateForSoftwarePlanes(std::move(video_frame));
672 }
673
YuvResourceFormat(int bits_per_channel)674 viz::ResourceFormat VideoResourceUpdater::YuvResourceFormat(
675 int bits_per_channel) {
676 DCHECK(raster_context_provider_ || context_provider_);
677 const auto& caps = raster_context_provider_
678 ? raster_context_provider_->ContextCapabilities()
679 : context_provider_->ContextCapabilities();
680 if (caps.disable_one_component_textures)
681 return viz::RGBA_8888;
682 if (bits_per_channel <= 8)
683 return caps.texture_rg ? viz::RED_8 : viz::LUMINANCE_8;
684 if (use_r16_texture_ && caps.texture_norm16)
685 return viz::R16_EXT;
686 if (caps.texture_half_float_linear)
687 return viz::LUMINANCE_F16;
688 return viz::LUMINANCE_8;
689 }
690
691 VideoResourceUpdater::PlaneResource*
RecycleOrAllocateResource(const gfx::Size & resource_size,viz::ResourceFormat resource_format,const gfx::ColorSpace & color_space,int unique_id,int plane_index)692 VideoResourceUpdater::RecycleOrAllocateResource(
693 const gfx::Size& resource_size,
694 viz::ResourceFormat resource_format,
695 const gfx::ColorSpace& color_space,
696 int unique_id,
697 int plane_index) {
698 PlaneResource* recyclable_resource = nullptr;
699 for (auto& resource : all_resources_) {
700 // If the plane index is valid (positive, or 0, meaning all planes)
701 // then we are allowed to return a referenced resource that already
702 // contains the right frame data. It's safe to reuse it even if
703 // resource_provider_ holds some references to it, because those
704 // references are read-only.
705 if (plane_index != -1 && resource->Matches(unique_id, plane_index)) {
706 DCHECK(resource->resource_size() == resource_size);
707 DCHECK(resource->resource_format() == resource_format);
708 return resource.get();
709 }
710
711 // Otherwise check whether this is an unreferenced resource of the right
712 // format that we can recycle. Remember it, but don't return immediately,
713 // because we still want to find any reusable resources.
714 const bool in_use = resource->has_refs();
715
716 if (!in_use && resource->resource_size() == resource_size &&
717 resource->resource_format() == resource_format) {
718 recyclable_resource = resource.get();
719 }
720 }
721
722 if (recyclable_resource)
723 return recyclable_resource;
724
725 // There was nothing available to reuse or recycle. Allocate a new resource.
726 return AllocateResource(resource_size, resource_format, color_space);
727 }
728
AllocateResource(const gfx::Size & plane_size,viz::ResourceFormat format,const gfx::ColorSpace & color_space)729 VideoResourceUpdater::PlaneResource* VideoResourceUpdater::AllocateResource(
730 const gfx::Size& plane_size,
731 viz::ResourceFormat format,
732 const gfx::ColorSpace& color_space) {
733 const uint32_t plane_resource_id = next_plane_resource_id_++;
734
735 if (software_compositor()) {
736 DCHECK_EQ(format, viz::ResourceFormat::RGBA_8888);
737
738 all_resources_.push_back(std::make_unique<SoftwarePlaneResource>(
739 plane_resource_id, plane_size, shared_bitmap_reporter_));
740 } else {
741 all_resources_.push_back(std::make_unique<HardwarePlaneResource>(
742 plane_resource_id, plane_size, format, color_space,
743 use_gpu_memory_buffer_resources_, context_provider_,
744 raster_context_provider_));
745 }
746 return all_resources_.back().get();
747 }
748
CopyHardwarePlane(VideoFrame * video_frame,const gfx::ColorSpace & resource_color_space,const gpu::MailboxHolder & mailbox_holder,VideoFrameExternalResources * external_resources)749 void VideoResourceUpdater::CopyHardwarePlane(
750 VideoFrame* video_frame,
751 const gfx::ColorSpace& resource_color_space,
752 const gpu::MailboxHolder& mailbox_holder,
753 VideoFrameExternalResources* external_resources) {
754 const gfx::Size output_plane_resource_size = video_frame->coded_size();
755 // The copy needs to be a direct transfer of pixel data, so we use an RGBA8
756 // target to avoid loss of precision or dropping any alpha component.
757 constexpr viz::ResourceFormat copy_resource_format =
758 viz::ResourceFormat::RGBA_8888;
759
760 const int no_unique_id = 0;
761 const int no_plane_index = -1; // Do not recycle referenced textures.
762 PlaneResource* plane_resource = RecycleOrAllocateResource(
763 output_plane_resource_size, copy_resource_format, resource_color_space,
764 no_unique_id, no_plane_index);
765 HardwarePlaneResource* hardware_resource = plane_resource->AsHardware();
766 hardware_resource->add_ref();
767
768 DCHECK_EQ(hardware_resource->texture_target(),
769 static_cast<GLenum>(GL_TEXTURE_2D));
770
771 auto* gl = raster_context_provider_ ? raster_context_provider_->ContextGL()
772 : context_provider_->ContextGL();
773
774 gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
775
776 // This is only used on Android where all video mailboxes already use shared
777 // images.
778 DCHECK(mailbox_holder.mailbox.IsSharedImage());
779
780 // TODO(vikassoni): Use raster interface instead of gl interface eventually.
781 GLuint src_texture_id =
782 gl->CreateAndTexStorage2DSharedImageCHROMIUM(mailbox_holder.mailbox.name);
783 gl->BeginSharedImageAccessDirectCHROMIUM(
784 src_texture_id, GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM);
785 {
786 HardwarePlaneResource::ScopedTexture scope(gl, hardware_resource);
787 gl->CopySubTextureCHROMIUM(
788 src_texture_id, 0, GL_TEXTURE_2D, scope.texture_id(), 0, 0, 0, 0, 0,
789 output_plane_resource_size.width(), output_plane_resource_size.height(),
790 false, false, false);
791 }
792 gl->EndSharedImageAccessDirectCHROMIUM(src_texture_id);
793 gl->DeleteTextures(1, &src_texture_id);
794
795 // Pass an empty sync token to force generation of a new sync token.
796 SyncTokenClientImpl client(gl, gpu::SyncToken());
797 gpu::SyncToken sync_token = video_frame->UpdateReleaseSyncToken(&client);
798
799 auto transferable_resource = viz::TransferableResource::MakeGL(
800 hardware_resource->mailbox(), GL_LINEAR, GL_TEXTURE_2D, sync_token,
801 output_plane_resource_size, false /* is_overlay_candidate */);
802 transferable_resource.color_space = resource_color_space;
803 transferable_resource.format = copy_resource_format;
804 external_resources->resources.push_back(std::move(transferable_resource));
805
806 external_resources->release_callbacks.push_back(base::BindOnce(
807 &VideoResourceUpdater::RecycleResource, weak_ptr_factory_.GetWeakPtr(),
808 hardware_resource->plane_resource_id()));
809 }
810
CreateForHardwarePlanes(scoped_refptr<VideoFrame> video_frame)811 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
812 scoped_refptr<VideoFrame> video_frame) {
813 TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
814 DCHECK(video_frame->HasTextures());
815 if (!context_provider_ && !raster_context_provider_)
816 return VideoFrameExternalResources();
817
818 VideoFrameExternalResources external_resources;
819 gfx::ColorSpace resource_color_space = video_frame->ColorSpace();
820
821 bool copy_required =
822 video_frame->metadata()->IsTrue(VideoFrameMetadata::COPY_REQUIRED);
823
824 GLuint target = video_frame->mailbox_holder(0).texture_target;
825 // If |copy_required| then we will copy into a GL_TEXTURE_2D target.
826 if (copy_required)
827 target = GL_TEXTURE_2D;
828
829 gfx::BufferFormat buffer_formats[VideoFrame::kMaxPlanes];
830 external_resources.type = ExternalResourceTypeForHardwarePlanes(
831 video_frame->format(), target, video_frame->NumTextures(), buffer_formats,
832 use_stream_video_draw_quad_);
833
834 if (external_resources.type == VideoFrameResourceType::NONE) {
835 DLOG(ERROR) << "Unsupported Texture format"
836 << VideoPixelFormatToString(video_frame->format());
837 return external_resources;
838 }
839 if (external_resources.type == VideoFrameResourceType::RGB ||
840 external_resources.type == VideoFrameResourceType::RGBA ||
841 external_resources.type == VideoFrameResourceType::RGBA_PREMULTIPLIED) {
842 resource_color_space = resource_color_space.GetAsFullRangeRGB();
843 }
844
845 const size_t num_textures = video_frame->NumTextures();
846 for (size_t i = 0; i < num_textures; ++i) {
847 const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
848 if (mailbox_holder.mailbox.IsZero())
849 break;
850
851 if (copy_required) {
852 CopyHardwarePlane(video_frame.get(), resource_color_space, mailbox_holder,
853 &external_resources);
854 } else {
855 const gfx::Size& coded_size = video_frame->coded_size();
856 const size_t width =
857 VideoFrame::Columns(i, video_frame->format(), coded_size.width());
858 const size_t height =
859 VideoFrame::Rows(i, video_frame->format(), coded_size.height());
860 const gfx::Size plane_size(width, height);
861 auto transfer_resource = viz::TransferableResource::MakeGL(
862 mailbox_holder.mailbox, GL_LINEAR, mailbox_holder.texture_target,
863 mailbox_holder.sync_token, plane_size,
864 video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY));
865 transfer_resource.color_space = resource_color_space;
866 transfer_resource.read_lock_fences_enabled =
867 video_frame->metadata()->IsTrue(
868 VideoFrameMetadata::READ_LOCK_FENCES_ENABLED);
869 transfer_resource.format = viz::GetResourceFormat(buffer_formats[i]);
870 transfer_resource.ycbcr_info = video_frame->ycbcr_info();
871
872 #if defined(OS_ANDROID)
873 transfer_resource.is_backed_by_surface_texture =
874 video_frame->metadata()->IsTrue(VideoFrameMetadata::TEXTURE_OWNER);
875 transfer_resource.wants_promotion_hint = video_frame->metadata()->IsTrue(
876 VideoFrameMetadata::WANTS_PROMOTION_HINT);
877 #endif
878 external_resources.resources.push_back(std::move(transfer_resource));
879 external_resources.release_callbacks.push_back(
880 base::BindOnce(&VideoResourceUpdater::ReturnTexture,
881 weak_ptr_factory_.GetWeakPtr(), video_frame));
882 }
883 }
884 return external_resources;
885 }
886
CreateForSoftwarePlanes(scoped_refptr<VideoFrame> video_frame)887 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
888 scoped_refptr<VideoFrame> video_frame) {
889 TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
890 const VideoPixelFormat input_frame_format = video_frame->format();
891
892 size_t bits_per_channel = video_frame->BitDepth();
893
894 // Only YUV and Y16 software video frames are supported.
895 DCHECK(IsYuvPlanar(input_frame_format) ||
896 input_frame_format == PIXEL_FORMAT_Y16);
897
898 viz::ResourceFormat output_resource_format;
899 gfx::ColorSpace output_color_space = video_frame->ColorSpace();
900 if (input_frame_format == PIXEL_FORMAT_Y16) {
901 // Unable to display directly as yuv planes so convert it to RGBA for
902 // compositing.
903 output_resource_format = viz::RGBA_8888;
904 output_color_space = output_color_space.GetAsFullRangeRGB();
905 } else if (!software_compositor()) {
906 // Can be composited directly from yuv planes.
907 output_resource_format = YuvResourceFormat(bits_per_channel);
908 }
909
910 // If GPU compositing is enabled, but the output resource format
911 // returned by the resource provider is viz::RGBA_8888, then a GPU driver
912 // bug workaround requires that YUV frames must be converted to RGB
913 // before texture upload.
914 bool texture_needs_rgb_conversion =
915 !software_compositor() &&
916 output_resource_format == viz::ResourceFormat::RGBA_8888;
917
918 size_t output_plane_count = VideoFrame::NumPlanes(input_frame_format);
919
920 // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
921 // conversion here. That involves an extra copy of each frame to a bitmap.
922 // Obviously, this is suboptimal and should be addressed once ubercompositor
923 // starts shaping up.
924 if (software_compositor() || texture_needs_rgb_conversion) {
925 output_resource_format = viz::RGBA_8888;
926 output_plane_count = 1;
927 bits_per_channel = 8;
928
929 // The YUV to RGB conversion will be performed when we convert
930 // from single-channel textures to an RGBA texture via
931 // ConvertVideoFrameToRGBPixels below.
932 output_color_space = output_color_space.GetAsFullRangeRGB();
933 }
934
935 std::vector<gfx::Size> outplane_plane_sizes;
936 outplane_plane_sizes.reserve(output_plane_count);
937 for (size_t i = 0; i < output_plane_count; ++i) {
938 outplane_plane_sizes.push_back(
939 SoftwarePlaneDimension(video_frame.get(), software_compositor(), i));
940 const gfx::Size& output_plane_resource_size = outplane_plane_sizes.back();
941 if (output_plane_resource_size.IsEmpty() ||
942 output_plane_resource_size.width() > max_resource_size_ ||
943 output_plane_resource_size.height() > max_resource_size_) {
944 // This output plane has invalid geometry so return an empty external
945 // resources.
946 return VideoFrameExternalResources();
947 }
948 }
949
950 // Delete recycled resources that are the wrong format or wrong size.
951 auto can_delete_resource_fn =
952 [output_resource_format,
953 &outplane_plane_sizes](const std::unique_ptr<PlaneResource>& resource) {
954 // Resources that are still being used can't be deleted.
955 if (resource->has_refs())
956 return false;
957
958 return resource->resource_format() != output_resource_format ||
959 !base::Contains(outplane_plane_sizes, resource->resource_size());
960 };
961 base::EraseIf(all_resources_, can_delete_resource_fn);
962
963 // Recycle or allocate resources for each video plane.
964 std::vector<PlaneResource*> plane_resources;
965 plane_resources.reserve(output_plane_count);
966 for (size_t i = 0; i < output_plane_count; ++i) {
967 plane_resources.push_back(RecycleOrAllocateResource(
968 outplane_plane_sizes[i], output_resource_format, output_color_space,
969 video_frame->unique_id(), i));
970 plane_resources.back()->add_ref();
971 }
972
973 VideoFrameExternalResources external_resources;
974
975 external_resources.bits_per_channel = bits_per_channel;
976
977 if (software_compositor() || texture_needs_rgb_conversion) {
978 DCHECK_EQ(plane_resources.size(), 1u);
979 PlaneResource* plane_resource = plane_resources[0];
980 DCHECK_EQ(plane_resource->resource_format(), viz::RGBA_8888);
981
982 if (!plane_resource->Matches(video_frame->unique_id(), 0)) {
983 // We need to transfer data from |video_frame| to the plane resource.
984 if (software_compositor()) {
985 if (!video_renderer_)
986 video_renderer_ = std::make_unique<PaintCanvasVideoRenderer>();
987
988 SoftwarePlaneResource* software_resource = plane_resource->AsSoftware();
989
990 // We know the format is RGBA_8888 from check above.
991 SkImageInfo info = SkImageInfo::MakeN32Premul(
992 gfx::SizeToSkISize(software_resource->resource_size()));
993
994 SkBitmap sk_bitmap;
995 sk_bitmap.installPixels(info, software_resource->pixels(),
996 info.minRowBytes());
997 cc::SkiaPaintCanvas canvas(sk_bitmap);
998
999 // This is software path, so canvas and video_frame are always backed
1000 // by software.
1001 video_renderer_->Copy(video_frame, &canvas, nullptr);
1002 } else {
1003 HardwarePlaneResource* hardware_resource = plane_resource->AsHardware();
1004 size_t bytes_per_row = viz::ResourceSizes::CheckedWidthInBytes<size_t>(
1005 video_frame->coded_size().width(), viz::ResourceFormat::RGBA_8888);
1006 size_t needed_size = bytes_per_row * video_frame->coded_size().height();
1007 if (upload_pixels_size_ < needed_size) {
1008 // Free the existing data first so that the memory can be reused,
1009 // if possible. Note that the new array is purposely not initialized.
1010 upload_pixels_.reset();
1011 upload_pixels_.reset(new uint8_t[needed_size]);
1012 upload_pixels_size_ = needed_size;
1013 }
1014
1015 PaintCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
1016 video_frame.get(), upload_pixels_.get(), bytes_per_row);
1017
1018 // Copy pixels into texture.
1019 auto* gl = raster_context_provider_
1020 ? raster_context_provider_->ContextGL()
1021 : context_provider_->ContextGL();
1022
1023 const gfx::Size& plane_size = hardware_resource->resource_size();
1024 {
1025 HardwarePlaneResource::ScopedTexture scope(gl, hardware_resource);
1026 gl->BindTexture(hardware_resource->texture_target(),
1027 scope.texture_id());
1028 gl->TexSubImage2D(
1029 hardware_resource->texture_target(), 0, 0, 0, plane_size.width(),
1030 plane_size.height(), GLDataFormat(viz::ResourceFormat::RGBA_8888),
1031 GLDataType(viz::ResourceFormat::RGBA_8888), upload_pixels_.get());
1032 }
1033 }
1034 plane_resource->SetUniqueId(video_frame->unique_id(), 0);
1035 }
1036
1037 viz::TransferableResource transferable_resource;
1038 if (software_compositor()) {
1039 SoftwarePlaneResource* software_resource = plane_resource->AsSoftware();
1040 external_resources.type = VideoFrameResourceType::RGBA_PREMULTIPLIED;
1041 transferable_resource = viz::TransferableResource::MakeSoftware(
1042 software_resource->shared_bitmap_id(),
1043 software_resource->resource_size(),
1044 plane_resource->resource_format());
1045 } else {
1046 HardwarePlaneResource* hardware_resource = plane_resource->AsHardware();
1047 external_resources.type = VideoFrameResourceType::RGBA;
1048 gpu::SyncToken sync_token;
1049 auto* gl = raster_context_provider_
1050 ? raster_context_provider_->ContextGL()
1051 : context_provider_->ContextGL();
1052 GenerateCompositorSyncToken(gl, &sync_token);
1053 transferable_resource = viz::TransferableResource::MakeGL(
1054 hardware_resource->mailbox(), GL_LINEAR,
1055 hardware_resource->texture_target(), sync_token,
1056 hardware_resource->resource_size(),
1057 hardware_resource->overlay_candidate());
1058 }
1059
1060 transferable_resource.color_space = output_color_space;
1061 transferable_resource.format = viz::ResourceFormat::RGBA_8888;
1062 external_resources.resources.push_back(std::move(transferable_resource));
1063 external_resources.release_callbacks.push_back(base::BindOnce(
1064 &VideoResourceUpdater::RecycleResource, weak_ptr_factory_.GetWeakPtr(),
1065 plane_resource->plane_resource_id()));
1066
1067 return external_resources;
1068 }
1069
1070 const viz::ResourceFormat yuv_resource_format =
1071 YuvResourceFormat(bits_per_channel);
1072 DCHECK(yuv_resource_format == viz::LUMINANCE_F16 ||
1073 yuv_resource_format == viz::R16_EXT ||
1074 yuv_resource_format == viz::LUMINANCE_8 ||
1075 yuv_resource_format == viz::RED_8)
1076 << yuv_resource_format;
1077
1078 std::unique_ptr<HalfFloatMaker> half_float_maker;
1079 if (yuv_resource_format == viz::LUMINANCE_F16) {
1080 half_float_maker = HalfFloatMaker::NewHalfFloatMaker(bits_per_channel);
1081 external_resources.offset = half_float_maker->Offset();
1082 external_resources.multiplier = half_float_maker->Multiplier();
1083 } else if (yuv_resource_format == viz::R16_EXT) {
1084 external_resources.multiplier = 65535.0f / ((1 << bits_per_channel) - 1);
1085 external_resources.offset = 0;
1086 }
1087
1088 // We need to transfer data from |video_frame| to the plane resources.
1089 for (size_t i = 0; i < plane_resources.size(); ++i) {
1090 HardwarePlaneResource* plane_resource = plane_resources[i]->AsHardware();
1091
1092 // Skip the transfer if this |video_frame|'s plane has been processed.
1093 if (plane_resource->Matches(video_frame->unique_id(), i))
1094 continue;
1095
1096 const viz::ResourceFormat plane_resource_format =
1097 plane_resource->resource_format();
1098 DCHECK_EQ(plane_resource_format, yuv_resource_format);
1099
1100 // TODO(hubbe): Move upload code to media/.
1101 // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
1102
1103 // |video_stride_bytes| is the width of the |video_frame| we are uploading
1104 // (including non-frame data to fill in the stride).
1105 const int video_stride_bytes = video_frame->stride(i);
1106
1107 // |resource_size_pixels| is the size of the destination resource.
1108 const gfx::Size resource_size_pixels = plane_resource->resource_size();
1109
1110 const size_t bytes_per_row =
1111 viz::ResourceSizes::CheckedWidthInBytes<size_t>(
1112 resource_size_pixels.width(), plane_resource_format);
1113 // Use 4-byte row alignment (OpenGL default) for upload performance.
1114 // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
1115 const size_t upload_image_stride =
1116 cc::MathUtil::CheckedRoundUp<size_t>(bytes_per_row, 4u);
1117
1118 const size_t resource_bit_depth =
1119 static_cast<size_t>(viz::BitsPerPixel(plane_resource_format));
1120
1121 // Data downshifting is needed if the resource bit depth is not enough.
1122 const bool needs_bit_downshifting = bits_per_channel > resource_bit_depth;
1123
1124 // A copy to adjust strides is needed if those are different and both source
1125 // and destination have the same bit depth.
1126 const bool needs_stride_adaptation =
1127 (bits_per_channel == resource_bit_depth) &&
1128 (upload_image_stride != static_cast<size_t>(video_stride_bytes));
1129
1130 // We need to convert the incoming data if we're transferring to half float,
1131 // if the need a bit downshift or if the strides need to be reconciled.
1132 const bool needs_conversion = plane_resource_format == viz::LUMINANCE_F16 ||
1133 needs_bit_downshifting ||
1134 needs_stride_adaptation;
1135
1136 const uint8_t* pixels;
1137 if (!needs_conversion) {
1138 pixels = video_frame->data(i);
1139 } else {
1140 // Avoid malloc for each frame/plane if possible.
1141 const size_t needed_size =
1142 upload_image_stride * resource_size_pixels.height();
1143 if (upload_pixels_size_ < needed_size) {
1144 // Free the existing data first so that the memory can be reused,
1145 // if possible. Note that the new array is purposely not initialized.
1146 upload_pixels_.reset();
1147 upload_pixels_.reset(new uint8_t[needed_size]);
1148 upload_pixels_size_ = needed_size;
1149 }
1150
1151 if (plane_resource_format == viz::LUMINANCE_F16) {
1152 for (int row = 0; row < resource_size_pixels.height(); ++row) {
1153 uint16_t* dst = reinterpret_cast<uint16_t*>(
1154 &upload_pixels_[upload_image_stride * row]);
1155 const uint16_t* src = reinterpret_cast<uint16_t*>(
1156 video_frame->data(i) + (video_stride_bytes * row));
1157 half_float_maker->MakeHalfFloats(src, bytes_per_row / 2, dst);
1158 }
1159 } else if (needs_bit_downshifting) {
1160 DCHECK(plane_resource_format == viz::LUMINANCE_8 ||
1161 plane_resource_format == viz::RED_8);
1162 const int scale = 0x10000 >> (bits_per_channel - 8);
1163 libyuv::Convert16To8Plane(
1164 reinterpret_cast<uint16_t*>(video_frame->data(i)),
1165 video_stride_bytes / 2, upload_pixels_.get(), upload_image_stride,
1166 scale, bytes_per_row, resource_size_pixels.height());
1167 } else {
1168 // Make a copy to reconcile stride, size and format being equal.
1169 DCHECK(needs_stride_adaptation);
1170 DCHECK(plane_resource_format == viz::LUMINANCE_8 ||
1171 plane_resource_format == viz::RED_8);
1172 libyuv::CopyPlane(video_frame->data(i), video_stride_bytes,
1173 upload_pixels_.get(), upload_image_stride,
1174 resource_size_pixels.width(),
1175 resource_size_pixels.height());
1176 }
1177
1178 pixels = upload_pixels_.get();
1179 }
1180
1181 // Copy pixels into texture. TexSubImage2D() is applicable because
1182 // |yuv_resource_format| is LUMINANCE_F16, R16_EXT, LUMINANCE_8 or RED_8.
1183 auto* gl = raster_context_provider_ ? raster_context_provider_->ContextGL()
1184 : context_provider_->ContextGL();
1185 DCHECK(GLSupportsFormat(plane_resource_format));
1186 {
1187 HardwarePlaneResource::ScopedTexture scope(gl, plane_resource);
1188 gl->BindTexture(plane_resource->texture_target(), scope.texture_id());
1189 gl->TexSubImage2D(plane_resource->texture_target(), 0, 0, 0,
1190 resource_size_pixels.width(),
1191 resource_size_pixels.height(),
1192 GLDataFormat(plane_resource_format),
1193 GLDataType(plane_resource_format), pixels);
1194 }
1195
1196 plane_resource->SetUniqueId(video_frame->unique_id(), i);
1197 }
1198
1199 // Set the sync token otherwise resource is assumed to be synchronized.
1200 gpu::SyncToken sync_token;
1201 auto* gl = raster_context_provider_ ? raster_context_provider_->ContextGL()
1202 : context_provider_->ContextGL();
1203 GenerateCompositorSyncToken(gl, &sync_token);
1204
1205 for (size_t i = 0; i < plane_resources.size(); ++i) {
1206 HardwarePlaneResource* plane_resource = plane_resources[i]->AsHardware();
1207 auto transferable_resource = viz::TransferableResource::MakeGL(
1208 plane_resource->mailbox(), GL_LINEAR, plane_resource->texture_target(),
1209 sync_token, plane_resource->resource_size(),
1210 plane_resource->overlay_candidate());
1211 transferable_resource.color_space = output_color_space;
1212 transferable_resource.format = output_resource_format;
1213 external_resources.resources.push_back(std::move(transferable_resource));
1214 external_resources.release_callbacks.push_back(base::BindOnce(
1215 &VideoResourceUpdater::RecycleResource, weak_ptr_factory_.GetWeakPtr(),
1216 plane_resource->plane_resource_id()));
1217 }
1218
1219 external_resources.type = VideoFrameResourceType::YUV;
1220 return external_resources;
1221 }
1222
ReturnTexture(scoped_refptr<VideoFrame> video_frame,const gpu::SyncToken & sync_token,bool lost_resource)1223 void VideoResourceUpdater::ReturnTexture(scoped_refptr<VideoFrame> video_frame,
1224 const gpu::SyncToken& sync_token,
1225 bool lost_resource) {
1226 // TODO(dshwang): Forward to the decoder as a lost resource.
1227 if (lost_resource)
1228 return;
1229
1230 // The video frame will insert a wait on the previous release sync token.
1231 auto* gl = raster_context_provider_ ? raster_context_provider_->ContextGL()
1232 : context_provider_->ContextGL();
1233 SyncTokenClientImpl client(gl, sync_token);
1234 video_frame->UpdateReleaseSyncToken(&client);
1235 }
1236
RecycleResource(uint32_t plane_resource_id,const gpu::SyncToken & sync_token,bool lost_resource)1237 void VideoResourceUpdater::RecycleResource(uint32_t plane_resource_id,
1238 const gpu::SyncToken& sync_token,
1239 bool lost_resource) {
1240 auto matches_id_fn =
1241 [plane_resource_id](const std::unique_ptr<PlaneResource>& resource) {
1242 return resource->plane_resource_id() == plane_resource_id;
1243 };
1244 auto resource_it =
1245 std::find_if(all_resources_.begin(), all_resources_.end(), matches_id_fn);
1246 if (resource_it == all_resources_.end())
1247 return;
1248
1249 if (context_provider_ && sync_token.HasData()) {
1250 auto* gl = raster_context_provider_ ? raster_context_provider_->ContextGL()
1251 : context_provider_->ContextGL();
1252 gl->WaitSyncTokenCHROMIUM(sync_token.GetConstData());
1253 }
1254
1255 if (lost_resource) {
1256 all_resources_.erase(resource_it);
1257 } else {
1258 (*resource_it)->remove_ref();
1259 }
1260 }
1261
OnMemoryDump(const base::trace_event::MemoryDumpArgs & args,base::trace_event::ProcessMemoryDump * pmd)1262 bool VideoResourceUpdater::OnMemoryDump(
1263 const base::trace_event::MemoryDumpArgs& args,
1264 base::trace_event::ProcessMemoryDump* pmd) {
1265 for (auto& resource : all_resources_) {
1266 std::string dump_name =
1267 base::StringPrintf("cc/video_memory/updater_%d/resource_%d",
1268 tracing_id_, resource->plane_resource_id());
1269 base::trace_event::MemoryAllocatorDump* dump =
1270 pmd->CreateAllocatorDump(dump_name);
1271
1272 const uint64_t total_bytes =
1273 viz::ResourceSizes::UncheckedSizeInBytesAligned<uint64_t>(
1274 resource->resource_size(), resource->resource_format());
1275 dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
1276 base::trace_event::MemoryAllocatorDump::kUnitsBytes,
1277 total_bytes);
1278
1279 // The importance value assigned to the GUID here must be greater than the
1280 // importance value assigned elsewhere so that resource ownership is
1281 // attributed to VideoResourceUpdater.
1282 constexpr int kImportance = 2;
1283
1284 // Resources are shared across processes and require a shared GUID to
1285 // prevent double counting the memory.
1286 if (software_compositor()) {
1287 base::UnguessableToken shm_guid =
1288 resource->AsSoftware()->GetSharedMemoryGuid();
1289 pmd->CreateSharedMemoryOwnershipEdge(dump->guid(), shm_guid, kImportance);
1290 } else {
1291 base::trace_event::MemoryAllocatorDumpGuid guid =
1292 gpu::GetSharedImageGUIDForTracing(resource->AsHardware()->mailbox());
1293 pmd->CreateSharedGlobalAllocatorDump(guid);
1294 pmd->AddOwnershipEdge(dump->guid(), guid, kImportance);
1295 }
1296 }
1297
1298 return true;
1299 }
1300
1301 } // namespace media
1302