1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/video/gpu_memory_buffer_video_frame_pool.h"
6
7 #include <GLES2/gl2.h>
8 #include <GLES2/gl2ext.h>
9 #include <stddef.h>
10 #include <stdint.h>
11
12 #include <algorithm>
13 #include <list>
14 #include <memory>
15 #include <utility>
16
17 #include "base/barrier_closure.h"
18 #include "base/bind.h"
19 #include "base/callback_helpers.h"
20 #include "base/containers/circular_deque.h"
21 #include "base/containers/stack_container.h"
22 #include "base/location.h"
23 #include "base/logging.h"
24 #include "base/macros.h"
25 #include "base/metrics/histogram_macros.h"
26 #include "base/strings/stringprintf.h"
27 #include "base/sys_byteorder.h"
28 #include "base/time/default_tick_clock.h"
29 #include "base/trace_event/memory_dump_manager.h"
30 #include "base/trace_event/memory_dump_provider.h"
31 #include "base/trace_event/trace_event.h"
32 #include "build/build_config.h"
33 #include "gpu/GLES2/gl2extchromium.h"
34 #include "gpu/command_buffer/client/shared_image_interface.h"
35 #include "gpu/command_buffer/common/shared_image_usage.h"
36 #include "media/base/bind_to_current_loop.h"
37 #include "media/video/gpu_video_accelerator_factories.h"
38 #include "third_party/libyuv/include/libyuv.h"
39 #include "ui/gfx/buffer_format_util.h"
40 #include "ui/gfx/color_space.h"
41 #include "ui/gl/trace_util.h"
42
43 #if defined(OS_MAC)
44 #include "base/mac/mac_util.h"
45 #include "ui/gfx/mac/io_surface.h"
46 #endif
47
48 namespace media {
49
50 // Implementation of a pool of GpuMemoryBuffers used to back VideoFrames.
51 class GpuMemoryBufferVideoFramePool::PoolImpl
52 : public base::RefCountedThreadSafe<
53 GpuMemoryBufferVideoFramePool::PoolImpl>,
54 public base::trace_event::MemoryDumpProvider {
55 public:
56 // |media_task_runner| is the media task runner associated with the
57 // GL context provided by |gpu_factories|
58 // |worker_task_runner| is a task runner used to asynchronously copy
59 // video frame's planes.
60 // |gpu_factories| is an interface to GPU related operation and can be
61 // null if a GL context is not available.
PoolImpl(const scoped_refptr<base::SingleThreadTaskRunner> & media_task_runner,const scoped_refptr<base::TaskRunner> & worker_task_runner,GpuVideoAcceleratorFactories * const gpu_factories)62 PoolImpl(const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
63 const scoped_refptr<base::TaskRunner>& worker_task_runner,
64 GpuVideoAcceleratorFactories* const gpu_factories)
65 : media_task_runner_(media_task_runner),
66 worker_task_runner_(worker_task_runner),
67 gpu_factories_(gpu_factories),
68 output_format_(GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED),
69 tick_clock_(base::DefaultTickClock::GetInstance()),
70 in_shutdown_(false) {
71 DCHECK(media_task_runner_);
72 DCHECK(worker_task_runner_);
73 }
74
75 // Takes a software VideoFrame and calls |frame_ready_cb| with a VideoFrame
76 // backed by native textures if possible.
77 // The data contained in |video_frame| is copied into the returned frame
78 // asynchronously posting tasks to |worker_task_runner_|, while
79 // |frame_ready_cb| will be called on |media_task_runner_| once all the data
80 // has been copied.
81 void CreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,
82 FrameReadyCB cb);
83
84 bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
85 base::trace_event::ProcessMemoryDump* pmd) override;
86
87 // Aborts any pending copies.
88 void Abort();
89
90 // Shuts down the frame pool and releases all frames in |frames_|.
91 // Once this is called frames will no longer be inserted back into
92 // |frames_|.
93 void Shutdown();
94
95 void SetTickClockForTesting(const base::TickClock* tick_clock);
96
97 private:
98 friend class base::RefCountedThreadSafe<
99 GpuMemoryBufferVideoFramePool::PoolImpl>;
100 ~PoolImpl() override;
101
102 // Resource to represent a plane.
103 struct PlaneResource {
104 gfx::Size size;
105 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer;
106 gpu::Mailbox mailbox;
107 };
108
109 // All the resources needed to compose a frame.
110 // TODO(dalecurtis): The method of use marking used is very brittle
111 // and prone to leakage. Switch this to pass around std::unique_ptr
112 // such that callers own resources explicitly.
113 struct FrameResources {
FrameResourcesmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources114 explicit FrameResources(const gfx::Size& size) : size(size) {}
MarkUsedmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources115 void MarkUsed() {
116 is_used_ = true;
117 last_use_time_ = base::TimeTicks();
118 }
MarkUnusedmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources119 void MarkUnused(base::TimeTicks last_use_time) {
120 is_used_ = false;
121 last_use_time_ = last_use_time;
122 }
is_usedmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources123 bool is_used() const { return is_used_; }
last_use_timemedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources124 base::TimeTicks last_use_time() const { return last_use_time_; }
125
126 const gfx::Size size;
127 PlaneResource plane_resources[VideoFrame::kMaxPlanes];
128 // The sync token used to recycle or destroy the resources. It is set when
129 // the resources are returned from the VideoFrame (via
130 // MailboxHoldersReleased).
131 gpu::SyncToken sync_token;
132
133 private:
134 bool is_used_ = true;
135 base::TimeTicks last_use_time_;
136 };
137
138 // Struct to keep track of requested videoframe copies.
139 struct VideoFrameCopyRequest {
VideoFrameCopyRequestmedia::GpuMemoryBufferVideoFramePool::PoolImpl::VideoFrameCopyRequest140 VideoFrameCopyRequest(scoped_refptr<VideoFrame> video_frame,
141 FrameReadyCB frame_ready_cb,
142 bool passthrough)
143 : video_frame(std::move(video_frame)),
144 frame_ready_cb(std::move(frame_ready_cb)),
145 passthrough(passthrough) {}
146 scoped_refptr<VideoFrame> video_frame;
147 FrameReadyCB frame_ready_cb;
148 bool passthrough;
149 };
150
151 // Start the copy of a video_frame on the worker_task_runner_.
152 // It assumes there are currently no in-flight copies and works on the request
153 // in the front of |frame_copy_requests_| queue.
154 void StartCopy();
155
156 // Copy |video_frame| data into |frame_resources| and calls |frame_ready_cb|
157 // when done.
158 void CopyVideoFrameToGpuMemoryBuffers(scoped_refptr<VideoFrame> video_frame,
159 FrameResources* frame_resources);
160
161 // Called when all the data has been copied.
162 void OnCopiesDone(scoped_refptr<VideoFrame> video_frame,
163 FrameResources* frame_resources);
164
165 // Prepares GL resources, mailboxes and calls |frame_ready_cb| with the new
166 // VideoFrame. This has to be run on |media_task_runner_| where
167 // |frame_ready_cb| associated with video_frame will also be run.
168 void BindAndCreateMailboxesHardwareFrameResources(
169 scoped_refptr<VideoFrame> video_frame,
170 FrameResources* frame_resources);
171
172 // Return true if |resources| can be used to represent a frame for
173 // specific |format| and |size|.
AreFrameResourcesCompatible(const FrameResources * resources,const gfx::Size & size)174 static bool AreFrameResourcesCompatible(const FrameResources* resources,
175 const gfx::Size& size) {
176 return size == resources->size;
177 }
178
179 // Get the resources needed for a frame out of the pool, or create them if
180 // necessary.
181 // This also drops the LRU resources that can't be reuse for this frame.
182 FrameResources* GetOrCreateFrameResources(
183 const gfx::Size& size,
184 GpuVideoAcceleratorFactories::OutputFormat format);
185
186 // Calls the FrameReadyCB of the first entry in |frame_copy_requests_|, with
187 // the provided |video_frame|, then deletes the entry from
188 // |frame_copy_requests_| and attempts to start another copy if there are
189 // other |frame_copy_requests_| elements.
190 void CompleteCopyRequestAndMaybeStartNextCopy(
191 scoped_refptr<VideoFrame> video_frame);
192
193 // Callback called when a VideoFrame generated with GetFrameResources is no
194 // longer referenced.
195 void MailboxHoldersReleased(FrameResources* frame_resources,
196 const gpu::SyncToken& sync_token);
197
198 // Delete resources. This has to be called on the thread where |task_runner|
199 // is current.
200 static void DeleteFrameResources(
201 GpuVideoAcceleratorFactories* const gpu_factories,
202 FrameResources* frame_resources);
203
204 // Task runner associated to the GL context provided by |gpu_factories_|.
205 const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
206 // Task runner used to asynchronously copy planes.
207 const scoped_refptr<base::TaskRunner> worker_task_runner_;
208
209 // Interface to GPU related operations.
210 GpuVideoAcceleratorFactories* const gpu_factories_;
211
212 // Pool of resources.
213 std::list<FrameResources*> resources_pool_;
214
215 GpuVideoAcceleratorFactories::OutputFormat output_format_;
216
217 // |tick_clock_| is always a DefaultTickClock outside of testing.
218 const base::TickClock* tick_clock_;
219
220 // Queued up video frames for copies. The front is the currently
221 // in-flight copy, new copies are added at the end.
222 base::circular_deque<VideoFrameCopyRequest> frame_copy_requests_;
223 bool in_shutdown_;
224
225 DISALLOW_COPY_AND_ASSIGN(PoolImpl);
226 };
227
228 namespace {
229
230 // VideoFrame copies to GpuMemoryBuffers will be split in copies where the
231 // output size is |kBytesPerCopyTarget| bytes and run in parallel.
232 constexpr size_t kBytesPerCopyTarget = 1024 * 1024; // 1MB
233
234 // Return the GpuMemoryBuffer format to use for a specific VideoPixelFormat
235 // and plane.
GpuMemoryBufferFormat(GpuVideoAcceleratorFactories::OutputFormat format,size_t plane)236 gfx::BufferFormat GpuMemoryBufferFormat(
237 GpuVideoAcceleratorFactories::OutputFormat format,
238 size_t plane) {
239 switch (format) {
240 case GpuVideoAcceleratorFactories::OutputFormat::I420:
241 DCHECK_LE(plane, 2u);
242 return gfx::BufferFormat::R_8;
243 case GpuVideoAcceleratorFactories::OutputFormat::P010:
244 DCHECK_LE(plane, 1u);
245 return gfx::BufferFormat::P010;
246 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
247 DCHECK_LE(plane, 1u);
248 return gfx::BufferFormat::YUV_420_BIPLANAR;
249 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
250 DCHECK_LE(plane, 1u);
251 return plane == 0 ? gfx::BufferFormat::R_8 : gfx::BufferFormat::RG_88;
252 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
253 DCHECK_EQ(0u, plane);
254 return gfx::BufferFormat::BGRA_1010102;
255 case GpuVideoAcceleratorFactories::OutputFormat::XB30:
256 DCHECK_EQ(0u, plane);
257 return gfx::BufferFormat::RGBA_1010102;
258 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
259 DCHECK_EQ(0u, plane);
260 return gfx::BufferFormat::RGBA_8888;
261 case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
262 DCHECK_EQ(0u, plane);
263 return gfx::BufferFormat::BGRA_8888;
264 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
265 NOTREACHED();
266 break;
267 }
268 return gfx::BufferFormat::BGRA_8888;
269 }
270
271 // The number of output planes to be copied in each iteration.
PlanesPerCopy(GpuVideoAcceleratorFactories::OutputFormat format)272 size_t PlanesPerCopy(GpuVideoAcceleratorFactories::OutputFormat format) {
273 switch (format) {
274 case GpuVideoAcceleratorFactories::OutputFormat::I420:
275 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
276 case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
277 return 1;
278 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
279 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
280 case GpuVideoAcceleratorFactories::OutputFormat::P010:
281 return 2;
282 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
283 case GpuVideoAcceleratorFactories::OutputFormat::XB30:
284 return 3;
285 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
286 NOTREACHED();
287 break;
288 }
289 return 0;
290 }
291
VideoFormat(GpuVideoAcceleratorFactories::OutputFormat format)292 VideoPixelFormat VideoFormat(
293 GpuVideoAcceleratorFactories::OutputFormat format) {
294 switch (format) {
295 case GpuVideoAcceleratorFactories::OutputFormat::I420:
296 return PIXEL_FORMAT_I420;
297 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
298 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
299 return PIXEL_FORMAT_NV12;
300 case GpuVideoAcceleratorFactories::OutputFormat::P010:
301 return PIXEL_FORMAT_P016LE;
302 case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
303 return PIXEL_FORMAT_ARGB;
304 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
305 return PIXEL_FORMAT_ABGR;
306 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
307 return PIXEL_FORMAT_XR30;
308 case GpuVideoAcceleratorFactories::OutputFormat::XB30:
309 return PIXEL_FORMAT_XB30;
310 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
311 NOTREACHED();
312 break;
313 }
314 return PIXEL_FORMAT_UNKNOWN;
315 }
316
317 // The number of output planes to be copied in each iteration.
NumGpuMemoryBuffers(GpuVideoAcceleratorFactories::OutputFormat format)318 size_t NumGpuMemoryBuffers(GpuVideoAcceleratorFactories::OutputFormat format) {
319 switch (format) {
320 case GpuVideoAcceleratorFactories::OutputFormat::I420:
321 return 3;
322 case GpuVideoAcceleratorFactories::OutputFormat::P010:
323 return 1;
324 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
325 return 1;
326 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
327 return 2;
328 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
329 case GpuVideoAcceleratorFactories::OutputFormat::XB30:
330 return 1;
331 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
332 case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
333 return 1;
334 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
335 NOTREACHED();
336 break;
337 }
338 NOTREACHED();
339 return 0;
340 }
341
342 // The number of output rows to be copied in each iteration.
RowsPerCopy(size_t plane,VideoPixelFormat format,int width)343 int RowsPerCopy(size_t plane, VideoPixelFormat format, int width) {
344 int bytes_per_row = VideoFrame::RowBytes(plane, format, width);
345 if (format == PIXEL_FORMAT_NV12) {
346 DCHECK_EQ(0u, plane);
347 bytes_per_row += VideoFrame::RowBytes(1, format, width);
348 }
349 // Copy an even number of lines, and at least one.
350 return std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
351 }
352
CopyRowsToI420Buffer(int first_row,int rows,int bytes_per_row,size_t bit_depth,const uint8_t * source,int source_stride,uint8_t * output,int dest_stride,base::OnceClosure done)353 void CopyRowsToI420Buffer(int first_row,
354 int rows,
355 int bytes_per_row,
356 size_t bit_depth,
357 const uint8_t* source,
358 int source_stride,
359 uint8_t* output,
360 int dest_stride,
361 base::OnceClosure done) {
362 base::ScopedClosureRunner done_runner(std::move(done));
363 TRACE_EVENT2("media", "CopyRowsToI420Buffer", "bytes_per_row", bytes_per_row,
364 "rows", rows);
365
366 if (!output)
367 return;
368
369 DCHECK_NE(dest_stride, 0);
370 DCHECK_LE(bytes_per_row, std::abs(dest_stride));
371 DCHECK_LE(bytes_per_row, source_stride);
372 DCHECK_GE(bit_depth, 8u);
373
374 if (bit_depth == 8) {
375 libyuv::CopyPlane(source + source_stride * first_row, source_stride,
376 output + dest_stride * first_row, dest_stride,
377 bytes_per_row, rows);
378 } else {
379 const int scale = 0x10000 >> (bit_depth - 8);
380 libyuv::Convert16To8Plane(
381 reinterpret_cast<const uint16_t*>(source + source_stride * first_row),
382 source_stride / 2, output + dest_stride * first_row, dest_stride, scale,
383 bytes_per_row, rows);
384 }
385 }
386
CopyRowsToP010Buffer(int first_row,int rows,int bytes_per_row,const VideoFrame * source_frame,uint8_t * dest_y,int dest_stride_y,uint8_t * dest_uv,int dest_stride_uv,base::OnceClosure done)387 void CopyRowsToP010Buffer(int first_row,
388 int rows,
389 int bytes_per_row,
390 const VideoFrame* source_frame,
391 uint8_t* dest_y,
392 int dest_stride_y,
393 uint8_t* dest_uv,
394 int dest_stride_uv,
395 base::OnceClosure done) {
396 base::ScopedClosureRunner done_runner(std::move(done));
397 TRACE_EVENT2("media", "CopyRowsToP010Buffer", "bytes_per_row", bytes_per_row,
398 "rows", rows);
399
400 if (!dest_y || !dest_uv)
401 return;
402
403 DCHECK_NE(dest_stride_y, 0);
404 DCHECK_NE(dest_stride_uv, 0);
405 DCHECK_LE(bytes_per_row, std::abs(dest_stride_y));
406 DCHECK_LE(bytes_per_row, std::abs(dest_stride_uv));
407 DCHECK_EQ(0, first_row % 2);
408 DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_YUV420P10);
409 DCHECK_LE(bytes_per_row, source_frame->stride(VideoFrame::kYPlane));
410
411 // TODO(crbug.com/libyuv/873): Replace this with a libyuv optimized path or at
412 // least add a SIMD variant.
413 for (int r = first_row; r < first_row + rows; ++r) {
414 const uint16_t* src = reinterpret_cast<const uint16_t*>(
415 source_frame->visible_data(VideoFrame::kYPlane) +
416 r * source_frame->stride(VideoFrame::kYPlane));
417 uint16_t* dest = reinterpret_cast<uint16_t*>(dest_y + r * dest_stride_y);
418 for (int c = 0; c < bytes_per_row / 2; ++c)
419 *dest++ = *src++ << 6;
420 }
421
422 for (int r = first_row / 2; r < (first_row + rows) / 2; ++r) {
423 const uint16_t* u_src = reinterpret_cast<const uint16_t*>(
424 source_frame->visible_data(VideoFrame::kUPlane) +
425 r * source_frame->stride(VideoFrame::kUPlane));
426 const uint16_t* v_src = reinterpret_cast<const uint16_t*>(
427 source_frame->visible_data(VideoFrame::kVPlane) +
428 r * source_frame->stride(VideoFrame::kVPlane));
429 uint16_t* dest = reinterpret_cast<uint16_t*>(dest_uv + r * dest_stride_uv);
430 for (int c = 0; c < bytes_per_row / 4; ++c) {
431 *dest++ = *u_src++ << 6;
432 *dest++ = *v_src++ << 6;
433 }
434 }
435 }
436
CopyRowsToNV12Buffer(int first_row,int rows,int bytes_per_row,const VideoFrame * source_frame,uint8_t * dest_y,int dest_stride_y,uint8_t * dest_uv,int dest_stride_uv,base::OnceClosure done)437 void CopyRowsToNV12Buffer(int first_row,
438 int rows,
439 int bytes_per_row,
440 const VideoFrame* source_frame,
441 uint8_t* dest_y,
442 int dest_stride_y,
443 uint8_t* dest_uv,
444 int dest_stride_uv,
445 base::OnceClosure done) {
446 base::ScopedClosureRunner done_runner(std::move(done));
447 TRACE_EVENT2("media", "CopyRowsToNV12Buffer", "bytes_per_row", bytes_per_row,
448 "rows", rows);
449
450 if (!dest_y || !dest_uv)
451 return;
452
453 DCHECK_NE(dest_stride_y, 0);
454 DCHECK_NE(dest_stride_uv, 0);
455 DCHECK_LE(bytes_per_row, std::abs(dest_stride_y));
456 DCHECK_LE(bytes_per_row, std::abs(dest_stride_uv));
457 DCHECK_EQ(0, first_row % 2);
458 DCHECK(source_frame->format() == PIXEL_FORMAT_I420 ||
459 source_frame->format() == PIXEL_FORMAT_YV12);
460 libyuv::I420ToNV12(
461 source_frame->visible_data(VideoFrame::kYPlane) +
462 first_row * source_frame->stride(VideoFrame::kYPlane),
463 source_frame->stride(VideoFrame::kYPlane),
464 source_frame->visible_data(VideoFrame::kUPlane) +
465 first_row / 2 * source_frame->stride(VideoFrame::kUPlane),
466 source_frame->stride(VideoFrame::kUPlane),
467 source_frame->visible_data(VideoFrame::kVPlane) +
468 first_row / 2 * source_frame->stride(VideoFrame::kVPlane),
469 source_frame->stride(VideoFrame::kVPlane),
470 dest_y + first_row * dest_stride_y, dest_stride_y,
471 dest_uv + first_row / 2 * dest_stride_uv, dest_stride_uv, bytes_per_row,
472 rows);
473 }
474
CopyRowsToRGB10Buffer(bool is_argb,int first_row,int rows,int width,const VideoFrame * source_frame,uint8_t * output,int dest_stride,base::OnceClosure done)475 void CopyRowsToRGB10Buffer(bool is_argb,
476 int first_row,
477 int rows,
478 int width,
479 const VideoFrame* source_frame,
480 uint8_t* output,
481 int dest_stride,
482 base::OnceClosure done) {
483 base::ScopedClosureRunner done_runner(std::move(done));
484 TRACE_EVENT2("media", "CopyRowsToXR30Buffer", "bytes_per_row", width * 2,
485 "rows", rows);
486 if (!output)
487 return;
488
489 DCHECK_NE(dest_stride, 0);
490 DCHECK_LE(width, std::abs(dest_stride / 2));
491 DCHECK_EQ(0, first_row % 2);
492 DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_YUV420P10);
493
494 const uint16_t* y_plane = reinterpret_cast<const uint16_t*>(
495 source_frame->visible_data(VideoFrame::kYPlane) +
496 first_row * source_frame->stride(VideoFrame::kYPlane));
497 const size_t y_plane_stride = source_frame->stride(VideoFrame::kYPlane) / 2;
498 const uint16_t* v_plane = reinterpret_cast<const uint16_t*>(
499 source_frame->visible_data(VideoFrame::kVPlane) +
500 first_row / 2 * source_frame->stride(VideoFrame::kVPlane));
501 const size_t v_plane_stride = source_frame->stride(VideoFrame::kVPlane) / 2;
502 const uint16_t* u_plane = reinterpret_cast<const uint16_t*>(
503 source_frame->visible_data(VideoFrame::kUPlane) +
504 first_row / 2 * source_frame->stride(VideoFrame::kUPlane));
505 const size_t u_plane_stride = source_frame->stride(VideoFrame::kUPlane) / 2;
506 uint8_t* dest_rgb10 = output + first_row * dest_stride;
507
508 SkYUVColorSpace skyuv = kRec709_SkYUVColorSpace;
509 source_frame->ColorSpace().ToSkYUVColorSpace(&skyuv);
510
511 if (skyuv == kRec601_SkYUVColorSpace) {
512 if (is_argb) {
513 libyuv::I010ToAR30(y_plane, y_plane_stride, u_plane, u_plane_stride,
514 v_plane, v_plane_stride, dest_rgb10, dest_stride,
515 width, rows);
516 } else {
517 libyuv::I010ToAB30(y_plane, y_plane_stride, u_plane, u_plane_stride,
518 v_plane, v_plane_stride, dest_rgb10, dest_stride,
519 width, rows);
520 }
521 } else if (skyuv == kBT2020_SkYUVColorSpace) {
522 if (is_argb) {
523 libyuv::U010ToAR30(y_plane, y_plane_stride, u_plane, u_plane_stride,
524 v_plane, v_plane_stride, dest_rgb10, dest_stride,
525 width, rows);
526 } else {
527 libyuv::U010ToAB30(y_plane, y_plane_stride, u_plane, u_plane_stride,
528 v_plane, v_plane_stride, dest_rgb10, dest_stride,
529 width, rows);
530 }
531 } else { // BT.709
532 if (is_argb) {
533 libyuv::H010ToAR30(y_plane, y_plane_stride, u_plane, u_plane_stride,
534 v_plane, v_plane_stride, dest_rgb10, dest_stride,
535 width, rows);
536 } else {
537 libyuv::H010ToAB30(y_plane, y_plane_stride, u_plane, u_plane_stride,
538 v_plane, v_plane_stride, dest_rgb10, dest_stride,
539 width, rows);
540 }
541 }
542 }
543
CopyRowsToRGBABuffer(bool is_rgba,int first_row,int rows,int width,const VideoFrame * source_frame,uint8_t * output,int dest_stride,base::OnceClosure done)544 void CopyRowsToRGBABuffer(bool is_rgba,
545 int first_row,
546 int rows,
547 int width,
548 const VideoFrame* source_frame,
549 uint8_t* output,
550 int dest_stride,
551 base::OnceClosure done) {
552 base::ScopedClosureRunner done_runner(std::move(done));
553 TRACE_EVENT2("media", "CopyRowsToRGBABuffer", "bytes_per_row", width * 2,
554 "rows", rows);
555
556 if (!output)
557 return;
558
559 DCHECK_NE(dest_stride, 0);
560 DCHECK_LE(width, std::abs(dest_stride / 2));
561 DCHECK_EQ(0, first_row % 2);
562 DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_I420A);
563
564 // libyuv uses little-endian for RGBx formats, whereas here we use big endian.
565 auto* func_ptr = is_rgba ? libyuv::I420AlphaToABGR : libyuv::I420AlphaToARGB;
566
567 func_ptr(source_frame->visible_data(VideoFrame::kYPlane) +
568 first_row * source_frame->stride(VideoFrame::kYPlane),
569 source_frame->stride(VideoFrame::kYPlane),
570 source_frame->visible_data(VideoFrame::kUPlane) +
571 first_row / 2 * source_frame->stride(VideoFrame::kUPlane),
572 source_frame->stride(VideoFrame::kUPlane),
573 source_frame->visible_data(VideoFrame::kVPlane) +
574 first_row / 2 * source_frame->stride(VideoFrame::kVPlane),
575 source_frame->stride(VideoFrame::kVPlane),
576 source_frame->visible_data(VideoFrame::kAPlane) +
577 first_row * source_frame->stride(VideoFrame::kAPlane),
578 source_frame->stride(VideoFrame::kAPlane),
579 output + first_row * dest_stride, dest_stride, width, rows,
580 // Textures are expected to be premultiplied by GL and compositors.
581 1 /* attenuate, meaning premultiply */);
582 }
583
CodedSize(const VideoFrame * video_frame,GpuVideoAcceleratorFactories::OutputFormat output_format)584 gfx::Size CodedSize(const VideoFrame* video_frame,
585 GpuVideoAcceleratorFactories::OutputFormat output_format) {
586 DCHECK(gfx::Rect(video_frame->coded_size())
587 .Contains(video_frame->visible_rect()));
588 DCHECK((video_frame->visible_rect().x() & 1) == 0);
589 gfx::Size output;
590 switch (output_format) {
591 case GpuVideoAcceleratorFactories::OutputFormat::I420:
592 case GpuVideoAcceleratorFactories::OutputFormat::P010:
593 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
594 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
595 DCHECK((video_frame->visible_rect().y() & 1) == 0);
596 output = gfx::Size((video_frame->visible_rect().width() + 1) & ~1,
597 (video_frame->visible_rect().height() + 1) & ~1);
598 break;
599 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
600 case GpuVideoAcceleratorFactories::OutputFormat::XB30:
601 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
602 case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
603 output = gfx::Size((video_frame->visible_rect().width() + 1) & ~1,
604 video_frame->visible_rect().height());
605 break;
606 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
607 NOTREACHED();
608 }
609 DCHECK(gfx::Rect(video_frame->coded_size()).Contains(gfx::Rect(output)));
610 return output;
611 }
612 } // unnamed namespace
613
614 // Creates a VideoFrame backed by native textures starting from a software
615 // VideoFrame.
616 // The data contained in |video_frame| is copied into the VideoFrame passed to
617 // |frame_ready_cb|.
618 // This has to be called on the thread where |media_task_runner_| is current.
CreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,FrameReadyCB frame_ready_cb)619 void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
620 scoped_refptr<VideoFrame> video_frame,
621 FrameReadyCB frame_ready_cb) {
622 DCHECK(media_task_runner_->BelongsToCurrentThread());
623 // Lazily initialize |output_format_| since VideoFrameOutputFormat() has to be
624 // called on the media_thread while this object might be instantiated on any.
625 const VideoPixelFormat pixel_format = video_frame->format();
626 if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED)
627 output_format_ = gpu_factories_->VideoFrameOutputFormat(pixel_format);
628 // Bail if we have a change of GpuVideoAcceleratorFactories::OutputFormat;
629 // such changes should not happen in general (see https://crbug.com/875158).
630 if (output_format_ != gpu_factories_->VideoFrameOutputFormat(pixel_format)) {
631 std::move(frame_ready_cb).Run(std::move(video_frame));
632 return;
633 }
634
635 bool is_software_backed_video_frame = !video_frame->HasTextures();
636 #if defined(OS_LINUX) || defined(OS_CHROMEOS) || defined(OS_BSD)
637 is_software_backed_video_frame &= !video_frame->HasDmaBufs();
638 #endif
639
640 bool passthrough = false;
641 #if defined(OS_MAC)
642 if (!IOSurfaceCanSetColorSpace(video_frame->ColorSpace()))
643 passthrough = true;
644 #endif
645 if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED)
646 passthrough = true;
647 switch (pixel_format) {
648 // Supported cases.
649 case PIXEL_FORMAT_YV12:
650 case PIXEL_FORMAT_I420:
651 case PIXEL_FORMAT_YUV420P10:
652 case PIXEL_FORMAT_I420A:
653 break;
654 // Unsupported cases.
655 case PIXEL_FORMAT_I422:
656 case PIXEL_FORMAT_I444:
657 case PIXEL_FORMAT_NV12:
658 case PIXEL_FORMAT_NV21:
659 case PIXEL_FORMAT_UYVY:
660 case PIXEL_FORMAT_YUY2:
661 case PIXEL_FORMAT_ARGB:
662 case PIXEL_FORMAT_BGRA:
663 case PIXEL_FORMAT_XRGB:
664 case PIXEL_FORMAT_RGB24:
665 case PIXEL_FORMAT_MJPEG:
666 case PIXEL_FORMAT_YUV422P9:
667 case PIXEL_FORMAT_YUV420P9:
668 case PIXEL_FORMAT_YUV444P9:
669 case PIXEL_FORMAT_YUV422P10:
670 case PIXEL_FORMAT_YUV444P10:
671 case PIXEL_FORMAT_YUV420P12:
672 case PIXEL_FORMAT_YUV422P12:
673 case PIXEL_FORMAT_YUV444P12:
674 case PIXEL_FORMAT_Y16:
675 case PIXEL_FORMAT_ABGR:
676 case PIXEL_FORMAT_XBGR:
677 case PIXEL_FORMAT_P016LE:
678 case PIXEL_FORMAT_XR30:
679 case PIXEL_FORMAT_XB30:
680 case PIXEL_FORMAT_UNKNOWN:
681 if (is_software_backed_video_frame) {
682 UMA_HISTOGRAM_ENUMERATION(
683 "Media.GpuMemoryBufferVideoFramePool.UnsupportedFormat",
684 pixel_format, PIXEL_FORMAT_MAX + 1);
685 }
686 passthrough = true;
687 }
688 // TODO(dcastagna): Handle odd positioned video frame input, see
689 // https://crbug.com/638906.
690 // TODO(emircan): Eliminate odd size video frame input cases as they are not
691 // valid, see https://crbug.com/webrtc/9033.
692 if ((video_frame->visible_rect().x() & 1) ||
693 (video_frame->visible_rect().y() & 1) ||
694 (video_frame->coded_size().width() & 1) ||
695 (video_frame->coded_size().height() & 1)) {
696 passthrough = true;
697 }
698
699 frame_copy_requests_.emplace_back(std::move(video_frame),
700 std::move(frame_ready_cb), passthrough);
701 if (frame_copy_requests_.size() == 1u)
702 StartCopy();
703 }
704
OnMemoryDump(const base::trace_event::MemoryDumpArgs & args,base::trace_event::ProcessMemoryDump * pmd)705 bool GpuMemoryBufferVideoFramePool::PoolImpl::OnMemoryDump(
706 const base::trace_event::MemoryDumpArgs& args,
707 base::trace_event::ProcessMemoryDump* pmd) {
708 const uint64_t tracing_process_id =
709 base::trace_event::MemoryDumpManager::GetInstance()
710 ->GetTracingProcessId();
711 const int kImportance = 2;
712 for (const FrameResources* frame_resources : resources_pool_) {
713 for (const PlaneResource& plane_resource :
714 frame_resources->plane_resources) {
715 if (plane_resource.gpu_memory_buffer) {
716 gfx::GpuMemoryBufferId buffer_id =
717 plane_resource.gpu_memory_buffer->GetId();
718 std::string dump_name = base::StringPrintf(
719 "media/video_frame_memory/buffer_%d", buffer_id.id);
720 base::trace_event::MemoryAllocatorDump* dump =
721 pmd->CreateAllocatorDump(dump_name);
722 size_t buffer_size_in_bytes = gfx::BufferSizeForBufferFormat(
723 plane_resource.size, plane_resource.gpu_memory_buffer->GetFormat());
724 dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
725 base::trace_event::MemoryAllocatorDump::kUnitsBytes,
726 buffer_size_in_bytes);
727 dump->AddScalar("free_size",
728 base::trace_event::MemoryAllocatorDump::kUnitsBytes,
729 frame_resources->is_used() ? 0 : buffer_size_in_bytes);
730 plane_resource.gpu_memory_buffer->OnMemoryDump(
731 pmd, dump->guid(), tracing_process_id, kImportance);
732 }
733 }
734 }
735 return true;
736 }
737
Abort()738 void GpuMemoryBufferVideoFramePool::PoolImpl::Abort() {
739 DCHECK(media_task_runner_->BelongsToCurrentThread());
740 // Abort any pending copy requests. If one is already in flight, we can't do
741 // anything about it.
742 if (frame_copy_requests_.size() <= 1u)
743 return;
744 frame_copy_requests_.erase(frame_copy_requests_.begin() + 1,
745 frame_copy_requests_.end());
746 }
747
OnCopiesDone(scoped_refptr<VideoFrame> video_frame,FrameResources * frame_resources)748 void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDone(
749 scoped_refptr<VideoFrame> video_frame,
750 FrameResources* frame_resources) {
751 for (const auto& plane_resource : frame_resources->plane_resources) {
752 if (plane_resource.gpu_memory_buffer) {
753 plane_resource.gpu_memory_buffer->Unmap();
754 plane_resource.gpu_memory_buffer->SetColorSpace(
755 video_frame->ColorSpace());
756 }
757 }
758
759 TRACE_EVENT_ASYNC_END0("media", "CopyVideoFrameToGpuMemoryBuffers",
760 video_frame->timestamp().InNanoseconds() /* id */);
761
762 media_task_runner_->PostTask(
763 FROM_HERE,
764 base::BindOnce(&PoolImpl::BindAndCreateMailboxesHardwareFrameResources,
765 this, std::move(video_frame), frame_resources));
766 }
767
StartCopy()768 void GpuMemoryBufferVideoFramePool::PoolImpl::StartCopy() {
769 DCHECK(media_task_runner_->BelongsToCurrentThread());
770 DCHECK(!frame_copy_requests_.empty());
771
772 while (!frame_copy_requests_.empty()) {
773 VideoFrameCopyRequest& request = frame_copy_requests_.front();
774 // Acquire resources. Incompatible ones will be dropped from the pool.
775 FrameResources* frame_resources =
776 request.passthrough
777 ? nullptr
778 : GetOrCreateFrameResources(
779 CodedSize(request.video_frame.get(), output_format_),
780 output_format_);
781 if (!frame_resources) {
782 std::move(request.frame_ready_cb).Run(std::move(request.video_frame));
783 frame_copy_requests_.pop_front();
784 continue;
785 }
786
787 worker_task_runner_->PostTask(
788 FROM_HERE, base::BindOnce(&PoolImpl::CopyVideoFrameToGpuMemoryBuffers,
789 this, request.video_frame, frame_resources));
790 break;
791 }
792 }
793
794 // Copies |video_frame| into |frame_resources| asynchronously, posting n tasks
795 // that will be synchronized by a barrier.
796 // After the barrier is passed OnCopiesDone will be called.
CopyVideoFrameToGpuMemoryBuffers(scoped_refptr<VideoFrame> video_frame,FrameResources * frame_resources)797 void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
798 scoped_refptr<VideoFrame> video_frame,
799 FrameResources* frame_resources) {
800 // Compute the number of tasks to post and create the barrier.
801 const size_t num_planes = VideoFrame::NumPlanes(VideoFormat(output_format_));
802 const size_t planes_per_copy = PlanesPerCopy(output_format_);
803 const gfx::Size coded_size = CodedSize(video_frame.get(), output_format_);
804 size_t copies = 0;
805 for (size_t i = 0; i < num_planes; i += planes_per_copy) {
806 const int rows =
807 VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
808 const int rows_per_copy =
809 RowsPerCopy(i, VideoFormat(output_format_), coded_size.width());
810 copies += rows / rows_per_copy;
811 if (rows % rows_per_copy)
812 ++copies;
813 }
814
815 // |barrier| keeps refptr of |video_frame| until all copy tasks are done.
816 const base::RepeatingClosure barrier = base::BarrierClosure(
817 copies, base::BindOnce(&PoolImpl::OnCopiesDone, this, video_frame,
818 frame_resources));
819
820 // Map the buffers.
821 for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
822 gfx::GpuMemoryBuffer* buffer =
823 frame_resources->plane_resources[i].gpu_memory_buffer.get();
824
825 if (!buffer || !buffer->Map()) {
826 DLOG(ERROR) << "Could not get or Map() buffer";
827 frame_resources->MarkUnused(tick_clock_->NowTicks());
828 return;
829 }
830 }
831
832 TRACE_EVENT_ASYNC_BEGIN0("media", "CopyVideoFrameToGpuMemoryBuffers",
833 video_frame->timestamp().InNanoseconds() /* id */);
834 // Post all the async tasks.
835 for (size_t i = 0; i < num_planes; i += planes_per_copy) {
836 gfx::GpuMemoryBuffer* buffer =
837 frame_resources->plane_resources[i].gpu_memory_buffer.get();
838 const int rows =
839 VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
840 const int rows_per_copy =
841 RowsPerCopy(i, VideoFormat(output_format_), coded_size.width());
842
843 for (int row = 0; row < rows; row += rows_per_copy) {
844 const int rows_to_copy = std::min(rows_per_copy, rows - row);
845 switch (output_format_) {
846 case GpuVideoAcceleratorFactories::OutputFormat::I420: {
847 const int bytes_per_row = VideoFrame::RowBytes(
848 i, VideoFormat(output_format_), coded_size.width());
849 worker_task_runner_->PostTask(
850 FROM_HERE,
851 base::BindOnce(&CopyRowsToI420Buffer, row, rows_to_copy,
852 bytes_per_row, video_frame->BitDepth(),
853 video_frame->visible_data(i),
854 video_frame->stride(i),
855 static_cast<uint8_t*>(buffer->memory(0)),
856 buffer->stride(0), barrier));
857 break;
858 }
859 case GpuVideoAcceleratorFactories::OutputFormat::P010:
860 // Using base::Unretained(video_frame) here is safe because |barrier|
861 // keeps refptr of |video_frame| until all copy tasks are done.
862 worker_task_runner_->PostTask(
863 FROM_HERE,
864 base::BindOnce(
865 &CopyRowsToP010Buffer, row, rows_to_copy,
866 coded_size.width() * 2, base::Unretained(video_frame.get()),
867 static_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
868 static_cast<uint8_t*>(buffer->memory(1)), buffer->stride(1),
869 barrier));
870 break;
871 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
872 // Using base::Unretained(video_frame) here is safe because |barrier|
873 // keeps refptr of |video_frame| until all copy tasks are done.
874 worker_task_runner_->PostTask(
875 FROM_HERE,
876 base::BindOnce(
877 &CopyRowsToNV12Buffer, row, rows_to_copy, coded_size.width(),
878 base::Unretained(video_frame.get()),
879 static_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
880 static_cast<uint8_t*>(buffer->memory(1)), buffer->stride(1),
881 barrier));
882 break;
883 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB: {
884 gfx::GpuMemoryBuffer* buffer2 =
885 frame_resources->plane_resources[1].gpu_memory_buffer.get();
886 // Using base::Unretained(video_frame) here is safe because |barrier|
887 // keeps refptr of |video_frame| until all copy tasks are done.
888 worker_task_runner_->PostTask(
889 FROM_HERE,
890 base::BindOnce(
891 &CopyRowsToNV12Buffer, row, rows_to_copy, coded_size.width(),
892 base::Unretained(video_frame.get()),
893 static_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
894 static_cast<uint8_t*>(buffer2->memory(0)), buffer2->stride(0),
895 barrier));
896 break;
897 }
898
899 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
900 case GpuVideoAcceleratorFactories::OutputFormat::XB30: {
901 const bool is_argb = output_format_ ==
902 GpuVideoAcceleratorFactories::OutputFormat::XR30;
903 // Using base::Unretained(video_frame) here is safe because |barrier|
904 // keeps refptr of |video_frame| until all copy tasks are done.
905 worker_task_runner_->PostTask(
906 FROM_HERE,
907 base::BindOnce(&CopyRowsToRGB10Buffer, is_argb, row, rows_to_copy,
908 coded_size.width(),
909 base::Unretained(video_frame.get()),
910 static_cast<uint8_t*>(buffer->memory(0)),
911 buffer->stride(0), barrier));
912 break;
913 }
914
915 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
916 case GpuVideoAcceleratorFactories::OutputFormat::BGRA: {
917 const bool is_rgba = output_format_ ==
918 GpuVideoAcceleratorFactories::OutputFormat::RGBA;
919 // Using base::Unretained(video_frame) here is safe because |barrier|
920 // keeps refptr of |video_frame| until all copy tasks are done.
921 worker_task_runner_->PostTask(
922 FROM_HERE,
923 base::BindOnce(&CopyRowsToRGBABuffer, is_rgba, row, rows_to_copy,
924 coded_size.width(),
925 base::Unretained(video_frame.get()),
926 static_cast<uint8_t*>(buffer->memory(0)),
927 buffer->stride(0), barrier));
928 break;
929 }
930
931 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
932 NOTREACHED();
933 }
934 }
935 }
936 }
937
938 void GpuMemoryBufferVideoFramePool::PoolImpl::
BindAndCreateMailboxesHardwareFrameResources(scoped_refptr<VideoFrame> video_frame,FrameResources * frame_resources)939 BindAndCreateMailboxesHardwareFrameResources(
940 scoped_refptr<VideoFrame> video_frame,
941 FrameResources* frame_resources) {
942 gpu::SharedImageInterface* sii = gpu_factories_->SharedImageInterface();
943 if (!sii) {
944 frame_resources->MarkUnused(tick_clock_->NowTicks());
945 CompleteCopyRequestAndMaybeStartNextCopy(std::move(video_frame));
946 return;
947 }
948
949 const gfx::Size coded_size = CodedSize(video_frame.get(), output_format_);
950 gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];
951 // Set up the planes creating the mailboxes needed to refer to the textures.
952 for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
953 PlaneResource& plane_resource = frame_resources->plane_resources[i];
954 const gfx::BufferFormat buffer_format =
955 GpuMemoryBufferFormat(output_format_, i);
956 unsigned texture_target = gpu_factories_->ImageTextureTarget(buffer_format);
957 // Bind the texture and create or rebind the image.
958 if (plane_resource.gpu_memory_buffer && plane_resource.mailbox.IsZero()) {
959 uint32_t usage =
960 gpu::SHARED_IMAGE_USAGE_GLES2 | gpu::SHARED_IMAGE_USAGE_RASTER |
961 gpu::SHARED_IMAGE_USAGE_DISPLAY | gpu::SHARED_IMAGE_USAGE_SCANOUT;
962 plane_resource.mailbox = sii->CreateSharedImage(
963 plane_resource.gpu_memory_buffer.get(),
964 gpu_factories_->GpuMemoryBufferManager(), video_frame->ColorSpace(),
965 kTopLeft_GrSurfaceOrigin, kPremul_SkAlphaType, usage);
966 } else if (!plane_resource.mailbox.IsZero()) {
967 sii->UpdateSharedImage(frame_resources->sync_token,
968 plane_resource.mailbox);
969 }
970 mailbox_holders[i] = gpu::MailboxHolder(plane_resource.mailbox,
971 gpu::SyncToken(), texture_target);
972 }
973
974 // Insert a sync_token, this is needed to make sure that the textures the
975 // mailboxes refer to will be used only after all the previous commands posted
976 // in the SharedImageInterface have been processed.
977 gpu::SyncToken sync_token = sii->GenUnverifiedSyncToken();
978 for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++)
979 mailbox_holders[i].sync_token = sync_token;
980
981 VideoPixelFormat frame_format = VideoFormat(output_format_);
982
983 // Create the VideoFrame backed by native textures.
984 gfx::Size visible_size = video_frame->visible_rect().size();
985 scoped_refptr<VideoFrame> frame = VideoFrame::WrapNativeTextures(
986 frame_format, mailbox_holders, VideoFrame::ReleaseMailboxCB(), coded_size,
987 gfx::Rect(visible_size), video_frame->natural_size(),
988 video_frame->timestamp());
989
990 if (!frame) {
991 frame_resources->MarkUnused(tick_clock_->NowTicks());
992 MailboxHoldersReleased(frame_resources, sync_token);
993 CompleteCopyRequestAndMaybeStartNextCopy(std::move(video_frame));
994 return;
995 }
996 frame->SetReleaseMailboxCB(
997 base::BindOnce(&PoolImpl::MailboxHoldersReleased, this, frame_resources));
998
999 frame->set_color_space(video_frame->ColorSpace());
1000
1001 bool allow_overlay = false;
1002 #if defined(OS_WIN)
1003 // Windows direct composition path only supports dual GMB NV12 video overlays.
1004 allow_overlay = (output_format_ ==
1005 GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB);
1006 #else
1007 switch (output_format_) {
1008 case GpuVideoAcceleratorFactories::OutputFormat::I420:
1009 allow_overlay = video_frame->metadata()->allow_overlay;
1010 break;
1011 case GpuVideoAcceleratorFactories::OutputFormat::P010:
1012 case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
1013 allow_overlay = true;
1014 break;
1015 case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
1016 // Only used on Windows where we can't use single NV12 textures.
1017 break;
1018 case GpuVideoAcceleratorFactories::OutputFormat::XR30:
1019 case GpuVideoAcceleratorFactories::OutputFormat::XB30:
1020 // TODO(mcasas): Enable this for ChromeOS https://crbug.com/776093.
1021 allow_overlay = false;
1022 #if defined(OS_MAC)
1023 allow_overlay = IOSurfaceCanSetColorSpace(video_frame->ColorSpace());
1024 #endif
1025 // We've converted the YUV to RGB, fix the color space.
1026 // TODO(hubbe): The libyuv YUV to RGB conversion may not have
1027 // honored the color space conversion 100%. We should either fix
1028 // libyuv or find a way for later passes to make up the difference.
1029 frame->set_color_space(video_frame->ColorSpace().GetAsRGB());
1030 break;
1031 case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
1032 case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
1033 allow_overlay = true;
1034 break;
1035 case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
1036 break;
1037 }
1038 #endif // OS_WIN
1039 frame->metadata()->MergeMetadataFrom(video_frame->metadata());
1040 frame->metadata()->allow_overlay = allow_overlay;
1041 frame->metadata()->read_lock_fences_enabled = true;
1042
1043 CompleteCopyRequestAndMaybeStartNextCopy(std::move(frame));
1044 }
1045
1046 // Destroy all the resources posting one task per FrameResources
1047 // to the |media_task_runner_|.
~PoolImpl()1048 GpuMemoryBufferVideoFramePool::PoolImpl::~PoolImpl() {
1049 DCHECK(in_shutdown_);
1050 }
1051
Shutdown()1052 void GpuMemoryBufferVideoFramePool::PoolImpl::Shutdown() {
1053 DCHECK(media_task_runner_->BelongsToCurrentThread());
1054 // Clients don't care about copies once shutdown has started, so abort them.
1055 Abort();
1056
1057 // Delete all the resources on the media thread.
1058 in_shutdown_ = true;
1059 for (auto* frame_resources : resources_pool_) {
1060 // Will be deleted later upon return to pool.
1061 if (frame_resources->is_used())
1062 continue;
1063
1064 media_task_runner_->PostTask(
1065 FROM_HERE,
1066 base::BindOnce(&PoolImpl::DeleteFrameResources, gpu_factories_,
1067 base::Owned(frame_resources)));
1068 }
1069 resources_pool_.clear();
1070 }
1071
SetTickClockForTesting(const base::TickClock * tick_clock)1072 void GpuMemoryBufferVideoFramePool::PoolImpl::SetTickClockForTesting(
1073 const base::TickClock* tick_clock) {
1074 tick_clock_ = tick_clock;
1075 }
1076
1077 // Tries to find the resources in the pool or create them.
1078 // Incompatible resources will be dropped.
1079 GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources*
GetOrCreateFrameResources(const gfx::Size & size,GpuVideoAcceleratorFactories::OutputFormat format)1080 GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
1081 const gfx::Size& size,
1082 GpuVideoAcceleratorFactories::OutputFormat format) {
1083 DCHECK(media_task_runner_->BelongsToCurrentThread());
1084
1085 auto it = resources_pool_.begin();
1086 while (it != resources_pool_.end()) {
1087 FrameResources* frame_resources = *it;
1088 if (!frame_resources->is_used()) {
1089 if (AreFrameResourcesCompatible(frame_resources, size)) {
1090 frame_resources->MarkUsed();
1091 return frame_resources;
1092 } else {
1093 resources_pool_.erase(it++);
1094 DeleteFrameResources(gpu_factories_, frame_resources);
1095 delete frame_resources;
1096 }
1097 } else {
1098 it++;
1099 }
1100 }
1101
1102 // Create the resources.
1103 FrameResources* frame_resources = new FrameResources(size);
1104 resources_pool_.push_back(frame_resources);
1105 for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
1106 PlaneResource& plane_resource = frame_resources->plane_resources[i];
1107 const size_t width =
1108 VideoFrame::Columns(i, VideoFormat(format), size.width());
1109 const size_t height =
1110 VideoFrame::Rows(i, VideoFormat(format), size.height());
1111 plane_resource.size = gfx::Size(width, height);
1112
1113 const gfx::BufferFormat buffer_format = GpuMemoryBufferFormat(format, i);
1114 plane_resource.gpu_memory_buffer = gpu_factories_->CreateGpuMemoryBuffer(
1115 plane_resource.size, buffer_format,
1116 gfx::BufferUsage::SCANOUT_CPU_READ_WRITE);
1117 }
1118 return frame_resources;
1119 }
1120
1121 void GpuMemoryBufferVideoFramePool::PoolImpl::
CompleteCopyRequestAndMaybeStartNextCopy(scoped_refptr<VideoFrame> video_frame)1122 CompleteCopyRequestAndMaybeStartNextCopy(
1123 scoped_refptr<VideoFrame> video_frame) {
1124 DCHECK(!frame_copy_requests_.empty());
1125
1126 std::move(frame_copy_requests_.front().frame_ready_cb)
1127 .Run(std::move(video_frame));
1128 frame_copy_requests_.pop_front();
1129 if (!frame_copy_requests_.empty())
1130 StartCopy();
1131 }
1132
1133 // static
DeleteFrameResources(GpuVideoAcceleratorFactories * const gpu_factories,FrameResources * frame_resources)1134 void GpuMemoryBufferVideoFramePool::PoolImpl::DeleteFrameResources(
1135 GpuVideoAcceleratorFactories* const gpu_factories,
1136 FrameResources* frame_resources) {
1137 // TODO(dcastagna): As soon as the context lost is dealt with in media,
1138 // make sure that we won't execute this callback (use a weak pointer to
1139 // the old context).
1140 gpu::SharedImageInterface* sii = gpu_factories->SharedImageInterface();
1141 if (!sii)
1142 return;
1143
1144 for (PlaneResource& plane_resource : frame_resources->plane_resources) {
1145 if (!plane_resource.mailbox.IsZero()) {
1146 sii->DestroySharedImage(frame_resources->sync_token,
1147 plane_resource.mailbox);
1148 }
1149 }
1150 }
1151
1152 // Called when a VideoFrame is no longer referenced.
1153 // Put back the resources in the pool.
MailboxHoldersReleased(FrameResources * frame_resources,const gpu::SyncToken & release_sync_token)1154 void GpuMemoryBufferVideoFramePool::PoolImpl::MailboxHoldersReleased(
1155 FrameResources* frame_resources,
1156 const gpu::SyncToken& release_sync_token) {
1157 if (!media_task_runner_->BelongsToCurrentThread()) {
1158 media_task_runner_->PostTask(
1159 FROM_HERE, base::BindOnce(&PoolImpl::MailboxHoldersReleased, this,
1160 frame_resources, release_sync_token));
1161 return;
1162 }
1163 frame_resources->sync_token = release_sync_token;
1164
1165 if (in_shutdown_) {
1166 DeleteFrameResources(gpu_factories_, frame_resources);
1167 delete frame_resources;
1168 return;
1169 }
1170
1171 const base::TimeTicks now = tick_clock_->NowTicks();
1172 frame_resources->MarkUnused(now);
1173 auto it = resources_pool_.begin();
1174 while (it != resources_pool_.end()) {
1175 FrameResources* frame_resources = *it;
1176
1177 constexpr base::TimeDelta kStaleFrameLimit =
1178 base::TimeDelta::FromSeconds(10);
1179 if (!frame_resources->is_used() &&
1180 now - frame_resources->last_use_time() > kStaleFrameLimit) {
1181 resources_pool_.erase(it++);
1182 DeleteFrameResources(gpu_factories_, frame_resources);
1183 delete frame_resources;
1184 } else {
1185 it++;
1186 }
1187 }
1188 }
1189
1190 GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool() = default;
1191
GpuMemoryBufferVideoFramePool(const scoped_refptr<base::SingleThreadTaskRunner> & media_task_runner,const scoped_refptr<base::TaskRunner> & worker_task_runner,GpuVideoAcceleratorFactories * gpu_factories)1192 GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool(
1193 const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
1194 const scoped_refptr<base::TaskRunner>& worker_task_runner,
1195 GpuVideoAcceleratorFactories* gpu_factories)
1196 : pool_impl_(
1197 new PoolImpl(media_task_runner, worker_task_runner, gpu_factories)) {
1198 base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
1199 pool_impl_.get(), "GpuMemoryBufferVideoFramePool", media_task_runner);
1200 }
1201
~GpuMemoryBufferVideoFramePool()1202 GpuMemoryBufferVideoFramePool::~GpuMemoryBufferVideoFramePool() {
1203 // May be nullptr in tests.
1204 if (!pool_impl_)
1205 return;
1206
1207 pool_impl_->Shutdown();
1208 base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
1209 pool_impl_.get());
1210 }
1211
MaybeCreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,FrameReadyCB frame_ready_cb)1212 void GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame(
1213 scoped_refptr<VideoFrame> video_frame,
1214 FrameReadyCB frame_ready_cb) {
1215 DCHECK(video_frame);
1216 pool_impl_->CreateHardwareFrame(std::move(video_frame),
1217 std::move(frame_ready_cb));
1218 }
1219
Abort()1220 void GpuMemoryBufferVideoFramePool::Abort() {
1221 pool_impl_->Abort();
1222 }
1223
SetTickClockForTesting(const base::TickClock * tick_clock)1224 void GpuMemoryBufferVideoFramePool::SetTickClockForTesting(
1225 const base::TickClock* tick_clock) {
1226 pool_impl_->SetTickClockForTesting(tick_clock);
1227 }
1228
1229 } // namespace media
1230