1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "media/video/gpu_memory_buffer_video_frame_pool.h"
6 
7 #include <GLES2/gl2.h>
8 #include <GLES2/gl2ext.h>
9 #include <stddef.h>
10 #include <stdint.h>
11 
12 #include <algorithm>
13 #include <list>
14 #include <memory>
15 #include <utility>
16 
17 #include "base/barrier_closure.h"
18 #include "base/bind.h"
19 #include "base/callback_helpers.h"
20 #include "base/containers/circular_deque.h"
21 #include "base/containers/stack_container.h"
22 #include "base/location.h"
23 #include "base/logging.h"
24 #include "base/macros.h"
25 #include "base/metrics/histogram_macros.h"
26 #include "base/strings/stringprintf.h"
27 #include "base/sys_byteorder.h"
28 #include "base/time/default_tick_clock.h"
29 #include "base/trace_event/memory_dump_manager.h"
30 #include "base/trace_event/memory_dump_provider.h"
31 #include "base/trace_event/trace_event.h"
32 #include "build/build_config.h"
33 #include "gpu/GLES2/gl2extchromium.h"
34 #include "gpu/command_buffer/client/shared_image_interface.h"
35 #include "gpu/command_buffer/common/shared_image_usage.h"
36 #include "media/base/bind_to_current_loop.h"
37 #include "media/video/gpu_video_accelerator_factories.h"
38 #include "third_party/libyuv/include/libyuv.h"
39 #include "ui/gfx/buffer_format_util.h"
40 #include "ui/gfx/color_space.h"
41 #include "ui/gl/trace_util.h"
42 
43 #if defined(OS_MAC)
44 #include "base/mac/mac_util.h"
45 #include "ui/gfx/mac/io_surface.h"
46 #endif
47 
48 namespace media {
49 
50 // Implementation of a pool of GpuMemoryBuffers used to back VideoFrames.
51 class GpuMemoryBufferVideoFramePool::PoolImpl
52     : public base::RefCountedThreadSafe<
53           GpuMemoryBufferVideoFramePool::PoolImpl>,
54       public base::trace_event::MemoryDumpProvider {
55  public:
56   // |media_task_runner| is the media task runner associated with the
57   // GL context provided by |gpu_factories|
58   // |worker_task_runner| is a task runner used to asynchronously copy
59   // video frame's planes.
60   // |gpu_factories| is an interface to GPU related operation and can be
61   // null if a GL context is not available.
PoolImpl(const scoped_refptr<base::SingleThreadTaskRunner> & media_task_runner,const scoped_refptr<base::TaskRunner> & worker_task_runner,GpuVideoAcceleratorFactories * const gpu_factories)62   PoolImpl(const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
63            const scoped_refptr<base::TaskRunner>& worker_task_runner,
64            GpuVideoAcceleratorFactories* const gpu_factories)
65       : media_task_runner_(media_task_runner),
66         worker_task_runner_(worker_task_runner),
67         gpu_factories_(gpu_factories),
68         output_format_(GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED),
69         tick_clock_(base::DefaultTickClock::GetInstance()),
70         in_shutdown_(false) {
71     DCHECK(media_task_runner_);
72     DCHECK(worker_task_runner_);
73   }
74 
75   // Takes a software VideoFrame and calls |frame_ready_cb| with a VideoFrame
76   // backed by native textures if possible.
77   // The data contained in |video_frame| is copied into the returned frame
78   // asynchronously posting tasks to |worker_task_runner_|, while
79   // |frame_ready_cb| will be called on |media_task_runner_| once all the data
80   // has been copied.
81   void CreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,
82                            FrameReadyCB cb);
83 
84   bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
85                     base::trace_event::ProcessMemoryDump* pmd) override;
86 
87   // Aborts any pending copies.
88   void Abort();
89 
90   // Shuts down the frame pool and releases all frames in |frames_|.
91   // Once this is called frames will no longer be inserted back into
92   // |frames_|.
93   void Shutdown();
94 
95   void SetTickClockForTesting(const base::TickClock* tick_clock);
96 
97  private:
98   friend class base::RefCountedThreadSafe<
99       GpuMemoryBufferVideoFramePool::PoolImpl>;
100   ~PoolImpl() override;
101 
102   // Resource to represent a plane.
103   struct PlaneResource {
104     gfx::Size size;
105     std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer;
106     gpu::Mailbox mailbox;
107   };
108 
109   // All the resources needed to compose a frame.
110   // TODO(dalecurtis): The method of use marking used is very brittle
111   // and prone to leakage. Switch this to pass around std::unique_ptr
112   // such that callers own resources explicitly.
113   struct FrameResources {
FrameResourcesmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources114     explicit FrameResources(const gfx::Size& size) : size(size) {}
MarkUsedmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources115     void MarkUsed() {
116       is_used_ = true;
117       last_use_time_ = base::TimeTicks();
118     }
MarkUnusedmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources119     void MarkUnused(base::TimeTicks last_use_time) {
120       is_used_ = false;
121       last_use_time_ = last_use_time;
122     }
is_usedmedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources123     bool is_used() const { return is_used_; }
last_use_timemedia::GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources124     base::TimeTicks last_use_time() const { return last_use_time_; }
125 
126     const gfx::Size size;
127     PlaneResource plane_resources[VideoFrame::kMaxPlanes];
128     // The sync token used to recycle or destroy the resources. It is set when
129     // the resources are returned from the VideoFrame (via
130     // MailboxHoldersReleased).
131     gpu::SyncToken sync_token;
132 
133    private:
134     bool is_used_ = true;
135     base::TimeTicks last_use_time_;
136   };
137 
138   // Struct to keep track of requested videoframe copies.
139   struct VideoFrameCopyRequest {
VideoFrameCopyRequestmedia::GpuMemoryBufferVideoFramePool::PoolImpl::VideoFrameCopyRequest140     VideoFrameCopyRequest(scoped_refptr<VideoFrame> video_frame,
141                           FrameReadyCB frame_ready_cb,
142                           bool passthrough)
143         : video_frame(std::move(video_frame)),
144           frame_ready_cb(std::move(frame_ready_cb)),
145           passthrough(passthrough) {}
146     scoped_refptr<VideoFrame> video_frame;
147     FrameReadyCB frame_ready_cb;
148     bool passthrough;
149   };
150 
151   // Start the copy of a video_frame on the worker_task_runner_.
152   // It assumes there are currently no in-flight copies and works on the request
153   // in the front of |frame_copy_requests_| queue.
154   void StartCopy();
155 
156   // Copy |video_frame| data into |frame_resources| and calls |frame_ready_cb|
157   // when done.
158   void CopyVideoFrameToGpuMemoryBuffers(scoped_refptr<VideoFrame> video_frame,
159                                         FrameResources* frame_resources);
160 
161   // Called when all the data has been copied.
162   void OnCopiesDone(scoped_refptr<VideoFrame> video_frame,
163                     FrameResources* frame_resources);
164 
165   // Prepares GL resources, mailboxes and calls |frame_ready_cb| with the new
166   // VideoFrame. This has to be run on |media_task_runner_| where
167   // |frame_ready_cb| associated with video_frame will also be run.
168   void BindAndCreateMailboxesHardwareFrameResources(
169       scoped_refptr<VideoFrame> video_frame,
170       FrameResources* frame_resources);
171 
172   // Return true if |resources| can be used to represent a frame for
173   // specific |format| and |size|.
AreFrameResourcesCompatible(const FrameResources * resources,const gfx::Size & size)174   static bool AreFrameResourcesCompatible(const FrameResources* resources,
175                                           const gfx::Size& size) {
176     return size == resources->size;
177   }
178 
179   // Get the resources needed for a frame out of the pool, or create them if
180   // necessary.
181   // This also drops the LRU resources that can't be reuse for this frame.
182   FrameResources* GetOrCreateFrameResources(
183       const gfx::Size& size,
184       GpuVideoAcceleratorFactories::OutputFormat format);
185 
186   // Calls the FrameReadyCB of the first entry in |frame_copy_requests_|, with
187   // the provided |video_frame|, then deletes the entry from
188   // |frame_copy_requests_| and attempts to start another copy if there are
189   // other |frame_copy_requests_| elements.
190   void CompleteCopyRequestAndMaybeStartNextCopy(
191       scoped_refptr<VideoFrame> video_frame);
192 
193   // Callback called when a VideoFrame generated with GetFrameResources is no
194   // longer referenced.
195   void MailboxHoldersReleased(FrameResources* frame_resources,
196                               const gpu::SyncToken& sync_token);
197 
198   // Delete resources. This has to be called on the thread where |task_runner|
199   // is current.
200   static void DeleteFrameResources(
201       GpuVideoAcceleratorFactories* const gpu_factories,
202       FrameResources* frame_resources);
203 
204   // Task runner associated to the GL context provided by |gpu_factories_|.
205   const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
206   // Task runner used to asynchronously copy planes.
207   const scoped_refptr<base::TaskRunner> worker_task_runner_;
208 
209   // Interface to GPU related operations.
210   GpuVideoAcceleratorFactories* const gpu_factories_;
211 
212   // Pool of resources.
213   std::list<FrameResources*> resources_pool_;
214 
215   GpuVideoAcceleratorFactories::OutputFormat output_format_;
216 
217   // |tick_clock_| is always a DefaultTickClock outside of testing.
218   const base::TickClock* tick_clock_;
219 
220   // Queued up video frames for copies. The front is the currently
221   // in-flight copy, new copies are added at the end.
222   base::circular_deque<VideoFrameCopyRequest> frame_copy_requests_;
223   bool in_shutdown_;
224 
225   DISALLOW_COPY_AND_ASSIGN(PoolImpl);
226 };
227 
228 namespace {
229 
230 // VideoFrame copies to GpuMemoryBuffers will be split in copies where the
231 // output size is |kBytesPerCopyTarget| bytes and run in parallel.
232 constexpr size_t kBytesPerCopyTarget = 1024 * 1024;  // 1MB
233 
234 // Return the GpuMemoryBuffer format to use for a specific VideoPixelFormat
235 // and plane.
GpuMemoryBufferFormat(GpuVideoAcceleratorFactories::OutputFormat format,size_t plane)236 gfx::BufferFormat GpuMemoryBufferFormat(
237     GpuVideoAcceleratorFactories::OutputFormat format,
238     size_t plane) {
239   switch (format) {
240     case GpuVideoAcceleratorFactories::OutputFormat::I420:
241       DCHECK_LE(plane, 2u);
242       return gfx::BufferFormat::R_8;
243     case GpuVideoAcceleratorFactories::OutputFormat::P010:
244       DCHECK_LE(plane, 1u);
245       return gfx::BufferFormat::P010;
246     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
247       DCHECK_LE(plane, 1u);
248       return gfx::BufferFormat::YUV_420_BIPLANAR;
249     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
250       DCHECK_LE(plane, 1u);
251       return plane == 0 ? gfx::BufferFormat::R_8 : gfx::BufferFormat::RG_88;
252     case GpuVideoAcceleratorFactories::OutputFormat::XR30:
253       DCHECK_EQ(0u, plane);
254       return gfx::BufferFormat::BGRA_1010102;
255     case GpuVideoAcceleratorFactories::OutputFormat::XB30:
256       DCHECK_EQ(0u, plane);
257       return gfx::BufferFormat::RGBA_1010102;
258     case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
259       DCHECK_EQ(0u, plane);
260       return gfx::BufferFormat::RGBA_8888;
261     case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
262       DCHECK_EQ(0u, plane);
263       return gfx::BufferFormat::BGRA_8888;
264     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
265       NOTREACHED();
266       break;
267   }
268   return gfx::BufferFormat::BGRA_8888;
269 }
270 
271 // The number of output planes to be copied in each iteration.
PlanesPerCopy(GpuVideoAcceleratorFactories::OutputFormat format)272 size_t PlanesPerCopy(GpuVideoAcceleratorFactories::OutputFormat format) {
273   switch (format) {
274     case GpuVideoAcceleratorFactories::OutputFormat::I420:
275     case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
276     case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
277       return 1;
278     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
279     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
280     case GpuVideoAcceleratorFactories::OutputFormat::P010:
281       return 2;
282     case GpuVideoAcceleratorFactories::OutputFormat::XR30:
283     case GpuVideoAcceleratorFactories::OutputFormat::XB30:
284       return 3;
285     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
286       NOTREACHED();
287       break;
288   }
289   return 0;
290 }
291 
VideoFormat(GpuVideoAcceleratorFactories::OutputFormat format)292 VideoPixelFormat VideoFormat(
293     GpuVideoAcceleratorFactories::OutputFormat format) {
294   switch (format) {
295     case GpuVideoAcceleratorFactories::OutputFormat::I420:
296       return PIXEL_FORMAT_I420;
297     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
298     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
299       return PIXEL_FORMAT_NV12;
300     case GpuVideoAcceleratorFactories::OutputFormat::P010:
301       return PIXEL_FORMAT_P016LE;
302     case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
303       return PIXEL_FORMAT_ARGB;
304     case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
305       return PIXEL_FORMAT_ABGR;
306     case GpuVideoAcceleratorFactories::OutputFormat::XR30:
307       return PIXEL_FORMAT_XR30;
308     case GpuVideoAcceleratorFactories::OutputFormat::XB30:
309       return PIXEL_FORMAT_XB30;
310     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
311       NOTREACHED();
312       break;
313   }
314   return PIXEL_FORMAT_UNKNOWN;
315 }
316 
317 // The number of output planes to be copied in each iteration.
NumGpuMemoryBuffers(GpuVideoAcceleratorFactories::OutputFormat format)318 size_t NumGpuMemoryBuffers(GpuVideoAcceleratorFactories::OutputFormat format) {
319   switch (format) {
320     case GpuVideoAcceleratorFactories::OutputFormat::I420:
321       return 3;
322     case GpuVideoAcceleratorFactories::OutputFormat::P010:
323       return 1;
324     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
325       return 1;
326     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
327       return 2;
328     case GpuVideoAcceleratorFactories::OutputFormat::XR30:
329     case GpuVideoAcceleratorFactories::OutputFormat::XB30:
330       return 1;
331     case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
332     case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
333       return 1;
334     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
335       NOTREACHED();
336       break;
337   }
338   NOTREACHED();
339   return 0;
340 }
341 
342 // The number of output rows to be copied in each iteration.
RowsPerCopy(size_t plane,VideoPixelFormat format,int width)343 int RowsPerCopy(size_t plane, VideoPixelFormat format, int width) {
344   int bytes_per_row = VideoFrame::RowBytes(plane, format, width);
345   if (format == PIXEL_FORMAT_NV12) {
346     DCHECK_EQ(0u, plane);
347     bytes_per_row += VideoFrame::RowBytes(1, format, width);
348   }
349   // Copy an even number of lines, and at least one.
350   return std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
351 }
352 
CopyRowsToI420Buffer(int first_row,int rows,int bytes_per_row,size_t bit_depth,const uint8_t * source,int source_stride,uint8_t * output,int dest_stride,base::OnceClosure done)353 void CopyRowsToI420Buffer(int first_row,
354                           int rows,
355                           int bytes_per_row,
356                           size_t bit_depth,
357                           const uint8_t* source,
358                           int source_stride,
359                           uint8_t* output,
360                           int dest_stride,
361                           base::OnceClosure done) {
362   base::ScopedClosureRunner done_runner(std::move(done));
363   TRACE_EVENT2("media", "CopyRowsToI420Buffer", "bytes_per_row", bytes_per_row,
364                "rows", rows);
365 
366   if (!output)
367     return;
368 
369   DCHECK_NE(dest_stride, 0);
370   DCHECK_LE(bytes_per_row, std::abs(dest_stride));
371   DCHECK_LE(bytes_per_row, source_stride);
372   DCHECK_GE(bit_depth, 8u);
373 
374   if (bit_depth == 8) {
375     libyuv::CopyPlane(source + source_stride * first_row, source_stride,
376                       output + dest_stride * first_row, dest_stride,
377                       bytes_per_row, rows);
378   } else {
379     const int scale = 0x10000 >> (bit_depth - 8);
380     libyuv::Convert16To8Plane(
381         reinterpret_cast<const uint16_t*>(source + source_stride * first_row),
382         source_stride / 2, output + dest_stride * first_row, dest_stride, scale,
383         bytes_per_row, rows);
384   }
385 }
386 
CopyRowsToP010Buffer(int first_row,int rows,int bytes_per_row,const VideoFrame * source_frame,uint8_t * dest_y,int dest_stride_y,uint8_t * dest_uv,int dest_stride_uv,base::OnceClosure done)387 void CopyRowsToP010Buffer(int first_row,
388                           int rows,
389                           int bytes_per_row,
390                           const VideoFrame* source_frame,
391                           uint8_t* dest_y,
392                           int dest_stride_y,
393                           uint8_t* dest_uv,
394                           int dest_stride_uv,
395                           base::OnceClosure done) {
396   base::ScopedClosureRunner done_runner(std::move(done));
397   TRACE_EVENT2("media", "CopyRowsToP010Buffer", "bytes_per_row", bytes_per_row,
398                "rows", rows);
399 
400   if (!dest_y || !dest_uv)
401     return;
402 
403   DCHECK_NE(dest_stride_y, 0);
404   DCHECK_NE(dest_stride_uv, 0);
405   DCHECK_LE(bytes_per_row, std::abs(dest_stride_y));
406   DCHECK_LE(bytes_per_row, std::abs(dest_stride_uv));
407   DCHECK_EQ(0, first_row % 2);
408   DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_YUV420P10);
409   DCHECK_LE(bytes_per_row, source_frame->stride(VideoFrame::kYPlane));
410 
411   // TODO(crbug.com/libyuv/873): Replace this with a libyuv optimized path or at
412   // least add a SIMD variant.
413   for (int r = first_row; r < first_row + rows; ++r) {
414     const uint16_t* src = reinterpret_cast<const uint16_t*>(
415         source_frame->visible_data(VideoFrame::kYPlane) +
416         r * source_frame->stride(VideoFrame::kYPlane));
417     uint16_t* dest = reinterpret_cast<uint16_t*>(dest_y + r * dest_stride_y);
418     for (int c = 0; c < bytes_per_row / 2; ++c)
419       *dest++ = *src++ << 6;
420   }
421 
422   for (int r = first_row / 2; r < (first_row + rows) / 2; ++r) {
423     const uint16_t* u_src = reinterpret_cast<const uint16_t*>(
424         source_frame->visible_data(VideoFrame::kUPlane) +
425         r * source_frame->stride(VideoFrame::kUPlane));
426     const uint16_t* v_src = reinterpret_cast<const uint16_t*>(
427         source_frame->visible_data(VideoFrame::kVPlane) +
428         r * source_frame->stride(VideoFrame::kVPlane));
429     uint16_t* dest = reinterpret_cast<uint16_t*>(dest_uv + r * dest_stride_uv);
430     for (int c = 0; c < bytes_per_row / 4; ++c) {
431       *dest++ = *u_src++ << 6;
432       *dest++ = *v_src++ << 6;
433     }
434   }
435 }
436 
CopyRowsToNV12Buffer(int first_row,int rows,int bytes_per_row,const VideoFrame * source_frame,uint8_t * dest_y,int dest_stride_y,uint8_t * dest_uv,int dest_stride_uv,base::OnceClosure done)437 void CopyRowsToNV12Buffer(int first_row,
438                           int rows,
439                           int bytes_per_row,
440                           const VideoFrame* source_frame,
441                           uint8_t* dest_y,
442                           int dest_stride_y,
443                           uint8_t* dest_uv,
444                           int dest_stride_uv,
445                           base::OnceClosure done) {
446   base::ScopedClosureRunner done_runner(std::move(done));
447   TRACE_EVENT2("media", "CopyRowsToNV12Buffer", "bytes_per_row", bytes_per_row,
448                "rows", rows);
449 
450   if (!dest_y || !dest_uv)
451     return;
452 
453   DCHECK_NE(dest_stride_y, 0);
454   DCHECK_NE(dest_stride_uv, 0);
455   DCHECK_LE(bytes_per_row, std::abs(dest_stride_y));
456   DCHECK_LE(bytes_per_row, std::abs(dest_stride_uv));
457   DCHECK_EQ(0, first_row % 2);
458   DCHECK(source_frame->format() == PIXEL_FORMAT_I420 ||
459          source_frame->format() == PIXEL_FORMAT_YV12);
460   libyuv::I420ToNV12(
461       source_frame->visible_data(VideoFrame::kYPlane) +
462           first_row * source_frame->stride(VideoFrame::kYPlane),
463       source_frame->stride(VideoFrame::kYPlane),
464       source_frame->visible_data(VideoFrame::kUPlane) +
465           first_row / 2 * source_frame->stride(VideoFrame::kUPlane),
466       source_frame->stride(VideoFrame::kUPlane),
467       source_frame->visible_data(VideoFrame::kVPlane) +
468           first_row / 2 * source_frame->stride(VideoFrame::kVPlane),
469       source_frame->stride(VideoFrame::kVPlane),
470       dest_y + first_row * dest_stride_y, dest_stride_y,
471       dest_uv + first_row / 2 * dest_stride_uv, dest_stride_uv, bytes_per_row,
472       rows);
473 }
474 
CopyRowsToRGB10Buffer(bool is_argb,int first_row,int rows,int width,const VideoFrame * source_frame,uint8_t * output,int dest_stride,base::OnceClosure done)475 void CopyRowsToRGB10Buffer(bool is_argb,
476                            int first_row,
477                            int rows,
478                            int width,
479                            const VideoFrame* source_frame,
480                            uint8_t* output,
481                            int dest_stride,
482                            base::OnceClosure done) {
483   base::ScopedClosureRunner done_runner(std::move(done));
484   TRACE_EVENT2("media", "CopyRowsToXR30Buffer", "bytes_per_row", width * 2,
485                "rows", rows);
486   if (!output)
487     return;
488 
489   DCHECK_NE(dest_stride, 0);
490   DCHECK_LE(width, std::abs(dest_stride / 2));
491   DCHECK_EQ(0, first_row % 2);
492   DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_YUV420P10);
493 
494   const uint16_t* y_plane = reinterpret_cast<const uint16_t*>(
495       source_frame->visible_data(VideoFrame::kYPlane) +
496       first_row * source_frame->stride(VideoFrame::kYPlane));
497   const size_t y_plane_stride = source_frame->stride(VideoFrame::kYPlane) / 2;
498   const uint16_t* v_plane = reinterpret_cast<const uint16_t*>(
499       source_frame->visible_data(VideoFrame::kVPlane) +
500       first_row / 2 * source_frame->stride(VideoFrame::kVPlane));
501   const size_t v_plane_stride = source_frame->stride(VideoFrame::kVPlane) / 2;
502   const uint16_t* u_plane = reinterpret_cast<const uint16_t*>(
503       source_frame->visible_data(VideoFrame::kUPlane) +
504       first_row / 2 * source_frame->stride(VideoFrame::kUPlane));
505   const size_t u_plane_stride = source_frame->stride(VideoFrame::kUPlane) / 2;
506   uint8_t* dest_rgb10 = output + first_row * dest_stride;
507 
508   SkYUVColorSpace skyuv = kRec709_SkYUVColorSpace;
509   source_frame->ColorSpace().ToSkYUVColorSpace(&skyuv);
510 
511   if (skyuv == kRec601_SkYUVColorSpace) {
512     if (is_argb) {
513       libyuv::I010ToAR30(y_plane, y_plane_stride, u_plane, u_plane_stride,
514                          v_plane, v_plane_stride, dest_rgb10, dest_stride,
515                          width, rows);
516     } else {
517       libyuv::I010ToAB30(y_plane, y_plane_stride, u_plane, u_plane_stride,
518                          v_plane, v_plane_stride, dest_rgb10, dest_stride,
519                          width, rows);
520     }
521   } else if (skyuv == kBT2020_SkYUVColorSpace) {
522     if (is_argb) {
523       libyuv::U010ToAR30(y_plane, y_plane_stride, u_plane, u_plane_stride,
524                          v_plane, v_plane_stride, dest_rgb10, dest_stride,
525                          width, rows);
526     } else {
527       libyuv::U010ToAB30(y_plane, y_plane_stride, u_plane, u_plane_stride,
528                          v_plane, v_plane_stride, dest_rgb10, dest_stride,
529                          width, rows);
530     }
531   } else {  // BT.709
532     if (is_argb) {
533       libyuv::H010ToAR30(y_plane, y_plane_stride, u_plane, u_plane_stride,
534                          v_plane, v_plane_stride, dest_rgb10, dest_stride,
535                          width, rows);
536     } else {
537       libyuv::H010ToAB30(y_plane, y_plane_stride, u_plane, u_plane_stride,
538                          v_plane, v_plane_stride, dest_rgb10, dest_stride,
539                          width, rows);
540     }
541   }
542 }
543 
CopyRowsToRGBABuffer(bool is_rgba,int first_row,int rows,int width,const VideoFrame * source_frame,uint8_t * output,int dest_stride,base::OnceClosure done)544 void CopyRowsToRGBABuffer(bool is_rgba,
545                           int first_row,
546                           int rows,
547                           int width,
548                           const VideoFrame* source_frame,
549                           uint8_t* output,
550                           int dest_stride,
551                           base::OnceClosure done) {
552   base::ScopedClosureRunner done_runner(std::move(done));
553   TRACE_EVENT2("media", "CopyRowsToRGBABuffer", "bytes_per_row", width * 2,
554                "rows", rows);
555 
556   if (!output)
557     return;
558 
559   DCHECK_NE(dest_stride, 0);
560   DCHECK_LE(width, std::abs(dest_stride / 2));
561   DCHECK_EQ(0, first_row % 2);
562   DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_I420A);
563 
564   // libyuv uses little-endian for RGBx formats, whereas here we use big endian.
565   auto* func_ptr = is_rgba ? libyuv::I420AlphaToABGR : libyuv::I420AlphaToARGB;
566 
567   func_ptr(source_frame->visible_data(VideoFrame::kYPlane) +
568                first_row * source_frame->stride(VideoFrame::kYPlane),
569            source_frame->stride(VideoFrame::kYPlane),
570            source_frame->visible_data(VideoFrame::kUPlane) +
571                first_row / 2 * source_frame->stride(VideoFrame::kUPlane),
572            source_frame->stride(VideoFrame::kUPlane),
573            source_frame->visible_data(VideoFrame::kVPlane) +
574                first_row / 2 * source_frame->stride(VideoFrame::kVPlane),
575            source_frame->stride(VideoFrame::kVPlane),
576            source_frame->visible_data(VideoFrame::kAPlane) +
577                first_row * source_frame->stride(VideoFrame::kAPlane),
578            source_frame->stride(VideoFrame::kAPlane),
579            output + first_row * dest_stride, dest_stride, width, rows,
580            // Textures are expected to be premultiplied by GL and compositors.
581            1 /* attenuate, meaning premultiply */);
582 }
583 
CodedSize(const VideoFrame * video_frame,GpuVideoAcceleratorFactories::OutputFormat output_format)584 gfx::Size CodedSize(const VideoFrame* video_frame,
585                     GpuVideoAcceleratorFactories::OutputFormat output_format) {
586   DCHECK(gfx::Rect(video_frame->coded_size())
587              .Contains(video_frame->visible_rect()));
588   DCHECK((video_frame->visible_rect().x() & 1) == 0);
589   gfx::Size output;
590   switch (output_format) {
591     case GpuVideoAcceleratorFactories::OutputFormat::I420:
592     case GpuVideoAcceleratorFactories::OutputFormat::P010:
593     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
594     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
595       DCHECK((video_frame->visible_rect().y() & 1) == 0);
596       output = gfx::Size((video_frame->visible_rect().width() + 1) & ~1,
597                          (video_frame->visible_rect().height() + 1) & ~1);
598       break;
599     case GpuVideoAcceleratorFactories::OutputFormat::XR30:
600     case GpuVideoAcceleratorFactories::OutputFormat::XB30:
601     case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
602     case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
603       output = gfx::Size((video_frame->visible_rect().width() + 1) & ~1,
604                          video_frame->visible_rect().height());
605       break;
606     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
607       NOTREACHED();
608   }
609   DCHECK(gfx::Rect(video_frame->coded_size()).Contains(gfx::Rect(output)));
610   return output;
611 }
612 }  // unnamed namespace
613 
614 // Creates a VideoFrame backed by native textures starting from a software
615 // VideoFrame.
616 // The data contained in |video_frame| is copied into the VideoFrame passed to
617 // |frame_ready_cb|.
618 // This has to be called on the thread where |media_task_runner_| is current.
CreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,FrameReadyCB frame_ready_cb)619 void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
620     scoped_refptr<VideoFrame> video_frame,
621     FrameReadyCB frame_ready_cb) {
622   DCHECK(media_task_runner_->BelongsToCurrentThread());
623   // Lazily initialize |output_format_| since VideoFrameOutputFormat() has to be
624   // called on the media_thread while this object might be instantiated on any.
625   const VideoPixelFormat pixel_format = video_frame->format();
626   if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED)
627     output_format_ = gpu_factories_->VideoFrameOutputFormat(pixel_format);
628   // Bail if we have a change of GpuVideoAcceleratorFactories::OutputFormat;
629   // such changes should not happen in general (see https://crbug.com/875158).
630   if (output_format_ != gpu_factories_->VideoFrameOutputFormat(pixel_format)) {
631     std::move(frame_ready_cb).Run(std::move(video_frame));
632     return;
633   }
634 
635   bool is_software_backed_video_frame = !video_frame->HasTextures();
636 #if defined(OS_LINUX) || defined(OS_CHROMEOS) || defined(OS_BSD)
637   is_software_backed_video_frame &= !video_frame->HasDmaBufs();
638 #endif
639 
640   bool passthrough = false;
641 #if defined(OS_MAC)
642   if (!IOSurfaceCanSetColorSpace(video_frame->ColorSpace()))
643     passthrough = true;
644 #endif
645   if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED)
646     passthrough = true;
647   switch (pixel_format) {
648     // Supported cases.
649     case PIXEL_FORMAT_YV12:
650     case PIXEL_FORMAT_I420:
651     case PIXEL_FORMAT_YUV420P10:
652     case PIXEL_FORMAT_I420A:
653       break;
654     // Unsupported cases.
655     case PIXEL_FORMAT_I422:
656     case PIXEL_FORMAT_I444:
657     case PIXEL_FORMAT_NV12:
658     case PIXEL_FORMAT_NV21:
659     case PIXEL_FORMAT_UYVY:
660     case PIXEL_FORMAT_YUY2:
661     case PIXEL_FORMAT_ARGB:
662     case PIXEL_FORMAT_BGRA:
663     case PIXEL_FORMAT_XRGB:
664     case PIXEL_FORMAT_RGB24:
665     case PIXEL_FORMAT_MJPEG:
666     case PIXEL_FORMAT_YUV422P9:
667     case PIXEL_FORMAT_YUV420P9:
668     case PIXEL_FORMAT_YUV444P9:
669     case PIXEL_FORMAT_YUV422P10:
670     case PIXEL_FORMAT_YUV444P10:
671     case PIXEL_FORMAT_YUV420P12:
672     case PIXEL_FORMAT_YUV422P12:
673     case PIXEL_FORMAT_YUV444P12:
674     case PIXEL_FORMAT_Y16:
675     case PIXEL_FORMAT_ABGR:
676     case PIXEL_FORMAT_XBGR:
677     case PIXEL_FORMAT_P016LE:
678     case PIXEL_FORMAT_XR30:
679     case PIXEL_FORMAT_XB30:
680     case PIXEL_FORMAT_UNKNOWN:
681       if (is_software_backed_video_frame) {
682         UMA_HISTOGRAM_ENUMERATION(
683             "Media.GpuMemoryBufferVideoFramePool.UnsupportedFormat",
684             pixel_format, PIXEL_FORMAT_MAX + 1);
685       }
686       passthrough = true;
687   }
688   // TODO(dcastagna): Handle odd positioned video frame input, see
689   // https://crbug.com/638906.
690   // TODO(emircan): Eliminate odd size video frame input cases as they are not
691   // valid, see https://crbug.com/webrtc/9033.
692   if ((video_frame->visible_rect().x() & 1) ||
693       (video_frame->visible_rect().y() & 1) ||
694       (video_frame->coded_size().width() & 1) ||
695       (video_frame->coded_size().height() & 1)) {
696     passthrough = true;
697   }
698 
699   frame_copy_requests_.emplace_back(std::move(video_frame),
700                                     std::move(frame_ready_cb), passthrough);
701   if (frame_copy_requests_.size() == 1u)
702     StartCopy();
703 }
704 
OnMemoryDump(const base::trace_event::MemoryDumpArgs & args,base::trace_event::ProcessMemoryDump * pmd)705 bool GpuMemoryBufferVideoFramePool::PoolImpl::OnMemoryDump(
706     const base::trace_event::MemoryDumpArgs& args,
707     base::trace_event::ProcessMemoryDump* pmd) {
708   const uint64_t tracing_process_id =
709       base::trace_event::MemoryDumpManager::GetInstance()
710           ->GetTracingProcessId();
711   const int kImportance = 2;
712   for (const FrameResources* frame_resources : resources_pool_) {
713     for (const PlaneResource& plane_resource :
714          frame_resources->plane_resources) {
715       if (plane_resource.gpu_memory_buffer) {
716         gfx::GpuMemoryBufferId buffer_id =
717             plane_resource.gpu_memory_buffer->GetId();
718         std::string dump_name = base::StringPrintf(
719             "media/video_frame_memory/buffer_%d", buffer_id.id);
720         base::trace_event::MemoryAllocatorDump* dump =
721             pmd->CreateAllocatorDump(dump_name);
722         size_t buffer_size_in_bytes = gfx::BufferSizeForBufferFormat(
723             plane_resource.size, plane_resource.gpu_memory_buffer->GetFormat());
724         dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
725                         base::trace_event::MemoryAllocatorDump::kUnitsBytes,
726                         buffer_size_in_bytes);
727         dump->AddScalar("free_size",
728                         base::trace_event::MemoryAllocatorDump::kUnitsBytes,
729                         frame_resources->is_used() ? 0 : buffer_size_in_bytes);
730         plane_resource.gpu_memory_buffer->OnMemoryDump(
731             pmd, dump->guid(), tracing_process_id, kImportance);
732       }
733     }
734   }
735   return true;
736 }
737 
Abort()738 void GpuMemoryBufferVideoFramePool::PoolImpl::Abort() {
739   DCHECK(media_task_runner_->BelongsToCurrentThread());
740   // Abort any pending copy requests. If one is already in flight, we can't do
741   // anything about it.
742   if (frame_copy_requests_.size() <= 1u)
743     return;
744   frame_copy_requests_.erase(frame_copy_requests_.begin() + 1,
745                              frame_copy_requests_.end());
746 }
747 
OnCopiesDone(scoped_refptr<VideoFrame> video_frame,FrameResources * frame_resources)748 void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDone(
749     scoped_refptr<VideoFrame> video_frame,
750     FrameResources* frame_resources) {
751   for (const auto& plane_resource : frame_resources->plane_resources) {
752     if (plane_resource.gpu_memory_buffer) {
753       plane_resource.gpu_memory_buffer->Unmap();
754       plane_resource.gpu_memory_buffer->SetColorSpace(
755           video_frame->ColorSpace());
756     }
757   }
758 
759   TRACE_EVENT_ASYNC_END0("media", "CopyVideoFrameToGpuMemoryBuffers",
760                          video_frame->timestamp().InNanoseconds() /* id */);
761 
762   media_task_runner_->PostTask(
763       FROM_HERE,
764       base::BindOnce(&PoolImpl::BindAndCreateMailboxesHardwareFrameResources,
765                      this, std::move(video_frame), frame_resources));
766 }
767 
StartCopy()768 void GpuMemoryBufferVideoFramePool::PoolImpl::StartCopy() {
769   DCHECK(media_task_runner_->BelongsToCurrentThread());
770   DCHECK(!frame_copy_requests_.empty());
771 
772   while (!frame_copy_requests_.empty()) {
773     VideoFrameCopyRequest& request = frame_copy_requests_.front();
774     // Acquire resources. Incompatible ones will be dropped from the pool.
775     FrameResources* frame_resources =
776         request.passthrough
777             ? nullptr
778             : GetOrCreateFrameResources(
779                   CodedSize(request.video_frame.get(), output_format_),
780                   output_format_);
781     if (!frame_resources) {
782       std::move(request.frame_ready_cb).Run(std::move(request.video_frame));
783       frame_copy_requests_.pop_front();
784       continue;
785     }
786 
787     worker_task_runner_->PostTask(
788         FROM_HERE, base::BindOnce(&PoolImpl::CopyVideoFrameToGpuMemoryBuffers,
789                                   this, request.video_frame, frame_resources));
790     break;
791   }
792 }
793 
794 // Copies |video_frame| into |frame_resources| asynchronously, posting n tasks
795 // that will be synchronized by a barrier.
796 // After the barrier is passed OnCopiesDone will be called.
CopyVideoFrameToGpuMemoryBuffers(scoped_refptr<VideoFrame> video_frame,FrameResources * frame_resources)797 void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
798     scoped_refptr<VideoFrame> video_frame,
799     FrameResources* frame_resources) {
800   // Compute the number of tasks to post and create the barrier.
801   const size_t num_planes = VideoFrame::NumPlanes(VideoFormat(output_format_));
802   const size_t planes_per_copy = PlanesPerCopy(output_format_);
803   const gfx::Size coded_size = CodedSize(video_frame.get(), output_format_);
804   size_t copies = 0;
805   for (size_t i = 0; i < num_planes; i += planes_per_copy) {
806     const int rows =
807         VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
808     const int rows_per_copy =
809         RowsPerCopy(i, VideoFormat(output_format_), coded_size.width());
810     copies += rows / rows_per_copy;
811     if (rows % rows_per_copy)
812       ++copies;
813   }
814 
815   // |barrier| keeps refptr of |video_frame| until all copy tasks are done.
816   const base::RepeatingClosure barrier = base::BarrierClosure(
817       copies, base::BindOnce(&PoolImpl::OnCopiesDone, this, video_frame,
818                              frame_resources));
819 
820   // Map the buffers.
821   for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
822     gfx::GpuMemoryBuffer* buffer =
823         frame_resources->plane_resources[i].gpu_memory_buffer.get();
824 
825     if (!buffer || !buffer->Map()) {
826       DLOG(ERROR) << "Could not get or Map() buffer";
827       frame_resources->MarkUnused(tick_clock_->NowTicks());
828       return;
829     }
830   }
831 
832   TRACE_EVENT_ASYNC_BEGIN0("media", "CopyVideoFrameToGpuMemoryBuffers",
833                            video_frame->timestamp().InNanoseconds() /* id */);
834   // Post all the async tasks.
835   for (size_t i = 0; i < num_planes; i += planes_per_copy) {
836     gfx::GpuMemoryBuffer* buffer =
837         frame_resources->plane_resources[i].gpu_memory_buffer.get();
838     const int rows =
839         VideoFrame::Rows(i, VideoFormat(output_format_), coded_size.height());
840     const int rows_per_copy =
841         RowsPerCopy(i, VideoFormat(output_format_), coded_size.width());
842 
843     for (int row = 0; row < rows; row += rows_per_copy) {
844       const int rows_to_copy = std::min(rows_per_copy, rows - row);
845       switch (output_format_) {
846         case GpuVideoAcceleratorFactories::OutputFormat::I420: {
847           const int bytes_per_row = VideoFrame::RowBytes(
848               i, VideoFormat(output_format_), coded_size.width());
849           worker_task_runner_->PostTask(
850               FROM_HERE,
851               base::BindOnce(&CopyRowsToI420Buffer, row, rows_to_copy,
852                              bytes_per_row, video_frame->BitDepth(),
853                              video_frame->visible_data(i),
854                              video_frame->stride(i),
855                              static_cast<uint8_t*>(buffer->memory(0)),
856                              buffer->stride(0), barrier));
857           break;
858         }
859         case GpuVideoAcceleratorFactories::OutputFormat::P010:
860           // Using base::Unretained(video_frame) here is safe because |barrier|
861           // keeps refptr of |video_frame| until all copy tasks are done.
862           worker_task_runner_->PostTask(
863               FROM_HERE,
864               base::BindOnce(
865                   &CopyRowsToP010Buffer, row, rows_to_copy,
866                   coded_size.width() * 2, base::Unretained(video_frame.get()),
867                   static_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
868                   static_cast<uint8_t*>(buffer->memory(1)), buffer->stride(1),
869                   barrier));
870           break;
871         case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
872           // Using base::Unretained(video_frame) here is safe because |barrier|
873           // keeps refptr of |video_frame| until all copy tasks are done.
874           worker_task_runner_->PostTask(
875               FROM_HERE,
876               base::BindOnce(
877                   &CopyRowsToNV12Buffer, row, rows_to_copy, coded_size.width(),
878                   base::Unretained(video_frame.get()),
879                   static_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
880                   static_cast<uint8_t*>(buffer->memory(1)), buffer->stride(1),
881                   barrier));
882           break;
883         case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB: {
884           gfx::GpuMemoryBuffer* buffer2 =
885               frame_resources->plane_resources[1].gpu_memory_buffer.get();
886           // Using base::Unretained(video_frame) here is safe because |barrier|
887           // keeps refptr of |video_frame| until all copy tasks are done.
888           worker_task_runner_->PostTask(
889               FROM_HERE,
890               base::BindOnce(
891                   &CopyRowsToNV12Buffer, row, rows_to_copy, coded_size.width(),
892                   base::Unretained(video_frame.get()),
893                   static_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
894                   static_cast<uint8_t*>(buffer2->memory(0)), buffer2->stride(0),
895                   barrier));
896           break;
897         }
898 
899         case GpuVideoAcceleratorFactories::OutputFormat::XR30:
900         case GpuVideoAcceleratorFactories::OutputFormat::XB30: {
901           const bool is_argb = output_format_ ==
902                                GpuVideoAcceleratorFactories::OutputFormat::XR30;
903           // Using base::Unretained(video_frame) here is safe because |barrier|
904           // keeps refptr of |video_frame| until all copy tasks are done.
905           worker_task_runner_->PostTask(
906               FROM_HERE,
907               base::BindOnce(&CopyRowsToRGB10Buffer, is_argb, row, rows_to_copy,
908                              coded_size.width(),
909                              base::Unretained(video_frame.get()),
910                              static_cast<uint8_t*>(buffer->memory(0)),
911                              buffer->stride(0), barrier));
912           break;
913         }
914 
915         case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
916         case GpuVideoAcceleratorFactories::OutputFormat::BGRA: {
917           const bool is_rgba = output_format_ ==
918                                GpuVideoAcceleratorFactories::OutputFormat::RGBA;
919           // Using base::Unretained(video_frame) here is safe because |barrier|
920           // keeps refptr of |video_frame| until all copy tasks are done.
921           worker_task_runner_->PostTask(
922               FROM_HERE,
923               base::BindOnce(&CopyRowsToRGBABuffer, is_rgba, row, rows_to_copy,
924                              coded_size.width(),
925                              base::Unretained(video_frame.get()),
926                              static_cast<uint8_t*>(buffer->memory(0)),
927                              buffer->stride(0), barrier));
928           break;
929         }
930 
931         case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
932           NOTREACHED();
933       }
934     }
935   }
936 }
937 
938 void GpuMemoryBufferVideoFramePool::PoolImpl::
BindAndCreateMailboxesHardwareFrameResources(scoped_refptr<VideoFrame> video_frame,FrameResources * frame_resources)939     BindAndCreateMailboxesHardwareFrameResources(
940         scoped_refptr<VideoFrame> video_frame,
941         FrameResources* frame_resources) {
942   gpu::SharedImageInterface* sii = gpu_factories_->SharedImageInterface();
943   if (!sii) {
944     frame_resources->MarkUnused(tick_clock_->NowTicks());
945     CompleteCopyRequestAndMaybeStartNextCopy(std::move(video_frame));
946     return;
947   }
948 
949   const gfx::Size coded_size = CodedSize(video_frame.get(), output_format_);
950   gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];
951   // Set up the planes creating the mailboxes needed to refer to the textures.
952   for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
953     PlaneResource& plane_resource = frame_resources->plane_resources[i];
954     const gfx::BufferFormat buffer_format =
955         GpuMemoryBufferFormat(output_format_, i);
956     unsigned texture_target = gpu_factories_->ImageTextureTarget(buffer_format);
957     // Bind the texture and create or rebind the image.
958     if (plane_resource.gpu_memory_buffer && plane_resource.mailbox.IsZero()) {
959       uint32_t usage =
960           gpu::SHARED_IMAGE_USAGE_GLES2 | gpu::SHARED_IMAGE_USAGE_RASTER |
961           gpu::SHARED_IMAGE_USAGE_DISPLAY | gpu::SHARED_IMAGE_USAGE_SCANOUT;
962       plane_resource.mailbox = sii->CreateSharedImage(
963           plane_resource.gpu_memory_buffer.get(),
964           gpu_factories_->GpuMemoryBufferManager(), video_frame->ColorSpace(),
965           kTopLeft_GrSurfaceOrigin, kPremul_SkAlphaType, usage);
966     } else if (!plane_resource.mailbox.IsZero()) {
967       sii->UpdateSharedImage(frame_resources->sync_token,
968                              plane_resource.mailbox);
969     }
970     mailbox_holders[i] = gpu::MailboxHolder(plane_resource.mailbox,
971                                             gpu::SyncToken(), texture_target);
972   }
973 
974   // Insert a sync_token, this is needed to make sure that the textures the
975   // mailboxes refer to will be used only after all the previous commands posted
976   // in the SharedImageInterface have been processed.
977   gpu::SyncToken sync_token = sii->GenUnverifiedSyncToken();
978   for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++)
979     mailbox_holders[i].sync_token = sync_token;
980 
981   VideoPixelFormat frame_format = VideoFormat(output_format_);
982 
983   // Create the VideoFrame backed by native textures.
984   gfx::Size visible_size = video_frame->visible_rect().size();
985   scoped_refptr<VideoFrame> frame = VideoFrame::WrapNativeTextures(
986       frame_format, mailbox_holders, VideoFrame::ReleaseMailboxCB(), coded_size,
987       gfx::Rect(visible_size), video_frame->natural_size(),
988       video_frame->timestamp());
989 
990   if (!frame) {
991     frame_resources->MarkUnused(tick_clock_->NowTicks());
992     MailboxHoldersReleased(frame_resources, sync_token);
993     CompleteCopyRequestAndMaybeStartNextCopy(std::move(video_frame));
994     return;
995   }
996   frame->SetReleaseMailboxCB(
997       base::BindOnce(&PoolImpl::MailboxHoldersReleased, this, frame_resources));
998 
999   frame->set_color_space(video_frame->ColorSpace());
1000 
1001   bool allow_overlay = false;
1002 #if defined(OS_WIN)
1003   // Windows direct composition path only supports dual GMB NV12 video overlays.
1004   allow_overlay = (output_format_ ==
1005                    GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB);
1006 #else
1007   switch (output_format_) {
1008     case GpuVideoAcceleratorFactories::OutputFormat::I420:
1009       allow_overlay = video_frame->metadata()->allow_overlay;
1010       break;
1011     case GpuVideoAcceleratorFactories::OutputFormat::P010:
1012     case GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB:
1013       allow_overlay = true;
1014       break;
1015     case GpuVideoAcceleratorFactories::OutputFormat::NV12_DUAL_GMB:
1016       // Only used on Windows where we can't use single NV12 textures.
1017       break;
1018     case GpuVideoAcceleratorFactories::OutputFormat::XR30:
1019     case GpuVideoAcceleratorFactories::OutputFormat::XB30:
1020       // TODO(mcasas): Enable this for ChromeOS https://crbug.com/776093.
1021       allow_overlay = false;
1022 #if defined(OS_MAC)
1023       allow_overlay = IOSurfaceCanSetColorSpace(video_frame->ColorSpace());
1024 #endif
1025       // We've converted the YUV to RGB, fix the color space.
1026       // TODO(hubbe): The libyuv YUV to RGB conversion may not have
1027       // honored the color space conversion 100%. We should either fix
1028       // libyuv or find a way for later passes to make up the difference.
1029       frame->set_color_space(video_frame->ColorSpace().GetAsRGB());
1030       break;
1031     case GpuVideoAcceleratorFactories::OutputFormat::RGBA:
1032     case GpuVideoAcceleratorFactories::OutputFormat::BGRA:
1033       allow_overlay = true;
1034       break;
1035     case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
1036       break;
1037   }
1038 #endif  // OS_WIN
1039   frame->metadata()->MergeMetadataFrom(video_frame->metadata());
1040   frame->metadata()->allow_overlay = allow_overlay;
1041   frame->metadata()->read_lock_fences_enabled = true;
1042 
1043   CompleteCopyRequestAndMaybeStartNextCopy(std::move(frame));
1044 }
1045 
1046 // Destroy all the resources posting one task per FrameResources
1047 // to the |media_task_runner_|.
~PoolImpl()1048 GpuMemoryBufferVideoFramePool::PoolImpl::~PoolImpl() {
1049   DCHECK(in_shutdown_);
1050 }
1051 
Shutdown()1052 void GpuMemoryBufferVideoFramePool::PoolImpl::Shutdown() {
1053   DCHECK(media_task_runner_->BelongsToCurrentThread());
1054   // Clients don't care about copies once shutdown has started, so abort them.
1055   Abort();
1056 
1057   // Delete all the resources on the media thread.
1058   in_shutdown_ = true;
1059   for (auto* frame_resources : resources_pool_) {
1060     // Will be deleted later upon return to pool.
1061     if (frame_resources->is_used())
1062       continue;
1063 
1064     media_task_runner_->PostTask(
1065         FROM_HERE,
1066         base::BindOnce(&PoolImpl::DeleteFrameResources, gpu_factories_,
1067                        base::Owned(frame_resources)));
1068   }
1069   resources_pool_.clear();
1070 }
1071 
SetTickClockForTesting(const base::TickClock * tick_clock)1072 void GpuMemoryBufferVideoFramePool::PoolImpl::SetTickClockForTesting(
1073     const base::TickClock* tick_clock) {
1074   tick_clock_ = tick_clock;
1075 }
1076 
1077 // Tries to find the resources in the pool or create them.
1078 // Incompatible resources will be dropped.
1079 GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources*
GetOrCreateFrameResources(const gfx::Size & size,GpuVideoAcceleratorFactories::OutputFormat format)1080 GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
1081     const gfx::Size& size,
1082     GpuVideoAcceleratorFactories::OutputFormat format) {
1083   DCHECK(media_task_runner_->BelongsToCurrentThread());
1084 
1085   auto it = resources_pool_.begin();
1086   while (it != resources_pool_.end()) {
1087     FrameResources* frame_resources = *it;
1088     if (!frame_resources->is_used()) {
1089       if (AreFrameResourcesCompatible(frame_resources, size)) {
1090         frame_resources->MarkUsed();
1091         return frame_resources;
1092       } else {
1093         resources_pool_.erase(it++);
1094         DeleteFrameResources(gpu_factories_, frame_resources);
1095         delete frame_resources;
1096       }
1097     } else {
1098       it++;
1099     }
1100   }
1101 
1102   // Create the resources.
1103   FrameResources* frame_resources = new FrameResources(size);
1104   resources_pool_.push_back(frame_resources);
1105   for (size_t i = 0; i < NumGpuMemoryBuffers(output_format_); i++) {
1106     PlaneResource& plane_resource = frame_resources->plane_resources[i];
1107     const size_t width =
1108         VideoFrame::Columns(i, VideoFormat(format), size.width());
1109     const size_t height =
1110         VideoFrame::Rows(i, VideoFormat(format), size.height());
1111     plane_resource.size = gfx::Size(width, height);
1112 
1113     const gfx::BufferFormat buffer_format = GpuMemoryBufferFormat(format, i);
1114     plane_resource.gpu_memory_buffer = gpu_factories_->CreateGpuMemoryBuffer(
1115         plane_resource.size, buffer_format,
1116         gfx::BufferUsage::SCANOUT_CPU_READ_WRITE);
1117   }
1118   return frame_resources;
1119 }
1120 
1121 void GpuMemoryBufferVideoFramePool::PoolImpl::
CompleteCopyRequestAndMaybeStartNextCopy(scoped_refptr<VideoFrame> video_frame)1122     CompleteCopyRequestAndMaybeStartNextCopy(
1123         scoped_refptr<VideoFrame> video_frame) {
1124   DCHECK(!frame_copy_requests_.empty());
1125 
1126   std::move(frame_copy_requests_.front().frame_ready_cb)
1127       .Run(std::move(video_frame));
1128   frame_copy_requests_.pop_front();
1129   if (!frame_copy_requests_.empty())
1130     StartCopy();
1131 }
1132 
1133 // static
DeleteFrameResources(GpuVideoAcceleratorFactories * const gpu_factories,FrameResources * frame_resources)1134 void GpuMemoryBufferVideoFramePool::PoolImpl::DeleteFrameResources(
1135     GpuVideoAcceleratorFactories* const gpu_factories,
1136     FrameResources* frame_resources) {
1137   // TODO(dcastagna): As soon as the context lost is dealt with in media,
1138   // make sure that we won't execute this callback (use a weak pointer to
1139   // the old context).
1140   gpu::SharedImageInterface* sii = gpu_factories->SharedImageInterface();
1141   if (!sii)
1142     return;
1143 
1144   for (PlaneResource& plane_resource : frame_resources->plane_resources) {
1145     if (!plane_resource.mailbox.IsZero()) {
1146       sii->DestroySharedImage(frame_resources->sync_token,
1147                               plane_resource.mailbox);
1148     }
1149   }
1150 }
1151 
1152 // Called when a VideoFrame is no longer referenced.
1153 // Put back the resources in the pool.
MailboxHoldersReleased(FrameResources * frame_resources,const gpu::SyncToken & release_sync_token)1154 void GpuMemoryBufferVideoFramePool::PoolImpl::MailboxHoldersReleased(
1155     FrameResources* frame_resources,
1156     const gpu::SyncToken& release_sync_token) {
1157   if (!media_task_runner_->BelongsToCurrentThread()) {
1158     media_task_runner_->PostTask(
1159         FROM_HERE, base::BindOnce(&PoolImpl::MailboxHoldersReleased, this,
1160                                   frame_resources, release_sync_token));
1161     return;
1162   }
1163   frame_resources->sync_token = release_sync_token;
1164 
1165   if (in_shutdown_) {
1166     DeleteFrameResources(gpu_factories_, frame_resources);
1167     delete frame_resources;
1168     return;
1169   }
1170 
1171   const base::TimeTicks now = tick_clock_->NowTicks();
1172   frame_resources->MarkUnused(now);
1173   auto it = resources_pool_.begin();
1174   while (it != resources_pool_.end()) {
1175     FrameResources* frame_resources = *it;
1176 
1177     constexpr base::TimeDelta kStaleFrameLimit =
1178         base::TimeDelta::FromSeconds(10);
1179     if (!frame_resources->is_used() &&
1180         now - frame_resources->last_use_time() > kStaleFrameLimit) {
1181       resources_pool_.erase(it++);
1182       DeleteFrameResources(gpu_factories_, frame_resources);
1183       delete frame_resources;
1184     } else {
1185       it++;
1186     }
1187   }
1188 }
1189 
1190 GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool() = default;
1191 
GpuMemoryBufferVideoFramePool(const scoped_refptr<base::SingleThreadTaskRunner> & media_task_runner,const scoped_refptr<base::TaskRunner> & worker_task_runner,GpuVideoAcceleratorFactories * gpu_factories)1192 GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool(
1193     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
1194     const scoped_refptr<base::TaskRunner>& worker_task_runner,
1195     GpuVideoAcceleratorFactories* gpu_factories)
1196     : pool_impl_(
1197           new PoolImpl(media_task_runner, worker_task_runner, gpu_factories)) {
1198   base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
1199       pool_impl_.get(), "GpuMemoryBufferVideoFramePool", media_task_runner);
1200 }
1201 
~GpuMemoryBufferVideoFramePool()1202 GpuMemoryBufferVideoFramePool::~GpuMemoryBufferVideoFramePool() {
1203   // May be nullptr in tests.
1204   if (!pool_impl_)
1205     return;
1206 
1207   pool_impl_->Shutdown();
1208   base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
1209       pool_impl_.get());
1210 }
1211 
MaybeCreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,FrameReadyCB frame_ready_cb)1212 void GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame(
1213     scoped_refptr<VideoFrame> video_frame,
1214     FrameReadyCB frame_ready_cb) {
1215   DCHECK(video_frame);
1216   pool_impl_->CreateHardwareFrame(std::move(video_frame),
1217                                   std::move(frame_ready_cb));
1218 }
1219 
Abort()1220 void GpuMemoryBufferVideoFramePool::Abort() {
1221   pool_impl_->Abort();
1222 }
1223 
SetTickClockForTesting(const base::TickClock * tick_clock)1224 void GpuMemoryBufferVideoFramePool::SetTickClockForTesting(
1225     const base::TickClock* tick_clock) {
1226   pool_impl_->SetTickClockForTesting(tick_clock);
1227 }
1228 
1229 }  // namespace media
1230