1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/base/video_frame.h"
6
7 #include <algorithm>
8 #include <climits>
9 #include <numeric>
10 #include <utility>
11
12 #include "base/atomic_sequence_num.h"
13 #include "base/bind.h"
14 #include "base/bits.h"
15 #include "base/callback_helpers.h"
16 #include "base/logging.h"
17 #include "base/memory/aligned_memory.h"
18 #include "base/stl_util.h"
19 #include "base/strings/string_piece.h"
20 #include "base/strings/stringprintf.h"
21 #include "build/build_config.h"
22 #include "media/base/color_plane_layout.h"
23 #include "media/base/format_utils.h"
24 #include "media/base/limits.h"
25 #include "media/base/timestamp_constants.h"
26 #include "media/base/video_util.h"
27 #include "ui/gfx/buffer_format_util.h"
28 #include "ui/gfx/geometry/point.h"
29 #include "ui/gfx/gpu_memory_buffer.h"
30
31 namespace media {
32
33 namespace {
34
35 // Helper to privide gfx::Rect::Intersect() as an expression.
Intersection(gfx::Rect a,const gfx::Rect & b)36 gfx::Rect Intersection(gfx::Rect a, const gfx::Rect& b) {
37 a.Intersect(b);
38 return a;
39 }
40
41 } // namespace
42
43 // Static constexpr class for generating unique identifiers for each VideoFrame.
44 static base::AtomicSequenceNumber g_unique_id_generator;
45
46 // static
StorageTypeToString(const VideoFrame::StorageType storage_type)47 std::string VideoFrame::StorageTypeToString(
48 const VideoFrame::StorageType storage_type) {
49 switch (storage_type) {
50 case VideoFrame::STORAGE_UNKNOWN:
51 return "UNKNOWN";
52 case VideoFrame::STORAGE_OPAQUE:
53 return "OPAQUE";
54 case VideoFrame::STORAGE_UNOWNED_MEMORY:
55 return "UNOWNED_MEMORY";
56 case VideoFrame::STORAGE_OWNED_MEMORY:
57 return "OWNED_MEMORY";
58 case VideoFrame::STORAGE_SHMEM:
59 return "SHMEM";
60 #if defined(OS_LINUX) || defined(OS_BSD)
61 case VideoFrame::STORAGE_DMABUFS:
62 return "DMABUFS";
63 #endif
64 case VideoFrame::STORAGE_MOJO_SHARED_BUFFER:
65 return "MOJO_SHARED_BUFFER";
66 case VideoFrame::STORAGE_GPU_MEMORY_BUFFER:
67 return "GPU_MEMORY_BUFFER";
68 }
69
70 NOTREACHED() << "Invalid StorageType provided: " << storage_type;
71 return "INVALID";
72 }
73
74 // static
IsStorageTypeMappable(VideoFrame::StorageType storage_type)75 bool VideoFrame::IsStorageTypeMappable(VideoFrame::StorageType storage_type) {
76 return
77 #if defined(OS_LINUX) || defined(OS_BSD)
78 // This is not strictly needed but makes explicit that, at VideoFrame
79 // level, DmaBufs are not mappable from userspace.
80 storage_type != VideoFrame::STORAGE_DMABUFS &&
81 #endif
82 // GpuMemoryBuffer is not mappable at VideoFrame level. In most places
83 // GpuMemoryBuffer is opaque to the CPU, and for places that really need
84 // to access the data on CPU they can get the buffer with
85 // GetGpuMemoryBuffer() and call gfx::GpuMemoryBuffer::Map().
86 (storage_type == VideoFrame::STORAGE_UNOWNED_MEMORY ||
87 storage_type == VideoFrame::STORAGE_OWNED_MEMORY ||
88 storage_type == VideoFrame::STORAGE_SHMEM ||
89 storage_type == VideoFrame::STORAGE_MOJO_SHARED_BUFFER);
90 }
91
92 // static
IsValidPlane(VideoPixelFormat format,size_t plane)93 bool VideoFrame::IsValidPlane(VideoPixelFormat format, size_t plane) {
94 DCHECK_LE(NumPlanes(format), static_cast<size_t>(kMaxPlanes));
95 return plane < NumPlanes(format);
96 }
97
98 // static
SampleSize(VideoPixelFormat format,size_t plane)99 gfx::Size VideoFrame::SampleSize(VideoPixelFormat format, size_t plane) {
100 DCHECK(IsValidPlane(format, plane));
101
102 switch (plane) {
103 case kYPlane: // and kARGBPlane:
104 case kAPlane:
105 return gfx::Size(1, 1);
106
107 case kUPlane: // and kUVPlane:
108 case kVPlane:
109 switch (format) {
110 case PIXEL_FORMAT_I444:
111 case PIXEL_FORMAT_YUV444P9:
112 case PIXEL_FORMAT_YUV444P10:
113 case PIXEL_FORMAT_YUV444P12:
114 case PIXEL_FORMAT_Y16:
115 return gfx::Size(1, 1);
116
117 case PIXEL_FORMAT_I422:
118 case PIXEL_FORMAT_YUV422P9:
119 case PIXEL_FORMAT_YUV422P10:
120 case PIXEL_FORMAT_YUV422P12:
121 return gfx::Size(2, 1);
122
123 case PIXEL_FORMAT_YV12:
124 case PIXEL_FORMAT_I420:
125 case PIXEL_FORMAT_I420A:
126 case PIXEL_FORMAT_NV12:
127 case PIXEL_FORMAT_NV21:
128 case PIXEL_FORMAT_YUV420P9:
129 case PIXEL_FORMAT_YUV420P10:
130 case PIXEL_FORMAT_YUV420P12:
131 case PIXEL_FORMAT_P016LE:
132 return gfx::Size(2, 2);
133
134 case PIXEL_FORMAT_UYVY:
135 case PIXEL_FORMAT_UNKNOWN:
136 case PIXEL_FORMAT_YUY2:
137 case PIXEL_FORMAT_ARGB:
138 case PIXEL_FORMAT_XRGB:
139 case PIXEL_FORMAT_RGB24:
140 case PIXEL_FORMAT_MJPEG:
141 case PIXEL_FORMAT_ABGR:
142 case PIXEL_FORMAT_XBGR:
143 case PIXEL_FORMAT_XR30:
144 case PIXEL_FORMAT_XB30:
145 case PIXEL_FORMAT_BGRA:
146 break;
147 }
148 }
149 NOTREACHED();
150 return gfx::Size();
151 }
152
153 // Checks if |source_format| can be wrapped into a |target_format| frame.
AreValidPixelFormatsForWrap(VideoPixelFormat source_format,VideoPixelFormat target_format)154 static bool AreValidPixelFormatsForWrap(VideoPixelFormat source_format,
155 VideoPixelFormat target_format) {
156 if (source_format == target_format)
157 return true;
158
159 // It is possible to add other planar to planar format conversions here if the
160 // use case is there.
161 return source_format == PIXEL_FORMAT_I420A &&
162 target_format == PIXEL_FORMAT_I420;
163 }
164
165 // If it is required to allocate aligned to multiple-of-two size overall for the
166 // frame of pixel |format|.
RequiresEvenSizeAllocation(VideoPixelFormat format)167 static bool RequiresEvenSizeAllocation(VideoPixelFormat format) {
168 switch (format) {
169 case PIXEL_FORMAT_ARGB:
170 case PIXEL_FORMAT_XRGB:
171 case PIXEL_FORMAT_RGB24:
172 case PIXEL_FORMAT_Y16:
173 case PIXEL_FORMAT_ABGR:
174 case PIXEL_FORMAT_XBGR:
175 case PIXEL_FORMAT_XR30:
176 case PIXEL_FORMAT_XB30:
177 case PIXEL_FORMAT_BGRA:
178 return false;
179 case PIXEL_FORMAT_NV12:
180 case PIXEL_FORMAT_NV21:
181 case PIXEL_FORMAT_I420:
182 case PIXEL_FORMAT_MJPEG:
183 case PIXEL_FORMAT_YUY2:
184 case PIXEL_FORMAT_YV12:
185 case PIXEL_FORMAT_I422:
186 case PIXEL_FORMAT_I444:
187 case PIXEL_FORMAT_YUV420P9:
188 case PIXEL_FORMAT_YUV422P9:
189 case PIXEL_FORMAT_YUV444P9:
190 case PIXEL_FORMAT_YUV420P10:
191 case PIXEL_FORMAT_YUV422P10:
192 case PIXEL_FORMAT_YUV444P10:
193 case PIXEL_FORMAT_YUV420P12:
194 case PIXEL_FORMAT_YUV422P12:
195 case PIXEL_FORMAT_YUV444P12:
196 case PIXEL_FORMAT_I420A:
197 case PIXEL_FORMAT_UYVY:
198 case PIXEL_FORMAT_P016LE:
199 return true;
200 case PIXEL_FORMAT_UNKNOWN:
201 break;
202 }
203 NOTREACHED() << "Unsupported video frame format: " << format;
204 return false;
205 }
206
207 // Creates VideoFrameLayout for tightly packed frame.
GetDefaultLayout(VideoPixelFormat format,const gfx::Size & coded_size)208 static base::Optional<VideoFrameLayout> GetDefaultLayout(
209 VideoPixelFormat format,
210 const gfx::Size& coded_size) {
211 std::vector<ColorPlaneLayout> planes;
212
213 switch (format) {
214 case PIXEL_FORMAT_I420: {
215 int uv_width = (coded_size.width() + 1) / 2;
216 int uv_height = (coded_size.height() + 1) / 2;
217 int uv_stride = uv_width;
218 int uv_size = uv_stride * uv_height;
219 planes = std::vector<ColorPlaneLayout>{
220 ColorPlaneLayout(coded_size.width(), 0, coded_size.GetArea()),
221 ColorPlaneLayout(uv_stride, coded_size.GetArea(), uv_size),
222 ColorPlaneLayout(uv_stride, coded_size.GetArea() + uv_size, uv_size),
223 };
224 break;
225 }
226
227 case PIXEL_FORMAT_Y16:
228 planes = std::vector<ColorPlaneLayout>{ColorPlaneLayout(
229 coded_size.width() * 2, 0, coded_size.GetArea() * 2)};
230 break;
231
232 case PIXEL_FORMAT_ARGB:
233 planes = std::vector<ColorPlaneLayout>{ColorPlaneLayout(
234 coded_size.width() * 4, 0, coded_size.GetArea() * 4)};
235 break;
236
237 case PIXEL_FORMAT_NV12: {
238 int uv_width = (coded_size.width() + 1) / 2;
239 int uv_height = (coded_size.height() + 1) / 2;
240 int uv_stride = uv_width * 2;
241 int uv_size = uv_stride * uv_height;
242 planes = std::vector<ColorPlaneLayout>{
243 ColorPlaneLayout(coded_size.width(), 0, coded_size.GetArea()),
244 ColorPlaneLayout(uv_stride, coded_size.GetArea(), uv_size),
245 };
246 break;
247 }
248
249 default:
250 // TODO(miu): This function should support any pixel format.
251 // http://crbug.com/555909 .
252 DLOG(ERROR)
253 << "Only PIXEL_FORMAT_I420, PIXEL_FORMAT_Y16, PIXEL_FORMAT_NV12, "
254 "and PIXEL_FORMAT_ARGB formats are supported: "
255 << VideoPixelFormatToString(format);
256 return base::nullopt;
257 }
258
259 return VideoFrameLayout::CreateWithPlanes(format, coded_size, planes);
260 }
261
262 #if defined(OS_LINUX) || defined(OS_BSD)
263 // This class allows us to embed a vector<ScopedFD> into a scoped_refptr, and
264 // thus to have several VideoFrames share the same set of DMABUF FDs.
265 class VideoFrame::DmabufHolder
266 : public base::RefCountedThreadSafe<DmabufHolder> {
267 public:
268 DmabufHolder() = default;
DmabufHolder(std::vector<base::ScopedFD> && fds)269 DmabufHolder(std::vector<base::ScopedFD>&& fds) : fds_(std::move(fds)) {}
270
fds() const271 const std::vector<base::ScopedFD>& fds() const { return fds_; }
size() const272 size_t size() const { return fds_.size(); }
273
274 private:
275 std::vector<base::ScopedFD> fds_;
276
277 friend class base::RefCountedThreadSafe<DmabufHolder>;
278 ~DmabufHolder() = default;
279 };
280 #endif // defined(OS_LINUX) || defined(OS_BSD)
281
282 // static
IsValidConfig(VideoPixelFormat format,StorageType storage_type,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size)283 bool VideoFrame::IsValidConfig(VideoPixelFormat format,
284 StorageType storage_type,
285 const gfx::Size& coded_size,
286 const gfx::Rect& visible_rect,
287 const gfx::Size& natural_size) {
288 // Check maximum limits for all formats.
289 int coded_size_area = coded_size.GetCheckedArea().ValueOrDefault(INT_MAX);
290 int natural_size_area = natural_size.GetCheckedArea().ValueOrDefault(INT_MAX);
291 static_assert(limits::kMaxCanvas < INT_MAX, "");
292 if (coded_size_area > limits::kMaxCanvas ||
293 coded_size.width() > limits::kMaxDimension ||
294 coded_size.height() > limits::kMaxDimension || visible_rect.x() < 0 ||
295 visible_rect.y() < 0 || visible_rect.right() > coded_size.width() ||
296 visible_rect.bottom() > coded_size.height() ||
297 natural_size_area > limits::kMaxCanvas ||
298 natural_size.width() > limits::kMaxDimension ||
299 natural_size.height() > limits::kMaxDimension) {
300 return false;
301 }
302
303 // TODO(mcasas): Remove parameter |storage_type| when the opaque storage types
304 // comply with the checks below. Right now we skip them.
305 if (!IsStorageTypeMappable(storage_type))
306 return true;
307
308 // Make sure new formats are properly accounted for in the method.
309 static_assert(PIXEL_FORMAT_MAX == 32,
310 "Added pixel format, please review IsValidConfig()");
311
312 if (format == PIXEL_FORMAT_UNKNOWN) {
313 return coded_size.IsEmpty() && visible_rect.IsEmpty() &&
314 natural_size.IsEmpty();
315 }
316
317 // Check that software-allocated buffer formats are not empty.
318 return !coded_size.IsEmpty() && !visible_rect.IsEmpty() &&
319 !natural_size.IsEmpty();
320 }
321
322 // static
CreateFrame(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp)323 scoped_refptr<VideoFrame> VideoFrame::CreateFrame(VideoPixelFormat format,
324 const gfx::Size& coded_size,
325 const gfx::Rect& visible_rect,
326 const gfx::Size& natural_size,
327 base::TimeDelta timestamp) {
328 return CreateFrameInternal(format, coded_size, visible_rect, natural_size,
329 timestamp, false);
330 }
331
332 // static
CreateVideoHoleFrame(const base::UnguessableToken & overlay_plane_id,const gfx::Size & natural_size,base::TimeDelta timestamp)333 scoped_refptr<VideoFrame> VideoFrame::CreateVideoHoleFrame(
334 const base::UnguessableToken& overlay_plane_id,
335 const gfx::Size& natural_size,
336 base::TimeDelta timestamp) {
337 auto layout = VideoFrameLayout::Create(PIXEL_FORMAT_UNKNOWN, natural_size);
338 scoped_refptr<VideoFrame> frame =
339 new VideoFrame(*layout, StorageType::STORAGE_OPAQUE,
340 gfx::Rect(natural_size), natural_size, timestamp);
341 frame->metadata()->SetUnguessableToken(VideoFrameMetadata::OVERLAY_PLANE_ID,
342 overlay_plane_id);
343 return frame;
344 }
345
346 // static
CreateZeroInitializedFrame(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp)347 scoped_refptr<VideoFrame> VideoFrame::CreateZeroInitializedFrame(
348 VideoPixelFormat format,
349 const gfx::Size& coded_size,
350 const gfx::Rect& visible_rect,
351 const gfx::Size& natural_size,
352 base::TimeDelta timestamp) {
353 return CreateFrameInternal(format, coded_size, visible_rect, natural_size,
354 timestamp, true);
355 }
356
357 // static
WrapNativeTextures(VideoPixelFormat format,const gpu::MailboxHolder (& mailbox_holders)[kMaxPlanes],ReleaseMailboxCB mailbox_holder_release_cb,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp)358 scoped_refptr<VideoFrame> VideoFrame::WrapNativeTextures(
359 VideoPixelFormat format,
360 const gpu::MailboxHolder (&mailbox_holders)[kMaxPlanes],
361 ReleaseMailboxCB mailbox_holder_release_cb,
362 const gfx::Size& coded_size,
363 const gfx::Rect& visible_rect,
364 const gfx::Size& natural_size,
365 base::TimeDelta timestamp) {
366 if (format != PIXEL_FORMAT_ARGB && format != PIXEL_FORMAT_XRGB &&
367 format != PIXEL_FORMAT_NV12 && format != PIXEL_FORMAT_I420 &&
368 format != PIXEL_FORMAT_ABGR && format != PIXEL_FORMAT_XR30 &&
369 format != PIXEL_FORMAT_XB30) {
370 DLOG(ERROR) << "Unsupported pixel format: "
371 << VideoPixelFormatToString(format);
372 return nullptr;
373 }
374 const StorageType storage = STORAGE_OPAQUE;
375 if (!IsValidConfig(format, storage, coded_size, visible_rect, natural_size)) {
376 DLOG(ERROR) << __func__ << " Invalid config."
377 << ConfigToString(format, storage, coded_size, visible_rect,
378 natural_size);
379 return nullptr;
380 }
381
382 auto layout = VideoFrameLayout::Create(format, coded_size);
383 if (!layout) {
384 DLOG(ERROR) << "Invalid layout.";
385 return nullptr;
386 }
387
388 scoped_refptr<VideoFrame> frame =
389 new VideoFrame(*layout, storage, visible_rect, natural_size, timestamp);
390 memcpy(&frame->mailbox_holders_, mailbox_holders,
391 sizeof(frame->mailbox_holders_));
392 frame->mailbox_holders_release_cb_ = std::move(mailbox_holder_release_cb);
393
394 // Wrapping native textures should... have textures. https://crbug.com/864145.
395 DCHECK(frame->HasTextures());
396
397 return frame;
398 }
399
400 // static
WrapExternalData(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,uint8_t * data,size_t data_size,base::TimeDelta timestamp)401 scoped_refptr<VideoFrame> VideoFrame::WrapExternalData(
402 VideoPixelFormat format,
403 const gfx::Size& coded_size,
404 const gfx::Rect& visible_rect,
405 const gfx::Size& natural_size,
406 uint8_t* data,
407 size_t data_size,
408 base::TimeDelta timestamp) {
409 auto layout = GetDefaultLayout(format, coded_size);
410 if (!layout)
411 return nullptr;
412 return WrapExternalDataWithLayout(*layout, visible_rect, natural_size, data,
413 data_size, timestamp);
414 }
415
416 // static
WrapExternalDataWithLayout(const VideoFrameLayout & layout,const gfx::Rect & visible_rect,const gfx::Size & natural_size,uint8_t * data,size_t data_size,base::TimeDelta timestamp)417 scoped_refptr<VideoFrame> VideoFrame::WrapExternalDataWithLayout(
418 const VideoFrameLayout& layout,
419 const gfx::Rect& visible_rect,
420 const gfx::Size& natural_size,
421 uint8_t* data,
422 size_t data_size,
423 base::TimeDelta timestamp) {
424 StorageType storage_type = STORAGE_UNOWNED_MEMORY;
425
426 if (!IsValidConfig(layout.format(), storage_type, layout.coded_size(),
427 visible_rect, natural_size)) {
428 DLOG(ERROR) << __func__ << " Invalid config."
429 << ConfigToString(layout.format(), storage_type,
430 layout.coded_size(), visible_rect,
431 natural_size);
432 return nullptr;
433 }
434
435 scoped_refptr<VideoFrame> frame = new VideoFrame(
436 layout, storage_type, visible_rect, natural_size, timestamp);
437
438 for (size_t i = 0; i < layout.planes().size(); ++i) {
439 frame->data_[i] = data + layout.planes()[i].offset;
440 }
441
442 return frame;
443 }
444
445 // static
WrapExternalYuvData(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,int32_t y_stride,int32_t u_stride,int32_t v_stride,uint8_t * y_data,uint8_t * u_data,uint8_t * v_data,base::TimeDelta timestamp)446 scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
447 VideoPixelFormat format,
448 const gfx::Size& coded_size,
449 const gfx::Rect& visible_rect,
450 const gfx::Size& natural_size,
451 int32_t y_stride,
452 int32_t u_stride,
453 int32_t v_stride,
454 uint8_t* y_data,
455 uint8_t* u_data,
456 uint8_t* v_data,
457 base::TimeDelta timestamp) {
458 auto layout = VideoFrameLayout::CreateWithStrides(
459 format, coded_size, {y_stride, u_stride, v_stride});
460 if (!layout) {
461 DLOG(ERROR) << "Invalid layout.";
462 return nullptr;
463 }
464
465 return WrapExternalYuvDataWithLayout(*layout, visible_rect, natural_size,
466 y_data, u_data, v_data, timestamp);
467 }
468
469 // static
WrapExternalYuvDataWithLayout(const VideoFrameLayout & layout,const gfx::Rect & visible_rect,const gfx::Size & natural_size,uint8_t * y_data,uint8_t * u_data,uint8_t * v_data,base::TimeDelta timestamp)470 scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvDataWithLayout(
471 const VideoFrameLayout& layout,
472 const gfx::Rect& visible_rect,
473 const gfx::Size& natural_size,
474 uint8_t* y_data,
475 uint8_t* u_data,
476 uint8_t* v_data,
477 base::TimeDelta timestamp) {
478 const StorageType storage = STORAGE_UNOWNED_MEMORY;
479 const VideoPixelFormat format = layout.format();
480 if (!IsValidConfig(format, storage, layout.coded_size(), visible_rect,
481 natural_size)) {
482 DLOG(ERROR) << __func__ << " Invalid config."
483 << ConfigToString(format, storage, layout.coded_size(),
484 visible_rect, natural_size);
485 return nullptr;
486 }
487 if (!IsYuvPlanar(format)) {
488 DLOG(ERROR) << __func__ << " Format is not YUV. " << format;
489 return nullptr;
490 }
491
492 DCHECK_LE(NumPlanes(format), 3u);
493 scoped_refptr<VideoFrame> frame(
494 new VideoFrame(layout, storage, visible_rect, natural_size, timestamp));
495 frame->data_[kYPlane] = y_data;
496 frame->data_[kUPlane] = u_data;
497 frame->data_[kVPlane] = v_data;
498 return frame;
499 }
500
501 // static
WrapExternalYuvaData(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,int32_t y_stride,int32_t u_stride,int32_t v_stride,int32_t a_stride,uint8_t * y_data,uint8_t * u_data,uint8_t * v_data,uint8_t * a_data,base::TimeDelta timestamp)502 scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvaData(
503 VideoPixelFormat format,
504 const gfx::Size& coded_size,
505 const gfx::Rect& visible_rect,
506 const gfx::Size& natural_size,
507 int32_t y_stride,
508 int32_t u_stride,
509 int32_t v_stride,
510 int32_t a_stride,
511 uint8_t* y_data,
512 uint8_t* u_data,
513 uint8_t* v_data,
514 uint8_t* a_data,
515 base::TimeDelta timestamp) {
516 const StorageType storage = STORAGE_UNOWNED_MEMORY;
517 if (!IsValidConfig(format, storage, coded_size, visible_rect, natural_size)) {
518 DLOG(ERROR) << __func__ << " Invalid config."
519 << ConfigToString(format, storage, coded_size, visible_rect,
520 natural_size);
521 return nullptr;
522 }
523
524 if (NumPlanes(format) != 4) {
525 DLOG(ERROR) << "Expecting Y, U, V and A planes to be present for the video"
526 << " format.";
527 return nullptr;
528 }
529
530 auto layout = VideoFrameLayout::CreateWithStrides(
531 format, coded_size, {y_stride, u_stride, v_stride, a_stride});
532 if (!layout) {
533 DLOG(ERROR) << "Invalid layout";
534 return nullptr;
535 }
536
537 scoped_refptr<VideoFrame> frame(
538 new VideoFrame(*layout, storage, visible_rect, natural_size, timestamp));
539 frame->data_[kYPlane] = y_data;
540 frame->data_[kUPlane] = u_data;
541 frame->data_[kVPlane] = v_data;
542 frame->data_[kAPlane] = a_data;
543 return frame;
544 }
545
546 // static
WrapExternalGpuMemoryBuffer(const gfx::Rect & visible_rect,const gfx::Size & natural_size,std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,const gpu::MailboxHolder (& mailbox_holders)[kMaxPlanes],ReleaseMailboxCB mailbox_holder_release_cb,base::TimeDelta timestamp)547 scoped_refptr<VideoFrame> VideoFrame::WrapExternalGpuMemoryBuffer(
548 const gfx::Rect& visible_rect,
549 const gfx::Size& natural_size,
550 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
551 const gpu::MailboxHolder (&mailbox_holders)[kMaxPlanes],
552 ReleaseMailboxCB mailbox_holder_release_cb,
553 base::TimeDelta timestamp) {
554 const base::Optional<VideoPixelFormat> format =
555 GfxBufferFormatToVideoPixelFormat(gpu_memory_buffer->GetFormat());
556 if (!format)
557 return nullptr;
558 constexpr StorageType storage = STORAGE_GPU_MEMORY_BUFFER;
559 const gfx::Size& coded_size = gpu_memory_buffer->GetSize();
560 if (!IsValidConfig(*format, storage, coded_size, visible_rect,
561 natural_size)) {
562 DLOG(ERROR) << __func__ << " Invalid config"
563 << ConfigToString(*format, storage, coded_size, visible_rect,
564 natural_size);
565 return nullptr;
566 }
567
568 const size_t num_planes =
569 NumberOfPlanesForLinearBufferFormat(gpu_memory_buffer->GetFormat());
570 std::vector<int32_t> strides;
571 for (size_t i = 0; i < num_planes; ++i)
572 strides.push_back(gpu_memory_buffer->stride(i));
573 const auto layout = VideoFrameLayout::CreateWithStrides(*format, coded_size,
574 std::move(strides));
575 if (!layout) {
576 DLOG(ERROR) << __func__ << " Invalid layout";
577 return nullptr;
578 }
579
580 scoped_refptr<VideoFrame> frame =
581 new VideoFrame(*layout, storage, visible_rect, natural_size, timestamp);
582 if (!frame) {
583 DLOG(ERROR) << __func__ << " Couldn't create VideoFrame instance";
584 return nullptr;
585 }
586 frame->gpu_memory_buffer_ = std::move(gpu_memory_buffer);
587 memcpy(&frame->mailbox_holders_, mailbox_holders,
588 sizeof(frame->mailbox_holders_));
589 frame->mailbox_holders_release_cb_ = std::move(mailbox_holder_release_cb);
590 return frame;
591 }
592
593 #if defined(OS_LINUX) || defined(OS_BSD)
594 // static
WrapExternalDmabufs(const VideoFrameLayout & layout,const gfx::Rect & visible_rect,const gfx::Size & natural_size,std::vector<base::ScopedFD> dmabuf_fds,base::TimeDelta timestamp)595 scoped_refptr<VideoFrame> VideoFrame::WrapExternalDmabufs(
596 const VideoFrameLayout& layout,
597 const gfx::Rect& visible_rect,
598 const gfx::Size& natural_size,
599 std::vector<base::ScopedFD> dmabuf_fds,
600 base::TimeDelta timestamp) {
601 const StorageType storage = STORAGE_DMABUFS;
602 const VideoPixelFormat format = layout.format();
603 const gfx::Size& coded_size = layout.coded_size();
604 if (!IsValidConfig(format, storage, coded_size, visible_rect, natural_size)) {
605 DLOG(ERROR) << __func__ << " Invalid config."
606 << ConfigToString(format, storage, coded_size, visible_rect,
607 natural_size);
608 return nullptr;
609 }
610
611 if (dmabuf_fds.empty() || dmabuf_fds.size() > NumPlanes(format)) {
612 DLOG(ERROR) << __func__ << " Incorrect number of dmabuf fds provided, got: "
613 << dmabuf_fds.size() << ", expected 1 to " << NumPlanes(format);
614 return nullptr;
615 }
616
617 gpu::MailboxHolder mailbox_holders[kMaxPlanes];
618 scoped_refptr<VideoFrame> frame =
619 new VideoFrame(layout, storage, visible_rect, natural_size, timestamp);
620 if (!frame) {
621 DLOG(ERROR) << __func__ << " Couldn't create VideoFrame instance.";
622 return nullptr;
623 }
624 memcpy(&frame->mailbox_holders_, mailbox_holders,
625 sizeof(frame->mailbox_holders_));
626 frame->mailbox_holders_release_cb_ = ReleaseMailboxCB();
627 frame->dmabuf_fds_ =
628 base::MakeRefCounted<DmabufHolder>(std::move(dmabuf_fds));
629 DCHECK(frame->HasDmaBufs());
630
631 return frame;
632 }
633 #endif
634
635 #if defined(OS_MACOSX)
636 // static
WrapCVPixelBuffer(CVPixelBufferRef cv_pixel_buffer,base::TimeDelta timestamp)637 scoped_refptr<VideoFrame> VideoFrame::WrapCVPixelBuffer(
638 CVPixelBufferRef cv_pixel_buffer,
639 base::TimeDelta timestamp) {
640 DCHECK(cv_pixel_buffer);
641 DCHECK(CFGetTypeID(cv_pixel_buffer) == CVPixelBufferGetTypeID());
642
643 const OSType cv_format = CVPixelBufferGetPixelFormatType(cv_pixel_buffer);
644 VideoPixelFormat format;
645 // There are very few compatible CV pixel formats, so just check each.
646 if (cv_format == kCVPixelFormatType_420YpCbCr8Planar) {
647 format = PIXEL_FORMAT_I420;
648 } else if (cv_format == kCVPixelFormatType_444YpCbCr8) {
649 format = PIXEL_FORMAT_I444;
650 } else if (cv_format == '420v') {
651 // TODO(jfroy): Use kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange when the
652 // minimum OS X and iOS SDKs permits it.
653 format = PIXEL_FORMAT_NV12;
654 } else {
655 DLOG(ERROR) << "CVPixelBuffer format not supported: " << cv_format;
656 return nullptr;
657 }
658
659 const gfx::Size coded_size(CVImageBufferGetEncodedSize(cv_pixel_buffer));
660 const gfx::Rect visible_rect(CVImageBufferGetCleanRect(cv_pixel_buffer));
661 const gfx::Size natural_size(CVImageBufferGetDisplaySize(cv_pixel_buffer));
662 const StorageType storage = STORAGE_UNOWNED_MEMORY;
663
664 if (!IsValidConfig(format, storage, coded_size, visible_rect, natural_size)) {
665 DLOG(ERROR) << __func__ << " Invalid config."
666 << ConfigToString(format, storage, coded_size, visible_rect,
667 natural_size);
668 return nullptr;
669 }
670
671 auto layout = VideoFrameLayout::Create(format, coded_size);
672 if (!layout) {
673 DLOG(ERROR) << "Invalid layout.";
674 return nullptr;
675 }
676
677 scoped_refptr<VideoFrame> frame(
678 new VideoFrame(*layout, storage, visible_rect, natural_size, timestamp));
679
680 frame->cv_pixel_buffer_.reset(cv_pixel_buffer, base::scoped_policy::RETAIN);
681 return frame;
682 }
683 #endif
684
685 // static
WrapVideoFrame(scoped_refptr<VideoFrame> frame,VideoPixelFormat format,const gfx::Rect & visible_rect,const gfx::Size & natural_size)686 scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame(
687 scoped_refptr<VideoFrame> frame,
688 VideoPixelFormat format,
689 const gfx::Rect& visible_rect,
690 const gfx::Size& natural_size) {
691 DCHECK(frame->visible_rect().Contains(visible_rect));
692
693 // The following storage type should not be wrapped as the shared region
694 // cannot be owned by both the wrapped frame and the wrapping frame.
695 //
696 // TODO: We can support this now since we have a reference to the wrapped
697 // frame through |wrapped_frame_|.
698 DCHECK(frame->storage_type() != STORAGE_MOJO_SHARED_BUFFER);
699
700 if (!AreValidPixelFormatsForWrap(frame->format(), format)) {
701 DLOG(ERROR) << __func__ << " Invalid format conversion."
702 << VideoPixelFormatToString(frame->format()) << " to "
703 << VideoPixelFormatToString(format);
704 return nullptr;
705 }
706
707 if (!IsValidConfig(format, frame->storage_type(), frame->coded_size(),
708 visible_rect, natural_size)) {
709 DLOG(ERROR) << __func__ << " Invalid config."
710 << ConfigToString(format, frame->storage_type(),
711 frame->coded_size(), visible_rect,
712 natural_size);
713 return nullptr;
714 }
715
716 scoped_refptr<VideoFrame> wrapping_frame(
717 new VideoFrame(frame->layout(), frame->storage_type(), visible_rect,
718 natural_size, frame->timestamp()));
719
720 // Copy all metadata to the wrapped frame->
721 wrapping_frame->metadata()->MergeMetadataFrom(frame->metadata());
722
723 if (frame->IsMappable()) {
724 for (size_t i = 0; i < NumPlanes(format); ++i) {
725 wrapping_frame->data_[i] = frame->data_[i];
726 }
727 }
728
729 #if defined(OS_LINUX) || defined(OS_BSD)
730 DCHECK(frame->dmabuf_fds_);
731 // If there are any |dmabuf_fds_| plugged in, we should refer them too.
732 wrapping_frame->dmabuf_fds_ = frame->dmabuf_fds_;
733 #endif
734
735 if (frame->storage_type() == STORAGE_SHMEM) {
736 DCHECK(frame->shm_region_ && frame->shm_region_->IsValid());
737 wrapping_frame->BackWithSharedMemory(frame->shm_region_);
738 }
739
740 wrapping_frame->wrapped_frame_ = std::move(frame);
741 return wrapping_frame;
742 }
743
744 // static
CreateEOSFrame()745 scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() {
746 auto layout = VideoFrameLayout::Create(PIXEL_FORMAT_UNKNOWN, gfx::Size());
747 if (!layout) {
748 DLOG(ERROR) << "Invalid layout.";
749 return nullptr;
750 }
751 scoped_refptr<VideoFrame> frame = new VideoFrame(
752 *layout, STORAGE_UNKNOWN, gfx::Rect(), gfx::Size(), kNoTimestamp);
753 frame->metadata()->SetBoolean(VideoFrameMetadata::END_OF_STREAM, true);
754 return frame;
755 }
756
757 // static
CreateColorFrame(const gfx::Size & size,uint8_t y,uint8_t u,uint8_t v,base::TimeDelta timestamp)758 scoped_refptr<VideoFrame> VideoFrame::CreateColorFrame(
759 const gfx::Size& size,
760 uint8_t y,
761 uint8_t u,
762 uint8_t v,
763 base::TimeDelta timestamp) {
764 scoped_refptr<VideoFrame> frame =
765 CreateFrame(PIXEL_FORMAT_I420, size, gfx::Rect(size), size, timestamp);
766 if (frame)
767 FillYUV(frame.get(), y, u, v);
768 return frame;
769 }
770
771 // static
CreateBlackFrame(const gfx::Size & size)772 scoped_refptr<VideoFrame> VideoFrame::CreateBlackFrame(const gfx::Size& size) {
773 const uint8_t kBlackY = 0x00;
774 const uint8_t kBlackUV = 0x80;
775 const base::TimeDelta kZero;
776 return CreateColorFrame(size, kBlackY, kBlackUV, kBlackUV, kZero);
777 }
778
779 // static
CreateTransparentFrame(const gfx::Size & size)780 scoped_refptr<VideoFrame> VideoFrame::CreateTransparentFrame(
781 const gfx::Size& size) {
782 const uint8_t kBlackY = 0x00;
783 const uint8_t kBlackUV = 0x00;
784 const uint8_t kTransparentA = 0x00;
785 const base::TimeDelta kZero;
786 scoped_refptr<VideoFrame> frame =
787 CreateFrame(PIXEL_FORMAT_I420A, size, gfx::Rect(size), size, kZero);
788 if (frame)
789 FillYUVA(frame.get(), kBlackY, kBlackUV, kBlackUV, kTransparentA);
790 return frame;
791 }
792
793 // static
NumPlanes(VideoPixelFormat format)794 size_t VideoFrame::NumPlanes(VideoPixelFormat format) {
795 return VideoFrameLayout::NumPlanes(format);
796 }
797
798 // static
AllocationSize(VideoPixelFormat format,const gfx::Size & coded_size)799 size_t VideoFrame::AllocationSize(VideoPixelFormat format,
800 const gfx::Size& coded_size) {
801 size_t total = 0;
802 for (size_t i = 0; i < NumPlanes(format); ++i)
803 total += PlaneSize(format, i, coded_size).GetArea();
804 return total;
805 }
806
807 // static
PlaneSize(VideoPixelFormat format,size_t plane,const gfx::Size & coded_size)808 gfx::Size VideoFrame::PlaneSize(VideoPixelFormat format,
809 size_t plane,
810 const gfx::Size& coded_size) {
811 DCHECK(IsValidPlane(format, plane));
812
813 int width = coded_size.width();
814 int height = coded_size.height();
815 if (RequiresEvenSizeAllocation(format)) {
816 // Align to multiple-of-two size overall. This ensures that non-subsampled
817 // planes can be addressed by pixel with the same scaling as the subsampled
818 // planes.
819 width = base::bits::Align(width, 2);
820 height = base::bits::Align(height, 2);
821 }
822
823 const gfx::Size subsample = SampleSize(format, plane);
824 DCHECK(width % subsample.width() == 0);
825 DCHECK(height % subsample.height() == 0);
826 return gfx::Size(BytesPerElement(format, plane) * width / subsample.width(),
827 height / subsample.height());
828 }
829
830 // static
PlaneHorizontalBitsPerPixel(VideoPixelFormat format,size_t plane)831 int VideoFrame::PlaneHorizontalBitsPerPixel(VideoPixelFormat format,
832 size_t plane) {
833 DCHECK(IsValidPlane(format, plane));
834 const int bits_per_element = 8 * BytesPerElement(format, plane);
835 const int horiz_pixels_per_element = SampleSize(format, plane).width();
836 DCHECK_EQ(bits_per_element % horiz_pixels_per_element, 0);
837 return bits_per_element / horiz_pixels_per_element;
838 }
839
840 // static
PlaneBitsPerPixel(VideoPixelFormat format,size_t plane)841 int VideoFrame::PlaneBitsPerPixel(VideoPixelFormat format, size_t plane) {
842 DCHECK(IsValidPlane(format, plane));
843 return PlaneHorizontalBitsPerPixel(format, plane) /
844 SampleSize(format, plane).height();
845 }
846
847 // static
RowBytes(size_t plane,VideoPixelFormat format,int width)848 size_t VideoFrame::RowBytes(size_t plane, VideoPixelFormat format, int width) {
849 DCHECK(IsValidPlane(format, plane));
850 return BytesPerElement(format, plane) * Columns(plane, format, width);
851 }
852
853 // static
BytesPerElement(VideoPixelFormat format,size_t plane)854 int VideoFrame::BytesPerElement(VideoPixelFormat format, size_t plane) {
855 DCHECK(IsValidPlane(format, plane));
856 switch (format) {
857 case PIXEL_FORMAT_ARGB:
858 case PIXEL_FORMAT_BGRA:
859 case PIXEL_FORMAT_XRGB:
860 case PIXEL_FORMAT_ABGR:
861 case PIXEL_FORMAT_XBGR:
862 case PIXEL_FORMAT_XR30:
863 case PIXEL_FORMAT_XB30:
864 return 4;
865 case PIXEL_FORMAT_RGB24:
866 return 3;
867 case PIXEL_FORMAT_Y16:
868 case PIXEL_FORMAT_UYVY:
869 case PIXEL_FORMAT_YUY2:
870 case PIXEL_FORMAT_YUV420P9:
871 case PIXEL_FORMAT_YUV422P9:
872 case PIXEL_FORMAT_YUV444P9:
873 case PIXEL_FORMAT_YUV420P10:
874 case PIXEL_FORMAT_YUV422P10:
875 case PIXEL_FORMAT_YUV444P10:
876 case PIXEL_FORMAT_YUV420P12:
877 case PIXEL_FORMAT_YUV422P12:
878 case PIXEL_FORMAT_YUV444P12:
879 case PIXEL_FORMAT_P016LE:
880 return 2;
881 case PIXEL_FORMAT_NV12:
882 case PIXEL_FORMAT_NV21: {
883 static const int bytes_per_element[] = {1, 2};
884 DCHECK_LT(plane, base::size(bytes_per_element));
885 return bytes_per_element[plane];
886 }
887 case PIXEL_FORMAT_YV12:
888 case PIXEL_FORMAT_I420:
889 case PIXEL_FORMAT_I422:
890 case PIXEL_FORMAT_I420A:
891 case PIXEL_FORMAT_I444:
892 return 1;
893 case PIXEL_FORMAT_MJPEG:
894 return 0;
895 case PIXEL_FORMAT_UNKNOWN:
896 break;
897 }
898 NOTREACHED();
899 return 0;
900 }
901
902 // static
ComputeStrides(VideoPixelFormat format,const gfx::Size & coded_size)903 std::vector<int32_t> VideoFrame::ComputeStrides(VideoPixelFormat format,
904 const gfx::Size& coded_size) {
905 std::vector<int32_t> strides;
906 const size_t num_planes = NumPlanes(format);
907 if (num_planes == 1) {
908 strides.push_back(RowBytes(0, format, coded_size.width()));
909 } else {
910 for (size_t plane = 0; plane < num_planes; ++plane) {
911 strides.push_back(base::bits::Align(
912 RowBytes(plane, format, coded_size.width()), kFrameAddressAlignment));
913 }
914 }
915 return strides;
916 }
917
918 // static
Rows(size_t plane,VideoPixelFormat format,int height)919 size_t VideoFrame::Rows(size_t plane, VideoPixelFormat format, int height) {
920 DCHECK(IsValidPlane(format, plane));
921 const int sample_height = SampleSize(format, plane).height();
922 return base::bits::Align(height, sample_height) / sample_height;
923 }
924
925 // static
Columns(size_t plane,VideoPixelFormat format,int width)926 size_t VideoFrame::Columns(size_t plane, VideoPixelFormat format, int width) {
927 DCHECK(IsValidPlane(format, plane));
928 const int sample_width = SampleSize(format, plane).width();
929 return base::bits::Align(width, sample_width) / sample_width;
930 }
931
932 // static
HashFrameForTesting(base::MD5Context * context,const VideoFrame & frame)933 void VideoFrame::HashFrameForTesting(base::MD5Context* context,
934 const VideoFrame& frame) {
935 DCHECK(context);
936 for (size_t plane = 0; plane < NumPlanes(frame.format()); ++plane) {
937 for (int row = 0; row < frame.rows(plane); ++row) {
938 base::MD5Update(context, base::StringPiece(reinterpret_cast<const char*>(
939 frame.data(plane) +
940 frame.stride(plane) * row),
941 frame.row_bytes(plane)));
942 }
943 }
944 }
945
BackWithSharedMemory(base::UnsafeSharedMemoryRegion * region)946 void VideoFrame::BackWithSharedMemory(base::UnsafeSharedMemoryRegion* region) {
947 DCHECK(!shm_region_);
948 DCHECK(!owned_shm_region_.IsValid());
949 // Either we should be backing a frame created with WrapExternal*, or we are
950 // wrapping an existing STORAGE_SHMEM, in which case the storage
951 // type has already been set to STORAGE_SHMEM.
952 DCHECK(storage_type_ == STORAGE_UNOWNED_MEMORY ||
953 storage_type_ == STORAGE_SHMEM);
954 DCHECK(region && region->IsValid());
955 storage_type_ = STORAGE_SHMEM;
956 shm_region_ = region;
957 }
958
BackWithOwnedSharedMemory(base::UnsafeSharedMemoryRegion region,base::WritableSharedMemoryMapping mapping)959 void VideoFrame::BackWithOwnedSharedMemory(
960 base::UnsafeSharedMemoryRegion region,
961 base::WritableSharedMemoryMapping mapping) {
962 DCHECK(!shm_region_);
963 DCHECK(!owned_shm_region_.IsValid());
964 // We should be backing a frame created with WrapExternal*. We cannot be
965 // wrapping an existing STORAGE_SHMEM, as the region is unowned in that case.
966 DCHECK(storage_type_ == STORAGE_UNOWNED_MEMORY);
967 storage_type_ = STORAGE_SHMEM;
968 owned_shm_region_ = std::move(region);
969 shm_region_ = &owned_shm_region_;
970 owned_shm_mapping_ = std::move(mapping);
971 }
972
IsMappable() const973 bool VideoFrame::IsMappable() const {
974 return IsStorageTypeMappable(storage_type_);
975 }
976
HasTextures() const977 bool VideoFrame::HasTextures() const {
978 return wrapped_frame_ ? wrapped_frame_->HasTextures()
979 : !mailbox_holders_[0].mailbox.IsZero();
980 }
981
NumTextures() const982 size_t VideoFrame::NumTextures() const {
983 if (!HasTextures())
984 return 0;
985
986 const auto& mailbox_holders =
987 wrapped_frame_ ? wrapped_frame_->mailbox_holders_ : mailbox_holders_;
988 size_t i = 0;
989 for (; i < NumPlanes(format()); ++i) {
990 if (mailbox_holders[i].mailbox.IsZero()) {
991 return i;
992 }
993 }
994 return i;
995 }
996
HasGpuMemoryBuffer() const997 bool VideoFrame::HasGpuMemoryBuffer() const {
998 return wrapped_frame_ ? wrapped_frame_->HasGpuMemoryBuffer()
999 : !!gpu_memory_buffer_;
1000 }
1001
GetGpuMemoryBuffer() const1002 gfx::GpuMemoryBuffer* VideoFrame::GetGpuMemoryBuffer() const {
1003 return wrapped_frame_ ? wrapped_frame_->GetGpuMemoryBuffer()
1004 : gpu_memory_buffer_.get();
1005 }
1006
IsSameAllocation(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size) const1007 bool VideoFrame::IsSameAllocation(VideoPixelFormat format,
1008 const gfx::Size& coded_size,
1009 const gfx::Rect& visible_rect,
1010 const gfx::Size& natural_size) const {
1011 // CreateFrameInternal() changes coded_size to new_coded_size. Match that
1012 // behavior here.
1013 const gfx::Size new_coded_size = DetermineAlignedSize(format, coded_size);
1014 return this->format() == format && this->coded_size() == new_coded_size &&
1015 visible_rect_ == visible_rect && natural_size_ == natural_size;
1016 }
1017
ColorSpace() const1018 gfx::ColorSpace VideoFrame::ColorSpace() const {
1019 return color_space_;
1020 }
1021
row_bytes(size_t plane) const1022 int VideoFrame::row_bytes(size_t plane) const {
1023 return RowBytes(plane, format(), coded_size().width());
1024 }
1025
rows(size_t plane) const1026 int VideoFrame::rows(size_t plane) const {
1027 return Rows(plane, format(), coded_size().height());
1028 }
1029
visible_data(size_t plane) const1030 const uint8_t* VideoFrame::visible_data(size_t plane) const {
1031 DCHECK(IsValidPlane(format(), plane));
1032 DCHECK(IsMappable());
1033
1034 // Calculate an offset that is properly aligned for all planes.
1035 const gfx::Size alignment = CommonAlignment(format());
1036 const gfx::Point offset(
1037 base::bits::AlignDown(visible_rect_.x(), alignment.width()),
1038 base::bits::AlignDown(visible_rect_.y(), alignment.height()));
1039
1040 const gfx::Size subsample = SampleSize(format(), plane);
1041 DCHECK(offset.x() % subsample.width() == 0);
1042 DCHECK(offset.y() % subsample.height() == 0);
1043 return data(plane) +
1044 stride(plane) * (offset.y() / subsample.height()) + // Row offset.
1045 BytesPerElement(format(), plane) * // Column offset.
1046 (offset.x() / subsample.width());
1047 }
1048
visible_data(size_t plane)1049 uint8_t* VideoFrame::visible_data(size_t plane) {
1050 return const_cast<uint8_t*>(
1051 static_cast<const VideoFrame*>(this)->visible_data(plane));
1052 }
1053
1054 const gpu::MailboxHolder&
mailbox_holder(size_t texture_index) const1055 VideoFrame::mailbox_holder(size_t texture_index) const {
1056 DCHECK(HasTextures());
1057 DCHECK(IsValidPlane(format(), texture_index));
1058 return wrapped_frame_ ? wrapped_frame_->mailbox_holders_[texture_index]
1059 : mailbox_holders_[texture_index];
1060 }
1061
1062 #if defined(OS_LINUX) || defined(OS_BSD)
DmabufFds() const1063 const std::vector<base::ScopedFD>& VideoFrame::DmabufFds() const {
1064 DCHECK_EQ(storage_type_, STORAGE_DMABUFS);
1065
1066 return dmabuf_fds_->fds();
1067 }
1068
HasDmaBufs() const1069 bool VideoFrame::HasDmaBufs() const {
1070 return dmabuf_fds_->size() > 0;
1071 }
1072
IsSameDmaBufsAs(const VideoFrame & frame) const1073 bool VideoFrame::IsSameDmaBufsAs(const VideoFrame& frame) const {
1074 return storage_type_ == STORAGE_DMABUFS &&
1075 frame.storage_type_ == STORAGE_DMABUFS &&
1076 &DmabufFds() == &frame.DmabufFds();
1077 }
1078 #endif
1079
1080 #if defined(OS_MACOSX)
CvPixelBuffer() const1081 CVPixelBufferRef VideoFrame::CvPixelBuffer() const {
1082 return cv_pixel_buffer_.get();
1083 }
1084 #endif
1085
SetReleaseMailboxCB(ReleaseMailboxCB release_mailbox_cb)1086 void VideoFrame::SetReleaseMailboxCB(ReleaseMailboxCB release_mailbox_cb) {
1087 DCHECK(release_mailbox_cb);
1088 DCHECK(!mailbox_holders_release_cb_);
1089 // We don't relay SetReleaseMailboxCB to |wrapped_frame_| because the method
1090 // is not thread safe. This method should only be called by the owner of
1091 // |wrapped_frame_| directly.
1092 DCHECK(!wrapped_frame_);
1093 mailbox_holders_release_cb_ = std::move(release_mailbox_cb);
1094 }
1095
HasReleaseMailboxCB() const1096 bool VideoFrame::HasReleaseMailboxCB() const {
1097 return wrapped_frame_ ? wrapped_frame_->HasReleaseMailboxCB()
1098 : !!mailbox_holders_release_cb_;
1099 }
1100
AddDestructionObserver(base::OnceClosure callback)1101 void VideoFrame::AddDestructionObserver(base::OnceClosure callback) {
1102 DCHECK(!callback.is_null());
1103 done_callbacks_.push_back(std::move(callback));
1104 }
1105
UpdateReleaseSyncToken(SyncTokenClient * client)1106 gpu::SyncToken VideoFrame::UpdateReleaseSyncToken(SyncTokenClient* client) {
1107 DCHECK(HasTextures());
1108 if (wrapped_frame_) {
1109 return wrapped_frame_->UpdateReleaseSyncToken(client);
1110 }
1111 base::AutoLock locker(release_sync_token_lock_);
1112 // Must wait on the previous sync point before inserting a new sync point so
1113 // that |mailbox_holders_release_cb_| guarantees the previous sync point
1114 // occurred when it waits on |release_sync_token_|.
1115 if (release_sync_token_.HasData())
1116 client->WaitSyncToken(release_sync_token_);
1117 client->GenerateSyncToken(&release_sync_token_);
1118 return release_sync_token_;
1119 }
1120
AsHumanReadableString() const1121 std::string VideoFrame::AsHumanReadableString() const {
1122 if (metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM))
1123 return "end of stream";
1124
1125 std::ostringstream s;
1126 s << ConfigToString(format(), storage_type_, coded_size(), visible_rect_,
1127 natural_size_)
1128 << " timestamp:" << timestamp_.InMicroseconds();
1129 return s.str();
1130 }
1131
BitDepth() const1132 size_t VideoFrame::BitDepth() const {
1133 return media::BitDepth(format());
1134 }
1135
VideoFrame(const VideoFrameLayout & layout,StorageType storage_type,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp)1136 VideoFrame::VideoFrame(const VideoFrameLayout& layout,
1137 StorageType storage_type,
1138 const gfx::Rect& visible_rect,
1139 const gfx::Size& natural_size,
1140 base::TimeDelta timestamp)
1141 : layout_(layout),
1142 storage_type_(storage_type),
1143 visible_rect_(Intersection(visible_rect, gfx::Rect(layout.coded_size()))),
1144 natural_size_(natural_size),
1145 #if defined(OS_LINUX) || defined(OS_BSD)
1146 dmabuf_fds_(base::MakeRefCounted<DmabufHolder>()),
1147 #endif
1148 timestamp_(timestamp),
1149 unique_id_(g_unique_id_generator.GetNext()) {
1150 DCHECK(IsValidConfig(format(), storage_type, coded_size(), visible_rect_,
1151 natural_size_));
1152 DCHECK(visible_rect_ == visible_rect)
1153 << "visible_rect " << visible_rect.ToString() << " exceeds coded_size "
1154 << coded_size().ToString();
1155 memset(&mailbox_holders_, 0, sizeof(mailbox_holders_));
1156 memset(&data_, 0, sizeof(data_));
1157 }
1158
~VideoFrame()1159 VideoFrame::~VideoFrame() {
1160 if (mailbox_holders_release_cb_) {
1161 gpu::SyncToken release_sync_token;
1162 {
1163 // To ensure that changes to |release_sync_token_| are visible on this
1164 // thread (imply a memory barrier).
1165 base::AutoLock locker(release_sync_token_lock_);
1166 release_sync_token = release_sync_token_;
1167 }
1168 std::move(mailbox_holders_release_cb_).Run(release_sync_token);
1169 }
1170
1171 for (auto& callback : done_callbacks_)
1172 std::move(callback).Run();
1173 }
1174
1175 // static
ConfigToString(const VideoPixelFormat format,const StorageType storage_type,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size)1176 std::string VideoFrame::ConfigToString(const VideoPixelFormat format,
1177 const StorageType storage_type,
1178 const gfx::Size& coded_size,
1179 const gfx::Rect& visible_rect,
1180 const gfx::Size& natural_size) {
1181 return base::StringPrintf(
1182 "format:%s storage_type:%s coded_size:%s visible_rect:%s natural_size:%s",
1183 VideoPixelFormatToString(format).c_str(),
1184 StorageTypeToString(storage_type).c_str(), coded_size.ToString().c_str(),
1185 visible_rect.ToString().c_str(), natural_size.ToString().c_str());
1186 }
1187
1188 // static
DetermineAlignedSize(VideoPixelFormat format,const gfx::Size & dimensions)1189 gfx::Size VideoFrame::DetermineAlignedSize(VideoPixelFormat format,
1190 const gfx::Size& dimensions) {
1191 const gfx::Size alignment = CommonAlignment(format);
1192 const gfx::Size adjusted =
1193 gfx::Size(base::bits::Align(dimensions.width(), alignment.width()),
1194 base::bits::Align(dimensions.height(), alignment.height()));
1195 DCHECK((adjusted.width() % alignment.width() == 0) &&
1196 (adjusted.height() % alignment.height() == 0));
1197 return adjusted;
1198 }
1199
1200 // static
CreateFrameInternal(VideoPixelFormat format,const gfx::Size & coded_size,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp,bool zero_initialize_memory)1201 scoped_refptr<VideoFrame> VideoFrame::CreateFrameInternal(
1202 VideoPixelFormat format,
1203 const gfx::Size& coded_size,
1204 const gfx::Rect& visible_rect,
1205 const gfx::Size& natural_size,
1206 base::TimeDelta timestamp,
1207 bool zero_initialize_memory) {
1208 // Since we're creating a new frame (and allocating memory for it ourselves),
1209 // we can pad the requested |coded_size| if necessary if the request does not
1210 // line up on sample boundaries. See discussion at http://crrev.com/1240833003
1211 const gfx::Size new_coded_size = DetermineAlignedSize(format, coded_size);
1212 auto layout = VideoFrameLayout::CreateWithStrides(
1213 format, new_coded_size, ComputeStrides(format, new_coded_size));
1214 if (!layout) {
1215 DLOG(ERROR) << "Invalid layout.";
1216 return nullptr;
1217 }
1218
1219 return CreateFrameWithLayout(*layout, visible_rect, natural_size, timestamp,
1220 zero_initialize_memory);
1221 }
1222
CreateFrameWithLayout(const VideoFrameLayout & layout,const gfx::Rect & visible_rect,const gfx::Size & natural_size,base::TimeDelta timestamp,bool zero_initialize_memory)1223 scoped_refptr<VideoFrame> VideoFrame::CreateFrameWithLayout(
1224 const VideoFrameLayout& layout,
1225 const gfx::Rect& visible_rect,
1226 const gfx::Size& natural_size,
1227 base::TimeDelta timestamp,
1228 bool zero_initialize_memory) {
1229 const StorageType storage = STORAGE_OWNED_MEMORY;
1230 if (!IsValidConfig(layout.format(), storage, layout.coded_size(),
1231 visible_rect, natural_size)) {
1232 DLOG(ERROR) << __func__ << " Invalid config."
1233 << ConfigToString(layout.format(), storage, layout.coded_size(),
1234 visible_rect, natural_size);
1235 return nullptr;
1236 }
1237
1238 scoped_refptr<VideoFrame> frame(new VideoFrame(
1239 std::move(layout), storage, visible_rect, natural_size, timestamp));
1240 frame->AllocateMemory(zero_initialize_memory);
1241 return frame;
1242 }
1243
1244 // static
CommonAlignment(VideoPixelFormat format)1245 gfx::Size VideoFrame::CommonAlignment(VideoPixelFormat format) {
1246 int max_sample_width = 0;
1247 int max_sample_height = 0;
1248 for (size_t plane = 0; plane < NumPlanes(format); ++plane) {
1249 const gfx::Size sample_size = SampleSize(format, plane);
1250 max_sample_width = std::max(max_sample_width, sample_size.width());
1251 max_sample_height = std::max(max_sample_height, sample_size.height());
1252 }
1253 return gfx::Size(max_sample_width, max_sample_height);
1254 }
1255
AllocateMemory(bool zero_initialize_memory)1256 void VideoFrame::AllocateMemory(bool zero_initialize_memory) {
1257 DCHECK_EQ(storage_type_, STORAGE_OWNED_MEMORY);
1258 static_assert(0 == kYPlane, "y plane data must be index 0");
1259
1260 std::vector<size_t> plane_size = CalculatePlaneSize();
1261 const size_t total_buffer_size =
1262 std::accumulate(plane_size.begin(), plane_size.end(), 0u);
1263
1264 uint8_t* data = reinterpret_cast<uint8_t*>(
1265 base::AlignedAlloc(total_buffer_size, layout_.buffer_addr_align()));
1266 if (zero_initialize_memory) {
1267 memset(data, 0, total_buffer_size);
1268 }
1269 AddDestructionObserver(base::BindOnce(&base::AlignedFree, data));
1270
1271 // Note that if layout.buffer_sizes is specified, color planes' layout is the
1272 // same as buffers'. See CalculatePlaneSize() for detail.
1273 for (size_t plane = 0, offset = 0; plane < NumPlanes(format()); ++plane) {
1274 data_[plane] = data + offset;
1275 offset += plane_size[plane];
1276 }
1277 }
1278
IsValidSharedMemoryFrame() const1279 bool VideoFrame::IsValidSharedMemoryFrame() const {
1280 if (storage_type_ == STORAGE_SHMEM)
1281 return shm_region_ && shm_region_->IsValid();
1282 return false;
1283 }
1284
CalculatePlaneSize() const1285 std::vector<size_t> VideoFrame::CalculatePlaneSize() const {
1286 // We have two cases for plane size mapping:
1287 // 1) If plane size is specified: use planes' size.
1288 // 2) VideoFrameLayout::size is unassigned: use legacy calculation formula.
1289
1290 const size_t num_planes = NumPlanes(format());
1291 const auto& planes = layout_.planes();
1292 std::vector<size_t> plane_size(num_planes);
1293 bool plane_size_assigned = true;
1294 DCHECK_EQ(planes.size(), num_planes);
1295 for (size_t i = 0; i < num_planes; ++i) {
1296 plane_size[i] = planes[i].size;
1297 plane_size_assigned &= plane_size[i] != 0;
1298 }
1299
1300 if (plane_size_assigned)
1301 return plane_size;
1302
1303 // Reset plane size.
1304 std::fill(plane_size.begin(), plane_size.end(), 0u);
1305 for (size_t plane = 0; plane < num_planes; ++plane) {
1306 // These values were chosen to mirror ffmpeg's get_video_buffer().
1307 // TODO(dalecurtis): This should be configurable; eventually ffmpeg wants
1308 // us to use av_cpu_max_align(), but... for now, they just hard-code 32.
1309 const size_t height =
1310 base::bits::Align(rows(plane), kFrameAddressAlignment);
1311 const size_t width = std::abs(stride(plane));
1312 plane_size[plane] = width * height;
1313 }
1314
1315 if (num_planes > 1) {
1316 // The extra line of UV being allocated is because h264 chroma MC
1317 // overreads by one line in some cases, see libavcodec/utils.c:
1318 // avcodec_align_dimensions2() and libavcodec/x86/h264_chromamc.asm:
1319 // put_h264_chroma_mc4_ssse3().
1320 DCHECK(IsValidPlane(format(), kUPlane));
1321 plane_size.back() += std::abs(stride(kUPlane)) + kFrameSizePadding;
1322 }
1323 return plane_size;
1324 }
1325
1326 } // namespace media
1327