1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/gpu/test/video_frame_helpers.h"
6
7 #include <utility>
8 #include <vector>
9
10 #include "base/callback_helpers.h"
11 #include "base/logging.h"
12 #include "base/memory/scoped_refptr.h"
13 #include "gpu/ipc/common/gpu_memory_buffer_support.h"
14 #include "gpu/ipc/service/gpu_memory_buffer_factory.h"
15 #include "media/base/color_plane_layout.h"
16 #include "media/base/format_utils.h"
17 #include "media/base/video_frame.h"
18 #include "media/gpu/test/image.h"
19 #include "media/media_buildflags.h"
20 #include "testing/gtest/include/gtest/gtest.h"
21 #include "third_party/libyuv/include/libyuv.h"
22 #include "ui/gfx/buffer_format_util.h"
23 #include "ui/gfx/gpu_memory_buffer.h"
24
25 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
26 #include "media/gpu/chromeos/platform_video_frame_utils.h"
27 #include "media/gpu/video_frame_mapper.h"
28 #include "media/gpu/video_frame_mapper_factory.h"
29 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
30
31 namespace media {
32 namespace test {
33
34 namespace {
35
36 #define ASSERT_TRUE_OR_RETURN(predicate, return_value) \
37 do { \
38 if (!(predicate)) { \
39 ADD_FAILURE(); \
40 return (return_value); \
41 } \
42 } while (0)
43
44 // Split 16-bit UV plane to 16bit U plane and 16 bit V plane.
SplitUVRow_16(const uint16_t * src_uv,uint16_t * dst_u,uint16_t * dst_v,int width_in_samples)45 void SplitUVRow_16(const uint16_t* src_uv,
46 uint16_t* dst_u,
47 uint16_t* dst_v,
48 int width_in_samples) {
49 for (int i = 0; i < width_in_samples; i++) {
50 dst_u[i] = src_uv[0];
51 dst_v[i] = src_uv[1];
52 src_uv += 2;
53 }
54 }
55
56 // Convert 16 bit NV12 to 16 bit I420. The strides in these arguments are in
57 // bytes.
P016LEToI420P016(const uint8_t * src_y,int src_stride_y,const uint8_t * src_uv,int src_stride_uv,uint8_t * dst_y,int dst_stride_y,uint8_t * dst_u,int dst_stride_u,uint8_t * dst_v,int dst_stride_v,int width,int height)58 void P016LEToI420P016(const uint8_t* src_y,
59 int src_stride_y,
60 const uint8_t* src_uv,
61 int src_stride_uv,
62 uint8_t* dst_y,
63 int dst_stride_y,
64 uint8_t* dst_u,
65 int dst_stride_u,
66 uint8_t* dst_v,
67 int dst_stride_v,
68 int width,
69 int height) {
70 libyuv::CopyPlane_16(reinterpret_cast<const uint16_t*>(src_y),
71 src_stride_y / 2, reinterpret_cast<uint16_t*>(dst_y),
72 dst_stride_y / 2, width, height);
73 const int half_width = (width + 1) / 2;
74 const int half_height = (height + 1) / 2;
75 for (int i = 0; i < half_height; i++) {
76 SplitUVRow_16(reinterpret_cast<const uint16_t*>(src_uv),
77 reinterpret_cast<uint16_t*>(dst_u),
78 reinterpret_cast<uint16_t*>(dst_v), half_width);
79 dst_u += dst_stride_u;
80 dst_v += dst_stride_v;
81 src_uv += src_stride_uv;
82 }
83 }
84
ConvertVideoFrameToI420(const VideoFrame * src_frame,VideoFrame * dst_frame)85 bool ConvertVideoFrameToI420(const VideoFrame* src_frame,
86 VideoFrame* dst_frame) {
87 ASSERT_TRUE_OR_RETURN(src_frame->visible_rect() == dst_frame->visible_rect(),
88 false);
89 ASSERT_TRUE_OR_RETURN(dst_frame->format() == PIXEL_FORMAT_I420, false);
90
91 const auto& visible_rect = src_frame->visible_rect();
92 const int width = visible_rect.width();
93 const int height = visible_rect.height();
94 uint8_t* const dst_y = dst_frame->data(VideoFrame::kYPlane);
95 uint8_t* const dst_u = dst_frame->data(VideoFrame::kUPlane);
96 uint8_t* const dst_v = dst_frame->data(VideoFrame::kVPlane);
97 const int dst_stride_y = dst_frame->stride(VideoFrame::kYPlane);
98 const int dst_stride_u = dst_frame->stride(VideoFrame::kUPlane);
99 const int dst_stride_v = dst_frame->stride(VideoFrame::kVPlane);
100
101 switch (src_frame->format()) {
102 case PIXEL_FORMAT_I420:
103 return libyuv::I420Copy(src_frame->data(VideoFrame::kYPlane),
104 src_frame->stride(VideoFrame::kYPlane),
105 src_frame->data(VideoFrame::kUPlane),
106 src_frame->stride(VideoFrame::kUPlane),
107 src_frame->data(VideoFrame::kVPlane),
108 src_frame->stride(VideoFrame::kVPlane), dst_y,
109 dst_stride_y, dst_u, dst_stride_u, dst_v,
110 dst_stride_v, width, height) == 0;
111 case PIXEL_FORMAT_NV12:
112 return libyuv::NV12ToI420(src_frame->data(VideoFrame::kYPlane),
113 src_frame->stride(VideoFrame::kYPlane),
114 src_frame->data(VideoFrame::kUVPlane),
115 src_frame->stride(VideoFrame::kUVPlane), dst_y,
116 dst_stride_y, dst_u, dst_stride_u, dst_v,
117 dst_stride_v, width, height) == 0;
118 case PIXEL_FORMAT_YV12:
119 // Swap U and V planes.
120 return libyuv::I420Copy(src_frame->data(VideoFrame::kYPlane),
121 src_frame->stride(VideoFrame::kYPlane),
122 src_frame->data(VideoFrame::kVPlane),
123 src_frame->stride(VideoFrame::kVPlane),
124 src_frame->data(VideoFrame::kUPlane),
125 src_frame->stride(VideoFrame::kUPlane), dst_y,
126 dst_stride_y, dst_u, dst_stride_u, dst_v,
127 dst_stride_v, width, height) == 0;
128 default:
129 LOG(ERROR) << "Unsupported input format: " << src_frame->format();
130 return false;
131 }
132 }
133
ConvertVideoFrameToYUV420P10(const VideoFrame * src_frame,VideoFrame * dst_frame)134 bool ConvertVideoFrameToYUV420P10(const VideoFrame* src_frame,
135 VideoFrame* dst_frame) {
136 if (src_frame->format() != PIXEL_FORMAT_P016LE) {
137 LOG(ERROR) << "Unsupported input format: "
138 << VideoPixelFormatToString(src_frame->format());
139 return false;
140 }
141
142 const auto& visible_rect = src_frame->visible_rect();
143 const int width = visible_rect.width();
144 const int height = visible_rect.height();
145 uint8_t* const dst_y = dst_frame->data(VideoFrame::kYPlane);
146 uint8_t* const dst_u = dst_frame->data(VideoFrame::kUPlane);
147 uint8_t* const dst_v = dst_frame->data(VideoFrame::kVPlane);
148 const int dst_stride_y = dst_frame->stride(VideoFrame::kYPlane);
149 const int dst_stride_u = dst_frame->stride(VideoFrame::kUPlane);
150 const int dst_stride_v = dst_frame->stride(VideoFrame::kVPlane);
151 P016LEToI420P016(src_frame->data(VideoFrame::kYPlane),
152 src_frame->stride(VideoFrame::kYPlane),
153 src_frame->data(VideoFrame::kUVPlane),
154 src_frame->stride(VideoFrame::kUVPlane), dst_y, dst_stride_y,
155 dst_u, dst_stride_u, dst_v, dst_stride_v, width, height);
156 return true;
157 }
158
ConvertVideoFrameToARGB(const VideoFrame * src_frame,VideoFrame * dst_frame)159 bool ConvertVideoFrameToARGB(const VideoFrame* src_frame,
160 VideoFrame* dst_frame) {
161 ASSERT_TRUE_OR_RETURN(src_frame->visible_rect() == dst_frame->visible_rect(),
162 false);
163 ASSERT_TRUE_OR_RETURN(dst_frame->format() == PIXEL_FORMAT_ARGB, false);
164
165 const auto& visible_rect = src_frame->visible_rect();
166 const int width = visible_rect.width();
167 const int height = visible_rect.height();
168 uint8_t* const dst_argb = dst_frame->data(VideoFrame::kARGBPlane);
169 const int dst_stride = dst_frame->stride(VideoFrame::kARGBPlane);
170
171 switch (src_frame->format()) {
172 case PIXEL_FORMAT_I420:
173 // Note that we use J420ToARGB instead of I420ToARGB so that the
174 // kYuvJPEGConstants YUV-to-RGB conversion matrix is used.
175 return libyuv::J420ToARGB(src_frame->data(VideoFrame::kYPlane),
176 src_frame->stride(VideoFrame::kYPlane),
177 src_frame->data(VideoFrame::kUPlane),
178 src_frame->stride(VideoFrame::kUPlane),
179 src_frame->data(VideoFrame::kVPlane),
180 src_frame->stride(VideoFrame::kVPlane),
181 dst_argb, dst_stride, width, height) == 0;
182 case PIXEL_FORMAT_NV12:
183 return libyuv::NV12ToARGB(src_frame->data(VideoFrame::kYPlane),
184 src_frame->stride(VideoFrame::kYPlane),
185 src_frame->data(VideoFrame::kUVPlane),
186 src_frame->stride(VideoFrame::kUVPlane),
187 dst_argb, dst_stride, width, height) == 0;
188 case PIXEL_FORMAT_YV12:
189 // Same as I420, but U and V planes are swapped.
190 return libyuv::J420ToARGB(src_frame->data(VideoFrame::kYPlane),
191 src_frame->stride(VideoFrame::kYPlane),
192 src_frame->data(VideoFrame::kVPlane),
193 src_frame->stride(VideoFrame::kVPlane),
194 src_frame->data(VideoFrame::kUPlane),
195 src_frame->stride(VideoFrame::kUPlane),
196 dst_argb, dst_stride, width, height) == 0;
197 break;
198 default:
199 LOG(ERROR) << "Unsupported input format: " << src_frame->format();
200 return false;
201 }
202 }
203
204 // Copy memory based |src_frame| buffer to |dst_frame| buffer.
CopyVideoFrame(const VideoFrame * src_frame,scoped_refptr<VideoFrame> dst_frame)205 bool CopyVideoFrame(const VideoFrame* src_frame,
206 scoped_refptr<VideoFrame> dst_frame) {
207 ASSERT_TRUE_OR_RETURN(src_frame->IsMappable(), false);
208 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
209 // If |dst_frame| is a Dmabuf-backed VideoFrame, we need to map its underlying
210 // buffer into memory. We use a VideoFrameMapper to create a memory-based
211 // VideoFrame that refers to the |dst_frame|'s buffer.
212 if (dst_frame->storage_type() == VideoFrame::STORAGE_DMABUFS) {
213 auto video_frame_mapper = VideoFrameMapperFactory::CreateMapper(
214 dst_frame->format(), VideoFrame::STORAGE_DMABUFS, true);
215 ASSERT_TRUE_OR_RETURN(video_frame_mapper, false);
216 dst_frame = video_frame_mapper->Map(std::move(dst_frame));
217 if (!dst_frame) {
218 LOG(ERROR) << "Failed to map DMABuf video frame.";
219 return false;
220 }
221 }
222 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
223 ASSERT_TRUE_OR_RETURN(dst_frame->IsMappable(), false);
224 ASSERT_TRUE_OR_RETURN(src_frame->format() == dst_frame->format(), false);
225
226 // Copy every plane's content from |src_frame| to |dst_frame|.
227 const size_t num_planes = VideoFrame::NumPlanes(dst_frame->format());
228 ASSERT_TRUE_OR_RETURN(dst_frame->layout().planes().size() == num_planes,
229 false);
230 ASSERT_TRUE_OR_RETURN(src_frame->layout().planes().size() == num_planes,
231 false);
232 for (size_t i = 0; i < num_planes; ++i) {
233 // |width| in libyuv::CopyPlane() is in bytes, not pixels.
234 gfx::Size plane_size =
235 VideoFrame::PlaneSize(dst_frame->format(), i, dst_frame->coded_size());
236 libyuv::CopyPlane(
237 src_frame->data(i), src_frame->layout().planes()[i].stride,
238 dst_frame->data(i), dst_frame->layout().planes()[i].stride,
239 plane_size.width(), plane_size.height());
240 }
241 return true;
242 }
243
244 } // namespace
245
ConvertVideoFrame(const VideoFrame * src_frame,VideoFrame * dst_frame)246 bool ConvertVideoFrame(const VideoFrame* src_frame, VideoFrame* dst_frame) {
247 ASSERT_TRUE_OR_RETURN(src_frame->visible_rect() == dst_frame->visible_rect(),
248 false);
249 ASSERT_TRUE_OR_RETURN(src_frame->IsMappable() && dst_frame->IsMappable(),
250 false);
251
252 // Writing into non-owned memory might produce some unexpected side effects.
253 if (dst_frame->storage_type() != VideoFrame::STORAGE_OWNED_MEMORY)
254 LOG(WARNING) << "writing into non-owned memory";
255
256 // Only I420, YUV420P10 and ARGB are currently supported as output formats.
257 switch (dst_frame->format()) {
258 case PIXEL_FORMAT_I420:
259 return ConvertVideoFrameToI420(src_frame, dst_frame);
260 case PIXEL_FORMAT_YUV420P10:
261 return ConvertVideoFrameToYUV420P10(src_frame, dst_frame);
262 case PIXEL_FORMAT_ARGB:
263 return ConvertVideoFrameToARGB(src_frame, dst_frame);
264 default:
265 LOG(ERROR) << "Unsupported output format: " << dst_frame->format();
266 return false;
267 }
268 }
269
ConvertVideoFrame(const VideoFrame * src_frame,VideoPixelFormat dst_pixel_format)270 scoped_refptr<VideoFrame> ConvertVideoFrame(const VideoFrame* src_frame,
271 VideoPixelFormat dst_pixel_format) {
272 gfx::Rect visible_rect = src_frame->visible_rect();
273 auto dst_frame = VideoFrame::CreateFrame(
274 dst_pixel_format, visible_rect.size(), visible_rect, visible_rect.size(),
275 base::TimeDelta());
276 if (!dst_frame) {
277 LOG(ERROR) << "Failed to convert video frame to " << dst_frame->format();
278 return nullptr;
279 }
280 bool conversion_success = ConvertVideoFrame(src_frame, dst_frame.get());
281 if (!conversion_success) {
282 LOG(ERROR) << "Failed to convert video frame to " << dst_frame->format();
283 return nullptr;
284 }
285 return dst_frame;
286 }
287
CloneVideoFrame(gpu::GpuMemoryBufferFactory * gpu_memory_buffer_factory,const VideoFrame * const src_frame,const VideoFrameLayout & dst_layout,VideoFrame::StorageType dst_storage_type,base::Optional<gfx::BufferUsage> dst_buffer_usage)288 scoped_refptr<VideoFrame> CloneVideoFrame(
289 gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory,
290 const VideoFrame* const src_frame,
291 const VideoFrameLayout& dst_layout,
292 VideoFrame::StorageType dst_storage_type,
293 base::Optional<gfx::BufferUsage> dst_buffer_usage) {
294 if (!src_frame)
295 return nullptr;
296 if (!src_frame->IsMappable()) {
297 LOG(ERROR) << "The source video frame must be memory-backed VideoFrame";
298 return nullptr;
299 }
300
301 scoped_refptr<VideoFrame> dst_frame;
302 switch (dst_storage_type) {
303 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
304 case VideoFrame::STORAGE_GPU_MEMORY_BUFFER:
305 case VideoFrame::STORAGE_DMABUFS:
306 if (!dst_buffer_usage) {
307 LOG(ERROR) << "Buffer usage is not specified for a graphic buffer";
308 return nullptr;
309 }
310 dst_frame = CreatePlatformVideoFrame(
311 gpu_memory_buffer_factory, dst_layout.format(),
312 dst_layout.coded_size(), src_frame->visible_rect(),
313 src_frame->natural_size(), src_frame->timestamp(), *dst_buffer_usage);
314 break;
315 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
316 case VideoFrame::STORAGE_OWNED_MEMORY:
317 // Create VideoFrame, which allocates and owns data.
318 dst_frame = VideoFrame::CreateFrameWithLayout(
319 dst_layout, src_frame->visible_rect(), src_frame->natural_size(),
320 src_frame->timestamp(), false /* zero_initialize_memory*/);
321 break;
322 default:
323 LOG(ERROR) << "Clone video frame must have the ownership of the buffer";
324 return nullptr;
325 }
326
327 if (!dst_frame) {
328 LOG(ERROR) << "Failed to create VideoFrame";
329 return nullptr;
330 }
331
332 if (!CopyVideoFrame(src_frame, dst_frame)) {
333 LOG(ERROR) << "Failed to copy VideoFrame";
334 return nullptr;
335 }
336
337 if (dst_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
338 // Here, the content in |src_frame| is already copied to |dst_frame|, which
339 // is a DMABUF based VideoFrame.
340 // Create GpuMemoryBuffer based VideoFrame from |dst_frame|.
341 dst_frame = CreateGpuMemoryBufferVideoFrame(
342 gpu_memory_buffer_factory, dst_frame.get(), *dst_buffer_usage);
343 }
344
345 return dst_frame;
346 }
347
CreateDmabufVideoFrame(const VideoFrame * const frame)348 scoped_refptr<VideoFrame> CreateDmabufVideoFrame(
349 const VideoFrame* const frame) {
350 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
351 if (!frame || frame->storage_type() != VideoFrame::STORAGE_GPU_MEMORY_BUFFER)
352 return nullptr;
353 gfx::GpuMemoryBuffer* gmb = frame->GetGpuMemoryBuffer();
354 gfx::GpuMemoryBufferHandle gmb_handle = gmb->CloneHandle();
355 DCHECK_EQ(gmb_handle.type, gfx::GpuMemoryBufferType::NATIVE_PIXMAP);
356 std::vector<ColorPlaneLayout> planes;
357 std::vector<base::ScopedFD> dmabuf_fds;
358 for (auto& plane : gmb_handle.native_pixmap_handle.planes) {
359 planes.emplace_back(plane.stride, plane.offset, plane.size);
360 dmabuf_fds.emplace_back(plane.fd.release());
361 }
362 return VideoFrame::WrapExternalDmabufs(
363 frame->layout(), frame->visible_rect(), frame->natural_size(),
364 std::move(dmabuf_fds), frame->timestamp());
365 #else
366 return nullptr;
367 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)}
368 }
369
CreateGpuMemoryBufferVideoFrame(gpu::GpuMemoryBufferFactory * gpu_memory_buffer_factory,const VideoFrame * const frame,gfx::BufferUsage buffer_usage)370 scoped_refptr<VideoFrame> CreateGpuMemoryBufferVideoFrame(
371 gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory,
372 const VideoFrame* const frame,
373 gfx::BufferUsage buffer_usage) {
374 gfx::GpuMemoryBufferHandle gmb_handle;
375 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
376 gmb_handle = CreateGpuMemoryBufferHandle(frame);
377 #endif
378 if (gmb_handle.is_null() || gmb_handle.type != gfx::NATIVE_PIXMAP) {
379 LOG(ERROR) << "Failed to create native GpuMemoryBufferHandle";
380 return nullptr;
381 }
382
383 base::Optional<gfx::BufferFormat> buffer_format =
384 VideoPixelFormatToGfxBufferFormat(frame->format());
385 if (!buffer_format) {
386 LOG(ERROR) << "Unexpected format: " << frame->format();
387 return nullptr;
388 }
389
390 // Create GpuMemoryBuffer from GpuMemoryBufferHandle.
391 gpu::GpuMemoryBufferSupport support;
392 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer =
393 support.CreateGpuMemoryBufferImplFromHandle(
394 std::move(gmb_handle), frame->coded_size(), *buffer_format,
395 buffer_usage, base::DoNothing());
396 if (!gpu_memory_buffer) {
397 LOG(ERROR) << "Failed to create GpuMemoryBuffer from GpuMemoryBufferHandle";
398 return nullptr;
399 }
400
401 gpu::MailboxHolder dummy_mailbox[media::VideoFrame::kMaxPlanes];
402 return media::VideoFrame::WrapExternalGpuMemoryBuffer(
403 frame->visible_rect(), frame->natural_size(),
404 std::move(gpu_memory_buffer), dummy_mailbox,
405 base::DoNothing() /* mailbox_holder_release_cb_ */, frame->timestamp());
406 }
407
CreateVideoFrameFromImage(const Image & image)408 scoped_refptr<const VideoFrame> CreateVideoFrameFromImage(const Image& image) {
409 DCHECK(image.IsLoaded());
410 const auto format = image.PixelFormat();
411 const auto& image_size = image.Size();
412 // Loaded image data must be tight.
413 DCHECK_EQ(image.DataSize(), VideoFrame::AllocationSize(format, image_size));
414
415 // Create planes for layout. We cannot use WrapExternalData() because it
416 // calls GetDefaultLayout() and it supports only a few pixel formats.
417 base::Optional<VideoFrameLayout> layout =
418 CreateVideoFrameLayout(format, image_size);
419 if (!layout) {
420 LOG(ERROR) << "Failed to create VideoFrameLayout";
421 return nullptr;
422 }
423
424 scoped_refptr<const VideoFrame> video_frame =
425 VideoFrame::WrapExternalDataWithLayout(
426 *layout, image.VisibleRect(), image.VisibleRect().size(),
427 image.Data(), image.DataSize(), base::TimeDelta());
428 if (!video_frame) {
429 LOG(ERROR) << "Failed to create VideoFrame";
430 return nullptr;
431 }
432
433 return video_frame;
434 }
435
CreateVideoFrameLayout(VideoPixelFormat pixel_format,const gfx::Size & dimension,const uint32_t alignment,std::vector<size_t> * plane_rows)436 base::Optional<VideoFrameLayout> CreateVideoFrameLayout(
437 VideoPixelFormat pixel_format,
438 const gfx::Size& dimension,
439 const uint32_t alignment,
440 std::vector<size_t>* plane_rows) {
441 const size_t num_planes = VideoFrame::NumPlanes(pixel_format);
442
443 std::vector<ColorPlaneLayout> planes(num_planes);
444 size_t offset = 0;
445 if (plane_rows)
446 plane_rows->resize(num_planes);
447 for (size_t i = 0; i < num_planes; ++i) {
448 const int32_t stride =
449 VideoFrame::RowBytes(i, pixel_format, dimension.width());
450 const size_t rows = VideoFrame::Rows(i, pixel_format, dimension.height());
451 const size_t plane_size = stride * rows;
452 const size_t aligned_size = base::bits::Align(plane_size, alignment);
453 planes[i].stride = stride;
454 planes[i].offset = offset;
455 planes[i].size = aligned_size;
456 offset += planes[i].size;
457 if (plane_rows)
458 (*plane_rows)[i] = rows;
459 }
460 return VideoFrameLayout::CreateWithPlanes(pixel_format, dimension,
461 std::move(planes));
462 }
463
464 } // namespace test
465 } // namespace media
466