1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/gpu/test/video_frame_validator.h"
6
7 #include "base/bind.h"
8 #include "base/cpu.h"
9 #include "base/files/file.h"
10 #include "base/hash/md5.h"
11 #include "base/memory/ptr_util.h"
12 #include "base/no_destructor.h"
13 #include "base/numerics/safe_conversions.h"
14 #include "base/strings/string_util.h"
15 #include "base/strings/stringprintf.h"
16 #include "base/system/sys_info.h"
17 #include "build/build_config.h"
18 #include "build/chromeos_buildflags.h"
19 #include "media/base/video_frame.h"
20 #include "media/gpu/buildflags.h"
21 #include "media/gpu/macros.h"
22 #include "media/gpu/test/image_quality_metrics.h"
23 #include "media/gpu/test/video_test_helpers.h"
24 #include "media/gpu/video_frame_mapper.h"
25 #include "media/gpu/video_frame_mapper_factory.h"
26 #include "media/media_buildflags.h"
27 #include "testing/gtest/include/gtest/gtest.h"
28 #include "ui/gfx/gpu_memory_buffer.h"
29
30 namespace media {
31 namespace test {
32
VideoFrameValidator(std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)33 VideoFrameValidator::VideoFrameValidator(
34 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)
35 : corrupt_frame_processor_(std::move(corrupt_frame_processor)),
36 num_frames_validating_(0),
37 frame_validator_thread_("FrameValidatorThread"),
38 frame_validator_cv_(&frame_validator_lock_) {
39 DETACH_FROM_SEQUENCE(validator_sequence_checker_);
40 DETACH_FROM_SEQUENCE(validator_thread_sequence_checker_);
41 }
42
~VideoFrameValidator()43 VideoFrameValidator::~VideoFrameValidator() {
44 Destroy();
45 }
46
Initialize()47 bool VideoFrameValidator::Initialize() {
48 if (!frame_validator_thread_.Start()) {
49 LOG(ERROR) << "Failed to start frame validator thread";
50 return false;
51 }
52 return true;
53 }
54
Destroy()55 void VideoFrameValidator::Destroy() {
56 frame_validator_thread_.Stop();
57 base::AutoLock auto_lock(frame_validator_lock_);
58 DCHECK_EQ(0u, num_frames_validating_);
59 }
60
PrintMismatchedFramesInfo() const61 void VideoFrameValidator::PrintMismatchedFramesInfo() const {
62 base::AutoLock auto_lock(frame_validator_lock_);
63 for (const auto& mismatched_frame_info : mismatched_frames_)
64 mismatched_frame_info->Print();
65 }
66
GetMismatchedFramesCount() const67 size_t VideoFrameValidator::GetMismatchedFramesCount() const {
68 base::AutoLock auto_lock(frame_validator_lock_);
69 return mismatched_frames_.size();
70 }
71
ProcessVideoFrame(scoped_refptr<const VideoFrame> video_frame,size_t frame_index)72 void VideoFrameValidator::ProcessVideoFrame(
73 scoped_refptr<const VideoFrame> video_frame,
74 size_t frame_index) {
75 DCHECK_CALLED_ON_VALID_SEQUENCE(validator_sequence_checker_);
76
77 if (!video_frame) {
78 LOG(ERROR) << "Video frame is nullptr";
79 return;
80 }
81
82 if (video_frame->visible_rect().IsEmpty()) {
83 // This occurs in bitstream buffer in webrtc scenario.
84 DLOG(WARNING) << "Skipping validation, frame_index=" << frame_index
85 << " because visible_rect is empty";
86 return;
87 }
88
89 base::AutoLock auto_lock(frame_validator_lock_);
90 num_frames_validating_++;
91
92 // Unretained is safe here, as we should not destroy the validator while there
93 // are still frames being validated.
94 frame_validator_thread_.task_runner()->PostTask(
95 FROM_HERE,
96 base::BindOnce(&VideoFrameValidator::ProcessVideoFrameTask,
97 base::Unretained(this), video_frame, frame_index));
98 }
99
WaitUntilDone()100 bool VideoFrameValidator::WaitUntilDone() {
101 {
102 base::AutoLock auto_lock(frame_validator_lock_);
103 while (num_frames_validating_ > 0) {
104 frame_validator_cv_.Wait();
105 }
106
107 if (corrupt_frame_processor_ && !corrupt_frame_processor_->WaitUntilDone())
108 return false;
109 }
110
111 if (!Passed()) {
112 LOG(ERROR) << GetMismatchedFramesCount() << " frames failed to validate.";
113 PrintMismatchedFramesInfo();
114 return false;
115 }
116 return true;
117 }
118
Passed() const119 bool VideoFrameValidator::Passed() const {
120 return GetMismatchedFramesCount() == 0u;
121 }
122
ProcessVideoFrameTask(const scoped_refptr<const VideoFrame> video_frame,size_t frame_index)123 void VideoFrameValidator::ProcessVideoFrameTask(
124 const scoped_refptr<const VideoFrame> video_frame,
125 size_t frame_index) {
126 DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_);
127
128 scoped_refptr<const VideoFrame> frame = video_frame;
129 // If this is a DMABuf-backed memory frame we need to map it before accessing.
130 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
131 if (frame->storage_type() == VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
132 // TODO(andrescj): This is a workaround. ClientNativePixmapFactoryDmabuf
133 // creates ClientNativePixmapOpaque for SCANOUT_VDA_WRITE buffers which does
134 // not allow us to map GpuMemoryBuffers easily for testing. Therefore, we
135 // extract the dma-buf FDs. Alternatively, we could consider creating our
136 // own ClientNativePixmapFactory for testing.
137 frame = CreateDmabufVideoFrame(frame.get());
138 if (!frame) {
139 LOG(ERROR) << "Failed to create Dmabuf-backed VideoFrame from "
140 << "GpuMemoryBuffer-based VideoFrame";
141 return;
142 }
143 }
144
145 if (frame->storage_type() == VideoFrame::STORAGE_DMABUFS) {
146 // Create VideoFrameMapper if not yet created. The decoder's output pixel
147 // format is not known yet when creating the VideoFrameValidator. We can
148 // only create the VideoFrameMapper upon receiving the first video frame.
149 if (!video_frame_mapper_) {
150 video_frame_mapper_ = VideoFrameMapperFactory::CreateMapper(
151 frame->format(), frame->storage_type());
152 ASSERT_TRUE(video_frame_mapper_) << "Failed to create VideoFrameMapper";
153 }
154
155 frame = video_frame_mapper_->Map(std::move(frame));
156 if (!frame) {
157 LOG(ERROR) << "Failed to map video frame";
158 return;
159 }
160 }
161 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
162
163 ASSERT_TRUE(frame->IsMappable());
164
165 auto mismatched_info = Validate(frame, frame_index);
166
167 base::AutoLock auto_lock(frame_validator_lock_);
168 if (mismatched_info) {
169 mismatched_frames_.push_back(std::move(mismatched_info));
170 // Perform additional processing on the corrupt video frame if requested.
171 if (corrupt_frame_processor_)
172 corrupt_frame_processor_->ProcessVideoFrame(frame, frame_index);
173 }
174
175 num_frames_validating_--;
176 frame_validator_cv_.Signal();
177 }
178
179 struct MD5VideoFrameValidator::MD5MismatchedFrameInfo
180 : public VideoFrameValidator::MismatchedFrameInfo {
MD5MismatchedFrameInfomedia::test::MD5VideoFrameValidator::MD5MismatchedFrameInfo181 MD5MismatchedFrameInfo(size_t frame_index,
182 const std::string& computed_md5,
183 const std::string& expected_md5)
184 : MismatchedFrameInfo(frame_index),
185 computed_md5(computed_md5),
186 expected_md5(expected_md5) {}
187 ~MD5MismatchedFrameInfo() override = default;
Printmedia::test::MD5VideoFrameValidator::MD5MismatchedFrameInfo188 void Print() const override {
189 LOG(ERROR) << "frame_index: " << frame_index
190 << ", computed_md5: " << computed_md5
191 << ", expected_md5: " << expected_md5;
192 }
193
194 const std::string computed_md5;
195 const std::string expected_md5;
196 };
197
198 // static
Create(const std::vector<std::string> & expected_frame_checksums,VideoPixelFormat validation_format,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)199 std::unique_ptr<MD5VideoFrameValidator> MD5VideoFrameValidator::Create(
200 const std::vector<std::string>& expected_frame_checksums,
201 VideoPixelFormat validation_format,
202 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor) {
203 auto video_frame_validator = base::WrapUnique(
204 new MD5VideoFrameValidator(expected_frame_checksums, validation_format,
205 std::move(corrupt_frame_processor)));
206 if (!video_frame_validator->Initialize()) {
207 LOG(ERROR) << "Failed to initialize MD5VideoFrameValidator.";
208 return nullptr;
209 }
210
211 return video_frame_validator;
212 }
213
MD5VideoFrameValidator(const std::vector<std::string> & expected_frame_checksums,VideoPixelFormat validation_format,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)214 MD5VideoFrameValidator::MD5VideoFrameValidator(
215 const std::vector<std::string>& expected_frame_checksums,
216 VideoPixelFormat validation_format,
217 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor)
218 : VideoFrameValidator(std::move(corrupt_frame_processor)),
219 expected_frame_checksums_(expected_frame_checksums),
220 validation_format_(validation_format) {}
221
222 MD5VideoFrameValidator::~MD5VideoFrameValidator() = default;
223
224 std::unique_ptr<VideoFrameValidator::MismatchedFrameInfo>
Validate(scoped_refptr<const VideoFrame> frame,size_t frame_index)225 MD5VideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
226 size_t frame_index) {
227 DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_);
228 #if BUILDFLAG(IS_ASH) || BUILDFLAG(IS_LACROS)
229 // b/149808895: There is a bug in the synchronization on mapped buffers, which
230 // causes the frame validation failure. The bug is due to some missing i915
231 // patches in kernel v3.18. The bug will be fixed if the kernel is upreved to
232 // v4.4 or newer. Inserts usleep as a short term workaround to the
233 // synchronization bug until the kernel uprev is complete for all the v3.18
234 // devices. Since this bug only occurs in Skylake just because they are 3.18
235 // devices, we also filter by the processor.
236 const static std::string kernel_version = base::SysInfo::KernelVersion();
237 if (base::StartsWith(kernel_version, "3.18")) {
238 constexpr int kPentiumAndLaterFamily = 0x06;
239 constexpr int kSkyLakeModelId = 0x5E;
240 constexpr int kSkyLake_LModelId = 0x4E;
241 static base::NoDestructor<base::CPU> cpuid;
242 static bool is_skylake = cpuid->family() == kPentiumAndLaterFamily &&
243 (cpuid->model() == kSkyLakeModelId ||
244 cpuid->model() == kSkyLake_LModelId);
245 if (is_skylake)
246 usleep(10);
247 }
248 #endif // BUILDFLAG(IS_ASH) || BUILDFLAG(IS_LACROS)
249 if (frame->format() != validation_format_) {
250 frame = ConvertVideoFrame(frame.get(), validation_format_);
251 }
252 CHECK(frame);
253
254 std::string computed_md5 = ComputeMD5FromVideoFrame(*frame);
255 if (expected_frame_checksums_.size() > 0) {
256 LOG_IF(FATAL, frame_index >= expected_frame_checksums_.size())
257 << "Frame number is over than the number of read md5 values in file.";
258 const auto& expected_md5 = expected_frame_checksums_[frame_index];
259 if (computed_md5 != expected_md5)
260 return std::make_unique<MD5MismatchedFrameInfo>(frame_index, computed_md5,
261 expected_md5);
262 }
263 return nullptr;
264 }
265
ComputeMD5FromVideoFrame(const VideoFrame & video_frame) const266 std::string MD5VideoFrameValidator::ComputeMD5FromVideoFrame(
267 const VideoFrame& video_frame) const {
268 DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_);
269 base::MD5Context context;
270 base::MD5Init(&context);
271
272 // VideoFrame::HashFrameForTesting() computes MD5 hash values of the coded
273 // area. However, MD5 hash values used in our test only use the visible area
274 // because they are computed from images output by decode tools like ffmpeg.
275 const VideoPixelFormat format = video_frame.format();
276 const gfx::Rect& visible_rect = video_frame.visible_rect();
277 for (size_t i = 0; i < VideoFrame::NumPlanes(format); ++i) {
278 const int visible_row_bytes =
279 VideoFrame::RowBytes(i, format, visible_rect.width());
280 const int visible_rows = VideoFrame::Rows(i, format, visible_rect.height());
281 const char* data = reinterpret_cast<const char*>(video_frame.data(i));
282 const size_t stride = video_frame.stride(i);
283 for (int row = 0; row < visible_rows; ++row) {
284 base::MD5Update(&context, base::StringPiece(data + (stride * row),
285 visible_row_bytes));
286 }
287 }
288 base::MD5Digest digest;
289 base::MD5Final(&digest, &context);
290 return MD5DigestToBase16(digest);
291 }
292
293 struct RawVideoFrameValidator::RawMismatchedFrameInfo
294 : public VideoFrameValidator::MismatchedFrameInfo {
RawMismatchedFrameInfomedia::test::RawVideoFrameValidator::RawMismatchedFrameInfo295 RawMismatchedFrameInfo(size_t frame_index, size_t diff_cnt)
296 : MismatchedFrameInfo(frame_index), diff_cnt(diff_cnt) {}
297 ~RawMismatchedFrameInfo() override = default;
Printmedia::test::RawVideoFrameValidator::RawMismatchedFrameInfo298 void Print() const override {
299 LOG(ERROR) << "frame_index: " << frame_index << ", diff_cnt: " << diff_cnt;
300 }
301
302 size_t diff_cnt;
303 };
304
305 // static
Create(const GetModelFrameCB & get_model_frame_cb,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,uint8_t tolerance)306 std::unique_ptr<RawVideoFrameValidator> RawVideoFrameValidator::Create(
307 const GetModelFrameCB& get_model_frame_cb,
308 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
309 uint8_t tolerance) {
310 auto video_frame_validator = base::WrapUnique(new RawVideoFrameValidator(
311 get_model_frame_cb, std::move(corrupt_frame_processor), tolerance));
312 if (!video_frame_validator->Initialize()) {
313 LOG(ERROR) << "Failed to initialize RawVideoFrameValidator.";
314 return nullptr;
315 }
316
317 return video_frame_validator;
318 }
319
RawVideoFrameValidator(const GetModelFrameCB & get_model_frame_cb,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,uint8_t tolerance)320 RawVideoFrameValidator::RawVideoFrameValidator(
321 const GetModelFrameCB& get_model_frame_cb,
322 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
323 uint8_t tolerance)
324 : VideoFrameValidator(std::move(corrupt_frame_processor)),
325 get_model_frame_cb_(get_model_frame_cb),
326 tolerance_(tolerance) {}
327
328 RawVideoFrameValidator::~RawVideoFrameValidator() = default;
329
330 std::unique_ptr<VideoFrameValidator::MismatchedFrameInfo>
Validate(scoped_refptr<const VideoFrame> frame,size_t frame_index)331 RawVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
332 size_t frame_index) {
333 SEQUENCE_CHECKER(validator_thread_sequence_checker_);
334 auto model_frame = get_model_frame_cb_.Run(frame_index);
335 CHECK(model_frame);
336 size_t diff_cnt =
337 CompareFramesWithErrorDiff(*frame, *model_frame, tolerance_);
338 if (diff_cnt > 0)
339 return std::make_unique<RawMismatchedFrameInfo>(frame_index, diff_cnt);
340 return nullptr;
341 }
342
343 struct PSNRVideoFrameValidator::PSNRMismatchedFrameInfo
344 : public VideoFrameValidator::MismatchedFrameInfo {
PSNRMismatchedFrameInfomedia::test::PSNRVideoFrameValidator::PSNRMismatchedFrameInfo345 PSNRMismatchedFrameInfo(size_t frame_index, double psnr)
346 : MismatchedFrameInfo(frame_index), psnr(psnr) {}
347 ~PSNRMismatchedFrameInfo() override = default;
Printmedia::test::PSNRVideoFrameValidator::PSNRMismatchedFrameInfo348 void Print() const override {
349 LOG(ERROR) << "frame_index: " << frame_index << ", psnr: " << psnr;
350 }
351
352 double psnr;
353 };
354
355 // static
Create(const GetModelFrameCB & get_model_frame_cb,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,ValidationMode validation_mode,double tolerance)356 std::unique_ptr<PSNRVideoFrameValidator> PSNRVideoFrameValidator::Create(
357 const GetModelFrameCB& get_model_frame_cb,
358 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
359 ValidationMode validation_mode,
360 double tolerance) {
361 auto video_frame_validator = base::WrapUnique(new PSNRVideoFrameValidator(
362 get_model_frame_cb, std::move(corrupt_frame_processor), validation_mode,
363 tolerance));
364 if (!video_frame_validator->Initialize()) {
365 LOG(ERROR) << "Failed to initialize PSNRVideoFrameValidator.";
366 return nullptr;
367 }
368
369 return video_frame_validator;
370 }
371
PSNRVideoFrameValidator(const GetModelFrameCB & get_model_frame_cb,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,ValidationMode validation_mode,double tolerance)372 PSNRVideoFrameValidator::PSNRVideoFrameValidator(
373 const GetModelFrameCB& get_model_frame_cb,
374 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
375 ValidationMode validation_mode,
376 double tolerance)
377 : VideoFrameValidator(std::move(corrupt_frame_processor)),
378 get_model_frame_cb_(get_model_frame_cb),
379 tolerance_(tolerance),
380 validation_mode_(validation_mode) {}
381
382 PSNRVideoFrameValidator::~PSNRVideoFrameValidator() = default;
383
384 std::unique_ptr<VideoFrameValidator::MismatchedFrameInfo>
Validate(scoped_refptr<const VideoFrame> frame,size_t frame_index)385 PSNRVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
386 size_t frame_index) {
387 SEQUENCE_CHECKER(validator_thread_sequence_checker_);
388 auto model_frame = get_model_frame_cb_.Run(frame_index);
389 CHECK(model_frame);
390 double psnr = ComputePSNR(*frame, *model_frame);
391 DVLOGF(4) << "frame_index: " << frame_index << ", psnr: " << psnr;
392 psnr_[frame_index] = psnr;
393 if (psnr < tolerance_)
394 return std::make_unique<PSNRMismatchedFrameInfo>(frame_index, psnr);
395 return nullptr;
396 }
397
Passed() const398 bool PSNRVideoFrameValidator::Passed() const {
399 if (validation_mode_ == ValidationMode::kThreshold)
400 return GetMismatchedFramesCount() == 0u;
401 if (psnr_.empty())
402 return true;
403
404 double average = 0;
405 for (const auto& psnr : psnr_) {
406 average += psnr.second;
407 }
408 average /= psnr_.size();
409 if (average < tolerance_) {
410 LOG(ERROR) << "Average PSNR is too low: " << average;
411 return false;
412 }
413 return true;
414 }
415
416 struct SSIMVideoFrameValidator::SSIMMismatchedFrameInfo
417 : public VideoFrameValidator::MismatchedFrameInfo {
SSIMMismatchedFrameInfomedia::test::SSIMVideoFrameValidator::SSIMMismatchedFrameInfo418 SSIMMismatchedFrameInfo(size_t frame_index, double ssim)
419 : MismatchedFrameInfo(frame_index), ssim(ssim) {}
420 ~SSIMMismatchedFrameInfo() override = default;
Printmedia::test::SSIMVideoFrameValidator::SSIMMismatchedFrameInfo421 void Print() const override {
422 LOG(ERROR) << "frame_index: " << frame_index << ", ssim: " << ssim;
423 }
424
425 double ssim;
426 };
427
428 // static
Create(const GetModelFrameCB & get_model_frame_cb,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,ValidationMode validation_mode,double tolerance)429 std::unique_ptr<SSIMVideoFrameValidator> SSIMVideoFrameValidator::Create(
430 const GetModelFrameCB& get_model_frame_cb,
431 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
432 ValidationMode validation_mode,
433 double tolerance) {
434 auto video_frame_validator = base::WrapUnique(new SSIMVideoFrameValidator(
435 get_model_frame_cb, std::move(corrupt_frame_processor), validation_mode,
436 tolerance));
437 if (!video_frame_validator->Initialize()) {
438 LOG(ERROR) << "Failed to initialize SSIMVideoFrameValidator.";
439 return nullptr;
440 }
441
442 return video_frame_validator;
443 }
444
SSIMVideoFrameValidator(const GetModelFrameCB & get_model_frame_cb,std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,ValidationMode validation_mode,double tolerance)445 SSIMVideoFrameValidator::SSIMVideoFrameValidator(
446 const GetModelFrameCB& get_model_frame_cb,
447 std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
448 ValidationMode validation_mode,
449 double tolerance)
450 : VideoFrameValidator(std::move(corrupt_frame_processor)),
451 get_model_frame_cb_(get_model_frame_cb),
452 tolerance_(tolerance),
453 validation_mode_(validation_mode) {}
454
455 SSIMVideoFrameValidator::~SSIMVideoFrameValidator() = default;
456
457 std::unique_ptr<VideoFrameValidator::MismatchedFrameInfo>
Validate(scoped_refptr<const VideoFrame> frame,size_t frame_index)458 SSIMVideoFrameValidator::Validate(scoped_refptr<const VideoFrame> frame,
459 size_t frame_index) {
460 SEQUENCE_CHECKER(validator_thread_sequence_checker_);
461 auto model_frame = get_model_frame_cb_.Run(frame_index);
462 CHECK(model_frame);
463 double ssim = ComputeSSIM(*frame, *model_frame);
464 DVLOGF(4) << "frame_index: " << frame_index << ", ssim: " << ssim;
465 ssim_[frame_index] = ssim;
466 if (ssim < tolerance_)
467 return std::make_unique<SSIMMismatchedFrameInfo>(frame_index, ssim);
468 return nullptr;
469 }
470
Passed() const471 bool SSIMVideoFrameValidator::Passed() const {
472 if (validation_mode_ == ValidationMode::kThreshold)
473 return GetMismatchedFramesCount() == 0u;
474 if (ssim_.empty())
475 return true;
476
477 double average = 0;
478 for (const auto& ssim : ssim_) {
479 average += ssim.second;
480 }
481 average /= ssim_.size();
482 if (average < tolerance_) {
483 LOG(ERROR) << "Average SSIM is too low: " << average;
484 return false;
485 }
486 return true;
487 }
488 } // namespace test
489 } // namespace media
490