1 // Copyright 2020 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "media/gpu/test/video_encoder/decoder_buffer_validator.h"
6 
7 #include "base/logging.h"
8 #include "media/base/decoder_buffer.h"
9 #include "media/gpu/h264_decoder.h"
10 #include "testing/gtest/include/gtest/gtest.h"
11 
12 namespace media {
13 namespace test {
14 namespace {
VideoCodecProfileToH264ProfileIDC(VideoCodecProfile profile)15 int VideoCodecProfileToH264ProfileIDC(VideoCodecProfile profile) {
16   switch (profile) {
17     case H264PROFILE_BASELINE:
18       return H264SPS::kProfileIDCBaseline;
19     case H264PROFILE_MAIN:
20       return H264SPS::kProfileIDCMain;
21     case H264PROFILE_HIGH:
22       return H264SPS::kProfileIDCHigh;
23     default:
24       LOG(ERROR) << "Unexpected video profile: " << GetProfileName(profile);
25   }
26   return H264SPS::kProfileIDCMain;
27 }
28 
VideoCodecProfileToVP9Profile(VideoCodecProfile profile)29 int VideoCodecProfileToVP9Profile(VideoCodecProfile profile) {
30   switch (profile) {
31     case VP9PROFILE_PROFILE0:
32       return 0;
33     default:
34       LOG(ERROR) << "Unexpected video profile: " << GetProfileName(profile);
35   }
36   return 0;
37 }
38 }  // namespace
39 
DecoderBufferValidator(const gfx::Rect & visible_rect)40 DecoderBufferValidator::DecoderBufferValidator(const gfx::Rect& visible_rect)
41     : visible_rect_(visible_rect) {}
42 
43 DecoderBufferValidator::~DecoderBufferValidator() = default;
44 
ProcessBitstream(scoped_refptr<BitstreamRef> bitstream,size_t frame_index)45 void DecoderBufferValidator::ProcessBitstream(
46     scoped_refptr<BitstreamRef> bitstream,
47     size_t frame_index) {
48   if (!Validate(*bitstream->buffer, bitstream->metadata))
49     num_errors_++;
50 }
51 
WaitUntilDone()52 bool DecoderBufferValidator::WaitUntilDone() {
53   return num_errors_ == 0;
54 }
55 
TemporalLayerValidator(size_t num_temporal_layers)56 TemporalLayerValidator::TemporalLayerValidator(size_t num_temporal_layers)
57     : num_temporal_layers_(num_temporal_layers) {
58   reference_frames_.fill(0);
59 }
60 
61 TemporalLayerValidator::~TemporalLayerValidator() = default;
62 
ValidateAndUpdate(bool keyframe,uint8_t temporal_index,uint8_t reference_index,uint8_t refresh_frame_index)63 bool TemporalLayerValidator::ValidateAndUpdate(bool keyframe,
64                                                uint8_t temporal_index,
65                                                uint8_t reference_index,
66                                                uint8_t refresh_frame_index) {
67   if (temporal_index >= num_temporal_layers_) {
68     LOG(ERROR) << "Temporal layer index is not less than the number of temporal"
69                << " layers, temporal_index=" << temporal_index
70                << ", num_temporal_layers=" << num_temporal_layers_;
71     return false;
72   }
73   if (keyframe) {
74     if (temporal_index != 0) {
75       LOG(ERROR) << "Key frame exists in non base layer, temporal_index="
76                  << temporal_index;
77       return false;
78     }
79     reference_frames_.fill(temporal_index);
80     return true;
81   }
82 
83   const std::bitset<kReferenceFramePoolSize> reference(reference_index);
84   for (size_t i = 0; i < kReferenceFramePoolSize; ++i) {
85     if (!reference[i])
86       continue;
87     const uint8_t referenced_index = reference_frames_[i];
88     if (referenced_index > temporal_index) {
89       LOG(ERROR) << "Frame in upper layer referenced, temporal_index="
90                  << temporal_index
91                  << ", referenced temporal index=" << referenced_index;
92       return false;
93     }
94   }
95   const std::bitset<kReferenceFramePoolSize> refresh(refresh_frame_index);
96   for (size_t i = 0; i < kReferenceFramePoolSize; ++i) {
97     if (refresh[i])
98       reference_frames_[i] = temporal_index;
99   }
100   return true;
101 }
102 
H264Validator(VideoCodecProfile profile,const gfx::Rect & visible_rect,base::Optional<uint8_t> level)103 H264Validator::H264Validator(VideoCodecProfile profile,
104                              const gfx::Rect& visible_rect,
105                              base::Optional<uint8_t> level)
106     : DecoderBufferValidator(visible_rect),
107       cur_pic_(new H264Picture),
108       profile_(VideoCodecProfileToH264ProfileIDC(profile)),
109       level_(level) {}
110 
111 H264Validator::~H264Validator() = default;
112 
Validate(const DecoderBuffer & decoder_buffer,const BitstreamBufferMetadata & metadata)113 bool H264Validator::Validate(const DecoderBuffer& decoder_buffer,
114                              const BitstreamBufferMetadata& metadata) {
115   parser_.SetStream(decoder_buffer.data(), decoder_buffer.data_size());
116 
117   size_t num_frames = 0;
118   H264NALU nalu;
119   H264Parser::Result result;
120   while ((result = parser_.AdvanceToNextNALU(&nalu)) != H264Parser::kEOStream) {
121     if (result != H264Parser::kOk) {
122       LOG(ERROR) << "Failed parsing";
123       return false;
124     }
125 
126     switch (nalu.nal_unit_type) {
127       case H264NALU::kIDRSlice:
128         if (!seen_sps_ || !seen_pps_) {
129           LOG(ERROR) << "IDR frame before SPS and PPS";
130           return false;
131         }
132         seen_idr_ = true;
133         FALLTHROUGH;
134       case H264NALU::kNonIDRSlice: {
135         if (!seen_idr_) {
136           LOG(ERROR) << "Non IDR frame before IDR frame";
137           return false;
138         }
139 
140         H264SliceHeader slice_hdr;
141         if (parser_.ParseSliceHeader(nalu, &slice_hdr) != H264Parser::kOk) {
142           LOG(ERROR) << "Failed parsing slice";
143           return false;
144         }
145         // TODO(hiroh): Add more checks.
146         if (IsNewPicture(slice_hdr)) {
147           // A new frame is found. Initialize |cur_pic|.
148           num_frames++;
149           if (!UpdateCurrentPicture(slice_hdr))
150             return false;
151         }
152         break;
153       }
154       case H264NALU::kSPS: {
155         int sps_id;
156         if (parser_.ParseSPS(&sps_id) != H264Parser::kOk) {
157           LOG(ERROR) << "Failed parsing SPS";
158           return false;
159         }
160 
161         // Check the visible rect.
162         const H264SPS* sps = parser_.GetSPS(sps_id);
163         const auto& visible_rect = sps->GetVisibleRect().value_or(gfx::Rect());
164         if (visible_rect != visible_rect_) {
165           LOG(ERROR) << "Visible rectangle mismatched. Actual visible_rect: "
166                      << visible_rect.ToString()
167                      << ", expected visible_rect: " << visible_rect_.ToString();
168           return false;
169         }
170         if (profile_ != sps->profile_idc) {
171           LOG(ERROR) << "Profile mismatched. Actual profile: "
172                      << sps->profile_idc << ", expected profile: " << profile_;
173           return false;
174         }
175         if (level_ && sps->GetIndicatedLevel() != *level_) {
176           LOG(ERROR) << "Level mismatched. Actual profile: "
177                      << static_cast<int>(sps->GetIndicatedLevel())
178                      << ", expected profile: " << static_cast<int>(*level_);
179           return false;
180         }
181 
182         seen_sps_ = true;
183         break;
184       }
185       case H264NALU::kPPS: {
186         if (!seen_sps_) {
187           LOG(ERROR) << "PPS before SPS";
188           return false;
189         }
190         int pps_id;
191         if (parser_.ParsePPS(&pps_id) != H264Parser::kOk) {
192           LOG(ERROR) << "Failed parsing PPS";
193           return false;
194         }
195         seen_pps_ = true;
196         break;
197       }
198       default:
199         break;
200     }
201   }
202 
203   return num_frames == 1u;
204 }
205 
IsNewPicture(const H264SliceHeader & slice_hdr)206 bool H264Validator::IsNewPicture(const H264SliceHeader& slice_hdr) {
207   if (!cur_pic_)
208     return true;
209   return H264Decoder::IsNewPrimaryCodedPicture(
210       cur_pic_.get(), cur_pps_id_, parser_.GetSPS(cur_sps_id_), slice_hdr);
211 }
212 
UpdateCurrentPicture(const H264SliceHeader & slice_hdr)213 bool H264Validator::UpdateCurrentPicture(const H264SliceHeader& slice_hdr) {
214   cur_pps_id_ = slice_hdr.pic_parameter_set_id;
215   const H264PPS* pps = parser_.GetPPS(cur_pps_id_);
216   if (!pps) {
217     LOG(ERROR) << "Cannot parse pps.";
218     return false;
219   }
220 
221   cur_sps_id_ = pps->seq_parameter_set_id;
222   const H264SPS* sps = parser_.GetSPS(cur_sps_id_);
223   if (!sps) {
224     LOG(ERROR) << "Cannot parse sps.";
225     return false;
226   }
227 
228   if (!H264Decoder::FillH264PictureFromSliceHeader(sps, slice_hdr,
229                                                    cur_pic_.get())) {
230     LOG(ERROR) << "Cannot initialize current frame.";
231     return false;
232   }
233   return true;
234 }
235 
VP8Validator(const gfx::Rect & visible_rect)236 VP8Validator::VP8Validator(const gfx::Rect& visible_rect)
237     : DecoderBufferValidator(visible_rect) {}
238 
239 VP8Validator::~VP8Validator() = default;
240 
Validate(const DecoderBuffer & decoder_buffer,const BitstreamBufferMetadata & metadata)241 bool VP8Validator::Validate(const DecoderBuffer& decoder_buffer,
242                             const BitstreamBufferMetadata& metadata) {
243   // TODO(hiroh): We could be getting more frames in the buffer, but there is
244   // no simple way to detect this. We'd need to parse the frames and go through
245   // partition numbers/sizes. For now assume one frame per buffer.
246   Vp8FrameHeader header;
247   if (!parser_.ParseFrame(decoder_buffer.data(), decoder_buffer.data_size(),
248                           &header)) {
249     LOG(ERROR) << "Failed parsing";
250     return false;
251   }
252 
253   if (header.IsKeyframe()) {
254     seen_keyframe_ = true;
255     if (gfx::Rect(header.width, header.height) != visible_rect_) {
256       LOG(ERROR) << "Visible rectangle mismatched. Actual visible_rect: "
257                  << gfx::Rect(header.width, header.height).ToString()
258                  << ", expected visible_rect: " << visible_rect_.ToString();
259       return false;
260     }
261   }
262 
263   return seen_keyframe_ && header.show_frame;
264 }
265 
VP9Validator(VideoCodecProfile profile,const gfx::Rect & visible_rect,size_t num_temporal_layers)266 VP9Validator::VP9Validator(VideoCodecProfile profile,
267                            const gfx::Rect& visible_rect,
268                            size_t num_temporal_layers)
269     : DecoderBufferValidator(visible_rect),
270       parser_(false /* parsing_compressed_header */),
271       profile_(VideoCodecProfileToVP9Profile(profile)),
272       temporal_layer_validator_(
273           num_temporal_layers > 1u
274               ? std::make_unique<TemporalLayerValidator>(num_temporal_layers)
275               : nullptr) {}
276 
277 VP9Validator::~VP9Validator() = default;
278 
Validate(const DecoderBuffer & decoder_buffer,const BitstreamBufferMetadata & metadata)279 bool VP9Validator::Validate(const DecoderBuffer& decoder_buffer,
280                             const BitstreamBufferMetadata& metadata) {
281   // TODO(hiroh): We could be getting more frames in the buffer, but there is
282   // no simple way to detect this. We'd need to parse the frames and go through
283   // partition numbers/sizes. For now assume one frame per buffer.
284   Vp9FrameHeader header;
285   gfx::Size allocate_size;
286   parser_.SetStream(decoder_buffer.data(), decoder_buffer.data_size(), nullptr);
287   if (parser_.ParseNextFrame(&header, &allocate_size, nullptr) ==
288       Vp9Parser::kInvalidStream) {
289     LOG(ERROR) << "Failed parsing";
290     return false;
291   }
292   if (metadata.key_frame != header.IsKeyframe()) {
293     LOG(ERROR) << "Keyframe info in metadata is wrong, metadata.keyframe="
294                << metadata.key_frame;
295     return false;
296   }
297 
298   if (header.IsKeyframe()) {
299     seen_keyframe_ = true;
300     if (header.profile != static_cast<uint8_t>(profile_)) {
301       LOG(ERROR) << "Profile mismatched. Actual profile: "
302                  << static_cast<int>(header.profile)
303                  << ", expected profile: " << profile_;
304       return false;
305     }
306     if (gfx::Rect(header.render_width, header.render_height) != visible_rect_) {
307       LOG(ERROR)
308           << "Visible rectangle mismatched. Actual visible_rect: "
309           << gfx::Rect(header.render_width, header.render_height).ToString()
310           << ", expected visible_rect: " << visible_rect_.ToString();
311       return false;
312     }
313   }
314 
315   if (!seen_keyframe_) {
316     LOG(ERROR) << "First frame is not key frame";
317     return false;
318   }
319 
320   if (!header.show_frame) {
321     LOG(ERROR) << "VideoEncodeAccelerator outputs non showable frame";
322     return false;
323   }
324 
325   if (!temporal_layer_validator_)
326     return true;
327 
328   if (!metadata.vp9.has_value()) {
329     LOG(ERROR) << "No metadata in temporal layer encoding";
330     return false;
331   }
332   uint8_t reference_index = 0;
333   for (size_t i = 0; i < kVp9NumRefsPerFrame; ++i) {
334     uint8_t ref_frame_index = header.ref_frame_idx[i];
335     if (ref_frame_index >= static_cast<uint8_t>(kVp9NumRefFrames)) {
336       LOG(ERROR) << "Invalid reference frame index: "
337                  << static_cast<int>(ref_frame_index);
338       return false;
339     }
340     reference_index |= (1u << ref_frame_index);
341   }
342   return temporal_layer_validator_->ValidateAndUpdate(
343       header.IsKeyframe(), metadata.vp9->temporal_idx, reference_index,
344       header.refresh_frame_flags);
345 }
346 }  // namespace test
347 }  // namespace media
348