1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <algorithm>
6 #include <limits>
7
8 #include "base/bind.h"
9 #include "base/callback_helpers.h"
10 #include "base/feature_list.h"
11 #include "base/logging.h"
12 #include "base/numerics/safe_conversions.h"
13 #include "base/optional.h"
14 #include "base/stl_util.h"
15 #include "media/base/media_switches.h"
16 #include "media/gpu/h264_decoder.h"
17 #include "media/video/h264_level_limits.h"
18
19 namespace media {
20
21 H264Decoder::H264Accelerator::H264Accelerator() = default;
22
23 H264Decoder::H264Accelerator::~H264Accelerator() = default;
24
SetStream(base::span<const uint8_t> stream,const DecryptConfig * decrypt_config)25 H264Decoder::H264Accelerator::Status H264Decoder::H264Accelerator::SetStream(
26 base::span<const uint8_t> stream,
27 const DecryptConfig* decrypt_config) {
28 return H264Decoder::H264Accelerator::Status::kNotSupported;
29 }
30
H264Decoder(std::unique_ptr<H264Accelerator> accelerator,VideoCodecProfile profile,const VideoColorSpace & container_color_space)31 H264Decoder::H264Decoder(std::unique_ptr<H264Accelerator> accelerator,
32 VideoCodecProfile profile,
33 const VideoColorSpace& container_color_space)
34 : state_(kNeedStreamMetadata),
35 container_color_space_(container_color_space),
36 max_frame_num_(0),
37 max_pic_num_(0),
38 max_long_term_frame_idx_(0),
39 max_num_reorder_frames_(0),
40 // TODO(hiroh): Set profile to UNKNOWN.
41 profile_(profile),
42 accelerator_(std::move(accelerator)) {
43 DCHECK(accelerator_);
44 decoder_buffer_is_complete_frame_ =
45 base::FeatureList::IsEnabled(media::kH264DecoderBufferIsCompleteFrame);
46 Reset();
47 }
48
49 H264Decoder::~H264Decoder() = default;
50
Reset()51 void H264Decoder::Reset() {
52 curr_pic_ = nullptr;
53 curr_nalu_ = nullptr;
54 curr_slice_hdr_ = nullptr;
55 curr_sps_id_ = -1;
56 curr_pps_id_ = -1;
57
58 prev_frame_num_ = -1;
59 prev_ref_frame_num_ = -1;
60 prev_frame_num_offset_ = -1;
61 prev_has_memmgmnt5_ = false;
62
63 prev_ref_has_memmgmnt5_ = false;
64 prev_ref_top_field_order_cnt_ = -1;
65 prev_ref_pic_order_cnt_msb_ = -1;
66 prev_ref_pic_order_cnt_lsb_ = -1;
67 prev_ref_field_ = H264Picture::FIELD_NONE;
68
69 ref_pic_list_p0_.clear();
70 ref_pic_list_b0_.clear();
71 ref_pic_list_b1_.clear();
72 dpb_.Clear();
73 parser_.Reset();
74 accelerator_->Reset();
75 last_output_poc_ = std::numeric_limits<int>::min();
76
77 // If we are in kDecoding, we can resume without processing an SPS.
78 if (state_ == kDecoding)
79 state_ = kAfterReset;
80 }
81
PrepareRefPicLists(const H264SliceHeader * slice_hdr)82 void H264Decoder::PrepareRefPicLists(const H264SliceHeader* slice_hdr) {
83 ConstructReferencePicListsP(slice_hdr);
84 ConstructReferencePicListsB(slice_hdr);
85 }
86
ModifyReferencePicLists(const H264SliceHeader * slice_hdr,H264Picture::Vector * ref_pic_list0,H264Picture::Vector * ref_pic_list1)87 bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr,
88 H264Picture::Vector* ref_pic_list0,
89 H264Picture::Vector* ref_pic_list1) {
90 ref_pic_list0->clear();
91 ref_pic_list1->clear();
92
93 // Fill reference picture lists for B and S/SP slices.
94 if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) {
95 *ref_pic_list0 = ref_pic_list_p0_;
96 return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0);
97 } else if (slice_hdr->IsBSlice()) {
98 *ref_pic_list0 = ref_pic_list_b0_;
99 *ref_pic_list1 = ref_pic_list_b1_;
100 return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) &&
101 ModifyReferencePicList(slice_hdr, 1, ref_pic_list1);
102 }
103
104 return true;
105 }
106
DecodePicture()107 H264Decoder::H264Accelerator::Status H264Decoder::DecodePicture() {
108 DCHECK(curr_pic_.get());
109
110 return accelerator_->SubmitDecode(curr_pic_);
111 }
112
InitNonexistingPicture(scoped_refptr<H264Picture> pic,int frame_num)113 bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic,
114 int frame_num) {
115 pic->nonexisting = true;
116 pic->nal_ref_idc = 1;
117 pic->frame_num = pic->pic_num = frame_num;
118 pic->adaptive_ref_pic_marking_mode_flag = false;
119 pic->ref = true;
120 pic->long_term_reference_flag = false;
121 pic->field = H264Picture::FIELD_NONE;
122
123 return CalculatePicOrderCounts(pic);
124 }
125
InitCurrPicture(const H264SliceHeader * slice_hdr)126 bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) {
127 if (!FillH264PictureFromSliceHeader(parser_.GetSPS(curr_sps_id_), *slice_hdr,
128 curr_pic_.get())) {
129 return false;
130 }
131
132 if (!CalculatePicOrderCounts(curr_pic_))
133 return false;
134
135 curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag;
136 curr_pic_->adaptive_ref_pic_marking_mode_flag =
137 slice_hdr->adaptive_ref_pic_marking_mode_flag;
138
139 // If the slice header indicates we will have to perform reference marking
140 // process after this picture is decoded, store required data for that
141 // purpose.
142 if (slice_hdr->adaptive_ref_pic_marking_mode_flag) {
143 static_assert(sizeof(curr_pic_->ref_pic_marking) ==
144 sizeof(slice_hdr->ref_pic_marking),
145 "Array sizes of ref pic marking do not match.");
146 memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking,
147 sizeof(curr_pic_->ref_pic_marking));
148 }
149
150 curr_pic_->set_visible_rect(visible_rect_);
151 curr_pic_->set_bitstream_id(stream_id_);
152
153 return true;
154 }
155
CalculatePicOrderCounts(scoped_refptr<H264Picture> pic)156 bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) {
157 const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
158 if (!sps)
159 return false;
160
161 switch (pic->pic_order_cnt_type) {
162 case 0: {
163 // See spec 8.2.1.1.
164 int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb;
165
166 if (pic->idr) {
167 prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0;
168 } else {
169 if (prev_ref_has_memmgmnt5_) {
170 if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) {
171 prev_pic_order_cnt_msb = 0;
172 prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_;
173 } else {
174 prev_pic_order_cnt_msb = 0;
175 prev_pic_order_cnt_lsb = 0;
176 }
177 } else {
178 prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_;
179 prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_;
180 }
181 }
182
183 int max_pic_order_cnt_lsb =
184 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
185 DCHECK_NE(max_pic_order_cnt_lsb, 0);
186 if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) &&
187 (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >=
188 max_pic_order_cnt_lsb / 2)) {
189 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb;
190 } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) &&
191 (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb >
192 max_pic_order_cnt_lsb / 2)) {
193 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb;
194 } else {
195 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb;
196 }
197
198 if (pic->field != H264Picture::FIELD_BOTTOM) {
199 pic->top_field_order_cnt =
200 pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
201 }
202
203 if (pic->field != H264Picture::FIELD_TOP) {
204 if (pic->field == H264Picture::FIELD_NONE) {
205 pic->bottom_field_order_cnt =
206 pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom;
207 } else {
208 pic->bottom_field_order_cnt =
209 pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
210 }
211 }
212 break;
213 }
214
215 case 1: {
216 // See spec 8.2.1.2.
217 if (prev_has_memmgmnt5_)
218 prev_frame_num_offset_ = 0;
219
220 if (pic->idr)
221 pic->frame_num_offset = 0;
222 else if (prev_frame_num_ > pic->frame_num)
223 pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
224 else
225 pic->frame_num_offset = prev_frame_num_offset_;
226
227 int abs_frame_num = 0;
228 if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
229 abs_frame_num = pic->frame_num_offset + pic->frame_num;
230 else
231 abs_frame_num = 0;
232
233 if (pic->nal_ref_idc == 0 && abs_frame_num > 0)
234 --abs_frame_num;
235
236 int expected_pic_order_cnt = 0;
237 if (abs_frame_num > 0) {
238 if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) {
239 DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle "
240 << "in stream";
241 return false;
242 }
243
244 int pic_order_cnt_cycle_cnt =
245 (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle;
246 int frame_num_in_pic_order_cnt_cycle =
247 (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle;
248
249 expected_pic_order_cnt = pic_order_cnt_cycle_cnt *
250 sps->expected_delta_per_pic_order_cnt_cycle;
251 // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser
252 for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i)
253 expected_pic_order_cnt += sps->offset_for_ref_frame[i];
254 }
255
256 if (!pic->nal_ref_idc)
257 expected_pic_order_cnt += sps->offset_for_non_ref_pic;
258
259 if (pic->field == H264Picture::FIELD_NONE) {
260 pic->top_field_order_cnt =
261 expected_pic_order_cnt + pic->delta_pic_order_cnt0;
262 pic->bottom_field_order_cnt = pic->top_field_order_cnt +
263 sps->offset_for_top_to_bottom_field +
264 pic->delta_pic_order_cnt1;
265 } else if (pic->field != H264Picture::FIELD_BOTTOM) {
266 pic->top_field_order_cnt =
267 expected_pic_order_cnt + pic->delta_pic_order_cnt0;
268 } else {
269 pic->bottom_field_order_cnt = expected_pic_order_cnt +
270 sps->offset_for_top_to_bottom_field +
271 pic->delta_pic_order_cnt0;
272 }
273 break;
274 }
275
276 case 2: {
277 // See spec 8.2.1.3.
278 if (prev_has_memmgmnt5_)
279 prev_frame_num_offset_ = 0;
280
281 if (pic->idr)
282 pic->frame_num_offset = 0;
283 else if (prev_frame_num_ > pic->frame_num)
284 pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
285 else
286 pic->frame_num_offset = prev_frame_num_offset_;
287
288 int temp_pic_order_cnt;
289 if (pic->idr) {
290 temp_pic_order_cnt = 0;
291 } else if (!pic->nal_ref_idc) {
292 temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1;
293 } else {
294 temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num);
295 }
296
297 if (pic->field == H264Picture::FIELD_NONE) {
298 pic->top_field_order_cnt = temp_pic_order_cnt;
299 pic->bottom_field_order_cnt = temp_pic_order_cnt;
300 } else if (pic->field == H264Picture::FIELD_BOTTOM) {
301 pic->bottom_field_order_cnt = temp_pic_order_cnt;
302 } else {
303 pic->top_field_order_cnt = temp_pic_order_cnt;
304 }
305 break;
306 }
307
308 default:
309 DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type;
310 return false;
311 }
312
313 switch (pic->field) {
314 case H264Picture::FIELD_NONE:
315 pic->pic_order_cnt =
316 std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt);
317 break;
318 case H264Picture::FIELD_TOP:
319 pic->pic_order_cnt = pic->top_field_order_cnt;
320 break;
321 case H264Picture::FIELD_BOTTOM:
322 pic->pic_order_cnt = pic->bottom_field_order_cnt;
323 break;
324 }
325
326 return true;
327 }
328
UpdatePicNums(int frame_num)329 void H264Decoder::UpdatePicNums(int frame_num) {
330 for (auto& pic : dpb_) {
331 if (!pic->ref)
332 continue;
333
334 // 8.2.4.1. Assumes non-interlaced stream.
335 DCHECK_EQ(pic->field, H264Picture::FIELD_NONE);
336 if (pic->long_term) {
337 pic->long_term_pic_num = pic->long_term_frame_idx;
338 } else {
339 if (pic->frame_num > frame_num)
340 pic->frame_num_wrap = pic->frame_num - max_frame_num_;
341 else
342 pic->frame_num_wrap = pic->frame_num;
343
344 pic->pic_num = pic->frame_num_wrap;
345 }
346 }
347 }
348
349 struct PicNumDescCompare {
operator ()media::PicNumDescCompare350 bool operator()(const scoped_refptr<H264Picture>& a,
351 const scoped_refptr<H264Picture>& b) const {
352 return a->pic_num > b->pic_num;
353 }
354 };
355
356 struct LongTermPicNumAscCompare {
operator ()media::LongTermPicNumAscCompare357 bool operator()(const scoped_refptr<H264Picture>& a,
358 const scoped_refptr<H264Picture>& b) const {
359 return a->long_term_pic_num < b->long_term_pic_num;
360 }
361 };
362
ConstructReferencePicListsP(const H264SliceHeader * slice_hdr)363 void H264Decoder::ConstructReferencePicListsP(
364 const H264SliceHeader* slice_hdr) {
365 // RefPicList0 (8.2.4.2.1) [[1] [2]], where:
366 // [1] shortterm ref pics sorted by descending pic_num,
367 // [2] longterm ref pics by ascending long_term_pic_num.
368 ref_pic_list_p0_.clear();
369
370 // First get the short ref pics...
371 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_);
372 size_t num_short_refs = ref_pic_list_p0_.size();
373
374 // and sort them to get [1].
375 std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(),
376 PicNumDescCompare());
377
378 // Now get long term pics and sort them by long_term_pic_num to get [2].
379 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_);
380 std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(),
381 LongTermPicNumAscCompare());
382 }
383
384 struct POCAscCompare {
operator ()media::POCAscCompare385 bool operator()(const scoped_refptr<H264Picture>& a,
386 const scoped_refptr<H264Picture>& b) const {
387 return a->pic_order_cnt < b->pic_order_cnt;
388 }
389 };
390
391 struct POCDescCompare {
operator ()media::POCDescCompare392 bool operator()(const scoped_refptr<H264Picture>& a,
393 const scoped_refptr<H264Picture>& b) const {
394 return a->pic_order_cnt > b->pic_order_cnt;
395 }
396 };
397
ConstructReferencePicListsB(const H264SliceHeader * slice_hdr)398 void H264Decoder::ConstructReferencePicListsB(
399 const H264SliceHeader* slice_hdr) {
400 // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where:
401 // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC,
402 // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC,
403 // [3] longterm ref pics by ascending long_term_pic_num.
404 ref_pic_list_b0_.clear();
405 ref_pic_list_b1_.clear();
406 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_);
407 size_t num_short_refs = ref_pic_list_b0_.size();
408
409 // First sort ascending, this will put [1] in right place and finish [2].
410 std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare());
411
412 // Find first with POC > curr_pic's POC to get first element in [2]...
413 H264Picture::Vector::iterator iter;
414 iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
415 curr_pic_.get(), POCAscCompare());
416
417 // and sort [1] descending, thus finishing sequence [1] [2].
418 std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare());
419
420 // Now add [3] and sort by ascending long_term_pic_num.
421 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_);
422 std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(),
423 LongTermPicNumAscCompare());
424
425 // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where:
426 // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC,
427 // [2] shortterm ref pics with POC < curr_pic's POC by descending POC,
428 // [3] longterm ref pics by ascending long_term_pic_num.
429
430 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_);
431 num_short_refs = ref_pic_list_b1_.size();
432
433 // First sort by descending POC.
434 std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare());
435
436 // Find first with POC < curr_pic's POC to get first element in [2]...
437 iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(),
438 curr_pic_.get(), POCDescCompare());
439
440 // and sort [1] ascending.
441 std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare());
442
443 // Now add [3] and sort by ascending long_term_pic_num
444 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b1_);
445 std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(),
446 LongTermPicNumAscCompare());
447
448 // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3)
449 if (ref_pic_list_b1_.size() > 1 &&
450 std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
451 ref_pic_list_b1_.begin()))
452 std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]);
453 }
454
455 // See 8.2.4
PicNumF(const H264Picture & pic)456 int H264Decoder::PicNumF(const H264Picture& pic) {
457 if (!pic.long_term)
458 return pic.pic_num;
459 else
460 return max_pic_num_;
461 }
462
463 // See 8.2.4
LongTermPicNumF(const H264Picture & pic)464 int H264Decoder::LongTermPicNumF(const H264Picture& pic) {
465 if (pic.ref && pic.long_term)
466 return pic.long_term_pic_num;
467 else
468 return 2 * (max_long_term_frame_idx_ + 1);
469 }
470
471 // Shift elements on the |v| starting from |from| to |to|, inclusive,
472 // one position to the right and insert pic at |from|.
ShiftRightAndInsert(H264Picture::Vector * v,int from,int to,scoped_refptr<H264Picture> pic)473 static void ShiftRightAndInsert(H264Picture::Vector* v,
474 int from,
475 int to,
476 scoped_refptr<H264Picture> pic) {
477 // Security checks, do not disable in Debug mode.
478 CHECK(from <= to);
479 CHECK(to <= std::numeric_limits<int>::max() - 2);
480 // Additional checks. Debug mode ok.
481 DCHECK(v);
482 DCHECK(pic);
483 DCHECK((to + 1 == static_cast<int>(v->size())) ||
484 (to + 2 == static_cast<int>(v->size())));
485
486 v->resize(to + 2);
487
488 for (int i = to + 1; i > from; --i)
489 (*v)[i] = (*v)[i - 1];
490
491 (*v)[from] = std::move(pic);
492 }
493
ModifyReferencePicList(const H264SliceHeader * slice_hdr,int list,H264Picture::Vector * ref_pic_listx)494 bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr,
495 int list,
496 H264Picture::Vector* ref_pic_listx) {
497 bool ref_pic_list_modification_flag_lX;
498 int num_ref_idx_lX_active_minus1;
499 const H264ModificationOfPicNum* list_mod;
500
501 // This can process either ref_pic_list0 or ref_pic_list1, depending on
502 // the list argument. Set up pointers to proper list to be processed here.
503 if (list == 0) {
504 ref_pic_list_modification_flag_lX =
505 slice_hdr->ref_pic_list_modification_flag_l0;
506 num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1;
507 list_mod = slice_hdr->ref_list_l0_modifications;
508 } else {
509 ref_pic_list_modification_flag_lX =
510 slice_hdr->ref_pic_list_modification_flag_l1;
511 num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1;
512 list_mod = slice_hdr->ref_list_l1_modifications;
513 }
514
515 // Resize the list to the size requested in the slice header.
516 // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to
517 // indicate there should be more ref pics on list than we constructed.
518 // Those superfluous ones should be treated as non-reference and will be
519 // initialized to nullptr, which must be handled by clients.
520 DCHECK_GE(num_ref_idx_lX_active_minus1, 0);
521 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
522
523 if (!ref_pic_list_modification_flag_lX)
524 return true;
525
526 // Spec 8.2.4.3:
527 // Reorder pictures on the list in a way specified in the stream.
528 int pic_num_lx_pred = curr_pic_->pic_num;
529 int ref_idx_lx = 0;
530 int pic_num_lx_no_wrap;
531 int pic_num_lx;
532 bool done = false;
533 scoped_refptr<H264Picture> pic;
534 for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) {
535 switch (list_mod->modification_of_pic_nums_idc) {
536 case 0:
537 case 1:
538 // Modify short reference picture position.
539 if (list_mod->modification_of_pic_nums_idc == 0) {
540 // Subtract given value from predicted PicNum.
541 pic_num_lx_no_wrap =
542 pic_num_lx_pred -
543 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
544 // Wrap around max_pic_num_ if it becomes < 0 as result
545 // of subtraction.
546 if (pic_num_lx_no_wrap < 0)
547 pic_num_lx_no_wrap += max_pic_num_;
548 } else {
549 // Add given value to predicted PicNum.
550 pic_num_lx_no_wrap =
551 pic_num_lx_pred +
552 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
553 // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result
554 // of the addition.
555 if (pic_num_lx_no_wrap >= max_pic_num_)
556 pic_num_lx_no_wrap -= max_pic_num_;
557 }
558
559 // For use in next iteration.
560 pic_num_lx_pred = pic_num_lx_no_wrap;
561
562 if (pic_num_lx_no_wrap > curr_pic_->pic_num)
563 pic_num_lx = pic_num_lx_no_wrap - max_pic_num_;
564 else
565 pic_num_lx = pic_num_lx_no_wrap;
566
567 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
568 H264SliceHeader::kRefListModSize);
569 pic = dpb_.GetShortRefPicByPicNum(pic_num_lx);
570 if (!pic) {
571 DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx;
572 return false;
573 }
574
575 if (ref_idx_lx > num_ref_idx_lX_active_minus1) {
576 DVLOG(1) << "Bounds mismatch: expected " << ref_idx_lx
577 << " <= " << num_ref_idx_lX_active_minus1;
578 return false;
579 }
580
581 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
582 num_ref_idx_lX_active_minus1, pic);
583 ref_idx_lx++;
584
585 for (int src = ref_idx_lx, dst = ref_idx_lx;
586 src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
587 auto* src_pic = (*ref_pic_listx)[src].get();
588 int src_pic_num_lx = src_pic ? PicNumF(*src_pic) : -1;
589 if (src_pic_num_lx != pic_num_lx)
590 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
591 }
592 break;
593
594 case 2:
595 // Modify long term reference picture position.
596 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
597 H264SliceHeader::kRefListModSize);
598 pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num);
599 if (!pic) {
600 DVLOG(1) << "Malformed stream, no pic num "
601 << list_mod->long_term_pic_num;
602 return false;
603 }
604 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
605 num_ref_idx_lX_active_minus1, pic);
606 ref_idx_lx++;
607
608 for (int src = ref_idx_lx, dst = ref_idx_lx;
609 src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
610 if (LongTermPicNumF(*(*ref_pic_listx)[src]) !=
611 static_cast<int>(list_mod->long_term_pic_num))
612 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
613 }
614 break;
615
616 case 3:
617 // End of modification list.
618 done = true;
619 break;
620
621 default:
622 // May be recoverable.
623 DVLOG(1) << "Invalid modification_of_pic_nums_idc="
624 << list_mod->modification_of_pic_nums_idc << " in position "
625 << i;
626 break;
627 }
628
629 ++list_mod;
630 }
631
632 // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is
633 // temporarily made one element longer than the required final list.
634 // Resize the list back to its required size.
635 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
636
637 return true;
638 }
639
OutputPic(scoped_refptr<H264Picture> pic)640 bool H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) {
641 DCHECK(!pic->outputted);
642 pic->outputted = true;
643
644 VideoColorSpace colorspace_for_frame = container_color_space_;
645 const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
646 if (sps && sps->GetColorSpace().IsSpecified())
647 colorspace_for_frame = sps->GetColorSpace();
648 pic->set_colorspace(colorspace_for_frame);
649
650 if (pic->nonexisting) {
651 DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num;
652 return true;
653 }
654
655 DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_)
656 << "Outputting out of order, likely a broken stream: " << last_output_poc_
657 << " -> " << pic->pic_order_cnt;
658 last_output_poc_ = pic->pic_order_cnt;
659
660 DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt;
661 return accelerator_->OutputPicture(pic);
662 }
663
ClearDPB()664 void H264Decoder::ClearDPB() {
665 // Clear DPB contents, marking the pictures as unused first.
666 dpb_.Clear();
667 last_output_poc_ = std::numeric_limits<int>::min();
668 }
669
OutputAllRemainingPics()670 bool H264Decoder::OutputAllRemainingPics() {
671 // Output all pictures that are waiting to be outputted.
672 if (FinishPrevFrameIfPresent() != H264Accelerator::Status::kOk)
673 return false;
674 H264Picture::Vector to_output;
675 dpb_.GetNotOutputtedPicsAppending(&to_output);
676 // Sort them by ascending POC to output in order.
677 std::sort(to_output.begin(), to_output.end(), POCAscCompare());
678
679 for (auto& pic : to_output) {
680 if (!OutputPic(pic))
681 return false;
682 }
683 return true;
684 }
685
Flush()686 bool H264Decoder::Flush() {
687 DVLOG(2) << "Decoder flush";
688
689 if (!OutputAllRemainingPics())
690 return false;
691
692 ClearDPB();
693 DVLOG(2) << "Decoder flush finished";
694 return true;
695 }
696
StartNewFrame(const H264SliceHeader * slice_hdr)697 H264Decoder::H264Accelerator::Status H264Decoder::StartNewFrame(
698 const H264SliceHeader* slice_hdr) {
699 // TODO posciak: add handling of max_num_ref_frames per spec.
700 CHECK(curr_pic_.get());
701 DCHECK(slice_hdr);
702
703 curr_pps_id_ = slice_hdr->pic_parameter_set_id;
704 const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
705 if (!pps)
706 return H264Accelerator::Status::kFail;
707
708 curr_sps_id_ = pps->seq_parameter_set_id;
709 const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
710 if (!sps)
711 return H264Accelerator::Status::kFail;
712
713 max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4);
714 int frame_num = slice_hdr->frame_num;
715 if (slice_hdr->idr_pic_flag)
716 prev_ref_frame_num_ = 0;
717
718 // 7.4.3
719 if (frame_num != prev_ref_frame_num_ &&
720 frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) {
721 if (!HandleFrameNumGap(frame_num))
722 return H264Accelerator::Status::kFail;
723 }
724
725 if (!InitCurrPicture(slice_hdr))
726 return H264Accelerator::Status::kFail;
727
728 UpdatePicNums(frame_num);
729 PrepareRefPicLists(slice_hdr);
730
731 return accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_,
732 ref_pic_list_b0_, ref_pic_list_b1_,
733 curr_pic_.get());
734 }
735
HandleMemoryManagementOps(scoped_refptr<H264Picture> pic)736 bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) {
737 // 8.2.5.4
738 for (size_t i = 0; i < base::size(pic->ref_pic_marking); ++i) {
739 // Code below does not support interlaced stream (per-field pictures).
740 H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i];
741 scoped_refptr<H264Picture> to_mark;
742 int pic_num_x;
743
744 switch (ref_pic_marking->memory_mgmnt_control_operation) {
745 case 0:
746 // Normal end of operations' specification.
747 return true;
748
749 case 1:
750 // Mark a short term reference picture as unused so it can be removed
751 // if outputted.
752 pic_num_x =
753 pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
754 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
755 if (to_mark) {
756 to_mark->ref = false;
757 } else {
758 DVLOG(1) << "Invalid short ref pic num to unmark";
759 return false;
760 }
761 break;
762
763 case 2:
764 // Mark a long term reference picture as unused so it can be removed
765 // if outputted.
766 to_mark = dpb_.GetLongRefPicByLongTermPicNum(
767 ref_pic_marking->long_term_pic_num);
768 if (to_mark) {
769 to_mark->ref = false;
770 } else {
771 DVLOG(1) << "Invalid long term ref pic num to unmark";
772 return false;
773 }
774 break;
775
776 case 3:
777 // Mark a short term reference picture as long term reference.
778 pic_num_x =
779 pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
780 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
781 if (to_mark) {
782 DCHECK(to_mark->ref && !to_mark->long_term);
783 to_mark->long_term = true;
784 to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
785 } else {
786 DVLOG(1) << "Invalid short term ref pic num to mark as long ref";
787 return false;
788 }
789 break;
790
791 case 4: {
792 // Unmark all reference pictures with long_term_frame_idx over new max.
793 max_long_term_frame_idx_ =
794 ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
795 H264Picture::Vector long_terms;
796 dpb_.GetLongTermRefPicsAppending(&long_terms);
797 for (size_t i = 0; i < long_terms.size(); ++i) {
798 scoped_refptr<H264Picture>& long_term_pic = long_terms[i];
799 DCHECK(long_term_pic->ref && long_term_pic->long_term);
800 // Ok to cast, max_long_term_frame_idx is much smaller than 16bit.
801 if (long_term_pic->long_term_frame_idx >
802 static_cast<int>(max_long_term_frame_idx_))
803 long_term_pic->ref = false;
804 }
805 break;
806 }
807
808 case 5:
809 // Unmark all reference pictures.
810 dpb_.MarkAllUnusedForRef();
811 max_long_term_frame_idx_ = -1;
812 pic->mem_mgmt_5 = true;
813 break;
814
815 case 6: {
816 // Replace long term reference pictures with current picture.
817 // First unmark if any existing with this long_term_frame_idx...
818 H264Picture::Vector long_terms;
819 dpb_.GetLongTermRefPicsAppending(&long_terms);
820 for (size_t i = 0; i < long_terms.size(); ++i) {
821 scoped_refptr<H264Picture>& long_term_pic = long_terms[i];
822 DCHECK(long_term_pic->ref && long_term_pic->long_term);
823 // Ok to cast, long_term_frame_idx is much smaller than 16bit.
824 if (long_term_pic->long_term_frame_idx ==
825 static_cast<int>(ref_pic_marking->long_term_frame_idx))
826 long_term_pic->ref = false;
827 }
828
829 // and mark the current one instead.
830 pic->ref = true;
831 pic->long_term = true;
832 pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
833 break;
834 }
835
836 default:
837 // Would indicate a bug in parser.
838 NOTREACHED();
839 }
840 }
841
842 return true;
843 }
844
845 // This method ensures that DPB does not overflow, either by removing
846 // reference pictures as specified in the stream, or using a sliding window
847 // procedure to remove the oldest one.
848 // It also performs marking and unmarking pictures as reference.
849 // See spac 8.2.5.1.
ReferencePictureMarking(scoped_refptr<H264Picture> pic)850 bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) {
851 // If the current picture is an IDR, all reference pictures are unmarked.
852 if (pic->idr) {
853 dpb_.MarkAllUnusedForRef();
854
855 if (pic->long_term_reference_flag) {
856 pic->long_term = true;
857 pic->long_term_frame_idx = 0;
858 max_long_term_frame_idx_ = 0;
859 } else {
860 pic->long_term = false;
861 max_long_term_frame_idx_ = -1;
862 }
863
864 return true;
865 }
866
867 // Not an IDR. If the stream contains instructions on how to discard pictures
868 // from DPB and how to mark/unmark existing reference pictures, do so.
869 // Otherwise, fall back to default sliding window process.
870 if (pic->adaptive_ref_pic_marking_mode_flag) {
871 DCHECK(!pic->nonexisting);
872 return HandleMemoryManagementOps(pic);
873 } else {
874 return SlidingWindowPictureMarking();
875 }
876 }
877
SlidingWindowPictureMarking()878 bool H264Decoder::SlidingWindowPictureMarking() {
879 const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
880 if (!sps)
881 return false;
882
883 // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture.
884 int num_ref_pics = dpb_.CountRefPics();
885 DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1));
886 if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) {
887 // Max number of reference pics reached, need to remove one of the short
888 // term ones. Find smallest frame_num_wrap short reference picture and mark
889 // it as unused.
890 scoped_refptr<H264Picture> to_unmark =
891 dpb_.GetLowestFrameNumWrapShortRefPic();
892 if (!to_unmark) {
893 DVLOG(1) << "Couldn't find a short ref picture to unmark";
894 return false;
895 }
896
897 to_unmark->ref = false;
898 }
899
900 return true;
901 }
902
FinishPicture(scoped_refptr<H264Picture> pic)903 bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) {
904 // Finish processing the picture.
905 // Start by storing previous picture data for later use.
906 if (pic->ref) {
907 ReferencePictureMarking(pic);
908 prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5;
909 prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt;
910 prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb;
911 prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb;
912 prev_ref_field_ = pic->field;
913 prev_ref_frame_num_ = pic->frame_num;
914 }
915 prev_frame_num_ = pic->frame_num;
916 prev_has_memmgmnt5_ = pic->mem_mgmt_5;
917 prev_frame_num_offset_ = pic->frame_num_offset;
918
919 // Remove unused (for reference or later output) pictures from DPB, marking
920 // them as such.
921 dpb_.DeleteUnused();
922
923 DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num
924 << ", entries in DPB: " << dpb_.size();
925
926 // The ownership of pic will either be transferred to DPB - if the picture is
927 // still needed (for output and/or reference) - or we will release it
928 // immediately if we manage to output it here and won't have to store it for
929 // future reference.
930
931 // Get all pictures that haven't been outputted yet.
932 H264Picture::Vector not_outputted;
933 dpb_.GetNotOutputtedPicsAppending(¬_outputted);
934 // Include the one we've just decoded.
935 not_outputted.push_back(pic);
936
937 // Sort in output order.
938 std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare());
939
940 // Try to output as many pictures as we can. A picture can be output,
941 // if the number of decoded and not yet outputted pictures that would remain
942 // in DPB afterwards would at least be equal to max_num_reorder_frames.
943 // If the outputted picture is not a reference picture, it doesn't have
944 // to remain in the DPB and can be removed.
945 auto output_candidate = not_outputted.begin();
946 size_t num_remaining = not_outputted.size();
947 while (num_remaining > max_num_reorder_frames_ ||
948 // If the condition below is used, this is an invalid stream. We should
949 // not be forced to output beyond max_num_reorder_frames in order to
950 // make room in DPB to store the current picture (if we need to do so).
951 // However, if this happens, ignore max_num_reorder_frames and try
952 // to output more. This may cause out-of-order output, but is not
953 // fatal, and better than failing instead.
954 ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) {
955 DVLOG_IF(1, num_remaining <= max_num_reorder_frames_)
956 << "Invalid stream: max_num_reorder_frames not preserved";
957
958 if (!OutputPic(*output_candidate))
959 return false;
960
961 if (!(*output_candidate)->ref) {
962 // Current picture hasn't been inserted into DPB yet, so don't remove it
963 // if we managed to output it immediately.
964 int outputted_poc = (*output_candidate)->pic_order_cnt;
965 if (outputted_poc != pic->pic_order_cnt)
966 dpb_.DeleteByPOC(outputted_poc);
967 }
968
969 ++output_candidate;
970 --num_remaining;
971 }
972
973 // If we haven't managed to output the picture that we just decoded, or if
974 // it's a reference picture, we have to store it in DPB.
975 if (!pic->outputted || pic->ref) {
976 if (dpb_.IsFull()) {
977 // If we haven't managed to output anything to free up space in DPB
978 // to store this picture, it's an error in the stream.
979 DVLOG(1) << "Could not free up space in DPB!";
980 return false;
981 }
982
983 dpb_.StorePic(std::move(pic));
984 }
985
986 return true;
987 }
988
UpdateMaxNumReorderFrames(const H264SPS * sps)989 bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) {
990 if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) {
991 max_num_reorder_frames_ =
992 base::checked_cast<size_t>(sps->max_num_reorder_frames);
993 if (max_num_reorder_frames_ > dpb_.max_num_pics()) {
994 DVLOG(1)
995 << "max_num_reorder_frames present, but larger than MaxDpbFrames ("
996 << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")";
997 max_num_reorder_frames_ = 0;
998 return false;
999 }
1000 return true;
1001 }
1002
1003 // max_num_reorder_frames not present, infer from profile/constraints
1004 // (see VUI semantics in spec).
1005 if (sps->constraint_set3_flag) {
1006 switch (sps->profile_idc) {
1007 case 44:
1008 case 86:
1009 case 100:
1010 case 110:
1011 case 122:
1012 case 244:
1013 max_num_reorder_frames_ = 0;
1014 break;
1015 default:
1016 max_num_reorder_frames_ = dpb_.max_num_pics();
1017 break;
1018 }
1019 } else {
1020 max_num_reorder_frames_ = dpb_.max_num_pics();
1021 }
1022
1023 return true;
1024 }
1025
ProcessSPS(int sps_id,bool * need_new_buffers)1026 bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) {
1027 DVLOG(4) << "Processing SPS id:" << sps_id;
1028
1029 const H264SPS* sps = parser_.GetSPS(sps_id);
1030 if (!sps)
1031 return false;
1032
1033 *need_new_buffers = false;
1034
1035 if (sps->frame_mbs_only_flag == 0) {
1036 DVLOG(1) << "frame_mbs_only_flag != 1 not supported";
1037 return false;
1038 }
1039
1040 gfx::Size new_pic_size = sps->GetCodedSize().value_or(gfx::Size());
1041 if (new_pic_size.IsEmpty()) {
1042 DVLOG(1) << "Invalid picture size";
1043 return false;
1044 }
1045
1046 int width_mb = new_pic_size.width() / 16;
1047 int height_mb = new_pic_size.height() / 16;
1048
1049 // Verify that the values are not too large before multiplying.
1050 if (std::numeric_limits<int>::max() / width_mb < height_mb) {
1051 DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString();
1052 return false;
1053 }
1054
1055 // Spec A.3.1 and A.3.2
1056 // For Baseline, Constrained Baseline and Main profile, the indicated level is
1057 // Level 1b if level_idc is equal to 11 and constraint_set3_flag is equal to 1
1058 uint8_t level = base::checked_cast<uint8_t>(sps->level_idc);
1059 if ((sps->profile_idc == H264SPS::kProfileIDCBaseline ||
1060 sps->profile_idc == H264SPS::kProfileIDCConstrainedBaseline ||
1061 sps->profile_idc == H264SPS::kProfileIDCMain) &&
1062 level == 11 && sps->constraint_set3_flag) {
1063 level = 9; // Level 1b
1064 }
1065 int max_dpb_mbs = base::checked_cast<int>(H264LevelToMaxDpbMbs(level));
1066 if (max_dpb_mbs == 0)
1067 return false;
1068
1069 // MaxDpbFrames from level limits per spec.
1070 size_t max_dpb_frames = std::min(max_dpb_mbs / (width_mb * height_mb),
1071 static_cast<int>(H264DPB::kDPBMaxSize));
1072 DVLOG(1) << "MaxDpbFrames: " << max_dpb_frames
1073 << ", max_num_ref_frames: " << sps->max_num_ref_frames
1074 << ", max_dec_frame_buffering: " << sps->max_dec_frame_buffering;
1075
1076 // Set DPB size to at least the level limit, or what the stream requires.
1077 size_t max_dpb_size =
1078 std::max(static_cast<int>(max_dpb_frames),
1079 std::max(sps->max_num_ref_frames, sps->max_dec_frame_buffering));
1080 // Some non-conforming streams specify more frames are needed than the current
1081 // level limit. Allow this, but only up to the maximum number of reference
1082 // frames allowed per spec.
1083 DVLOG_IF(1, max_dpb_size > max_dpb_frames)
1084 << "Invalid stream, DPB size > MaxDpbFrames";
1085 if (max_dpb_size == 0 || max_dpb_size > H264DPB::kDPBMaxSize) {
1086 DVLOG(1) << "Invalid DPB size: " << max_dpb_size;
1087 return false;
1088 }
1089
1090 VideoCodecProfile new_profile =
1091 H264Parser::ProfileIDCToVideoCodecProfile(sps->profile_idc);
1092 if (pic_size_ != new_pic_size || dpb_.max_num_pics() != max_dpb_size ||
1093 profile_ != new_profile) {
1094 if (!Flush())
1095 return false;
1096 DVLOG(1) << "Codec profile: " << GetProfileName(new_profile)
1097 << ", level: " << level << ", DPB size: " << max_dpb_size
1098 << ", Picture size: " << new_pic_size.ToString();
1099 *need_new_buffers = true;
1100 profile_ = new_profile;
1101 pic_size_ = new_pic_size;
1102 dpb_.set_max_num_pics(max_dpb_size);
1103 }
1104
1105 gfx::Rect new_visible_rect = sps->GetVisibleRect().value_or(gfx::Rect());
1106 if (visible_rect_ != new_visible_rect) {
1107 DVLOG(2) << "New visible rect: " << new_visible_rect.ToString();
1108 visible_rect_ = new_visible_rect;
1109 }
1110
1111 if (!UpdateMaxNumReorderFrames(sps))
1112 return false;
1113 DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_;
1114
1115 return true;
1116 }
1117
FinishPrevFrameIfPresent()1118 H264Decoder::H264Accelerator::Status H264Decoder::FinishPrevFrameIfPresent() {
1119 // If we already have a frame waiting to be decoded, decode it and finish.
1120 if (curr_pic_) {
1121 H264Accelerator::Status result = DecodePicture();
1122 if (result != H264Accelerator::Status::kOk)
1123 return result;
1124
1125 scoped_refptr<H264Picture> pic = curr_pic_;
1126 curr_pic_ = nullptr;
1127 if (!FinishPicture(pic))
1128 return H264Accelerator::Status::kFail;
1129 }
1130
1131 return H264Accelerator::Status::kOk;
1132 }
1133
HandleFrameNumGap(int frame_num)1134 bool H264Decoder::HandleFrameNumGap(int frame_num) {
1135 const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
1136 if (!sps)
1137 return false;
1138
1139 if (!sps->gaps_in_frame_num_value_allowed_flag) {
1140 DVLOG(1) << "Invalid frame_num: " << frame_num;
1141 // TODO(b:129119729, b:146914440): Youtube android app sometimes sends an
1142 // invalid frame number after a seek. The sequence goes like:
1143 // Seek, SPS, PPS, IDR-frame, non-IDR, ... non-IDR with invalid number.
1144 // The only way to work around this reliably is to ignore this error.
1145 // Video playback is not affected, no artefacts are visible.
1146 // return false;
1147 }
1148
1149 DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->"
1150 << frame_num;
1151
1152 // 7.4.3/7-23
1153 int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_;
1154 while (unused_short_term_frame_num != frame_num) {
1155 scoped_refptr<H264Picture> pic = new H264Picture();
1156 if (!InitNonexistingPicture(pic, unused_short_term_frame_num))
1157 return false;
1158
1159 UpdatePicNums(unused_short_term_frame_num);
1160
1161 if (!FinishPicture(pic))
1162 return false;
1163
1164 unused_short_term_frame_num++;
1165 unused_short_term_frame_num %= max_frame_num_;
1166 }
1167
1168 return true;
1169 }
1170
PreprocessCurrentSlice()1171 H264Decoder::H264Accelerator::Status H264Decoder::PreprocessCurrentSlice() {
1172 const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
1173 DCHECK(slice_hdr);
1174
1175 if (IsNewPrimaryCodedPicture(curr_pic_.get(), curr_pps_id_,
1176 parser_.GetSPS(curr_sps_id_), *slice_hdr)) {
1177 // New picture, so first finish the previous one before processing it.
1178 H264Accelerator::Status result = FinishPrevFrameIfPresent();
1179 if (result != H264Accelerator::Status::kOk)
1180 return result;
1181
1182 DCHECK(!curr_pic_);
1183
1184 if (slice_hdr->first_mb_in_slice != 0) {
1185 DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: "
1186 << slice_hdr->first_mb_in_slice;
1187 return H264Accelerator::Status::kFail;
1188 }
1189
1190 // If the new picture is an IDR, flush DPB.
1191 if (slice_hdr->idr_pic_flag) {
1192 // Output all remaining pictures, unless we are explicitly instructed
1193 // not to do so.
1194 if (!slice_hdr->no_output_of_prior_pics_flag) {
1195 if (!Flush())
1196 return H264Accelerator::Status::kFail;
1197 }
1198 dpb_.Clear();
1199 last_output_poc_ = std::numeric_limits<int>::min();
1200 }
1201 }
1202
1203 return H264Accelerator::Status::kOk;
1204 }
1205
ProcessCurrentSlice()1206 H264Decoder::H264Accelerator::Status H264Decoder::ProcessCurrentSlice() {
1207 DCHECK(curr_pic_);
1208
1209 const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
1210 DCHECK(slice_hdr);
1211
1212 if (slice_hdr->field_pic_flag == 0)
1213 max_pic_num_ = max_frame_num_;
1214 else
1215 max_pic_num_ = 2 * max_frame_num_;
1216
1217 H264Picture::Vector ref_pic_list0, ref_pic_list1;
1218 if (!ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1))
1219 return H264Accelerator::Status::kFail;
1220
1221 const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
1222 if (!pps)
1223 return H264Accelerator::Status::kFail;
1224
1225 return accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1,
1226 curr_pic_.get(), slice_hdr->nalu_data,
1227 slice_hdr->nalu_size,
1228 parser_.GetCurrentSubsamples());
1229 }
1230
1231 #define SET_ERROR_AND_RETURN() \
1232 do { \
1233 DVLOG(1) << "Error during decode"; \
1234 state_ = kError; \
1235 return H264Decoder::kDecodeError; \
1236 } while (0)
1237
1238 #define CHECK_ACCELERATOR_RESULT(func) \
1239 do { \
1240 H264Accelerator::Status result = (func); \
1241 switch (result) { \
1242 case H264Accelerator::Status::kOk: \
1243 break; \
1244 case H264Accelerator::Status::kTryAgain: \
1245 DVLOG(1) << #func " needs to try again"; \
1246 return H264Decoder::kTryAgain; \
1247 case H264Accelerator::Status::kFail: \
1248 case H264Accelerator::Status::kNotSupported: \
1249 SET_ERROR_AND_RETURN(); \
1250 } \
1251 } while (0)
1252
SetStream(int32_t id,const DecoderBuffer & decoder_buffer)1253 void H264Decoder::SetStream(int32_t id, const DecoderBuffer& decoder_buffer) {
1254 const uint8_t* ptr = decoder_buffer.data();
1255 const size_t size = decoder_buffer.data_size();
1256 const DecryptConfig* decrypt_config = decoder_buffer.decrypt_config();
1257
1258 DCHECK(ptr);
1259 DCHECK(size);
1260 DVLOG(4) << "New input stream id: " << id << " at: " << (void*)ptr
1261 << " size: " << size;
1262 stream_id_ = id;
1263 current_stream_ = ptr;
1264 current_stream_size_ = size;
1265 current_stream_has_been_changed_ = true;
1266 if (decrypt_config) {
1267 parser_.SetEncryptedStream(ptr, size, decrypt_config->subsamples());
1268 current_decrypt_config_ = decrypt_config->Clone();
1269 } else {
1270 parser_.SetStream(ptr, size);
1271 current_decrypt_config_ = nullptr;
1272 }
1273 }
1274
Decode()1275 H264Decoder::DecodeResult H264Decoder::Decode() {
1276 if (state_ == kError) {
1277 DVLOG(1) << "Decoder in error state";
1278 return kDecodeError;
1279 }
1280
1281 if (current_stream_has_been_changed_) {
1282 // Calling H264Accelerator::SetStream() here instead of when the stream is
1283 // originally set in case the accelerator needs to return kTryAgain.
1284 H264Accelerator::Status result = accelerator_->SetStream(
1285 base::span<const uint8_t>(current_stream_, current_stream_size_),
1286 current_decrypt_config_.get());
1287 switch (result) {
1288 case H264Accelerator::Status::kOk:
1289 case H264Accelerator::Status::kNotSupported:
1290 // kNotSupported means the accelerator can't handle this stream,
1291 // so everything will be done through the parser.
1292 break;
1293 case H264Accelerator::Status::kTryAgain:
1294 DVLOG(1) << "SetStream() needs to try again";
1295 return H264Decoder::kTryAgain;
1296 case H264Accelerator::Status::kFail:
1297 SET_ERROR_AND_RETURN();
1298 }
1299
1300 // Reset the flag so that this is only called again next time SetStream()
1301 // is called.
1302 current_stream_has_been_changed_ = false;
1303 }
1304
1305 while (1) {
1306 H264Parser::Result par_res;
1307
1308 if (!curr_nalu_) {
1309 curr_nalu_.reset(new H264NALU());
1310 par_res = parser_.AdvanceToNextNALU(curr_nalu_.get());
1311 if (par_res == H264Parser::kEOStream) {
1312 if (decoder_buffer_is_complete_frame_)
1313 CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
1314
1315 return kRanOutOfStreamData;
1316 } else if (par_res != H264Parser::kOk) {
1317 SET_ERROR_AND_RETURN();
1318 }
1319
1320 DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type);
1321 }
1322
1323 switch (curr_nalu_->nal_unit_type) {
1324 case H264NALU::kNonIDRSlice:
1325 // We can't resume from a non-IDR slice.
1326 if (state_ == kError || state_ == kAfterReset)
1327 break;
1328
1329 FALLTHROUGH;
1330 case H264NALU::kIDRSlice: {
1331 // TODO(posciak): the IDR may require an SPS that we don't have
1332 // available. For now we'd fail if that happens, but ideally we'd like
1333 // to keep going until the next SPS in the stream.
1334 if (state_ == kNeedStreamMetadata) {
1335 // We need an SPS, skip this IDR and keep looking.
1336 break;
1337 }
1338
1339 // If after reset or waiting for a key, we should be able to recover
1340 // from an IDR. |state_|, |curr_slice_hdr_|, and |curr_pic_| are used
1341 // to keep track of what has previously been attempted, so that after
1342 // a retryable result is returned, subsequent calls to Decode() retry
1343 // the call that failed previously. If it succeeds (it may not if no
1344 // additional key has been provided, for example), then the remaining
1345 // steps will be executed.
1346 if (!curr_slice_hdr_) {
1347 curr_slice_hdr_.reset(new H264SliceHeader());
1348 par_res =
1349 parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get());
1350 if (par_res != H264Parser::kOk)
1351 SET_ERROR_AND_RETURN();
1352 state_ = kTryPreprocessCurrentSlice;
1353 }
1354
1355 if (state_ == kTryPreprocessCurrentSlice) {
1356 CHECK_ACCELERATOR_RESULT(PreprocessCurrentSlice());
1357 state_ = kEnsurePicture;
1358 }
1359
1360 if (state_ == kEnsurePicture) {
1361 if (curr_pic_) {
1362 // |curr_pic_| already exists, so skip to ProcessCurrentSlice().
1363 state_ = kTryCurrentSlice;
1364 } else {
1365 // New picture/finished previous one, try to start a new one
1366 // or tell the client we need more surfaces.
1367 curr_pic_ = accelerator_->CreateH264Picture();
1368 if (!curr_pic_)
1369 return kRanOutOfSurfaces;
1370 if (current_decrypt_config_)
1371 curr_pic_->set_decrypt_config(current_decrypt_config_->Clone());
1372
1373 state_ = kTryNewFrame;
1374 }
1375 }
1376
1377 if (state_ == kTryNewFrame) {
1378 CHECK_ACCELERATOR_RESULT(StartNewFrame(curr_slice_hdr_.get()));
1379 state_ = kTryCurrentSlice;
1380 }
1381
1382 DCHECK_EQ(state_, kTryCurrentSlice);
1383 CHECK_ACCELERATOR_RESULT(ProcessCurrentSlice());
1384 curr_slice_hdr_.reset();
1385 state_ = kDecoding;
1386 break;
1387 }
1388
1389 case H264NALU::kSPS: {
1390 int sps_id;
1391
1392 CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
1393 par_res = parser_.ParseSPS(&sps_id);
1394 if (par_res != H264Parser::kOk)
1395 SET_ERROR_AND_RETURN();
1396
1397 bool need_new_buffers = false;
1398 if (!ProcessSPS(sps_id, &need_new_buffers))
1399 SET_ERROR_AND_RETURN();
1400
1401 if (state_ == kNeedStreamMetadata)
1402 state_ = kAfterReset;
1403
1404 if (need_new_buffers) {
1405 curr_pic_ = nullptr;
1406 curr_nalu_ = nullptr;
1407 ref_pic_list_p0_.clear();
1408 ref_pic_list_b0_.clear();
1409 ref_pic_list_b1_.clear();
1410
1411 return kConfigChange;
1412 }
1413 break;
1414 }
1415
1416 case H264NALU::kPPS: {
1417 int pps_id;
1418
1419 CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
1420 par_res = parser_.ParsePPS(&pps_id);
1421 if (par_res != H264Parser::kOk)
1422 SET_ERROR_AND_RETURN();
1423
1424 break;
1425 }
1426
1427 case H264NALU::kAUD:
1428 case H264NALU::kEOSeq:
1429 case H264NALU::kEOStream:
1430 if (state_ != kDecoding)
1431 break;
1432
1433 CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
1434 break;
1435
1436 default:
1437 DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type;
1438 break;
1439 }
1440
1441 DVLOG(4) << "NALU done";
1442 curr_nalu_.reset();
1443 }
1444 }
1445
GetPicSize() const1446 gfx::Size H264Decoder::GetPicSize() const {
1447 return pic_size_;
1448 }
1449
GetVisibleRect() const1450 gfx::Rect H264Decoder::GetVisibleRect() const {
1451 return visible_rect_;
1452 }
1453
GetProfile() const1454 VideoCodecProfile H264Decoder::GetProfile() const {
1455 return profile_;
1456 }
1457
GetRequiredNumOfPictures() const1458 size_t H264Decoder::GetRequiredNumOfPictures() const {
1459 constexpr size_t kPicsInPipeline = limits::kMaxVideoFrames + 1;
1460 return GetNumReferenceFrames() + kPicsInPipeline;
1461 }
1462
GetNumReferenceFrames() const1463 size_t H264Decoder::GetNumReferenceFrames() const {
1464 // Use the maximum number of pictures in the Decoded Picture Buffer.
1465 return dpb_.max_num_pics();
1466 }
1467
1468 // static
FillH264PictureFromSliceHeader(const H264SPS * sps,const H264SliceHeader & slice_hdr,H264Picture * pic)1469 bool H264Decoder::FillH264PictureFromSliceHeader(
1470 const H264SPS* sps,
1471 const H264SliceHeader& slice_hdr,
1472 H264Picture* pic) {
1473 DCHECK(pic);
1474
1475 pic->idr = slice_hdr.idr_pic_flag;
1476 if (pic->idr)
1477 pic->idr_pic_id = slice_hdr.idr_pic_id;
1478
1479 if (slice_hdr.field_pic_flag) {
1480 pic->field = slice_hdr.bottom_field_flag ? H264Picture::FIELD_BOTTOM
1481 : H264Picture::FIELD_TOP;
1482 } else {
1483 pic->field = H264Picture::FIELD_NONE;
1484 }
1485
1486 if (pic->field != H264Picture::FIELD_NONE) {
1487 DVLOG(1) << "Interlaced video not supported.";
1488 return false;
1489 }
1490
1491 pic->nal_ref_idc = slice_hdr.nal_ref_idc;
1492 pic->ref = slice_hdr.nal_ref_idc != 0;
1493 // This assumes non-interlaced stream.
1494 pic->frame_num = pic->pic_num = slice_hdr.frame_num;
1495
1496 if (!sps)
1497 return false;
1498
1499 pic->pic_order_cnt_type = sps->pic_order_cnt_type;
1500 switch (pic->pic_order_cnt_type) {
1501 case 0:
1502 pic->pic_order_cnt_lsb = slice_hdr.pic_order_cnt_lsb;
1503 pic->delta_pic_order_cnt_bottom = slice_hdr.delta_pic_order_cnt_bottom;
1504 break;
1505
1506 case 1:
1507 pic->delta_pic_order_cnt0 = slice_hdr.delta_pic_order_cnt0;
1508 pic->delta_pic_order_cnt1 = slice_hdr.delta_pic_order_cnt1;
1509 break;
1510
1511 case 2:
1512 break;
1513
1514 default:
1515 NOTREACHED();
1516 return false;
1517 }
1518 return true;
1519 }
1520
1521 // static
IsNewPrimaryCodedPicture(const H264Picture * curr_pic,int curr_pps_id,const H264SPS * sps,const H264SliceHeader & slice_hdr)1522 bool H264Decoder::IsNewPrimaryCodedPicture(const H264Picture* curr_pic,
1523 int curr_pps_id,
1524 const H264SPS* sps,
1525 const H264SliceHeader& slice_hdr) {
1526 if (!curr_pic)
1527 return true;
1528
1529 // 7.4.1.2.4, assumes non-interlaced.
1530 if (slice_hdr.frame_num != curr_pic->frame_num ||
1531 slice_hdr.pic_parameter_set_id != curr_pps_id ||
1532 slice_hdr.nal_ref_idc != curr_pic->nal_ref_idc ||
1533 slice_hdr.idr_pic_flag != curr_pic->idr ||
1534 (slice_hdr.idr_pic_flag &&
1535 (slice_hdr.idr_pic_id != curr_pic->idr_pic_id ||
1536 // If we have two consecutive IDR slices, and the second one has
1537 // first_mb_in_slice == 0, treat it as a new picture.
1538 // Per spec, idr_pic_id should not be equal in this case (and we should
1539 // have hit the condition above instead, see spec 7.4.3 on idr_pic_id),
1540 // but some encoders neglect changing idr_pic_id for two consecutive
1541 // IDRs. Work around this by checking if the next slice contains the
1542 // zeroth macroblock, i.e. data that belongs to the next picture.
1543 slice_hdr.first_mb_in_slice == 0)))
1544 return true;
1545
1546 if (!sps)
1547 return false;
1548
1549 if (sps->pic_order_cnt_type == curr_pic->pic_order_cnt_type) {
1550 if (curr_pic->pic_order_cnt_type == 0) {
1551 if (slice_hdr.pic_order_cnt_lsb != curr_pic->pic_order_cnt_lsb ||
1552 slice_hdr.delta_pic_order_cnt_bottom !=
1553 curr_pic->delta_pic_order_cnt_bottom)
1554 return true;
1555 } else if (curr_pic->pic_order_cnt_type == 1) {
1556 if (slice_hdr.delta_pic_order_cnt0 != curr_pic->delta_pic_order_cnt0 ||
1557 slice_hdr.delta_pic_order_cnt1 != curr_pic->delta_pic_order_cnt1)
1558 return true;
1559 }
1560 }
1561
1562 return false;
1563 }
1564
1565 } // namespace media
1566