1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
12 #include "modules/video_coding/codecs/h264/h264_decoder_impl.h"
13
14 #include <algorithm>
15 #include <limits>
16
17 extern "C" {
18 #include "third_party/ffmpeg/libavcodec/avcodec.h"
19 #include "third_party/ffmpeg/libavformat/avformat.h"
20 #include "third_party/ffmpeg/libavutil/imgutils.h"
21 } // extern "C"
22
23 #include "api/video/i420_buffer.h"
24 #include "common_video/include/video_frame_buffer.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/criticalsection.h"
27 #include "rtc_base/keep_ref_until_done.h"
28 #include "rtc_base/logging.h"
29 #include "system_wrappers/include/metrics.h"
30
31 namespace webrtc {
32
33 namespace {
34
35 const AVPixelFormat kPixelFormat = AV_PIX_FMT_YUV420P;
36 const size_t kYPlaneIndex = 0;
37 const size_t kUPlaneIndex = 1;
38 const size_t kVPlaneIndex = 2;
39
40 // Used by histograms. Values of entries should not be changed.
41 enum H264DecoderImplEvent {
42 kH264DecoderEventInit = 0,
43 kH264DecoderEventError = 1,
44 kH264DecoderEventMax = 16,
45 };
46
47 #if defined(WEBRTC_INITIALIZE_FFMPEG)
48
49 rtc::CriticalSection ffmpeg_init_lock;
50 bool ffmpeg_initialized = false;
51
52 // Called by FFmpeg to do mutex operations if initialized using
53 // |InitializeFFmpeg|. Disabling thread safety analysis because void** does not
54 // play nicely with thread_annotations.h macros.
LockManagerOperation(void ** lock,AVLockOp op)55 int LockManagerOperation(void** lock,
56 AVLockOp op) RTC_NO_THREAD_SAFETY_ANALYSIS {
57 switch (op) {
58 case AV_LOCK_CREATE:
59 *lock = new rtc::CriticalSection();
60 return 0;
61 case AV_LOCK_OBTAIN:
62 static_cast<rtc::CriticalSection*>(*lock)->Enter();
63 return 0;
64 case AV_LOCK_RELEASE:
65 static_cast<rtc::CriticalSection*>(*lock)->Leave();
66 return 0;
67 case AV_LOCK_DESTROY:
68 delete static_cast<rtc::CriticalSection*>(*lock);
69 *lock = nullptr;
70 return 0;
71 }
72 RTC_NOTREACHED() << "Unrecognized AVLockOp.";
73 return -1;
74 }
75
InitializeFFmpeg()76 void InitializeFFmpeg() {
77 rtc::CritScope cs(&ffmpeg_init_lock);
78 if (!ffmpeg_initialized) {
79 if (av_lockmgr_register(LockManagerOperation) < 0) {
80 RTC_NOTREACHED() << "av_lockmgr_register failed.";
81 return;
82 }
83 av_register_all();
84 ffmpeg_initialized = true;
85 }
86 }
87
88 #endif // defined(WEBRTC_INITIALIZE_FFMPEG)
89
90 } // namespace
91
AVGetBuffer2(AVCodecContext * context,AVFrame * av_frame,int flags)92 int H264DecoderImpl::AVGetBuffer2(
93 AVCodecContext* context, AVFrame* av_frame, int flags) {
94 // Set in |InitDecode|.
95 H264DecoderImpl* decoder = static_cast<H264DecoderImpl*>(context->opaque);
96 // DCHECK values set in |InitDecode|.
97 RTC_DCHECK(decoder);
98 RTC_DCHECK_EQ(context->pix_fmt, kPixelFormat);
99 // Necessary capability to be allowed to provide our own buffers.
100 RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
101
102 // |av_frame->width| and |av_frame->height| are set by FFmpeg. These are the
103 // actual image's dimensions and may be different from |context->width| and
104 // |context->coded_width| due to reordering.
105 int width = av_frame->width;
106 int height = av_frame->height;
107 // See |lowres|, if used the decoder scales the image by 1/2^(lowres). This
108 // has implications on which resolutions are valid, but we don't use it.
109 RTC_CHECK_EQ(context->lowres, 0);
110 // Adjust the |width| and |height| to values acceptable by the decoder.
111 // Without this, FFmpeg may overflow the buffer. If modified, |width| and/or
112 // |height| are larger than the actual image and the image has to be cropped
113 // (top-left corner) after decoding to avoid visible borders to the right and
114 // bottom of the actual image.
115 avcodec_align_dimensions(context, &width, &height);
116
117 RTC_CHECK_GE(width, 0);
118 RTC_CHECK_GE(height, 0);
119 int ret = av_image_check_size(static_cast<unsigned int>(width),
120 static_cast<unsigned int>(height), 0, nullptr);
121 if (ret < 0) {
122 RTC_LOG(LS_ERROR) << "Invalid picture size " << width << "x" << height;
123 decoder->ReportError();
124 return ret;
125 }
126
127 // The video frame is stored in |frame_buffer|. |av_frame| is FFmpeg's version
128 // of a video frame and will be set up to reference |frame_buffer|'s data.
129
130 // FFmpeg expects the initial allocation to be zero-initialized according to
131 // http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
132 // TODO(nisse): Delete that feature from the video pool, instead add
133 // an explicit call to InitializeData here.
134 rtc::scoped_refptr<I420Buffer> frame_buffer =
135 decoder->pool_.CreateBuffer(width, height);
136
137 int y_size = width * height;
138 int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight();
139 // DCHECK that we have a continuous buffer as is required.
140 RTC_DCHECK_EQ(frame_buffer->DataU(), frame_buffer->DataY() + y_size);
141 RTC_DCHECK_EQ(frame_buffer->DataV(), frame_buffer->DataU() + uv_size);
142 int total_size = y_size + 2 * uv_size;
143
144 av_frame->format = context->pix_fmt;
145 av_frame->reordered_opaque = context->reordered_opaque;
146
147 // Set |av_frame| members as required by FFmpeg.
148 av_frame->data[kYPlaneIndex] = frame_buffer->MutableDataY();
149 av_frame->linesize[kYPlaneIndex] = frame_buffer->StrideY();
150 av_frame->data[kUPlaneIndex] = frame_buffer->MutableDataU();
151 av_frame->linesize[kUPlaneIndex] = frame_buffer->StrideU();
152 av_frame->data[kVPlaneIndex] = frame_buffer->MutableDataV();
153 av_frame->linesize[kVPlaneIndex] = frame_buffer->StrideV();
154 RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
155
156 // Create a VideoFrame object, to keep a reference to the buffer.
157 // TODO(nisse): The VideoFrame's timestamp and rotation info is not used.
158 // Refactor to do not use a VideoFrame object at all.
159 av_frame->buf[0] = av_buffer_create(
160 av_frame->data[kYPlaneIndex],
161 total_size,
162 AVFreeBuffer2,
163 static_cast<void*>(new VideoFrame(frame_buffer,
164 0 /* timestamp */,
165 0 /* render_time_ms */,
166 kVideoRotation_0)),
167 0);
168 RTC_CHECK(av_frame->buf[0]);
169 return 0;
170 }
171
AVFreeBuffer2(void * opaque,uint8_t * data)172 void H264DecoderImpl::AVFreeBuffer2(void* opaque, uint8_t* data) {
173 // The buffer pool recycles the buffer used by |video_frame| when there are no
174 // more references to it. |video_frame| is a thin buffer holder and is not
175 // recycled.
176 VideoFrame* video_frame = static_cast<VideoFrame*>(opaque);
177 delete video_frame;
178 }
179
H264DecoderImpl()180 H264DecoderImpl::H264DecoderImpl() : pool_(true),
181 decoded_image_callback_(nullptr),
182 has_reported_init_(false),
183 has_reported_error_(false) {
184 }
185
~H264DecoderImpl()186 H264DecoderImpl::~H264DecoderImpl() {
187 Release();
188 }
189
InitDecode(const VideoCodec * codec_settings,int32_t number_of_cores)190 int32_t H264DecoderImpl::InitDecode(const VideoCodec* codec_settings,
191 int32_t number_of_cores) {
192 ReportInit();
193 if (codec_settings &&
194 codec_settings->codecType != kVideoCodecH264) {
195 ReportError();
196 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
197 }
198
199 // FFmpeg must have been initialized (with |av_lockmgr_register| and
200 // |av_register_all|) before we proceed. |InitializeFFmpeg| does this, which
201 // makes sense for WebRTC standalone. In other cases, such as Chromium, FFmpeg
202 // is initialized externally and calling |InitializeFFmpeg| would be
203 // thread-unsafe and result in FFmpeg being initialized twice, which could
204 // break other FFmpeg usage. See the |rtc_initialize_ffmpeg| flag.
205 #if defined(WEBRTC_INITIALIZE_FFMPEG)
206 // Make sure FFmpeg has been initialized. Subsequent |InitializeFFmpeg| calls
207 // do nothing.
208 InitializeFFmpeg();
209 #endif
210
211 // Release necessary in case of re-initializing.
212 int32_t ret = Release();
213 if (ret != WEBRTC_VIDEO_CODEC_OK) {
214 ReportError();
215 return ret;
216 }
217 RTC_DCHECK(!av_context_);
218
219 // Initialize AVCodecContext.
220 av_context_.reset(avcodec_alloc_context3(nullptr));
221
222 av_context_->codec_type = AVMEDIA_TYPE_VIDEO;
223 av_context_->codec_id = AV_CODEC_ID_H264;
224 if (codec_settings) {
225 av_context_->coded_width = codec_settings->width;
226 av_context_->coded_height = codec_settings->height;
227 }
228 av_context_->pix_fmt = kPixelFormat;
229 av_context_->extradata = nullptr;
230 av_context_->extradata_size = 0;
231
232 // If this is ever increased, look at |av_context_->thread_safe_callbacks| and
233 // make it possible to disable the thread checker in the frame buffer pool.
234 av_context_->thread_count = 1;
235 av_context_->thread_type = FF_THREAD_SLICE;
236
237 // Function used by FFmpeg to get buffers to store decoded frames in.
238 av_context_->get_buffer2 = AVGetBuffer2;
239 // |get_buffer2| is called with the context, there |opaque| can be used to get
240 // a pointer |this|.
241 av_context_->opaque = this;
242
243 AVCodec* codec = avcodec_find_decoder(av_context_->codec_id);
244 if (!codec) {
245 // This is an indication that FFmpeg has not been initialized or it has not
246 // been compiled/initialized with the correct set of codecs.
247 RTC_LOG(LS_ERROR) << "FFmpeg H.264 decoder not found.";
248 Release();
249 ReportError();
250 return WEBRTC_VIDEO_CODEC_ERROR;
251 }
252 int res = avcodec_open2(av_context_.get(), codec, nullptr);
253 if (res < 0) {
254 RTC_LOG(LS_ERROR) << "avcodec_open2 error: " << res;
255 Release();
256 ReportError();
257 return WEBRTC_VIDEO_CODEC_ERROR;
258 }
259
260 av_frame_.reset(av_frame_alloc());
261 return WEBRTC_VIDEO_CODEC_OK;
262 }
263
Release()264 int32_t H264DecoderImpl::Release() {
265 av_context_.reset();
266 av_frame_.reset();
267 return WEBRTC_VIDEO_CODEC_OK;
268 }
269
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)270 int32_t H264DecoderImpl::RegisterDecodeCompleteCallback(
271 DecodedImageCallback* callback) {
272 decoded_image_callback_ = callback;
273 return WEBRTC_VIDEO_CODEC_OK;
274 }
275
Decode(const EncodedImage & input_image,bool,const RTPFragmentationHeader *,const CodecSpecificInfo * codec_specific_info,int64_t)276 int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
277 bool /*missing_frames*/,
278 const RTPFragmentationHeader* /*fragmentation*/,
279 const CodecSpecificInfo* codec_specific_info,
280 int64_t /*render_time_ms*/) {
281 if (!IsInitialized()) {
282 ReportError();
283 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
284 }
285 if (!decoded_image_callback_) {
286 RTC_LOG(LS_WARNING)
287 << "InitDecode() has been called, but a callback function "
288 "has not been set with RegisterDecodeCompleteCallback()";
289 ReportError();
290 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
291 }
292 if (!input_image._buffer || !input_image._length) {
293 ReportError();
294 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
295 }
296 if (codec_specific_info &&
297 codec_specific_info->codecType != kVideoCodecH264) {
298 ReportError();
299 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
300 }
301
302 // FFmpeg requires padding due to some optimized bitstream readers reading 32
303 // or 64 bits at once and could read over the end. See avcodec_decode_video2.
304 RTC_CHECK_GE(input_image._size, input_image._length +
305 EncodedImage::GetBufferPaddingBytes(kVideoCodecH264));
306 // "If the first 23 bits of the additional bytes are not 0, then damaged MPEG
307 // bitstreams could cause overread and segfault." See
308 // AV_INPUT_BUFFER_PADDING_SIZE. We'll zero the entire padding just in case.
309 memset(input_image._buffer + input_image._length,
310 0,
311 EncodedImage::GetBufferPaddingBytes(kVideoCodecH264));
312
313 AVPacket packet;
314 av_init_packet(&packet);
315 packet.data = input_image._buffer;
316 if (input_image._length >
317 static_cast<size_t>(std::numeric_limits<int>::max())) {
318 ReportError();
319 return WEBRTC_VIDEO_CODEC_ERROR;
320 }
321 packet.size = static_cast<int>(input_image._length);
322 int64_t frame_timestamp_us = input_image.ntp_time_ms_ * 1000; // ms -> μs
323 av_context_->reordered_opaque = frame_timestamp_us;
324
325 int result = avcodec_send_packet(av_context_.get(), &packet);
326 if (result < 0) {
327 RTC_LOG(LS_ERROR) << "avcodec_send_packet error: " << result;
328 ReportError();
329 return WEBRTC_VIDEO_CODEC_ERROR;
330 }
331
332 result = avcodec_receive_frame(av_context_.get(), av_frame_.get());
333 if (result < 0) {
334 RTC_LOG(LS_ERROR) << "avcodec_receive_frame error: " << result;
335 ReportError();
336 return WEBRTC_VIDEO_CODEC_ERROR;
337 }
338
339 // We don't expect reordering. Decoded frame tamestamp should match
340 // the input one.
341 RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us);
342
343 // Obtain the |video_frame| containing the decoded image.
344 VideoFrame* video_frame = static_cast<VideoFrame*>(
345 av_buffer_get_opaque(av_frame_->buf[0]));
346 RTC_DCHECK(video_frame);
347 rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer =
348 video_frame->video_frame_buffer()->GetI420();
349 RTC_CHECK_EQ(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
350 RTC_CHECK_EQ(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
351 RTC_CHECK_EQ(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
352 video_frame->set_timestamp(input_image._timeStamp);
353
354 rtc::Optional<uint8_t> qp;
355 // TODO(sakal): Maybe it is possible to get QP directly from FFmpeg.
356 h264_bitstream_parser_.ParseBitstream(input_image._buffer,
357 input_image._length);
358 int qp_int;
359 if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) {
360 qp.emplace(qp_int);
361 }
362
363 // The decoded image may be larger than what is supposed to be visible, see
364 // |AVGetBuffer2|'s use of |avcodec_align_dimensions|. This crops the image
365 // without copying the underlying buffer.
366 if (av_frame_->width != i420_buffer->width() ||
367 av_frame_->height != i420_buffer->height()) {
368 rtc::scoped_refptr<VideoFrameBuffer> cropped_buf(
369 new rtc::RefCountedObject<WrappedI420Buffer>(
370 av_frame_->width, av_frame_->height,
371 i420_buffer->DataY(), i420_buffer->StrideY(),
372 i420_buffer->DataU(), i420_buffer->StrideU(),
373 i420_buffer->DataV(), i420_buffer->StrideV(),
374 rtc::KeepRefUntilDone(i420_buffer)));
375 VideoFrame cropped_frame(
376 cropped_buf, video_frame->timestamp(), video_frame->render_time_ms(),
377 video_frame->rotation());
378 // TODO(nisse): Timestamp and rotation are all zero here. Change decoder
379 // interface to pass a VideoFrameBuffer instead of a VideoFrame?
380 decoded_image_callback_->Decoded(cropped_frame, rtc::Optional<int32_t>(),
381 qp);
382 } else {
383 // Return decoded frame.
384 decoded_image_callback_->Decoded(*video_frame, rtc::Optional<int32_t>(),
385 qp);
386 }
387 // Stop referencing it, possibly freeing |video_frame|.
388 av_frame_unref(av_frame_.get());
389 video_frame = nullptr;
390
391 return WEBRTC_VIDEO_CODEC_OK;
392 }
393
ImplementationName() const394 const char* H264DecoderImpl::ImplementationName() const {
395 return "FFmpeg";
396 }
397
IsInitialized() const398 bool H264DecoderImpl::IsInitialized() const {
399 return av_context_ != nullptr;
400 }
401
ReportInit()402 void H264DecoderImpl::ReportInit() {
403 if (has_reported_init_)
404 return;
405 RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event",
406 kH264DecoderEventInit,
407 kH264DecoderEventMax);
408 has_reported_init_ = true;
409 }
410
ReportError()411 void H264DecoderImpl::ReportError() {
412 if (has_reported_error_)
413 return;
414 RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event",
415 kH264DecoderEventError,
416 kH264DecoderEventMax);
417 has_reported_error_ = true;
418 }
419
420 } // namespace webrtc
421