1 /* ============================================================
2 *
3 * This file is a part of digiKam project
4 * https://www.digikam.org
5 *
6 * Date : 2016-04-21
7 * Description : video thumbnails extraction based on ffmpeg
8 *
9 * Copyright (C) 2010 by Dirk Vanden Boer <dirk dot vdb at gmail dot com>
10 * Copyright (C) 2016-2018 by Maik Qualmann <metzpinguin at gmail dot com>
11 * Copyright (C) 2016-2021 by Gilles Caulier <caulier dot gilles at gmail dot com>
12 *
13 * This program is free software; you can redistribute it
14 * and/or modify it under the terms of the GNU General
15 * Public License as published by the Free Software Foundation;
16 * either version 2, or (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * ============================================================ */
24
25 #include "videodecoder_p.h"
26
27 // Local includes
28
29 #include "digikam_debug.h"
30
31 namespace Digikam
32 {
33
Private()34 VideoDecoder::Private::Private()
35 : videoStream (-1),
36 pFormatContext (nullptr),
37 pVideoCodecContext (nullptr),
38 pVideoCodecParameters (nullptr),
39 pVideoCodec (nullptr),
40 pVideoStream (nullptr),
41 pFrame (nullptr),
42 pFrameBuffer (nullptr),
43 pPacket (nullptr),
44 allowSeek (true),
45 initialized (false),
46 bufferSinkContext (nullptr),
47 bufferSourceContext (nullptr),
48 filterGraph (nullptr),
49 filterFrame (nullptr),
50 lastWidth (0),
51 lastHeight (0),
52 lastPixfmt (AV_PIX_FMT_NONE)
53 {
54 }
55
~Private()56 VideoDecoder::Private::~Private()
57 {
58 }
59
createAVFrame(AVFrame ** const avFrame,quint8 ** const frameBuffer,int width,int height,AVPixelFormat format)60 void VideoDecoder::Private::createAVFrame(AVFrame** const avFrame,
61 quint8** const frameBuffer,
62 int width,
63 int height,
64 AVPixelFormat format)
65 {
66 *avFrame = av_frame_alloc();
67 int numBytes = av_image_get_buffer_size(format, width, height, 1);
68 *frameBuffer = reinterpret_cast<quint8*>(av_malloc(numBytes));
69
70 av_image_fill_arrays((*avFrame)->data, (*avFrame)->linesize, *frameBuffer, format, width, height, 1);
71 }
72
initializeVideo()73 bool VideoDecoder::Private::initializeVideo()
74 {
75 for (unsigned int i = 0 ; i < pFormatContext->nb_streams ; ++i)
76 {
77 if (pFormatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
78 {
79 pVideoStream = pFormatContext->streams[i];
80 videoStream = i;
81 break;
82 }
83 }
84
85 if (videoStream == -1)
86 {
87 qDebug(DIGIKAM_GENERAL_LOG) << "Could not find video stream";
88
89 return false;
90 }
91
92 pVideoCodecParameters = pFormatContext->streams[videoStream]->codecpar;
93 pVideoCodec = avcodec_find_decoder(pVideoCodecParameters->codec_id);
94
95 if (pVideoCodec == nullptr)
96 {
97 // set to 0, otherwise avcodec_close(d->pVideoCodecContext) crashes
98
99 pVideoCodecContext = nullptr;
100 qDebug(DIGIKAM_GENERAL_LOG) << "Video Codec not found";
101
102 return false;
103 }
104
105 pVideoCodecContext = avcodec_alloc_context3(pVideoCodec);
106 avcodec_parameters_to_context(pVideoCodecContext, pVideoCodecParameters);
107
108 if (avcodec_open2(pVideoCodecContext, pVideoCodec, nullptr) < 0)
109 {
110 qDebug(DIGIKAM_GENERAL_LOG) << "Could not open video codec";
111
112 return false;
113 }
114
115 return true;
116 }
117
decodeVideoPacket() const118 bool VideoDecoder::Private::decodeVideoPacket() const
119 {
120 if (pPacket->stream_index != videoStream)
121 {
122 return false;
123 }
124
125 av_frame_unref(pFrame);
126
127 int frameFinished = 0;
128
129 #if LIBAVCODEC_VERSION_MAJOR < 53
130
131 int bytesDecoded = avcodec_decode_video(pVideoCodecContext,
132 pFrame,
133 &frameFinished,
134 pPacket->data,
135 pPacket->size);
136 #else
137
138 int bytesDecoded = decodeVideoNew(pVideoCodecContext,
139 pFrame,
140 &frameFinished,
141 pPacket);
142
143 #endif
144
145 if (bytesDecoded < 0)
146 {
147 qDebug(DIGIKAM_GENERAL_LOG) << "Failed to decode video frame: bytesDecoded < 0";
148 }
149
150 return (frameFinished > 0);
151 }
152
decodeVideoNew(AVCodecContext * const avContext,AVFrame * const avFrame,int * gotFrame,AVPacket * const avPacket) const153 int VideoDecoder::Private::decodeVideoNew(AVCodecContext* const avContext,
154 AVFrame* const avFrame,
155 int* gotFrame,
156 AVPacket* const avPacket) const
157 {
158 int ret = 0;
159 *gotFrame = 0;
160
161 if (avPacket)
162 {
163 ret = avcodec_send_packet(avContext, avPacket);
164
165 // In particular, we don't expect AVERROR(EAGAIN), because we read all
166 // decoded frames with avcodec_receive_frame() until done.
167
168 if (ret < 0)
169 {
170 return (ret == AVERROR_EOF ? 0 : ret);
171 }
172 }
173
174 ret = avcodec_receive_frame(avContext, avFrame);
175
176 if ((ret < 0) && (ret != AVERROR(EAGAIN)) && (ret != AVERROR_EOF))
177 {
178 return ret;
179 }
180
181 if (ret >= 0)
182 {
183 *gotFrame = 1;
184 }
185
186 return 0;
187 }
188
getVideoPacket()189 bool VideoDecoder::Private::getVideoPacket()
190 {
191 bool framesAvailable = true;
192 bool frameDecoded = false;
193 int attempts = 0;
194
195 if (pPacket)
196 {
197 av_packet_unref(pPacket);
198 delete pPacket;
199 }
200
201 pPacket = new AVPacket();
202
203 while (framesAvailable &&
204 !frameDecoded &&
205 (attempts++ < 1000))
206 {
207 framesAvailable = (av_read_frame(pFormatContext, pPacket) >= 0);
208
209 if (framesAvailable)
210 {
211 frameDecoded = (pPacket->stream_index == videoStream);
212
213 if (!frameDecoded)
214 {
215 av_packet_unref(pPacket);
216 }
217 }
218 }
219
220 return frameDecoded;
221 }
222
deleteFilterGraph()223 void VideoDecoder::Private::deleteFilterGraph()
224 {
225 if (filterGraph)
226 {
227 av_frame_free(&filterFrame);
228 avfilter_graph_free(&filterGraph);
229 filterGraph = nullptr;
230 }
231 }
232
initFilterGraph(enum AVPixelFormat pixfmt,int width,int height)233 bool VideoDecoder::Private::initFilterGraph(enum AVPixelFormat pixfmt,
234 int width, int height)
235 {
236 AVFilterInOut* inputs = nullptr;
237 AVFilterInOut* outputs = nullptr;
238
239 deleteFilterGraph();
240 filterGraph = avfilter_graph_alloc();
241
242 QByteArray arguments("buffer=");
243 arguments += "video_size=" + QByteArray::number(width) + 'x' + QByteArray::number(height) + ':';
244 arguments += "pix_fmt=" + QByteArray::number(pixfmt) + ':';
245 arguments += "time_base=1/1:pixel_aspect=0/1[in];";
246 arguments += "[in]yadif[out];";
247 arguments += "[out]buffersink";
248
249 int ret = avfilter_graph_parse2(filterGraph, arguments.constData(), &inputs, &outputs);
250
251 if (ret < 0)
252 {
253 qWarning(DIGIKAM_GENERAL_LOG) << "Unable to parse filter graph";
254
255 return false;
256 }
257
258 if (inputs || outputs)
259 {
260 return (-1);
261 }
262
263 ret = avfilter_graph_config(filterGraph, nullptr);
264
265 if (ret < 0)
266 {
267 qWarning(DIGIKAM_GENERAL_LOG) << "Unable to validate filter graph";
268
269 return false;
270 }
271
272 bufferSourceContext = avfilter_graph_get_filter(filterGraph, "Parsed_buffer_0");
273 bufferSinkContext = avfilter_graph_get_filter(filterGraph, "Parsed_buffersink_2");
274
275 if (!bufferSourceContext || !bufferSinkContext)
276 {
277 qWarning(DIGIKAM_GENERAL_LOG) << "Unable to get source or sink";
278
279 return false;
280 }
281
282 filterFrame = av_frame_alloc();
283 lastWidth = width;
284 lastHeight = height;
285 lastPixfmt = pixfmt;
286
287 return true;
288 }
289
processFilterGraph(AVFrame * const dst,const AVFrame * const src,enum AVPixelFormat pixfmt,int width,int height)290 bool VideoDecoder::Private::processFilterGraph(AVFrame* const dst,
291 const AVFrame* const src,
292 enum AVPixelFormat pixfmt,
293 int width, int height)
294 {
295 if (!filterGraph ||
296 (width != lastWidth) ||
297 (height != lastHeight) ||
298 (pixfmt != lastPixfmt))
299 {
300
301 if (!initFilterGraph(pixfmt, width, height))
302 {
303 return false;
304 }
305 }
306
307 memcpy(filterFrame->data, src->data, sizeof(src->data));
308 memcpy(filterFrame->linesize, src->linesize, sizeof(src->linesize));
309
310 filterFrame->width = width;
311 filterFrame->height = height;
312 filterFrame->format = pixfmt;
313 int ret = av_buffersrc_add_frame(bufferSourceContext, filterFrame);
314
315 if (ret < 0)
316 {
317 return false;
318 }
319
320 ret = av_buffersink_get_frame(bufferSinkContext, filterFrame);
321
322 if (ret < 0)
323 {
324 return false;
325 }
326
327 av_image_copy(dst->data, dst->linesize, (const uint8_t**)filterFrame->data, filterFrame->linesize, pixfmt, width, height);
328 av_frame_unref(filterFrame);
329
330 return true;
331 }
332
convertAndScaleFrame(AVPixelFormat format,int scaledSize,bool maintainAspectRatio,int & scaledWidth,int & scaledHeight)333 void VideoDecoder::Private::convertAndScaleFrame(AVPixelFormat format,
334 int scaledSize,
335 bool maintainAspectRatio,
336 int& scaledWidth,
337 int& scaledHeight)
338 {
339 AVPixelFormat pVideoCodecContextPixFormat;
340 pVideoCodecContextPixFormat = pVideoCodecContext->pix_fmt;
341
342 #if LIBAVUTIL_VERSION_MAJOR > 55
343
344 switch (pVideoCodecContextPixFormat)
345 {
346 case AV_PIX_FMT_YUVJ420P:
347 {
348 pVideoCodecContextPixFormat = AV_PIX_FMT_YUV420P;
349 break;
350 }
351
352 case AV_PIX_FMT_YUVJ422P:
353 {
354 pVideoCodecContextPixFormat = AV_PIX_FMT_YUV422P;
355 break;
356 }
357
358 case AV_PIX_FMT_YUVJ444P:
359 {
360 pVideoCodecContextPixFormat = AV_PIX_FMT_YUV444P;
361 break;
362 }
363
364 case AV_PIX_FMT_YUVJ440P:
365 {
366 pVideoCodecContextPixFormat = AV_PIX_FMT_YUV440P;
367 break;
368 }
369
370 default:
371 {
372 break;
373 }
374 }
375
376 #endif
377
378 calculateDimensions(scaledSize, maintainAspectRatio, scaledWidth, scaledHeight);
379
380 SwsContext* const scaleContext = sws_getContext(pVideoCodecContext->width,
381 pVideoCodecContext->height,
382 pVideoCodecContextPixFormat,
383 scaledWidth,
384 scaledHeight,
385 format,
386 SWS_BICUBIC,
387 nullptr,
388 nullptr,
389 nullptr);
390
391 if (!scaleContext)
392 {
393 qDebug(DIGIKAM_GENERAL_LOG) << "Failed to create resize context";
394 return;
395 }
396
397 AVFrame* convertedFrame = nullptr;
398 uint8_t* convertedFrameBuffer = nullptr;
399
400 createAVFrame(&convertedFrame,
401 &convertedFrameBuffer,
402 scaledWidth,
403 scaledHeight,
404 format);
405
406 sws_scale(scaleContext,
407 pFrame->data,
408 pFrame->linesize,
409 0,
410 pVideoCodecContext->height,
411 convertedFrame->data,
412 convertedFrame->linesize);
413
414 sws_freeContext(scaleContext);
415
416 av_frame_free(&pFrame);
417 av_free(pFrameBuffer);
418
419 pFrame = convertedFrame;
420 pFrameBuffer = convertedFrameBuffer;
421 }
422
calculateDimensions(int squareSize,bool maintainAspectRatio,int & destWidth,int & destHeight)423 void VideoDecoder::Private::calculateDimensions(int squareSize,
424 bool maintainAspectRatio,
425 int& destWidth,
426 int& destHeight)
427 {
428 if (!maintainAspectRatio)
429 {
430 destWidth = squareSize;
431 destHeight = squareSize;
432 }
433 else
434 {
435 int srcWidth = pVideoCodecContext->width;
436 int srcHeight = pVideoCodecContext->height;
437 int ascpectNominator = pVideoCodecContext->sample_aspect_ratio.num;
438 int ascpectDenominator = pVideoCodecContext->sample_aspect_ratio.den;
439
440 if ((ascpectNominator != 0) && (ascpectDenominator != 0))
441 {
442 srcWidth = srcWidth * ascpectNominator / ascpectDenominator;
443 }
444
445 if (srcWidth > srcHeight)
446 {
447 destWidth = squareSize;
448 destHeight = static_cast<int>(static_cast<float>(squareSize) / srcWidth * srcHeight);
449 }
450 else
451 {
452 destWidth = static_cast<int>(static_cast<float>(squareSize) / srcHeight * srcWidth);
453 destHeight = squareSize;
454 }
455 }
456 }
457
458 } // namespace Digikam
459