1 /* ***** BEGIN LICENSE BLOCK *****
2 * This file is part of openfx-io <https://github.com/MrKepzie/openfx-io>,
3 * Copyright (C) 2013-2018 INRIA
4 *
5 * openfx-io is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * openfx-io is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with openfx-io. If not, see <http://www.gnu.org/licenses/gpl-2.0.html>
17 * ***** END LICENSE BLOCK ***** */
18
19 /*
20 * OFX ffmpeg Reader plugin.
21 * Reads a video input file using the libav library.
22 * Synced with mov64Reader 11.1v3
23 *
24 * BUGS:
25 * - The last frames from long-GOP mp4 don't read, see:
26 * - https://github.com/NatronGitHub/Natron/issues/241
27 * - https://github.com/NatronGitHub/Natron/issues/231
28 * - MPEG1 files cannot be read, for example
29 * - https://github.com/NatronGitHub/Natron-Tests/raw/master/TestReadMPEG1/input.mpg
30 * - http://devernay.free.fr/vision/diffprop/herve3d.mpg
31 * This was already true before the 11.1v3 sync (e.g. at commit 4d0d3a5).
32 */
33 //#define TRACE_DECODE_PROCESS 1
34
35 #if (defined(_STDINT_H) || defined(_STDINT_H_) || defined(_MSC_STDINT_H_ ) ) && !defined(UINT64_C)
36 #warning "__STDC_CONSTANT_MACROS has to be defined before including <stdint.h>, this file will probably not compile."
37 #endif
38 #ifndef __STDC_CONSTANT_MACROS
39 #define __STDC_CONSTANT_MACROS // ...or stdint.h wont' define UINT64_C, needed by libavutil
40 #endif
41 #include "FFmpegFile.h"
42
43 #include <cmath>
44 #include <iostream>
45 #include <algorithm>
46
47 #include <ofxsImageEffect.h>
48 #include <ofxsMacros.h>
49
50 #if defined(_WIN32) || defined(WIN64)
51 # include <windows.h> // for GetSystemInfo()
52 #define strncasecmp _strnicmp
53 #else
54 # include <unistd.h> // for sysconf()
55 #endif
56
57 using namespace OFX;
58
59 using std::string;
60 using std::make_pair;
61
62 // FFMPEG 3.1
63 #define USE_NEW_FFMPEG_API ( LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 0) )
64
65 #define CHECK(x) \
66 { \
67 int error = x; \
68 if (error < 0) { \
69 setInternalError(error); \
70 return; \
71 } \
72 } \
73
74 //#define TRACE_DECODE_PROCESS 1
75 //#define TRACE_FILE_OPEN 1
76
77 // Use one decoding thread per processor for video decoding.
78 // source: http://git.savannah.gnu.org/cgit/bino.git/tree/src/media_object.cpp
79 #if 0
80 static int
81 video_decoding_threads()
82 {
83 static long n = -1;
84
85 if (n < 0) {
86 #if defined(WIN32) || defined(WIN64)
87 SYSTEM_INFO si;
88 GetSystemInfo(&si);
89 n = si.dwNumberOfProcessors;
90 #else
91 n = sysconf(_SC_NPROCESSORS_ONLN);
92 #endif
93 if (n < 1) {
94 n = 1;
95 } else if (n > 16) {
96 n = 16;
97 }
98 }
99
100 return n;
101 }
102
103 #endif
104
105 static bool
extensionCorrespondToImageFile(const string & ext)106 extensionCorrespondToImageFile(const string & ext)
107 {
108 return (ext == "bmp" ||
109 ext == "cin" ||
110 ext == "dpx" ||
111 ext == "exr" ||
112 /*ext == "gif" ||*/
113 ext == "jpeg" ||
114 ext == "jpg" ||
115 ext == "pix" ||
116 ext == "png" ||
117 ext == "ppm" ||
118 ext == "ptx" ||
119 ext == "rgb" ||
120 ext == "rgba" ||
121 ext == "tga" ||
122 ext == "tiff" ||
123 ext == "webp");
124 }
125
126 bool
isImageFile(const string & filename)127 FFmpegFile::isImageFile(const string & filename)
128 {
129 ///find the last index of the '.' character
130 size_t lastDot = filename.find_last_of('.');
131
132 if (lastDot == string::npos) { //we reached the start of the file, return false because we can't determine from the extension
133 return false;
134 }
135 ++lastDot;//< bypass the '.' character
136 string ext;
137 std::locale loc;
138 while ( lastDot < filename.size() ) {
139 ext.append( 1, std::tolower(filename.at(lastDot), loc) );
140 ++lastDot;
141 }
142
143 return extensionCorrespondToImageFile(ext);
144 }
145
146 namespace {
147 struct FilterEntry
148 {
149 const char* name;
150 bool enableReader;
151 bool enableWriter;
152 };
153
154 // Bug 11027 - Nuke write: ffmpeg codec fails has details on individual codecs
155
156 // For a full list of formats, define FN_FFMPEGWRITER_PRINT_CODECS in ffmpegWriter.cpp
157 const FilterEntry kFormatWhitelist[] =
158 {
159 { "3gp", true, true },
160 { "3g2", true, true },
161 { "avi", true, true },
162 { "dv", true, false }, // DV (Digital Video), no HD support
163 { "flv", true, true }, // FLV (Flash Video), only used with flv codec. cannot be read in official Qt
164 { "gif", true, true }, // GIF Animation
165 { "h264", true, false }, // raw H.264 video. prefer a proper container (mp4, mov, avi)
166 { "hevc", true, false }, // raw HEVC video. hevc codec cannot be read in official qt
167 { "m4v", true, false }, // raw MPEG-4 video. prefer a proper container (mp4, mov, avi)
168 { "matroska", true, true }, // not readable in Qt but may be used with other software
169 { "mov", true, true },
170 { "mp4", true, true },
171 { "mpeg", true, true },
172 { "mpegts", true, true },
173 { "mxf", true, false }, // not readable in Qt but may be used with other software, however MXF has too many constraints to be easyly writable (for H264 it requires avctx.profile = FF_PROFILE_H264_BASELINE, FF_PROFILE_H264_HIGH_10 or FF_PROFILE_H264_HIGH_422). it is better to transcode with an external tool
174 { "ogg", true, false }, // Ogg, for theora codec (use ogv for writing)
175 { "ogv", true, true }, // Ogg Video, for theora codec
176 { NULL, false, false}
177 };
178
179 // For a full list of formats, define FN_FFMPEGWRITER_PRINT_CODECS in ffmpegWriter.cpp
180 // A range of codecs are omitted for licensing reasons, or because they support obselete/unnecessary
181 // formats that confuse the interface.
182
183 #define UNSAFEQT0 true // set to true: not really harmful
184 #define UNSAFEQT false // set to false: we care about QuickTime, because it is used widely - mainly colorshift issues
185 #define UNSAFEVLC true // set to true: we don't care much about being playable in VLC
186 #define TERRIBLE false
187 //#define SHOULDWORK true
188 #define SHOULDWORK false
189 const FilterEntry kCodecWhitelist[] =
190 {
191 // Video codecs.
192 { "aic", true, false }, // Apple Intermediate Codec (no encoder)
193 { "avrp", true, UNSAFEQT0 && UNSAFEVLC }, // Avid 1:1 10-bit RGB Packer - write not supported as not official qt readable with relevant 3rd party codec.
194 { "avui", true, false }, // Avid Meridien Uncompressed - write not supported as this is an SD only codec. Only 720x486 and 720x576 are supported. experimental in ffmpeg 2.6.1.
195 { "ayuv", true, UNSAFEQT0 && UNSAFEVLC }, // Uncompressed packed MS 4:4:4:4 - write not supported as not official qt readable.
196 { "cfhd", true, false }, // Cineform HD.
197 { "cinepak", true, true }, // Cinepak.
198 { "dxv", true, false }, // Resolume DXV
199 { "dnxhd", true, true }, // VC3/DNxHD
200 { "ffv1", true, UNSAFEQT0 && UNSAFEVLC }, // FFmpeg video codec #1 - write not supported as not official qt readable.
201 { "ffvhuff", true, UNSAFEQT0 && UNSAFEVLC }, // Huffyuv FFmpeg variant - write not supported as not official qt readable.
202 { "flv", true, UNSAFEQT0 }, // FLV / Sorenson Spark / Sorenson H.263 (Flash Video) - write not supported as not official qt readable.
203 { "gif", true, true }, // GIF (Graphics Interchange Format) - write not supported as 8-bit only.
204 { "h263p", true, true }, // H.263+ / H.263-1998 / H.263 version 2
205 { "h264", true, false }, // H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 (the encoder is libx264)
206 { "hap", true, true }, // Vidvox Hap
207 { "hevc", true, false }, // H.265 / HEVC (High Efficiency Video Coding) (the encoder is libx265)
208 { "huffyuv", true, UNSAFEQT0 && UNSAFEVLC }, // HuffYUV - write not supported as not official qt readable.
209 { "jpeg2000", true, UNSAFEQT0 }, // JPEG 2000 - write not supported as not official qt readable.
210 { "jpegls", true, UNSAFEQT0 }, // JPEG-LS - write not supported as can't be read in in official qt.
211 { "libopenh264", true, true }, // Cisco libopenh264 H.264/MPEG-4 AVC encoder
212 { "libopenjpeg", true, true }, // OpenJPEG JPEG 2000
213 { "libschroedinger", true, UNSAFEQT0 && UNSAFEVLC }, // libschroedinger Dirac - write untested. VLC plays with a wrong format
214 { "libtheora", true, UNSAFEQT0 }, // libtheora Theora - write untested.
215 { "libvpx", true, UNSAFEQT0 }, // On2 VP8
216 { "libvpx-vp9", true, UNSAFEQT0 }, // Google VP9
217 { "libx264", true, UNSAFEQT0 }, // H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 (encoder)
218 { "libx264rgb", true, UNSAFEQT0 }, // H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 RGB (encoder)
219 { "libx265", true, UNSAFEQT0 }, // H.265 / HEVC (High Efficiency Video Coding) (encoder) - resizes the image
220 { "libxavs", true, false }, // Chinese AVS (Audio Video Standard) (encoder) - untested
221 { "libxvid", true, true }, // MPEG-4 part 2
222 { "ljpeg", true, UNSAFEQT0 }, // Lossless JPEG - write not supported as can't be read in in official qt.
223 { "mjpeg", true, true }, // MJPEG (Motion JPEG) - this looks to be MJPEG-A. MJPEG-B encoding is not supported by FFmpeg so is not included here. To avoid confusion over the MJPEG-A and MJPEG-B variants, this codec is displayed as 'Photo JPEG'. This is done to i) avoid the confusion of the naming, ii) be consistent with Apple QuickTime, iii) the term 'Photo JPEG' is recommend for progressive frames which is appropriate to Nuke/NukeStudio as it does not have interlaced support.
224 { "mpeg1video", true, TERRIBLE }, // MPEG-1 video - write not supported as it gives random 8x8 blocky errors
225 { "mpeg2video", true, true }, // MPEG-2 video
226 { "mpeg4", true, true }, // MPEG-4 part 2
227 { "msmpeg4v2", true, UNSAFEQT0 }, // MPEG-4 part 2 Microsoft variant version 2 - write not supported as doesn't read in official qt.
228 { "msmpeg4", true, UNSAFEQT0 }, // MPEG-4 part 2 Microsoft variant version 3 - write not supported as doesn't read in official qt.
229 { "png", true, true }, // PNG (Portable Network Graphics) image
230 { "prores", true, false }, // Apple ProRes (the encoder is prores_ks)
231 { "qtrle", true, true }, // QuickTime Animation (RLE) video
232 { "r10k", true, UNSAFEQT && UNSAFEVLC }, // AJA Kono 10-bit RGB - write not supported as not official qt readable without colourshifts.
233 { "r210", true, UNSAFEQT && UNSAFEVLC }, // Uncompressed RGB 10-bit - write not supported as not official qt readable with relevant 3rd party codec without colourshifts.
234 { "rawvideo", true, UNSAFEQT && UNSAFEVLC }, // raw video - write not supported as not official qt readable.
235 { "svq1", true, true }, // Sorenson Vector Quantizer 1 / Sorenson Video 1 / SVQ1
236 { "targa", true, true }, // Truevision Targa image.
237 { "theora", true, false }, // Theora (decoder).
238 { "tiff", true, true }, // TIFF Image
239 { "v210", true, UNSAFEQT }, // Uncompressed 4:2:2 10-bit- write not supported as not official qt readable without colourshifts.
240 { "v308", true, UNSAFEQT0 && UNSAFEVLC }, // Uncompressed packed 4:4:4 - write not supported as not official qt readable and 8-bit only.
241 { "v408", true, UNSAFEQT0 && UNSAFEVLC }, // Uncompressed packed QT 4:4:4:4 - write not supported as official qt can't write, so bad round trip choice and 8-bit only.
242 { "v410", true, UNSAFEQT0 && UNSAFEVLC }, // Uncompressed 4:4:4 10-bit - write not supported as not official qt readable with standard codecs.
243 { "vc2", true, UNSAFEQT0 && UNSAFEVLC }, // SMPTE VC-2 (previously BBC Dirac Pro).
244 { "vp8", true, false }, // On2 VP8 (decoder)
245 { "vp9", true, false }, // Google VP9 (decoder)
246
247 // Audio codecs.
248 { "pcm_alaw", true, true }, // PCM A-law / G.711 A-law
249 { "pcm_f32be", true, true }, // PCM 32-bit floating point big-endian
250 { "pcm_f32le", true, true }, // PCM 32-bit floating point little-endian
251 { "pcm_f64be", true, true }, // PCM 64-bit floating point big-endian
252 { "pcm_f64le", true, true }, // PCM 64-bit floating point little-endian
253 { "pcm_mulaw", true, true }, // PCM mu-law / G.711 mu-law
254 { "pcm_s16be", true, true }, // PCM signed 16-bit big-endian
255 { "pcm_s16le", true, true }, // PCM signed 16-bit little-endian
256 { "pcm_s24be", true, true }, // PCM signed 24-bit big-endian
257 { "pcm_s24le", true, true }, // PCM signed 24-bit little-endian
258 { "pcm_s32be", true, true }, // PCM signed 32-bit big-endian
259 { "pcm_s32le", true, true }, // PCM signed 32-bit little-endian
260 { "pcm_s8", true, true }, // PCM signed 8-bit
261 { "pcm_u16be", true, true }, // PCM unsigned 16-bit big-endian
262 { "pcm_u16le", true, true }, // PCM unsigned 16-bit little-endian
263 { "pcm_u24be", true, true }, // PCM unsigned 24-bit big-endian
264 { "pcm_u24le", true, true }, // PCM unsigned 24-bit little-endian
265 { "pcm_u32be", true, true }, // PCM unsigned 32-bit big-endian
266 { "pcm_u32le", true, true }, // PCM unsigned 32-bit little-endian
267 { "pcm_u8", true, true }, // PCM unsigned 8-bit
268 { NULL, false, false}
269 };
270
271 const FilterEntry*
getEntry(const char * name,const FilterEntry * whitelist,const FilterEntry * blacklist=NULL)272 getEntry(const char* name,
273 const FilterEntry* whitelist,
274 const FilterEntry* blacklist = NULL)
275 {
276 const FilterEntry* iterWhitelist = whitelist;
277 const size_t nameLength = strlen(name);
278
279 // check for normal mode
280 while (iterWhitelist->name != NULL) {
281 size_t iteNameLength = strlen(iterWhitelist->name);
282 size_t maxLength = (nameLength > iteNameLength) ? nameLength : iteNameLength;
283 if (strncmp(name, iterWhitelist->name, maxLength) == 0) {
284 // Found in whitelist, now check blacklist
285 if (blacklist) {
286 const FilterEntry* iterBlacklist = blacklist;
287
288 while (iterBlacklist->name != NULL) {
289 iteNameLength = strlen(iterBlacklist->name);
290 maxLength = (nameLength > iteNameLength) ? nameLength : iteNameLength;
291 if (strncmp(name, iterBlacklist->name, maxLength) == 0) {
292 // Found in codec whitelist but blacklisted too
293 return NULL;
294 }
295
296 ++iterBlacklist;
297 }
298 }
299
300 // Found in whitelist and not in blacklist
301 return iterWhitelist;
302 }
303
304 ++iterWhitelist;
305 }
306
307 return NULL;
308 }
309 } // namespace {
310
311 bool
isFormatWhitelistedForReading(const char * formatName)312 FFmpegFile::isFormatWhitelistedForReading(const char* formatName)
313 {
314 const FilterEntry* whitelistEntry = getEntry(formatName, kFormatWhitelist);
315
316 return (whitelistEntry && whitelistEntry->enableReader);
317 }
318
319 bool
isFormatWhitelistedForWriting(const char * formatName)320 FFmpegFile::isFormatWhitelistedForWriting(const char* formatName)
321 {
322 const FilterEntry* whitelistEntry = getEntry(formatName, kFormatWhitelist);
323
324 return (whitelistEntry && whitelistEntry->enableWriter);
325 }
326
327 bool
isCodecWhitelistedForReading(const char * codecName)328 FFmpegFile::isCodecWhitelistedForReading(const char* codecName)
329 {
330 const FilterEntry* whitelistEntry = getEntry(codecName, kCodecWhitelist);
331
332 return (whitelistEntry && whitelistEntry->enableReader);
333 }
334
335 bool
isCodecWhitelistedForWriting(const char * codecName)336 FFmpegFile::isCodecWhitelistedForWriting(const char* codecName)
337 {
338 const FilterEntry* whitelistEntry = getEntry(codecName, kCodecWhitelist);
339
340 return (whitelistEntry && whitelistEntry->enableWriter);
341 }
342
343 SwsContext*
getConvertCtx(AVPixelFormat srcPixelFormat,int srcWidth,int srcHeight,int srcColorRange,AVPixelFormat dstPixelFormat,int dstWidth,int dstHeight)344 FFmpegFile::Stream::getConvertCtx(AVPixelFormat srcPixelFormat,
345 int srcWidth,
346 int srcHeight,
347 int srcColorRange,
348 AVPixelFormat dstPixelFormat,
349 int dstWidth,
350 int dstHeight)
351 {
352 // Reset is flagged when the UI colour matrix selection is
353 // modified. This causes a new convert context to be created
354 // that reflects the UI selection.
355 if (_resetConvertCtx) {
356 _resetConvertCtx = false;
357 if (_convertCtx) {
358 sws_freeContext(_convertCtx);
359 _convertCtx = NULL;
360 }
361 }
362
363 if (!_convertCtx) {
364 //Preventing deprecated pixel format used error messages, see:
365 //https://libav.org/doxygen/master/pixfmt_8h.html#a9a8e335cf3be472042bc9f0cf80cd4c5
366 //This manually sets them to the new versions of equivalent types.
367 switch (srcPixelFormat) {
368 case AV_PIX_FMT_YUVJ420P:
369 srcPixelFormat = AV_PIX_FMT_YUV420P;
370 if (srcColorRange == AVCOL_RANGE_UNSPECIFIED) {
371 srcColorRange = AVCOL_RANGE_JPEG;
372 }
373 break;
374 case AV_PIX_FMT_YUVJ422P:
375 srcPixelFormat = AV_PIX_FMT_YUV422P;
376 if (srcColorRange == AVCOL_RANGE_UNSPECIFIED) {
377 srcColorRange = AVCOL_RANGE_JPEG;
378 }
379 break;
380 case AV_PIX_FMT_YUVJ444P:
381 srcPixelFormat = AV_PIX_FMT_YUV444P;
382 if (srcColorRange == AVCOL_RANGE_UNSPECIFIED) {
383 srcColorRange = AVCOL_RANGE_JPEG;
384 }
385 break;
386 case AV_PIX_FMT_YUVJ440P:
387 srcPixelFormat = AV_PIX_FMT_YUV440P;
388 if (srcColorRange == AVCOL_RANGE_UNSPECIFIED) {
389 srcColorRange = AVCOL_RANGE_JPEG;
390 }
391 default:
392 break;
393 }
394
395 _convertCtx = sws_getContext(srcWidth, srcHeight, srcPixelFormat, // src format
396 dstWidth, dstHeight, dstPixelFormat, // dest format
397 SWS_BICUBIC, NULL, NULL, NULL);
398
399 // Set up the SoftWareScaler to convert colorspaces correctly.
400 // Colorspace conversion makes no sense for RGB->RGB conversions
401 if ( !isYUV() ) {
402 return _convertCtx;
403 }
404
405 int colorspace = isRec709Format() ? SWS_CS_ITU709 : SWS_CS_ITU601;
406 // Optional color space override
407 if (_colorMatrixTypeOverride > 0) {
408 if (_colorMatrixTypeOverride == 1) {
409 colorspace = SWS_CS_ITU709;
410 } else {
411 colorspace = SWS_CS_ITU601;
412 }
413 }
414
415 // sws_setColorspaceDetails takes a flag indicating the white-black range of the input:
416 // 0 - mpeg, 16..235
417 // 1 - jpeg, 0..255
418 int srcRange;
419 // Set this flag according to the color_range reported by the codec context.
420 switch (srcColorRange) {
421 case AVCOL_RANGE_MPEG:
422 srcRange = 0;
423 break;
424 case AVCOL_RANGE_JPEG:
425 srcRange = 1;
426 break;
427 case AVCOL_RANGE_UNSPECIFIED:
428 default:
429 // If the colour range wasn't specified, set the flag according to
430 // whether the data is YUV or not.
431 srcRange = isYUV() ? 0 : 1;
432 break;
433 }
434
435 int result = sws_setColorspaceDetails(_convertCtx,
436 sws_getCoefficients(colorspace), // inv_table
437 srcRange, // srcRange -flag indicating the white-black range of the input (1=jpeg / 0=mpeg) 0 = 16..235, 1 = 0..255
438 sws_getCoefficients(SWS_CS_DEFAULT), // table
439 1, // dstRange - 0 = 16..235, 1 = 0..255
440 0, // brightness fixed point, with 0 meaning no change,
441 1 << 16, // contrast fixed point, with 1<<16 meaning no change,
442 1 << 16); // saturation fixed point, with 1<<16 meaning no change);
443
444 assert(result != -1);
445 }
446
447 return _convertCtx;
448 } // FFmpegFile::Stream::getConvertCtx
449
450 /*static*/ double
GetStreamAspectRatio(Stream * stream)451 FFmpegFile::Stream::GetStreamAspectRatio(Stream* stream)
452 {
453 if (stream->_avstream->sample_aspect_ratio.num) {
454 #if TRACE_FILE_OPEN
455 std::cout << " Aspect ratio (from stream)=" << av_q2d(stream->_avstream->sample_aspect_ratio) << std::endl;
456 #endif
457
458 return av_q2d(stream->_avstream->sample_aspect_ratio);
459 } else if (stream->_codecContext->sample_aspect_ratio.num) {
460 #if TRACE_FILE_OPEN
461 std::cout << " Aspect ratio (from codec)=" << av_q2d(stream->_codecContext->sample_aspect_ratio) << std::endl;
462 #endif
463
464 return av_q2d(stream->_codecContext->sample_aspect_ratio);
465 }
466 #if TRACE_FILE_OPEN
467 else {
468 std::cout << " Aspect ratio unspecified, assuming " << stream->_aspect << std::endl;
469 }
470 #endif
471
472 return stream->_aspect;
473 }
474
475 // get stream start time
476 int64_t
getStreamStartTime(Stream & stream)477 FFmpegFile::getStreamStartTime(Stream & stream)
478 {
479 #if TRACE_FILE_OPEN
480 std::cout << " Determining stream start PTS:" << std::endl;
481 #endif
482
483 AVPacket avPacket;
484 av_init_packet(&avPacket);
485
486 // Read from stream. If the value read isn't valid, get it from the first frame in the stream that provides such a
487 // value.
488 int64_t startPTS = stream._avstream->start_time;
489 int64_t startDTS = stream._avstream->start_time;
490 #if TRACE_FILE_OPEN
491 if ( startPTS != int64_t(AV_NOPTS_VALUE) ) {
492 std::cout << " Obtained from AVStream::start_time=";
493 }
494 #endif
495
496 if (startPTS < 0) {
497 #if TRACE_FILE_OPEN
498 std::cout << " Not specified by AVStream::start_time, searching frames..." << std::endl;
499 #endif
500
501 // Seek 1st key-frame in video stream.
502 avcodec_flush_buffers(stream._codecContext);
503
504 // Here, avPacket needs to be local as we don't need to keep any information outside this function context.
505 // Do not replace this with _avPacket, which is global, because _avPacket is associated with the playback process
506 // and that may produces inconsistencies. _avPacket is now used only in the |decode| method and we need to ensure
507 // that it remains valid even after we get out of the |decode| method context, because the information stored by
508 // _avPacket may be displayed on the screen for multiple frames. Please have a look at TP 162892 for more information
509 // https://foundry.tpondemand.com/entity/162892
510 if (av_seek_frame(_context, stream._idx, startPTS, AVSEEK_FLAG_BACKWARD) >= 0) {
511
512 // Read frames until we get one for the video stream that contains a valid PTS.
513 while (av_read_frame(_context, &avPacket) >= 0) {
514 if (avPacket.stream_index != stream._idx) {
515 continue;
516 }
517 // Packet read for video stream. Get its PTS.
518 startPTS = avPacket.pts;
519 startDTS = avPacket.dts;
520
521 // Loop will continue if the current packet doesn't end after 0
522 if (startPTS + avPacket.duration > 0) {
523 break;
524 }
525 }
526 }
527 #if TRACE_FILE_OPEN
528 else {
529 std::cout << " Seek error, aborted search" << std::endl;
530 }
531 #endif
532
533 #if TRACE_FILE_OPEN
534 if ( startPTS != int64_t(AV_NOPTS_VALUE) ) {
535 std::cout << " Found by searching frames=";
536 }
537 #endif
538 }
539
540 // If we still don't have a valid initial PTS, assume 0. (This really shouldn't happen for any real media file, as
541 // it would make meaningful playback presentation timing and seeking impossible.);
542 // TP 162519 - We discard the samples with a negative timestamp to make mov64 match the Quick Time Player;
543 // Video streams are usually inter-coded (most frames rely on other frames to be decoded). So, when such a stream
544 // is trimmed those reference frames, even if not within the trimmed time range, have to be included in the output
545 // in order for the video to be playable. These frames are assigned to a negative time-stamp;
546 // Based on the experimental work we concluded that for the streams that have included frames which are assigned
547 // to negative timestamps it doesn't make sense to pick a value grater than 0 for the first frame timestamp. The first
548 // frame timestamp is going to match the packet which starts just before 0 and ends after 0 if that exists. Otherwise
549 // it will be 0.
550 // For more information please have a look at TP 162519
551 const bool isStartPTSValid = (startPTS + avPacket.duration > 0);
552 if (!isStartPTSValid) {
553 #if TRACE_FILE_OPEN
554 std::cout << " Not found by searching frames, assuming ";
555 #endif
556 startPTS = 0;
557 startDTS = 0;
558 }
559
560 #if TRACE_FILE_OPEN
561 std::cout << startPTS << " ticks, " << double(startPTS) * double(stream._avstream->time_base.num) /
562 double(stream._avstream->time_base.den) << " s" << std::endl;
563 #endif
564
565 stream._startDTS = startDTS;
566 av_packet_unref(&avPacket);
567
568 return startPTS;
569 } // FFmpegFile::getStreamStartTime
570
571 // Get the video stream duration in frames...
572 int64_t
getStreamFrames(Stream & stream)573 FFmpegFile::getStreamFrames(Stream & stream)
574 {
575 #if TRACE_FILE_OPEN
576 std::cout << " Determining stream frame count:" << std::endl;
577 #endif
578
579 int64_t frames = 0;
580
581 // Obtain from movie duration if specified. This is preferred since mov/mp4 formats allow the media in
582 // tracks (=streams) to be remapped in time to the final movie presentation without needing to recode the
583 // underlying tracks content; the movie duration thus correctly describes the final presentation.
584 if (frames <= 0 && _context->duration > 0) {
585 // Annoyingly, FFmpeg exposes the movie duration converted (with round-to-nearest semantics) to units of
586 // AV_TIME_BASE (microseconds in practice) and does not expose the original rational number duration
587 // from a mov/mp4 file's "mvhd" atom/box. Accuracy may be lost in this conversion; a duration that was
588 // an exact number of frames as a rational may end up as a duration slightly over or under that number
589 // of frames in units of AV_TIME_BASE.
590 // Conversion to whole frames rounds up the resulting number of frames because a partial frame is still
591 // a frame. However, in an attempt to compensate for AVFormatContext's inaccurate representation of
592 // duration, with unknown rounding direction, the conversion to frames subtracts 1 unit (microsecond)
593 // from that duration. The rationale for this is thus:
594 // * If the stored duration exactly represents an exact number of frames, then that duration minus 1
595 // will result in that same number of frames once rounded up.
596 // * If the stored duration is for an exact number of frames that was rounded down, then that duration
597 // minus 1 will result in that number of frames once rounded up.
598 // * If the stored duration is for an exact number of frames that was rounded up, then that duration
599 // minus 1 will result in that number of frames once rounded up, while that duration unchanged would
600 // result in 1 more frame being counted after rounding up.
601 // * If the original duration in the file was not for an exact number of frames, then the movie timebase
602 // would have to be >= 10^6 for there to be any chance of this calculation resulting in the wrong
603 // number of frames. This isn't a case that I've seen. Even if that were to be the case, the original
604 // duration would have to be <= 1 microsecond greater than an exact number of frames in order to
605 // result in the wrong number of frames, which is highly improbable.
606 int64_t divisor = int64_t(AV_TIME_BASE) * stream._fpsDen;
607 frames = ( (_context->duration - 1) * stream._fpsNum + divisor - 1 ) / divisor;
608
609 // The above calculation is not reliable, because it seems in some situations (such as rendering out a mov
610 // with 5 frames at 24 fps from Nuke) the duration has been rounded up to the nearest millisecond, which
611 // leads to an extra frame being reported. To attempt to work around this, compare against the number of
612 // frames in the stream, and if they differ by one, use that value instead.
613 int64_t streamFrames = stream._avstream->nb_frames;
614 if ( (streamFrames > 0) && (std::abs( (double)(frames - streamFrames) ) <= 1) ) {
615 frames = streamFrames;
616 }
617 #if TRACE_FILE_OPEN
618 std::cout << " Obtained from AVFormatContext::duration & framerate=";
619 #endif
620 }
621
622 // If number of frames still unknown, obtain from stream's number of frames if specified. Will be 0 if
623 // unknown.
624 if (frames <= 0 && stream._avstream->nb_frames > 0) {
625 #if TRACE_FILE_OPEN
626 std::cout << " Not specified by AVFormatContext::duration, obtaining from AVStream::nb_frames..." << std::endl;
627 #endif
628 frames = stream._avstream->nb_frames;
629 #if TRACE_FILE_OPEN
630 if (frames) {
631 std::cout << " Obtained from AVStream::nb_frames=";
632 }
633 #endif
634 }
635
636 // If number of frames still unknown, attempt to calculate from stream's duration, fps and timebase.
637 if (frames <= 0 && stream._avstream->duration > 0) {
638 #if TRACE_FILE_OPEN
639 std::cout << " Not specified by AVStream::nb_frames, calculating from duration & framerate..." << std::endl;
640 #endif
641 frames = (int64_t(stream._avstream->duration) * stream._avstream->time_base.num * stream._fpsNum) /
642 (int64_t(stream._avstream->time_base.den) * stream._fpsDen);
643 #if TRACE_FILE_OPEN
644 if (frames > 0) {
645 std::cout << " Calculated from duration & framerate=";
646 }
647 #endif
648 }
649
650 // If the number of frames is still unknown, attempt to measure it from the last frame PTS for the stream in the
651 // file relative to first (which we know from earlier).
652 if (frames <= 0) {
653 #if TRACE_FILE_OPEN
654 std::cout << " Not specified by duration & framerate, searching frames for last PTS..." << std::endl;
655 #endif
656
657 int64_t maxPts = stream._startPTS;
658
659 // Seek last key-frame.
660 avcodec_flush_buffers(stream._codecContext);
661 av_seek_frame(_context, stream._idx, stream.frameToPts(1 << 29), AVSEEK_FLAG_BACKWARD);
662
663 // Read up to last frame, extending max PTS for every valid PTS value found for the video stream.
664 AVPacket avPacket;
665 // Here, avPacket needs to be local as we don't need to keep any information outside this function context.
666 // Do not replace this with _avPacket, which is global, because _avPacket is associated with the playback process
667 // and that may produces inconsistencies. _avPacket is now used only in the |decode| method and we need to ensure
668 // that it remains valid even after we get out of the |decode| method context, because the information stored by
669 // _avPacket may be displayed on the screen for multiple frames. Please have a look at TP 162892 for more information
670 // https://foundry.tpondemand.com/entity/162892
671 av_init_packet(&avPacket);
672
673 while (av_read_frame(_context, &avPacket) >= 0) {
674 if (avPacket.stream_index == stream._idx && avPacket.pts != int64_t(AV_NOPTS_VALUE) && avPacket.pts > maxPts)
675 maxPts = avPacket.pts;
676 av_packet_unref(&avPacket);
677 }
678 #if TRACE_FILE_OPEN
679 std::cout << " Start PTS=" << stream._startPTS << ", Max PTS found=" << maxPts << std::endl;
680 #endif
681
682 // Compute frame range from min to max PTS. Need to add 1 as both min and max are at starts of frames, so stream
683 // extends for 1 frame beyond this.
684 frames = 1 + stream.ptsToFrame(maxPts);
685 #if TRACE_FILE_OPEN
686 std::cout << " Calculated from frame PTS range=";
687 #endif
688 }
689
690 #if TRACE_FILE_OPEN
691 std::cout << frames << std::endl;
692 #endif
693
694 return frames;
695 } // FFmpegFile::getStreamFrames
696
697 // Returns true if the properties of the two streams are considered to match in terms of
698 // codec, resolution, frame rate, time base, etc. The motivation for this is that streams
699 // that match in this way are more likely to contain multiple views rather then unrelated
700 // content. This is somewhat arbitrary but seems like a reasonable starting point until
701 // users tell us otherwise.
702 static
703 bool
CheckStreamPropertiesMatch(const AVStream * streamA,const AVStream * streamB)704 CheckStreamPropertiesMatch(const AVStream* streamA,
705 const AVStream* streamB)
706 {
707 #if 0//FF_API_LAVF_AVCTX
708 const AVCodecContext* codecA = streamA->codec;
709 const AVCodecContext* codecB = streamB->codec;
710 #else
711 const AVCodecParameters* codecA = streamA->codecpar;
712 const AVCodecParameters* codecB = streamB->codecpar;
713 #endif
714
715 // Sanity check //
716 if (codecA == NULL || codecB == NULL) {
717 return false;
718 }
719
720 #if 0//FF_API_LAVF_AVCTX
721 AVPixelFormat codecAfmt = codecA->pix_fmt;
722 AVPixelFormat codecBfmt = codecB->pix_fmt;
723 #else
724 AVPixelFormat codecAfmt = (AVPixelFormat)codecA->format;
725 AVPixelFormat codecBfmt = (AVPixelFormat)codecB->format;
726 #endif
727
728 // Not matching even if both reads failed
729 if (codecAfmt == AV_PIX_FMT_NONE || codecBfmt == AV_PIX_FMT_NONE) {
730 return false;
731 }
732
733 const AVPixFmtDescriptor* pixFmtDescA = av_pix_fmt_desc_get(codecAfmt);
734 const AVPixFmtDescriptor* pixFmtDescB = av_pix_fmt_desc_get(codecBfmt);
735
736 return
737 (codecA->codec_id == codecB->codec_id) &&
738 (codecA->bits_per_raw_sample == codecB->bits_per_raw_sample) &&
739 (codecA->width == codecB->width) &&
740 (codecA->height == codecB->height) &&
741 (codecA->sample_aspect_ratio.num == codecB->sample_aspect_ratio.num) &&
742 (codecA->sample_aspect_ratio.den == codecB->sample_aspect_ratio.den) &&
743 (pixFmtDescA->nb_components == pixFmtDescB->nb_components) &&
744 (streamA->sample_aspect_ratio.num == streamB->sample_aspect_ratio.num) &&
745 (streamA->sample_aspect_ratio.den == streamB->sample_aspect_ratio.den) &&
746 (streamA->time_base.num == streamB->time_base.num) &&
747 (streamA->time_base.den == streamB->time_base.den) &&
748 (streamA->start_time == streamB->start_time) &&
749 (streamA->duration == streamB->duration) &&
750 (streamA->nb_frames == streamB->nb_frames) &&
751 (streamA->r_frame_rate.num == streamB->r_frame_rate.num) &&
752 (streamA->r_frame_rate.den == streamB->r_frame_rate.den);
753 }
754
755 // constructor
FFmpegFile(const string & filename)756 FFmpegFile::FFmpegFile(const string & filename)
757 : _filename(filename)
758 , _context(NULL)
759 , _format(NULL)
760 , _streams()
761 , _selectedStream(NULL)
762 , _errorMsg()
763 , _invalidState(false)
764 , _avPacket()
765 #ifdef OFX_IO_MT_FFMPEG
766 , _lock()
767 , _invalidStateLock()
768 #endif
769 {
770 #ifdef OFX_IO_MT_FFMPEG
771 //MultiThread::AutoMutex guard(_lock); // not needed in a constructor: we are the only owner
772 #endif
773
774 #if TRACE_FILE_OPEN
775 std::cout << "FFmpeg Reader=" << this << "::c'tor(): filename=" << filename << std::endl;
776 #endif
777
778 assert( !_filename.empty() );
779 CHECK( avformat_open_input(&_context, _filename.c_str(), _format, NULL) );
780 assert(_context);
781 // Bug 51016 - probesize is the maximum amount of data, in bytes, which is read to determine
782 // frame information in avformat_find_stream_info. It's important that when reading
783 // stereo quicktimes that the probe size be large enough to read data from both video tracks.
784 // Otherwise the streams could be reported as having different properties, when really they're
785 // the same but due to an insufficient probesize the second stream didn't have all the relevent data
786 // loaded. This defaults to 5meg. 100meg should be enough for large stereo quicktimes.
787 _context->probesize = 100000000;
788
789 CHECK( avformat_find_stream_info(_context, NULL) );
790
791 #if TRACE_FILE_OPEN
792 std::cout << " " << _context->nb_streams << " streams:" << std::endl;
793 #endif
794
795 // fill the array with all available video streams
796 bool unsuported_codec = false;
797
798 // find all streams that the library is able to decode
799 for (unsigned i = 0; i < _context->nb_streams; ++i) {
800 #if TRACE_FILE_OPEN
801 std::cout << " FFmpeg stream index " << i << ": ";
802 #endif
803 AVStream* avstream = _context->streams[i];
804
805 // be sure to have a valid stream
806 if (!avstream || !avstream->FFMSCODEC) {
807 #if TRACE_FILE_OPEN
808 std::cout << "No valid stream or codec, skipping..." << std::endl;
809 #endif
810 continue;
811 }
812
813 AVCodecContext *avctx;
814 avctx = avcodec_alloc_context3(NULL);
815 if (!avctx) {
816 setError( "cannot allocate codec context" );
817
818 return;
819 }
820
821 int ret = make_context(avctx, avstream);
822 if (ret < 0) {
823 #if TRACE_FILE_OPEN
824 std::cout << "Could not convert to context, skipping..." << std::endl;
825 #endif
826 continue;
827 }
828
829 // considering only video streams, skipping audio
830 if (avctx->codec_type != AVMEDIA_TYPE_VIDEO) {
831 #if TRACE_FILE_OPEN
832 std::cout << "Not a video stream, skipping..." << std::endl;
833 #endif
834 continue;
835 }
836 if (avctx->pix_fmt == AV_PIX_FMT_NONE) {
837 # if TRACE_FILE_OPEN
838 std::cout << "Unknown pixel format, skipping..." << std::endl;
839 # endif
840 continue;
841 }
842
843 // find the codec
844 AVCodec* videoCodec = avcodec_find_decoder(avctx->codec_id);
845 if (videoCodec == NULL) {
846 #if TRACE_FILE_OPEN
847 std::cout << "Decoder not found, skipping..." << std::endl;
848 #endif
849 continue;
850 }
851
852 // skip codecs not in the white list
853 //string reason;
854 if ( !isCodecWhitelistedForReading(videoCodec->name) ) {
855 # if TRACE_FILE_OPEN
856 std::cout << "Decoder \"" << videoCodec->name << "\" disallowed, skipping..." << std::endl;
857 # endif
858 unsuported_codec = true;
859 continue;
860 }
861
862 if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
863 // source: http://git.savannah.gnu.org/cgit/bino.git/tree/src/media_object.cpp
864
865 // Some codecs support multi-threaded decoding (eg mpeg). Its fast but causes problems when opening many readers
866 // simultaneously since each opens as many threads as you have cores. This leads to resource starvation and failed reads.
867 // Hopefully, getNumCPUs() will give us the right number of usable cores
868
869 // Activate multithreaded decoding. This must be done before opening the codec; see
870 // http://lists.gnu.org/archive/html/bino-list/2011-08/msg00019.html
871 # ifdef AV_CODEC_CAP_AUTO_THREADS
872 // Do not use AV_CODEC_CAP_AUTO_THREADS, since it may create too many threads
873 //if (avstream->codec->codec && (avstream->codec->codec->capabilities & AV_CODEC_CAP_AUTO_THREADS)) {
874 // avstream->codec->thread_count = 0;
875 //} else
876 # endif
877 {
878 avctx->thread_count = std::min( (int)MultiThread::getNumCPUs(), OFX_FFMPEG_MAX_THREADS ); // ask for the number of available cores for multithreading
879 # ifdef AV_CODEC_CAP_SLICE_THREADS
880 if ( avctx->codec && (avctx->codec->capabilities & AV_CODEC_CAP_SLICE_THREADS) ) {
881 // multiple threads are used to decode a single frame. Reduces delay
882 avctx->thread_type = FF_THREAD_SLICE;
883 }
884 # endif
885 //avstream->codec->thread_count = video_decoding_threads(); // bino's strategy (disabled)
886 }
887 // Set CODEC_FLAG_EMU_EDGE in the same situations in which ffplay sets it.
888 // I don't know what exactly this does, but it is necessary to fix the problem
889 // described in this thread: http://lists.nongnu.org/archive/html/bino-list/2012-02/msg00039.html
890 int lowres = 0;
891 #ifdef FF_API_LOWRES
892 lowres = avctx->lowres;
893 #endif
894 #ifdef CODEC_FLAG_EMU_EDGE // removed from ffmpeg 4.0
895 if ( lowres || ( videoCodec && (videoCodec->capabilities & CODEC_CAP_DR1) ) ) {
896 avctx->flags |= CODEC_FLAG_EMU_EDGE;
897 }
898 #endif
899 }
900
901 // skip if the codec can't be open
902 if (avcodec_open2(avctx, videoCodec, NULL) < 0) {
903 #if TRACE_FILE_OPEN
904 std::cout << "Decoder \"" << videoCodec->name << "\" failed to open, skipping..." << std::endl;
905 #endif
906 continue;
907 }
908
909 #if TRACE_FILE_OPEN
910 std::cout << "Video decoder \"" << videoCodec->name << "\" opened ok, getting stream properties:" << std::endl;
911 #endif
912
913 if (!_streams.empty()) {
914 // Assume that if this stream's properties don't match those of the zeroth stream then it doesn't
915 // correspond to an alternative view.
916 // This may turn out to be either too stringent or pointlessly lax, we'll have to see what users
917 // make of it. The way to handle this properly is to provide a knob allowing the user to map views
918 // to streams.
919 if (!CheckStreamPropertiesMatch(_streams[0]->_avstream, avstream)) {
920 #if TRACE_FILE_OPEN
921 std::cout << "Stream properties do not match those of first video stream, ignoring this stream." << std::endl;
922 #endif
923 continue;
924 }
925 }
926 Stream* stream = new Stream();
927 stream->_idx = i;
928 stream->_avstream = avstream;
929 stream->_codecContext = avctx;
930 stream->_videoCodec = videoCodec;
931 stream->_avFrame = av_frame_alloc(); // avcodec_alloc_frame();
932 {
933 // In |engine| the output bit depth was hard coded to 16-bits.
934 // Now it will use the bit depth reported by the decoder so
935 // that if a decoder outputs 10-bits then |engine| will convert
936 // this correctly. This means that the following change is
937 // requireded for FFmpeg decoders. Currently |_bitDepth| is used
938 // internally so this change has no side effects.
939 // [openfx-io note] when using insternal ffmpeg 8bits->16 bits conversion,
940 // (255 = 100%) becomes (65280 =99.6%)
941 stream->_bitDepth = avctx->bits_per_raw_sample; // disabled in Nuke's reader
942 //stream->_bitDepth = 16; // enabled in Nuke's reader
943
944 const AVPixFmtDescriptor* avPixFmtDescriptor = av_pix_fmt_desc_get(stream->_codecContext->pix_fmt);
945 if (avPixFmtDescriptor == NULL) {
946 throw std::runtime_error("av_pix_fmt_desc_get() failed");
947 }
948 // Sanity check the number of components.
949 // Only 3 or 4 components are supported by |engine|, that is
950 // Nuke/NukeStudio will only accept 3 or 4 component data.
951 // For a monochrome image (single channel) promote to 3
952 // channels. This is in keeping with all the assumptions
953 // throughout the code that if it is not 4 channels data
954 // then it must be three channel data. This ensures that
955 // all the buffer size calculations are correct.
956 stream->_numberOfComponents = avPixFmtDescriptor->nb_components;
957 if (3 > stream->_numberOfComponents) {
958 stream->_numberOfComponents = 3;
959 }
960 // AVCodecContext::bits_pre_raw_sample may not be set, if
961 // it's not set, try with the following utility function.
962 if (0 == stream->_bitDepth) {
963 stream->_bitDepth = av_get_bits_per_pixel(avPixFmtDescriptor) / stream->_numberOfComponents;
964 }
965 }
966
967 if (stream->_bitDepth > 8) {
968 # if VERSION_CHECK(LIBAVUTIL_VERSION_INT, <, 53, 6, 0, 53, 6, 0)
969 stream->_outputPixelFormat = (4 == stream->_numberOfComponents) ? AV_PIX_FMT_RGBA : AV_PIX_FMT_RGB48LE; // 16-bit.
970 # else
971 stream->_outputPixelFormat = (4 == stream->_numberOfComponents) ? AV_PIX_FMT_RGBA64LE : AV_PIX_FMT_RGB48LE; // 16-bit.
972 # endif
973 } else {
974 stream->_outputPixelFormat = (4 == stream->_numberOfComponents) ? AV_PIX_FMT_RGBA : AV_PIX_FMT_RGB24; // 8-bit
975 }
976 #if TRACE_FILE_OPEN
977 std::cout << " Timebase=" << avstream->time_base.num << "/" << avstream->time_base.den << " s/tick" << std::endl;
978 std::cout << " Duration=" << avstream->duration << " ticks, " <<
979 double(avstream->duration) * double(avstream->time_base.num) /
980 double(avstream->time_base.den) << " s" << std::endl;
981 std::cout << " BitDepth=" << stream->_bitDepth << std::endl;
982 std::cout << " NumberOfComponents=" << stream->_numberOfComponents << std::endl;
983 #endif
984
985 // If FPS is specified, record it.
986 // Otherwise assume 1 fps (default value).
987 if ( (avstream->r_frame_rate.num != 0) && (avstream->r_frame_rate.den != 0) ) {
988 stream->_fpsNum = avstream->r_frame_rate.num;
989 stream->_fpsDen = avstream->r_frame_rate.den;
990 #if TRACE_FILE_OPEN
991 std::cout << " Framerate=" << stream->_fpsNum << "/" << stream->_fpsDen << ", " <<
992 double(stream->_fpsNum) / double(stream->_fpsDen) << " fps" << std::endl;
993 #endif
994 }
995 #if TRACE_FILE_OPEN
996 else {
997 std::cout << " Framerate unspecified, assuming 1 fps" << std::endl;
998 }
999 #endif
1000
1001 stream->_width = avctx->width;
1002 stream->_height = avctx->height;
1003 #if TRACE_FILE_OPEN
1004 std::cout << " Image size=" << stream->_width << "x" << stream->_height << std::endl;
1005 #endif
1006
1007 // set aspect ratio
1008 stream->_aspect = Stream::GetStreamAspectRatio(stream);
1009
1010 // set stream start time and numbers of frames
1011 stream->_startPTS = getStreamStartTime(*stream);
1012 stream->_frames = getStreamFrames(*stream);
1013
1014 // save the stream
1015 _streams.push_back(stream);
1016 }
1017 if ( _streams.empty() ) {
1018 setError( unsuported_codec ? "unsupported codec..." : "unable to find video stream" );
1019 _selectedStream = NULL;
1020 } else {
1021 #pragma message WARN("should we build separate FFmpegfile for each view? see also FFmpegFileManager")
1022 const int viewIndex = 0; // TODO: pass as parameter?
1023 assert(viewIndex >= 0 && "Negative view index specified.");
1024
1025 if (static_cast<size_t>(viewIndex) < _streams.size()) {
1026 _selectedStream = _streams[viewIndex];
1027 }
1028 else {
1029 _selectedStream = _streams[0];
1030 }
1031 }
1032 }
1033
1034 // destructor
~FFmpegFile()1035 FFmpegFile::~FFmpegFile()
1036 {
1037 #ifdef OFX_IO_MT_FFMPEG
1038 AutoMutex guard(_lock);
1039 #endif
1040
1041 // force to close all resources needed for all streams
1042 for (unsigned int i = 0; i < _streams.size(); ++i) {
1043 delete _streams[i];
1044 }
1045 _streams.clear();
1046
1047 if (_context) {
1048 avformat_close_input(&_context);
1049 av_free(_context);
1050 }
1051 _filename.clear();
1052 _errorMsg.clear();
1053 }
1054
setSelectedStream(int streamIndex)1055 void FFmpegFile::setSelectedStream(int streamIndex)
1056 {
1057 if ((streamIndex >= 0) && (streamIndex < static_cast<int>(_streams.size()))) {
1058 _selectedStream = _streams[streamIndex];
1059 }
1060 else {
1061 assert(false && "setSelectedStream: Invalid streamIndex");
1062 _selectedStream = !_streams.empty() ? _streams[0] : NULL;
1063 }
1064 }
1065
1066 const char*
getColorspace() const1067 FFmpegFile::getColorspace() const
1068 {
1069 //The preferred colorspace is figured out from a number of sources - initially we look for a number
1070 //of different metadata sources that may be present in the file. If these fail we then fall back
1071 //to using the codec's underlying storage mechanism - if RGB we default to gamma 1.8, if YCbCr we
1072 //default to gamma 2.2 (note prores special case). Note we also ignore the NCLC atom for reading
1073 //purposes, as in practise it tends to be incorrect.
1074
1075 //First look for the meta keys that (recent) Nukes would've written, or special cases in Arri meta.
1076 //Doubles up searching for lower case keys as the ffmpeg searches are case sensitive, and the keys
1077 //have been seen to be lower cased (particularly in old Arri movs).
1078 if (_context && _context->metadata) {
1079 AVDictionaryEntry* t;
1080
1081 t = av_dict_get(_context->metadata, "uk.co.thefoundry.Colorspace", NULL, AV_DICT_IGNORE_SUFFIX);
1082 if (!t) {
1083 av_dict_get(_context->metadata, "uk.co.thefoundry.colorspace", NULL, AV_DICT_IGNORE_SUFFIX);
1084 }
1085 if (t) {
1086 #if 0
1087 //Validate t->value against root list, to make sure it's been written with a LUT
1088 //we have a matching conversion for.
1089 bool found = false;
1090 int i = 0;
1091 while (!found && LUT::builtin_names[i] != NULL) {
1092 found = !strcasecmp(t->value, LUT::builtin_names[i++]);
1093 }
1094 #else
1095 bool found = true;
1096 #endif
1097 if (found) {
1098 return t->value;
1099 }
1100 }
1101
1102 t = av_dict_get(_context->metadata, "com.arri.camera.ColorGammaSxS", NULL, AV_DICT_IGNORE_SUFFIX);
1103 if (!t) {
1104 av_dict_get(_context->metadata, "com.arri.camera.colorgammasxs", NULL, AV_DICT_IGNORE_SUFFIX);
1105 }
1106 if ( t && !strncasecmp(t->value, "LOG-C", 5) ) {
1107 return "AlexaV3LogC";
1108 }
1109 if ( t && !strncasecmp(t->value, "REC-709", 7) ) {
1110 return "rec709";
1111 }
1112 }
1113
1114 //Special case for prores - the util YUV will report RGB, due to pixel format support, but for
1115 //compatibility and consistency with official quicktime, we need to be using 2.2 for 422 material
1116 //and 1.8 for 4444. Protected to deal with ffmpeg vagaries.
1117 assert((_streams.empty() || _selectedStream) && "_streams not empty but null _selectedStream");
1118 if (!_streams.empty() && _selectedStream->_codecContext && _selectedStream->_codecContext->codec_id) {
1119 if (_selectedStream->_codecContext->codec_id == AV_CODEC_ID_PRORES) {
1120 if ( ( _streams[0]->_codecContext->codec_tag == MKTAG('a', 'p', '4', 'h') ) ||
1121 ( _streams[0]->_codecContext->codec_tag == MKTAG('a', 'p', '4', 'x') ) ) {
1122 return "Gamma1.8";
1123 } else {
1124 return "Gamma2.2";
1125 }
1126 }
1127 }
1128
1129 return isYUV() ? "Gamma2.2" : "Gamma1.8";
1130 } // FFmpegFile::getColorspace
1131
1132 void
setError(const char * msg,const char * prefix)1133 FFmpegFile::setError(const char* msg,
1134 const char* prefix)
1135 {
1136 #ifdef OFX_IO_MT_FFMPEG
1137 AutoMutex guard(_invalidStateLock);
1138 #endif
1139 if (prefix) {
1140 _errorMsg = prefix;
1141 _errorMsg += msg;
1142 #if TRACE_DECODE_PROCESS
1143 std::cout << "!!ERROR: " << prefix << msg << std::endl;
1144 #endif
1145 } else {
1146 _errorMsg = msg;
1147 #if TRACE_DECODE_PROCESS
1148 std::cout << "!!ERROR: " << msg << std::endl;
1149 #endif
1150 }
1151 _invalidState = true;
1152 }
1153
1154 const string &
getError() const1155 FFmpegFile::getError() const
1156 {
1157 #ifdef OFX_IO_MT_FFMPEG
1158 AutoMutex guard(_lock);
1159 #endif
1160
1161 return _errorMsg;
1162 }
1163
1164 // return true if the reader can't decode the frame
1165 bool
isInvalid() const1166 FFmpegFile::isInvalid() const
1167 {
1168 #ifdef OFX_IO_MT_FFMPEG
1169 AutoMutex guard(_invalidStateLock);
1170 #endif
1171
1172 return _invalidState;
1173 }
1174
1175 bool
seekFrame(int frame,Stream * stream)1176 FFmpegFile::seekFrame(int frame,
1177 Stream* stream)
1178 {
1179 ///Private should not lock
1180
1181 avcodec_flush_buffers(stream->_codecContext);
1182 int64_t timestamp = stream->frameToDts(frame);
1183 int error = av_seek_frame(_context, stream->_idx, timestamp, AVSEEK_FLAG_BACKWARD);
1184 if (error < 0) {
1185 // Seek error. Abort attempt to read and decode frames.
1186 setInternalError(error, "FFmpeg Reader failed to seek frame: ");
1187
1188 return false;
1189 }
1190
1191 // We can't be re-using the existing _avPacket data because we've just had to seek
1192 _avPacket.FreePacket();
1193
1194 return true;
1195 }
1196
1197 // decode a single frame into the buffer thread safe
1198 bool
decode(const ImageEffect * plugin,int frame,bool loadNearest,int maxRetries,unsigned char * buffer)1199 FFmpegFile::decode(const ImageEffect* plugin,
1200 int frame,
1201 bool loadNearest,
1202 int maxRetries,
1203 unsigned char* buffer)
1204 {
1205 #ifdef OFX_IO_MT_FFMPEG
1206 AutoMutex guard(_lock);
1207 #endif
1208
1209 if (_streams.empty()) {
1210 return false;
1211 }
1212
1213 assert(_selectedStream && "Null _selectedStream");
1214 if (!_selectedStream) {
1215 return false;
1216 }
1217 Stream* stream = _selectedStream;
1218
1219 // Translate from the 1-based frames expected to 0-based frame offsets for use in the rest of this code.
1220 int originalFrame = frame;
1221 frame = frame - 1;
1222
1223 // Early-out if out-of-range frame requested.
1224 if (frame < 0) {
1225 if (loadNearest) {
1226 frame = 0;
1227 } else {
1228 throw std::runtime_error("Missing frame");
1229 }
1230 } else if (frame >= stream->_frames) {
1231 if (loadNearest) {
1232 frame = (int)stream->_frames - 1;
1233 } else {
1234 throw std::runtime_error("Missing frame");
1235 }
1236 }
1237
1238 #if TRACE_DECODE_PROCESS
1239 std::cout << "FFmpeg Reader=" << this << "::decode(): frame=" << frame << /*", _viewIndex = " << _viewIndex <<*/ ", stream->_idx=" << stream->_idx << std::endl;
1240 #endif
1241
1242 // Number of read retries remaining when decode stall is detected before we give up (in the case of post-seek stalls,
1243 // such retries are applied only after we've searched all the way back to the start of the file and failed to find a
1244 // successful start point for playback)..
1245 //
1246 // We have a rather annoying case with a small subset of media files in which decode latency (between input and output
1247 // frames) will exceed the maximum above which we detect decode stall at certain frames on the first pass through the
1248 // file but those same frames will decode succesfully on a second attempt. The root cause of this is not understood but
1249 // it appears to be some oddity of FFmpeg. While I don't really like it, retrying decode enables us to successfully
1250 // decode those files rather than having to fail the read.
1251 int retriesRemaining = std::max(1, maxRetries);
1252
1253 // Whether we have just performed a seek and are still awaiting the first decoded frame after that seek. This controls
1254 // how we respond when a decode stall is detected.
1255 //
1256 // One cause of such stalls is when a file contains incorrect information indicating that a frame is a key-frame when it
1257 // is not; a seek may land at such a frame but the decoder will not be able to start decoding until a real key-frame is
1258 // reached, which may be a long way in the future. Once a frame has been decoded, we will expect it to be the first frame
1259 // input to decode but it will actually be the next real key-frame found, leading to subsequent frames appearing as
1260 // earlier frame numbers and the movie ending earlier than it should. To handle such cases, when a stall is detected
1261 // immediately after a seek, we seek to the frame before the previous seek's landing frame, allowing us to search back
1262 // through the movie for a valid key frame from which decode commences correctly; if this search reaches the beginning of
1263 // the movie, we give up and fail the read, thus ensuring that this method will exit at some point.
1264 //
1265 // Stalls once seeking is complete and frames are being decoded are handled differently; these result in immediate read
1266 // failure.
1267 bool awaitingFirstDecodeAfterSeek = false;
1268
1269 // If the frame we want is not the next one to be decoded, seek to the keyframe before/at our desired frame. Set the last
1270 // seeked frame to indicate that we need to synchronise frame indices once we've read the first frame of the video stream,
1271 // since we don't yet know which frame number the seek will land at. Also invalidate current indices, reset accumulated
1272 // decode latency and record that we're awaiting the first decoded frame after a seek.
1273 int lastSeekedFrame = -1; // 0-based index of the last frame to which we seeked when seek in progress / negative when no
1274 // seek in progress,
1275
1276 if (frame != stream->_decodeNextFrameOut) {
1277 #if TRACE_DECODE_PROCESS
1278 std::cout << " Next frame expected out=" << stream->_decodeNextFrameOut << ", Seeking to desired frame" << std::endl;
1279 #endif
1280
1281 lastSeekedFrame = frame;
1282 stream->_decodeNextFrameIn = -1;
1283 stream->_decodeNextFrameOut = -1;
1284 stream->_accumDecodeLatency = 0;
1285 awaitingFirstDecodeAfterSeek = true;
1286
1287 if ( !seekFrame(frame, stream) ) {
1288 return false;
1289 }
1290 }
1291 #if TRACE_DECODE_PROCESS
1292 else {
1293 std::cout << " Next frame expected out=" << stream->_decodeNextFrameOut << ", No seek required" << std::endl;
1294 }
1295 #endif
1296
1297 // Loop until the desired frame has been decoded. May also break from within loop on failure conditions where the
1298 // desired frame will never be decoded.
1299 bool hasPicture = false;
1300 do {
1301 bool decodeAttempted = false;
1302 int frameDecoded = 0;
1303 AVColorRange srcColourRange = stream->_codecContext->color_range;
1304 AVPixelFormat srcPixelFormat = stream->_codecContext->pix_fmt;
1305
1306 // If the next frame to decode is within range of frames (or negative implying invalid; we've just seeked), read
1307 // a new frame from the source file and feed it to the decoder if it's for the video stream.
1308 if (stream->_decodeNextFrameIn < stream->_frames) {
1309 #if TRACE_DECODE_PROCESS
1310 std::cout << " Next frame expected in=";
1311 if (stream->_decodeNextFrameIn >= 0) {
1312 std::cout << stream->_decodeNextFrameIn;
1313 } else {
1314 std::cout << "unknown";
1315 }
1316 #endif
1317
1318 int error = 0;
1319 // Check if the packet is within the decompression range (dts - decompression timestamp)
1320 const int64_t currentFramePts = stream->frameToPts(frame);
1321 const bool isPacketValid = (_avPacket.data != NULL) && (_avPacket.stream_index == stream->_idx);
1322 const bool isCurrentPacketInRange = isPacketValid && (_avPacket.dts + _avPacket.duration > currentFramePts) && (currentFramePts >= _avPacket.dts);
1323 // If the actual packet is invalid (data == NULL) or it's not within the current decompression range it means that we need to read the next packet;
1324 // This happens when decoding for the first time or after the _avPacket was invalidated by calling av_free_packet;
1325 if (_avPacket.data == NULL || !isCurrentPacketInRange) {
1326 // release the previous packet if this wasn't already released at this point
1327 if (_avPacket.data != NULL) {
1328 _avPacket.FreePacket();
1329 }
1330 _avPacket.InitPacket();
1331 error = av_read_frame(_context, &_avPacket);
1332 }
1333 // [FD] 2015/01/20
1334 // the following if() was not in Nuke's FFmpeg Reader.cpp
1335 if (error == (int)AVERROR_EOF) {
1336 // getStreamFrames() was probably wrong
1337 stream->_frames = stream->_decodeNextFrameIn;
1338 if (loadNearest) {
1339 // try again
1340 frame = (int)stream->_frames - 1;
1341 lastSeekedFrame = frame;
1342 stream->_decodeNextFrameIn = -1;
1343 stream->_decodeNextFrameOut = -1;
1344 stream->_accumDecodeLatency = 0;
1345 awaitingFirstDecodeAfterSeek = true;
1346
1347 if ( !seekFrame(frame, stream) ) {
1348 return false;
1349 }
1350 }
1351 continue;
1352 }
1353 if (error < 0) {
1354 // Read error. Abort attempt to read and decode frames.
1355 #if TRACE_DECODE_PROCESS
1356 std::cout << ", Read failed" << std::endl;
1357 #endif
1358 setInternalError(error, "FFmpeg Reader failed to read frame: ");
1359 break;
1360 }
1361 #if TRACE_DECODE_PROCESS
1362 std::cout << ", Read OK, Packet data:" << std::endl;
1363 std::cout << " PTS=" << _avPacket.pts <<
1364 ", DTS=" << _avPacket.dts <<
1365 ", Duration=" << _avPacket.duration <<
1366 ", KeyFrame=" << ( (_avPacket.flags & AV_PKT_FLAG_KEY) ? 1 : 0 ) <<
1367 ", Corrupt=" << ( (_avPacket.flags & AV_PKT_FLAG_CORRUPT) ? 1 : 0 ) <<
1368 ", StreamIdx=" << _avPacket.stream_index <<
1369 ", PktSize=" << _avPacket.size;
1370 #endif
1371
1372 // If the packet read belongs to the video stream, synchronise frame indices from it if required and feed it
1373 // into the decoder.
1374 if (_avPacket.stream_index == stream->_idx) {
1375 #if TRACE_DECODE_PROCESS
1376 std::cout << ", Relevant stream" << std::endl;
1377 #endif
1378
1379 // If the packet read has a valid PTS, record that we have seen a PTS for this stream.
1380 if ( _avPacket.pts != int64_t(AV_NOPTS_VALUE) ) {
1381 stream->_ptsSeen = true;
1382 }
1383
1384 // If a seek is in progress, we need to synchronise frame indices if we can...
1385 if (lastSeekedFrame >= 0) {
1386 #if TRACE_DECODE_PROCESS
1387 std::cout << " In seek (" << lastSeekedFrame << ")";
1388 #endif
1389
1390 // Determine which frame the seek landed at, using whichever kind of timestamp is currently selected for this
1391 // stream. If there's no timestamp available at that frame, we can't synchronise frame indices to know which
1392 // frame we're first going to decode, so we need to seek back to an earlier frame in hope of obtaining a
1393 // timestamp. Likewise, if the landing frame is after the seek target frame (this can happen, presumably a bug
1394 // in FFmpeg seeking), we need to seek back to an earlier frame so that we can start decoding at or before the
1395 // desired frame.
1396 bool noTimestamp = ( _avPacket.*stream->_timestampField == int64_t(AV_NOPTS_VALUE) );
1397 int landingFrame = noTimestamp ? -1 : stream->ptsToFrame(_avPacket.*stream->_timestampField);
1398
1399 if ( noTimestamp || (landingFrame > lastSeekedFrame) ) {
1400 #if TRACE_DECODE_PROCESS
1401 std::cout << ", landing frame not found";
1402 if ( noTimestamp ) {
1403 std::cout << " (no timestamp)";
1404 } else {
1405 std::cout << " (landed after target at " << landingFrame << ")";
1406 }
1407 #endif
1408
1409 // Wind back 1 frame from last seeked frame. If that takes us to before frame 0, we're never going to be
1410 // able to synchronise using the current timestamp source...
1411 if (--lastSeekedFrame < 0) {
1412 #if TRACE_DECODE_PROCESS
1413 std::cout << ", can't seek before start";
1414 #endif
1415
1416 // If we're currently using PTSs to determine the landing frame and we've never seen a valid PTS for any
1417 // frame from this stream, switch to using DTSs and retry the read from the initial desired frame.
1418 if ( (stream->_timestampField == &AVPacket::pts) && !stream->_ptsSeen ) {
1419 stream->_timestampField = &AVPacket::dts;
1420 lastSeekedFrame = frame;
1421 #if TRACE_DECODE_PROCESS
1422 std::cout << ", PTSs absent, switching to use DTSs";
1423 #endif
1424 }
1425 // Otherwise, failure to find a landing point isn't caused by an absence of PTSs from the file or isn't
1426 // recovered by using DTSs instead. Something is wrong with the file. Abort attempt to read and decode frames.
1427 else {
1428 #if TRACE_DECODE_PROCESS
1429 if (stream->_timestampField == &AVPacket::dts) {
1430 std::cout << ", search using DTSs failed";
1431 } else {
1432 std::cout << ", PTSs present";
1433 }
1434 std::cout << ", giving up" << std::endl;
1435 #endif
1436 setError("FFmpeg Reader failed to find timing reference frame, possible file corruption");
1437 break;
1438 }
1439 }
1440
1441 // Seek to the new frame. By leaving the seek in progress, we will seek backwards frame by frame until we
1442 // either successfully synchronise frame indices or give up having reached the beginning of the stream.
1443 #if TRACE_DECODE_PROCESS
1444 std::cout << ", seeking to " << lastSeekedFrame << std::endl;
1445 #endif
1446 if ( !seekFrame(lastSeekedFrame, stream) ) {
1447 break;
1448 }
1449 }
1450 // Otherwise, we have a valid landing frame, so set that as the next frame into and out of decode and set
1451 // no seek in progress.
1452 else {
1453 #if TRACE_DECODE_PROCESS
1454 std::cout << ", landed at " << landingFrame << std::endl;
1455 #endif
1456 // If the packet is within the decompression range but the landingFrame is different than the current frame it means that
1457 // we are on a packet that needs to be displayed for more than one frame. In that case landingFrame will be pointing at the
1458 // first frame of that packet which means that we need to update the _decodeNextFrameOut and _decodeNextFrameIn for the other frames
1459 // which are in the current packet.
1460 // If not then it means that we have jumped on a different frame so update that accordingly;
1461 //
1462 // In the below ASCII figure - landingFrame will end up being 2 whenever frame is in the range 2 - 7
1463 //
1464 // Nuke frame 0 1 2 3 4 5 6 7 8 9 10 ...
1465 // +---+---+-----------------------+---+---+---+-
1466 // data stream | | | | | | | ...
1467 // +---+---+-----------------------+---+---+---+-
1468 // QT/ffmpeg 0 1 2 3 4 5 ...
1469 // frame
1470 //
1471 // So in this example, whenever frame is 3-7 we actually need to set stream->_decodeNextFrameIn/Out to frame rather than landingFrame, which is still 2;
1472 // stream->_decodeNextFrameIn/Out is used by our logic to determined what Nuke frame is being decoded;
1473 const int64_t currentFramePts = stream->frameToPts(frame);
1474 const bool isNewPacketInRange = (_avPacket.dts + _avPacket.duration > currentFramePts) && (currentFramePts >= _avPacket.dts);
1475 if (isNewPacketInRange && (landingFrame != frame)) {
1476 stream->_decodeNextFrameOut = stream->_decodeNextFrameIn = frame;
1477 } else {
1478 stream->_decodeNextFrameOut = stream->_decodeNextFrameIn = landingFrame;
1479 }
1480 lastSeekedFrame = -1;
1481 }
1482 }
1483
1484 // If there's no seek in progress, feed this frame into the decoder.
1485 if (lastSeekedFrame < 0) {
1486 #if TRACE_DECODE_BITSTREAM
1487 // H.264 ONLY
1488 std::cout << " Decoding input frame " << stream->_decodeNextFrameIn << " bitstream:" << std::endl;
1489 uint8_t *data = _avPacket.data;
1490 uint32_t remain = _avPacket.size;
1491 while (remain > 0) {
1492 if (remain < 4) {
1493 std::cout << " Insufficient remaining bytes (" << remain << ") for block size at BlockOffset=" << (data - _avPacket.data) << std::endl;
1494 remain = 0;
1495 } else {
1496 uint32_t blockSize = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
1497 data += 4;
1498 remain -= 4;
1499 std::cout << " BlockOffset=" << (data - _avPacket.data) << ", Size=" << blockSize;
1500 if (remain < blockSize) {
1501 std::cout << ", Insufficient remaining bytes (" << remain << ")" << std::endl;
1502 remain = 0;
1503 } else {
1504 std::cout << ", Bytes:";
1505 int count = (blockSize > 16 ? 16 : blockSize);
1506 for (int offset = 0; offset < count; offset++) {
1507 static const char hexTable[] = "0123456789ABCDEF";
1508 uint8_t byte = data[offset];
1509 std::cout << ' ' << hexTable[byte >> 4] << hexTable[byte & 0xF];
1510 }
1511 std::cout << std::endl;
1512 data += blockSize;
1513 remain -= blockSize;
1514 }
1515 }
1516 }
1517 #elif TRACE_DECODE_PROCESS
1518 std::cout << " Decoding input frame " << stream->_decodeNextFrameIn << std::endl;
1519 #endif
1520
1521 // Advance next frame to input.
1522 ++stream->_decodeNextFrameIn;
1523
1524 // Decode the frame just read. frameDecoded indicates whether a decoded frame was output.
1525 decodeAttempted = true;
1526
1527 // if the packet wasn't decode yet then it doesn't really metter
1528 // if it's in the playback range or not; we still keep trying to
1529 // decode the information iside the packet
1530 if (_avPacket.wasPacketDecoded && isCurrentPacketInRange) {
1531 error = 0;
1532 frameDecoded = 1; // the image is already pre-cached in the _avFrame - don't do anything here
1533 }
1534 else {
1535 #if USE_NEW_FFMPEG_API
1536 error = avcodec_send_packet(stream->_codecContext, &_avPacket);
1537 if (error == 0) {
1538 error = avcodec_receive_frame(stream->_codecContext, stream->_avFrame);
1539 if ( error == AVERROR(EAGAIN) ) {
1540 frameDecoded = 0;
1541 error = 0;
1542 } else if (error < 0) {
1543 frameDecoded = 0;
1544 } else {
1545 frameDecoded = 1;
1546 }
1547 }
1548 #else
1549 error = avcodec_decode_video2(stream->_codecContext, stream->_avFrame, &frameDecoded, &_avPacket);
1550 #endif
1551 _avPacket.wasPacketDecoded = frameDecoded;
1552 }
1553 if (error < 0) {
1554 // Decode error. Abort attempt to read and decode frames.
1555 setInternalError(error, "FFmpeg Reader failed to decode frame: ");
1556 break;
1557 }
1558 }
1559 } //_avPacket.stream_index == stream->_idx
1560 #if TRACE_DECODE_PROCESS
1561 else {
1562 std::cout << ", Irrelevant stream" << std::endl;
1563 }
1564 #endif
1565 } //stream->_decodeNextFrameIn < stream->_frames
1566 // If the next frame to decode is out of frame range, there's nothing more to read and the decoder will be fed
1567 // null input frames in order to obtain any remaining output.
1568 else {
1569 #if TRACE_DECODE_PROCESS
1570 std::cout << " No more frames to read, pumping remaining decoder output" << std::endl;
1571 #endif
1572
1573 // Obtain remaining frames from the decoder. pkt_ contains NULL packet data pointer and size at this point,
1574 // required to pump out remaining frames with no more input. frameDecoded indicates whether a decoded frame
1575 // was output.
1576 decodeAttempted = true;
1577 int error = 0;
1578 if ( (AV_CODEC_ID_PRORES == stream->_codecContext->codec_id) ||
1579 (AV_CODEC_ID_DNXHD == stream->_codecContext->codec_id) ) {
1580 // Apple ProRes specific.
1581 // The ProRes codec is I-frame only so will not have any
1582 // remaining frames.
1583 } else {
1584 // Obtain remaining frames from the decoder. emptyAVPacket contains NULL packet data pointer and size at this point,
1585 // required to pump out remaining frames with no more input
1586 // That's based on the following FFmpeg's remark:
1587 // Codecs which have the CODEC_CAP_DELAY capability set have a delay between input and output, these need to be fed
1588 // with avpkt->data=NULL, avpkt->size=0 at the end to return the remaining frames.
1589 // Please have a look at the following TP item from more info: TP 24765 https://foundry.tpondemand.com/entity/246765
1590
1591 MyAVPacket emptyAVPacket;
1592 #if USE_NEW_FFMPEG_API
1593 error = avcodec_send_packet(stream->_codecContext, &emptyAVPacket);
1594 if (error == 0) {
1595 error = avcodec_receive_frame(stream->_codecContext, stream->_avFrame);
1596 if ( error == AVERROR(EAGAIN) ) {
1597 frameDecoded = 0;
1598 error = 0;
1599 } else if (error < 0) {
1600 frameDecoded = 0;
1601 // Decode error. Abort attempt to read and decode frames.
1602 } else {
1603 frameDecoded = 1;
1604 }
1605 }
1606 #else
1607 error = avcodec_decode_video2(stream->_codecContext, stream->_avFrame, &frameDecoded, &emptyAVPacket);
1608 #endif
1609 }
1610 if (error < 0) {
1611 // Decode error. Abort attempt to read and decode frames.
1612 setInternalError(error, "FFmpeg Reader failed to decode frame: ");
1613 break;
1614 }
1615 }
1616
1617 // If a frame was decoded, ...
1618 if (frameDecoded) {
1619 #if TRACE_DECODE_PROCESS
1620 std::cout << " Frame decoded=" << stream->_decodeNextFrameOut;
1621 #endif
1622
1623 // Now that we have had a frame decoded, we know that seek landed at a valid place to start decode. Any decode
1624 // stalls detected after this point will result in immediate decode failure.
1625 awaitingFirstDecodeAfterSeek = false;
1626
1627 // If the frame just output from decode is the desired one, get the decoded picture from it and set that we
1628 // have a picture.
1629 if (stream->_decodeNextFrameOut == frame) {
1630 #if TRACE_DECODE_PROCESS
1631 std::cout << ", is desired frame" << std::endl;
1632 #endif
1633
1634 SwsContext* context = NULL;
1635 {
1636 context = stream->getConvertCtx(srcPixelFormat, stream->_width, stream->_height,
1637 srcColourRange,
1638 stream->_outputPixelFormat, stream->_width, stream->_height);
1639 }
1640
1641 // Scale if any of the decoding path has provided a convert
1642 // context. Otherwise, no scaling/conversion is required after
1643 // decoding the frame.
1644 if (context) {
1645 uint8_t *data[4];
1646 int linesize[4];
1647 #if 0
1648 // Nuke's version
1649 {
1650 AVPicture output;
1651 avpicture_fill(&output, buffer, stream->_outputPixelFormat, stream->_width, stream->_height);
1652 data[0] = output.data[0];
1653 data[1] = output.data[1];
1654 data[2] = output.data[2];
1655 data[3] = output.data[3];
1656 linesize[0] = output.linesize[0];
1657 linesize[1] = output.linesize[1];
1658 linesize[2] = output.linesize[2];
1659 linesize[3] = output.linesize[3];
1660 }
1661 #else
1662 av_image_fill_arrays(data, linesize, buffer, stream->_outputPixelFormat, stream->_width, stream->_height, 1);
1663 #endif
1664 sws_scale(context,
1665 stream->_avFrame->data,
1666 stream->_avFrame->linesize,
1667 0,
1668 stream->_height,
1669 data,
1670 linesize);
1671 }
1672
1673 hasPicture = true;
1674 }
1675 #if TRACE_DECODE_PROCESS
1676 else {
1677 std::cout << ", is not desired frame (" << frame << ")" << std::endl;
1678 }
1679 #endif
1680
1681 // Advance next output frame expected from decode.
1682 ++stream->_decodeNextFrameOut;
1683 }
1684 // If no frame was decoded but decode was attempted, determine whether this constitutes a decode stall and handle if so.
1685 else if (decodeAttempted) {
1686 // Failure to get an output frame for an input frame increases the accumulated decode latency for this stream.
1687 ++stream->_accumDecodeLatency;
1688
1689 #if TRACE_DECODE_PROCESS
1690 std::cout << " No frame decoded, accumulated decode latency=" << stream->_accumDecodeLatency << ", max allowed latency=" << stream->getCodecDelay() << std::endl;
1691 #endif
1692
1693 // If the accumulated decode latency exceeds the maximum delay permitted for this codec at this time (the delay can
1694 // change dynamically if the codec discovers B-frames mid-stream), we've detected a decode stall.
1695 if ( stream->_accumDecodeLatency > stream->getCodecDelay() ) {
1696 int seekTargetFrame; // Target frame for any seek we might perform to attempt decode stall recovery.
1697
1698 // Handle a post-seek decode stall.
1699 if (awaitingFirstDecodeAfterSeek) {
1700 // If there's anywhere in the file to seek back to before the last seek's landing frame (which can be found in
1701 // stream->_decodeNextFrameOut, since we know we've not decoded any frames since landing), then set up a seek to
1702 // the frame before that landing point to try to find a valid decode start frame earlier in the file.
1703 if (stream->_decodeNextFrameOut > 0) {
1704 seekTargetFrame = stream->_decodeNextFrameOut - 1;
1705 #if TRACE_DECODE_PROCESS
1706 std::cout << " Post-seek stall detected, trying earlier decode start, seeking frame " << seekTargetFrame << std::endl;
1707 #endif
1708 }
1709 // Otherwise, there's nowhere to seek back. If we have any retries remaining, use one to attempt the read again,
1710 // starting from the desired frame.
1711 else if (retriesRemaining > 0) {
1712 --retriesRemaining;
1713 seekTargetFrame = frame;
1714 #if TRACE_DECODE_PROCESS
1715 std::cout << " Post-seek stall detected, at start of file, retrying from desired frame " << seekTargetFrame << std::endl;
1716 #endif
1717 }
1718 // Otherwise, all we can do is to fail the read so that this method exits safely.
1719 else {
1720 #if TRACE_DECODE_PROCESS
1721 std::cout << " Post-seek STALL DETECTED, at start of file, no more retries, failed read" << std::endl;
1722 #endif
1723 setError("FFmpeg Reader failed to find decode reference frame, possible file corruption");
1724 break;
1725 }
1726 }
1727 // Handle a mid-decode stall. All we can do is to fail the read so that this method exits safely.
1728 else {
1729 // If we have any retries remaining, use one to attempt the read again, starting from the desired frame.
1730 if (retriesRemaining > 0) {
1731 --retriesRemaining;
1732 seekTargetFrame = frame;
1733 #if TRACE_DECODE_PROCESS
1734 std::cout << " Mid-decode stall detected, retrying from desired frame " << seekTargetFrame << std::endl;
1735 #endif
1736 }
1737 // Otherwise, all we can do is to fail the read so that this method exits safely.
1738 else {
1739 #if TRACE_DECODE_PROCESS
1740 std::cout << " Mid-decode STALL DETECTED, no more retries, failed read" << std::endl;
1741 #endif
1742 setError("FFmpeg Reader detected decoding stall, possible file corruption");
1743 break;
1744 }
1745 } // if (awaitingFirstDecodeAfterSeek)
1746
1747 // If we reach here, seek to the target frame chosen above in an attempt to recover from the decode stall.
1748 lastSeekedFrame = seekTargetFrame;
1749 stream->_decodeNextFrameIn = -1;
1750 stream->_decodeNextFrameOut = -1;
1751 stream->_accumDecodeLatency = 0;
1752 awaitingFirstDecodeAfterSeek = true;
1753
1754 if ( !seekFrame(seekTargetFrame, stream) ) {
1755 break;
1756 }
1757 } // if (decodeAttempted)
1758 } // if (stream->_decodeNextFrameIn < stream->_frames)
1759 } while (!hasPicture && !plugin->abort());
1760
1761 #if TRACE_DECODE_PROCESS
1762 std::cout << "<-validPicture=" << hasPicture << " for frame " << frame << std::endl;
1763 #endif
1764
1765 // If read failed, reset the next frame expected out so that we seek and restart decode on next read attempt. Also free
1766 // the AVPacket, since it won't have been freed at the end of the above loop (we reach here by a break from the main
1767 // loop when hasPicture is false).
1768 if (!hasPicture) {
1769 // If the process failed to get a picture, but the packet contains information about the decompressed image, then
1770 // we need to dispose it;
1771 if (_avPacket.size > 0) {
1772 _avPacket.FreePacket();
1773 }
1774 stream->_decodeNextFrameOut = -1;
1775 }
1776
1777 return hasPicture;
1778 } // FFmpegFile::decode
1779
1780 bool
getFPS(double & fps,unsigned streamIdx)1781 FFmpegFile::getFPS(double & fps,
1782 unsigned streamIdx)
1783 {
1784 if ( streamIdx >= _streams.size() ) {
1785 return false;
1786 }
1787
1788 // get the stream
1789 Stream* stream = _streams[streamIdx];
1790 // guard against division by zero
1791 assert(stream->_fpsDen);
1792 fps = stream->_fpsDen ? ( (double)stream->_fpsNum / stream->_fpsDen ) : stream->_fpsNum;
1793
1794 return true;
1795 }
1796
1797 // get stream information
1798 bool
getInfo(int & width,int & height,double & aspect,int & frames)1799 FFmpegFile::getInfo(int & width,
1800 int & height,
1801 double & aspect,
1802 int & frames)
1803 {
1804 AutoMutex guard(_lock);
1805
1806 if (_streams.empty()) {
1807 return false;
1808 }
1809
1810 assert(_selectedStream && "Null _selectedStream");
1811 if (!_selectedStream) {
1812 return false;
1813 }
1814 width = _selectedStream->_width;
1815 height = _selectedStream->_height;
1816 aspect = _selectedStream->_aspect;
1817 frames = (int)_selectedStream->_frames;
1818
1819 return true;
1820 }
1821
1822 std::size_t
getBufferBytesCount() const1823 FFmpegFile::getBufferBytesCount() const
1824 {
1825 if ( _streams.empty() ) {
1826 return 0;
1827 }
1828
1829 Stream* stream = _streams[0];
1830 std::size_t pixelDepth = stream->_bitDepth > 8 ? sizeof(unsigned short) : sizeof(unsigned char);
1831
1832 // this is the first stream (in fact the only one we consider for now), allocate the output buffer according to the bitdepth
1833 return stream->_width * stream->_height * stream->_numberOfComponents * pixelDepth;
1834 }
1835
FFmpegFileManager()1836 FFmpegFileManager::FFmpegFileManager()
1837 : _files()
1838 , _lock(NULL)
1839 {
1840 }
1841
~FFmpegFileManager()1842 FFmpegFileManager::~FFmpegFileManager()
1843 {
1844 for (FilesMap::iterator it = _files.begin(); it != _files.end(); ++it) {
1845 for (std::list<FFmpegFile*>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
1846 delete *it2;
1847 }
1848 }
1849 _files.clear();
1850 delete _lock;
1851 }
1852
1853 void
init()1854 FFmpegFileManager::init()
1855 {
1856 _lock = new FFmpegFile::Mutex;
1857 }
1858
1859 void
clear(void const * plugin)1860 FFmpegFileManager::clear(void const * plugin)
1861 {
1862 assert(_lock);
1863 FFmpegFile::AutoMutex guard(*_lock);
1864 FilesMap::iterator found = _files.find(plugin);
1865 if ( found != _files.end() ) {
1866 for (std::list<FFmpegFile*>::iterator it = found->second.begin(); it != found->second.end(); ++it) {
1867 delete *it;
1868 }
1869 _files.erase(found);
1870 }
1871 }
1872
1873 FFmpegFile*
get(void const * plugin,const string & filename) const1874 FFmpegFileManager::get(void const * plugin,
1875 const string &filename) const
1876 {
1877 if (filename.empty() || !plugin) {
1878 return 0;
1879 }
1880 assert(_lock);
1881 FFmpegFile::AutoMutex guard(*_lock);
1882 FilesMap::iterator found = _files.find(plugin);
1883 if ( found != _files.end() ) {
1884 for (std::list<FFmpegFile*>::iterator it = found->second.begin(); it != found->second.end(); ++it) {
1885 if ( (*it)->getFilename() == filename ) {
1886 if ( (*it)->isInvalid() ) {
1887 delete *it;
1888 found->second.erase(it);
1889 break;
1890 } else {
1891 return *it;
1892 }
1893 }
1894 }
1895 }
1896
1897 return NULL;
1898 }
1899
1900 FFmpegFile*
getOrCreate(void const * plugin,const string & filename) const1901 FFmpegFileManager::getOrCreate(void const * plugin,
1902 const string &filename) const
1903 {
1904 if (filename.empty() || !plugin) {
1905 return 0;
1906 }
1907 assert(_lock);
1908 FFmpegFile::AutoMutex guard(*_lock);
1909 FilesMap::iterator found = _files.find(plugin);
1910 if ( found != _files.end() ) {
1911 for (std::list<FFmpegFile*>::iterator it = found->second.begin(); it != found->second.end(); ++it) {
1912 if ( (*it)->getFilename() == filename ) {
1913 if ( (*it)->isInvalid() ) {
1914 delete *it;
1915 found->second.erase(it);
1916 break;
1917 } else {
1918 return *it;
1919 }
1920 }
1921 }
1922 }
1923
1924 FFmpegFile* file = new FFmpegFile(filename);
1925 if ( found == _files.end() ) {
1926 std::list<FFmpegFile*> fileList;
1927 fileList.push_back(file);
1928 _files.insert( make_pair(plugin, fileList) );
1929 } else {
1930 found->second.push_back(file);
1931 }
1932
1933 return file;
1934 }
1935