1 /**
2 * @file
3 * @brief Source file for FFmpegWriter class
4 * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5 *
6 * @ref License
7 */
8
9 /* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12 * (http://www.openshotstudios.com). This file is part of
13 * OpenShot Library (http://www.openshot.org), an open-source project
14 * dedicated to delivering high quality video editing and animation solutions
15 * to the world.
16 *
17 * This file is originally based on the Libavformat API example, and then modified
18 * by the libopenshot project.
19 *
20 * OpenShot Library (libopenshot) is free software: you can redistribute it
21 * and/or modify it under the terms of the GNU Lesser General Public License
22 * as published by the Free Software Foundation, either version 3 of the
23 * License, or (at your option) any later version.
24 *
25 * OpenShot Library (libopenshot) is distributed in the hope that it will be
26 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU Lesser General Public License for more details.
29 *
30 * You should have received a copy of the GNU Lesser General Public License
31 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32 */
33
34 #include "FFmpegWriter.h"
35 #include "Exceptions.h"
36
37 #include <iostream>
38
39 using namespace openshot;
40
41 // Multiplexer parameters temporary storage
42 AVDictionary *mux_dict = NULL;
43
44 #if USE_HW_ACCEL
45 int hw_en_on = 1; // Is set in UI
46 int hw_en_supported = 0; // Is set by FFmpegWriter
47 AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
48 AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
49 static AVBufferRef *hw_device_ctx = NULL;
50 AVFrame *hw_frame = NULL;
51
set_hwframe_ctx(AVCodecContext * ctx,AVBufferRef * hw_device_ctx,int64_t width,int64_t height)52 static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height)
53 {
54 AVBufferRef *hw_frames_ref;
55 AVHWFramesContext *frames_ctx = NULL;
56 int err = 0;
57
58 if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
59 std::clog << "Failed to create HW frame context.\n";
60 return -1;
61 }
62 frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
63 frames_ctx->format = hw_en_av_pix_fmt;
64 frames_ctx->sw_format = AV_PIX_FMT_NV12;
65 frames_ctx->width = width;
66 frames_ctx->height = height;
67 frames_ctx->initial_pool_size = 20;
68 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
69 std::clog << "Failed to initialize HW frame context. " <<
70 "Error code: " << av_err2string(err) << "\n";
71 av_buffer_unref(&hw_frames_ref);
72 return err;
73 }
74 ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
75 if (!ctx->hw_frames_ctx)
76 err = AVERROR(ENOMEM);
77
78 av_buffer_unref(&hw_frames_ref);
79 return err;
80 }
81 #endif // USE_HW_ACCEL
82
FFmpegWriter(const std::string & path)83 FFmpegWriter::FFmpegWriter(const std::string& path) :
84 path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL),
85 audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
86 initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32),
87 rescaler_position(0), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0),
88 original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
89 write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) {
90
91 // Disable audio & video (so they can be independently enabled)
92 info.has_audio = false;
93 info.has_video = false;
94
95 // Initialize FFMpeg, and register all formats and codecs
96 AV_REGISTER_ALL
97
98 // auto detect format
99 auto_detect_format();
100 }
101
102 // Open the writer
Open()103 void FFmpegWriter::Open() {
104 if (!is_open) {
105 // Open the writer
106 is_open = true;
107
108 // Prepare streams (if needed)
109 if (!prepare_streams)
110 PrepareStreams();
111
112 // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
113 if (info.has_video && video_st)
114 open_video(oc, video_st);
115 if (info.has_audio && audio_st)
116 open_audio(oc, audio_st);
117
118 // Write header (if needed)
119 if (!write_header)
120 WriteHeader();
121 }
122 }
123
124 // auto detect format (from path)
auto_detect_format()125 void FFmpegWriter::auto_detect_format() {
126 // Auto detect the output format from the name. default is mpeg.
127 fmt = av_guess_format(NULL, path.c_str(), NULL);
128 if (!fmt)
129 throw InvalidFormat("Could not deduce output format from file extension.", path);
130
131 // Allocate the output media context
132 AV_OUTPUT_CONTEXT(&oc, path.c_str());
133 if (!oc)
134 throw OutOfMemory("Could not allocate memory for AVFormatContext.", path);
135
136 // Set the AVOutputFormat for the current AVFormatContext
137 oc->oformat = fmt;
138
139 // Update codec names
140 if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
141 // Update video codec name
142 info.vcodec = avcodec_find_encoder(fmt->video_codec)->name;
143
144 if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
145 // Update audio codec name
146 info.acodec = avcodec_find_encoder(fmt->audio_codec)->name;
147 }
148
149 // initialize streams
initialize_streams()150 void FFmpegWriter::initialize_streams() {
151 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE);
152
153 // Add the audio and video streams using the default format codecs and initialize the codecs
154 video_st = NULL;
155 audio_st = NULL;
156 if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
157 // Add video stream
158 video_st = add_video_stream();
159
160 if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
161 // Add audio stream
162 audio_st = add_audio_stream();
163 }
164
165 // Set video export options
SetVideoOptions(bool has_video,std::string codec,Fraction fps,int width,int height,Fraction pixel_ratio,bool interlaced,bool top_field_first,int bit_rate)166 void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) {
167 // Set the video options
168 if (codec.length() > 0) {
169 AVCodec *new_codec;
170 // Check if the codec selected is a hardware accelerated codec
171 #if USE_HW_ACCEL
172 #if defined(__unix__)
173 if (strstr(codec.c_str(), "_vaapi") != NULL) {
174 new_codec = avcodec_find_encoder_by_name(codec.c_str());
175 hw_en_on = 1;
176 hw_en_supported = 1;
177 hw_en_av_pix_fmt = AV_PIX_FMT_VAAPI;
178 hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
179 } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
180 new_codec = avcodec_find_encoder_by_name(codec.c_str());
181 hw_en_on = 1;
182 hw_en_supported = 1;
183 hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
184 hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
185 } else {
186 new_codec = avcodec_find_encoder_by_name(codec.c_str());
187 hw_en_on = 0;
188 hw_en_supported = 0;
189 }
190 #elif defined(_WIN32)
191 if (strstr(codec.c_str(), "_dxva2") != NULL) {
192 new_codec = avcodec_find_encoder_by_name(codec.c_str());
193 hw_en_on = 1;
194 hw_en_supported = 1;
195 hw_en_av_pix_fmt = AV_PIX_FMT_DXVA2_VLD;
196 hw_en_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
197 } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
198 new_codec = avcodec_find_encoder_by_name(codec.c_str());
199 hw_en_on = 1;
200 hw_en_supported = 1;
201 hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
202 hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
203 } else {
204 new_codec = avcodec_find_encoder_by_name(codec.c_str());
205 hw_en_on = 0;
206 hw_en_supported = 0;
207 }
208 #elif defined(__APPLE__)
209 if (strstr(codec.c_str(), "_videotoolbox") != NULL) {
210 new_codec = avcodec_find_encoder_by_name(codec.c_str());
211 hw_en_on = 1;
212 hw_en_supported = 1;
213 hw_en_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
214 hw_en_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
215 } else {
216 new_codec = avcodec_find_encoder_by_name(codec.c_str());
217 hw_en_on = 0;
218 hw_en_supported = 0;
219 }
220 #else // unknown OS
221 new_codec = avcodec_find_encoder_by_name(codec.c_str());
222 #endif //__unix__/_WIN32/__APPLE__
223 #else // USE_HW_ACCEL
224 new_codec = avcodec_find_encoder_by_name(codec.c_str());
225 #endif // USE_HW_ACCEL
226 if (new_codec == NULL)
227 throw InvalidCodec("A valid video codec could not be found for this file.", path);
228 else {
229 // Set video codec
230 info.vcodec = new_codec->name;
231
232 // Update video codec in fmt
233 fmt->video_codec = new_codec->id;
234 }
235 }
236 if (fps.num > 0) {
237 // Set frames per second (if provided)
238 info.fps.num = fps.num;
239 info.fps.den = fps.den;
240
241 // Set the timebase (inverse of fps)
242 info.video_timebase.num = info.fps.den;
243 info.video_timebase.den = info.fps.num;
244 }
245 if (width >= 1)
246 info.width = width;
247 if (height >= 1)
248 info.height = height;
249 if (pixel_ratio.num > 0) {
250 info.pixel_ratio.num = pixel_ratio.num;
251 info.pixel_ratio.den = pixel_ratio.den;
252 }
253 if (bit_rate >= 1000) // bit_rate is the bitrate in b/s
254 info.video_bit_rate = bit_rate;
255 if ((bit_rate >= 0) && (bit_rate < 256)) // bit_rate is the bitrate in crf
256 info.video_bit_rate = bit_rate;
257
258 info.interlaced_frame = interlaced;
259 info.top_field_first = top_field_first;
260
261 // Calculate the DAR (display aspect ratio)
262 Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
263
264 // Reduce size fraction
265 size.Reduce();
266
267 // Set the ratio based on the reduced fraction
268 info.display_ratio.num = size.num;
269 info.display_ratio.den = size.den;
270
271 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetVideoOptions (" + codec + ")", "width", width, "height", height, "size.num", size.num, "size.den", size.den, "fps.num", fps.num, "fps.den", fps.den);
272
273 // Enable / Disable video
274 info.has_video = has_video;
275 }
276
277 // Set video export options (overloaded function)
SetVideoOptions(std::string codec,int width,int height,Fraction fps,int bit_rate)278 void FFmpegWriter::SetVideoOptions(std::string codec, int width, int height, Fraction fps, int bit_rate) {
279 // Call full signature with some default parameters
280 FFmpegWriter::SetVideoOptions(
281 true, codec, fps, width, height,
282 openshot::Fraction(1, 1), false, true, bit_rate
283 );
284 }
285
286
287 // Set audio export options
SetAudioOptions(bool has_audio,std::string codec,int sample_rate,int channels,ChannelLayout channel_layout,int bit_rate)288 void FFmpegWriter::SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate) {
289 // Set audio options
290 if (codec.length() > 0) {
291 AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
292 if (new_codec == NULL)
293 throw InvalidCodec("A valid audio codec could not be found for this file.", path);
294 else {
295 // Set audio codec
296 info.acodec = new_codec->name;
297
298 // Update audio codec in fmt
299 fmt->audio_codec = new_codec->id;
300 }
301 }
302 if (sample_rate > 7999)
303 info.sample_rate = sample_rate;
304 if (channels > 0)
305 info.channels = channels;
306 if (bit_rate > 999)
307 info.audio_bit_rate = bit_rate;
308 info.channel_layout = channel_layout;
309
310 // init resample options (if zero)
311 if (original_sample_rate == 0)
312 original_sample_rate = info.sample_rate;
313 if (original_channels == 0)
314 original_channels = info.channels;
315
316 ZmqLogger::Instance()->AppendDebugMethod(
317 "FFmpegWriter::SetAudioOptions (" + codec + ")",
318 "sample_rate", sample_rate,
319 "channels", channels,
320 "bit_rate", bit_rate);
321
322 // Enable / Disable audio
323 info.has_audio = has_audio;
324 }
325
326
327 // Set audio export options (overloaded function)
SetAudioOptions(std::string codec,int sample_rate,int bit_rate)328 void FFmpegWriter::SetAudioOptions(std::string codec, int sample_rate, int bit_rate) {
329 // Call full signature with some default parameters
330 FFmpegWriter::SetAudioOptions(
331 true, codec, sample_rate, 2,
332 openshot::LAYOUT_STEREO, bit_rate
333 );
334 }
335
336
337 // Set custom options (some codecs accept additional params)
SetOption(StreamType stream,std::string name,std::string value)338 void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string value) {
339 // Declare codec context
340 AVCodecContext *c = NULL;
341 AVStream *st = NULL;
342 std::stringstream convert(value);
343
344 if (info.has_video && stream == VIDEO_STREAM && video_st) {
345 st = video_st;
346 // Get codec context
347 c = AV_GET_CODEC_PAR_CONTEXT(st, video_codec_ctx);
348 // Was a codec / stream found?
349 if (c) {
350 if (info.interlaced_frame) {
351 c->field_order = info.top_field_first ? AV_FIELD_TT : AV_FIELD_BB;
352 // We only use these two version and ignore AV_FIELD_TB and AV_FIELD_BT
353 // Otherwise we would need to change the whole export window
354 }
355 }
356 } else if (info.has_audio && stream == AUDIO_STREAM && audio_st) {
357 st = audio_st;
358 // Get codec context
359 c = AV_GET_CODEC_PAR_CONTEXT(st, audio_codec_ctx);
360 } else
361 throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
362
363 // Init AVOption
364 const AVOption *option = NULL;
365
366 // Was a codec / stream found?
367 if (c)
368 // Find AVOption (if it exists)
369 option = AV_OPTION_FIND(c->priv_data, name.c_str());
370
371 // Was option found?
372 if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
373 name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" ||
374 name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp")) {
375 // Check for specific named options
376 if (name == "g")
377 // Set gop_size
378 convert >> c->gop_size;
379
380 else if (name == "qmin")
381 // Minimum quantizer
382 convert >> c->qmin;
383
384 else if (name == "qmax")
385 // Maximum quantizer
386 convert >> c->qmax;
387
388 else if (name == "max_b_frames")
389 // Maximum number of B-frames between non-B-frames
390 convert >> c->max_b_frames;
391
392 else if (name == "mb_decision")
393 // Macroblock decision mode
394 convert >> c->mb_decision;
395
396 else if (name == "level")
397 // Set codec level
398 convert >> c->level;
399
400 else if (name == "profile")
401 // Set codec profile
402 convert >> c->profile;
403
404 else if (name == "slices")
405 // Indicates number of picture subdivisions
406 convert >> c->slices;
407
408 else if (name == "rc_min_rate")
409 // Minimum bitrate
410 convert >> c->rc_min_rate;
411
412 else if (name == "rc_max_rate")
413 // Maximum bitrate
414 convert >> c->rc_max_rate;
415
416 else if (name == "rc_buffer_size")
417 // Buffer size
418 convert >> c->rc_buffer_size;
419
420 else if (name == "cqp") {
421 // encode quality and special settings like lossless
422 // This might be better in an extra methods as more options
423 // and way to set quality are possible
424 #if USE_HW_ACCEL
425 if (hw_en_on) {
426 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
427 } else
428 #endif // USE_HW_ACCEL
429 {
430 switch (c->codec_id) {
431 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
432 // FFmpeg 4.0+
433 case AV_CODEC_ID_AV1 :
434 c->bit_rate = 0;
435 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
436 break;
437 #endif
438 case AV_CODEC_ID_VP8 :
439 c->bit_rate = 10000000;
440 av_opt_set_int(c->priv_data, "qp", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
441 break;
442 case AV_CODEC_ID_VP9 :
443 c->bit_rate = 0; // Must be zero!
444 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
445 if (std::stoi(value) == 0) {
446 av_opt_set(c->priv_data, "preset", "veryslow", 0);
447 av_opt_set_int(c->priv_data, "lossless", 1, 0);
448 }
449 break;
450 case AV_CODEC_ID_H264 :
451 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
452 if (std::stoi(value) == 0) {
453 av_opt_set(c->priv_data, "preset", "veryslow", 0);
454 c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
455 }
456 break;
457 case AV_CODEC_ID_HEVC :
458 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
459 if (std::stoi(value) == 0) {
460 av_opt_set(c->priv_data, "preset", "veryslow", 0);
461 av_opt_set_int(c->priv_data, "lossless", 1, 0);
462 }
463 break;
464 default:
465 // For all other codecs assume a range of 0-63
466 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
467 c->bit_rate = 0;
468 }
469 }
470 } else if (name == "crf") {
471 // encode quality and special settings like lossless
472 // This might be better in an extra methods as more options
473 // and way to set quality are possible
474 #if USE_HW_ACCEL
475 if (hw_en_on) {
476 double mbs = 15000000.0;
477 if (info.video_bit_rate > 0) {
478 if (info.video_bit_rate > 42) {
479 mbs = 380000.0;
480 }
481 else {
482 mbs *= std::pow(0.912,info.video_bit_rate);
483 }
484 }
485 c->bit_rate = (int)(mbs);
486 } else
487 #endif // USE_HW_ACCEL
488 {
489 switch (c->codec_id) {
490 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
491 // FFmpeg 4.0+
492 case AV_CODEC_ID_AV1 :
493 c->bit_rate = 0;
494 // AV1 only supports "crf" quality values
495 av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
496 break;
497 #endif
498 case AV_CODEC_ID_VP8 :
499 c->bit_rate = 10000000;
500 av_opt_set_int(c->priv_data, "crf", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
501 break;
502 case AV_CODEC_ID_VP9 :
503 c->bit_rate = 0; // Must be zero!
504 av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 63), 0); // 0-63
505 if (std::stoi(value) == 0) {
506 av_opt_set(c->priv_data, "preset", "veryslow", 0);
507 av_opt_set_int(c->priv_data, "lossless", 1, 0);
508 }
509 break;
510 case AV_CODEC_ID_H264 :
511 av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
512 if (std::stoi(value) == 0) {
513 av_opt_set(c->priv_data, "preset", "veryslow", 0);
514 c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
515 }
516 break;
517 case AV_CODEC_ID_HEVC :
518 if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
519 av_opt_set_int(c->priv_data, "preset", 7, 0);
520 av_opt_set_int(c->priv_data, "forced-idr",1,0);
521 av_opt_set_int(c->priv_data, "qp",std::min(std::stoi(value), 51),0);
522 }
523 else {
524 av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
525 }
526 if (std::stoi(value) == 0) {
527 av_opt_set(c->priv_data, "preset", "veryslow", 0);
528 av_opt_set_int(c->priv_data, "lossless", 1, 0);
529 }
530 break;
531 default:
532 // If this codec doesn't support crf calculate a bitrate
533 // TODO: find better formula
534 double mbs = 15000000.0;
535 if (info.video_bit_rate > 0) {
536 if (info.video_bit_rate > 42) {
537 mbs = 380000.0;
538 } else {
539 mbs *= std::pow(0.912, info.video_bit_rate);
540 }
541 }
542 c->bit_rate = (int) (mbs);
543 }
544 }
545 } else if (name == "qp") {
546 // encode quality and special settings like lossless
547 // This might be better in an extra methods as more options
548 // and way to set quality are possible
549 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
550 // FFmpeg 4.0+
551 switch (c->codec_id) {
552 case AV_CODEC_ID_AV1 :
553 c->bit_rate = 0;
554 if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
555 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0);
556 }
557 else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
558 // Set number of tiles to a fixed value
559 // TODO Let user choose number of tiles
560 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),255), 0);
561 }
562 else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
563 // Set number of tiles to a fixed value
564 // TODO Let user choose number of tiles
565 // libaom doesn't have qp only crf
566 av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
567 }
568 else {
569 av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
570 }
571 break;
572 case AV_CODEC_ID_HEVC :
573 c->bit_rate = 0;
574 if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
575 av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),51), 0);
576 av_opt_set_int(c->priv_data, "preset", 7, 0);
577 av_opt_set_int(c->priv_data, "forced-idr",1,0);
578 }
579 break;
580 default:
581 break;
582 }
583 #endif // FFmpeg 4.0+
584 } else {
585 // Set AVOption
586 AV_OPTION_SET(st, c->priv_data, name.c_str(), value.c_str(), c);
587 }
588
589 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (std::string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM);
590
591 // Muxing dictionary is not part of the codec context.
592 // Just reusing SetOption function to set popular multiplexing presets.
593 } else if (name == "muxing_preset") {
594 if (value == "mp4_faststart") {
595 // 'moov' box to the beginning; only for MOV, MP4
596 av_dict_set(&mux_dict, "movflags", "faststart", 0);
597 } else if (value == "mp4_fragmented") {
598 // write selfcontained fragmented file, minimum length of the fragment 8 sec; only for MOV, MP4
599 av_dict_set(&mux_dict, "movflags", "frag_keyframe", 0);
600 av_dict_set(&mux_dict, "min_frag_duration", "8000000", 0);
601 }
602 } else {
603 throw InvalidOptions("The option is not valid for this codec.", path);
604 }
605
606 }
607
608 /// Determine if codec name is valid
IsValidCodec(std::string codec_name)609 bool FFmpegWriter::IsValidCodec(std::string codec_name) {
610 // Initialize FFMpeg, and register all formats and codecs
611 AV_REGISTER_ALL
612
613 // Find the codec (if any)
614 if (avcodec_find_encoder_by_name(codec_name.c_str()) == NULL)
615 return false;
616 else
617 return true;
618 }
619
620 // Prepare & initialize streams and open codecs
PrepareStreams()621 void FFmpegWriter::PrepareStreams() {
622 if (!info.has_audio && !info.has_video)
623 throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
624
625 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::PrepareStreams [" + path + "]", "info.has_audio", info.has_audio, "info.has_video", info.has_video);
626
627 // Initialize the streams (i.e. add the streams)
628 initialize_streams();
629
630 // Mark as 'prepared'
631 prepare_streams = true;
632 }
633
634 // Write the file header (after the options are set)
WriteHeader()635 void FFmpegWriter::WriteHeader() {
636 if (!info.has_audio && !info.has_video)
637 throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
638
639 // Open the output file, if needed
640 if (!(fmt->flags & AVFMT_NOFILE)) {
641 if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
642 throw InvalidFile("Could not open or write file.", path);
643 }
644
645 // Force the output filename (which doesn't always happen for some reason)
646 AV_SET_FILENAME(oc, path.c_str());
647
648 // Add general metadata (if any)
649 for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
650 av_dict_set(&oc->metadata, iter->first.c_str(), iter->second.c_str(), 0);
651 }
652
653 // Set multiplexing parameters
654 AVDictionary *dict = NULL;
655
656 bool is_mp4 = strcmp(oc->oformat->name, "mp4");
657 bool is_mov = strcmp(oc->oformat->name, "mov");
658 // Set dictionary preset only for MP4 and MOV files
659 if (is_mp4 || is_mov)
660 av_dict_copy(&dict, mux_dict, 0);
661
662 // Write the stream header
663 if (avformat_write_header(oc, &dict) != 0) {
664 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader (avformat_write_header)");
665 throw InvalidFile("Could not write header to file.", path);
666 };
667
668 // Free multiplexing dictionaries sets
669 if (dict) av_dict_free(&dict);
670 if (mux_dict) av_dict_free(&mux_dict);
671
672 // Mark as 'written'
673 write_header = true;
674
675 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader");
676 }
677
678 // Add a frame to the queue waiting to be encoded.
WriteFrame(std::shared_ptr<Frame> frame)679 void FFmpegWriter::WriteFrame(std::shared_ptr<Frame> frame) {
680 // Check for open reader (or throw exception)
681 if (!is_open)
682 throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
683
684 // Add frame pointer to "queue", waiting to be processed the next
685 // time the WriteFrames() method is called.
686 if (info.has_video && video_st)
687 spooled_video_frames.push_back(frame);
688
689 if (info.has_audio && audio_st)
690 spooled_audio_frames.push_back(frame);
691
692 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing);
693
694 // Write the frames once it reaches the correct cache size
695 if ((int)spooled_video_frames.size() == cache_size || (int)spooled_audio_frames.size() == cache_size) {
696 // Write frames to video file
697 write_queued_frames();
698 }
699
700 // Keep track of the last frame added
701 last_frame = frame;
702 }
703
704 // Write all frames in the queue to the video file.
write_queued_frames()705 void FFmpegWriter::write_queued_frames() {
706 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size());
707
708 // Flip writing flag
709 is_writing = true;
710
711 // Transfer spool to queue
712 queued_video_frames = spooled_video_frames;
713 queued_audio_frames = spooled_audio_frames;
714
715 // Empty spool
716 spooled_video_frames.clear();
717 spooled_audio_frames.clear();
718
719 // Create blank exception
720 bool has_error_encoding_video = false;
721
722 // Process all audio frames (in a separate thread)
723 if (info.has_audio && audio_st && !queued_audio_frames.empty())
724 write_audio_packets(false);
725
726 // Loop through each queued image frame
727 while (!queued_video_frames.empty()) {
728 // Get front frame (from the queue)
729 std::shared_ptr<Frame> frame = queued_video_frames.front();
730
731 // Add to processed queue
732 processed_frames.push_back(frame);
733
734 // Encode and add the frame to the output file
735 if (info.has_video && video_st)
736 process_video_packet(frame);
737
738 // Remove front item
739 queued_video_frames.pop_front();
740
741 } // end while
742
743
744 // Loop back through the frames (in order), and write them to the video file
745 while (!processed_frames.empty()) {
746 // Get front frame (from the queue)
747 std::shared_ptr<Frame> frame = processed_frames.front();
748
749 if (info.has_video && video_st) {
750 // Add to deallocate queue (so we can remove the AVFrames when we are done)
751 deallocate_frames.push_back(frame);
752
753 // Does this frame's AVFrame still exist
754 if (av_frames.count(frame)) {
755 // Get AVFrame
756 AVFrame *frame_final = av_frames[frame];
757
758 // Write frame to video file
759 bool success = write_video_packet(frame, frame_final);
760 if (!success)
761 has_error_encoding_video = true;
762 }
763 }
764
765 // Remove front item
766 processed_frames.pop_front();
767 }
768
769 // Loop through, and deallocate AVFrames
770 while (!deallocate_frames.empty()) {
771 // Get front frame (from the queue)
772 std::shared_ptr<Frame> frame = deallocate_frames.front();
773
774 // Does this frame's AVFrame still exist
775 if (av_frames.count(frame)) {
776 // Get AVFrame
777 AVFrame *av_frame = av_frames[frame];
778
779 // Deallocate AVPicture and AVFrame
780 av_freep(&(av_frame->data[0]));
781 AV_FREE_FRAME(&av_frame);
782 av_frames.erase(frame);
783 }
784
785 // Remove front item
786 deallocate_frames.pop_front();
787 }
788
789 // Done writing
790 is_writing = false;
791
792 // Raise exception from main thread
793 if (has_error_encoding_video)
794 throw ErrorEncodingVideo("Error while writing raw video frame", -1);
795 }
796
797 // Write a block of frames from a reader
WriteFrame(ReaderBase * reader,int64_t start,int64_t length)798 void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) {
799 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length);
800
801 // Loop through each frame (and encoded it)
802 for (int64_t number = start; number <= length; number++) {
803 // Get the frame
804 std::shared_ptr<Frame> f = reader->GetFrame(number);
805
806 // Encode frame
807 WriteFrame(f);
808 }
809 }
810
811 // Write the file trailer (after all frames are written)
WriteTrailer()812 void FFmpegWriter::WriteTrailer() {
813 // Write any remaining queued frames to video file
814 write_queued_frames();
815
816 // Process final audio frame (if any)
817 if (info.has_audio && audio_st)
818 write_audio_packets(true);
819
820 // Flush encoders (who sometimes hold on to frames)
821 flush_encoders();
822
823 /* write the trailer, if any. The trailer must be written
824 * before you close the CodecContexts open when you wrote the
825 * header; otherwise write_trailer may try to use memory that
826 * was freed on av_codec_close() */
827 av_write_trailer(oc);
828
829 // Mark as 'written'
830 write_trailer = true;
831
832 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer");
833 }
834
835 // Flush encoders
flush_encoders()836 void FFmpegWriter::flush_encoders() {
837 if (info.has_audio && audio_codec_ctx && AV_GET_CODEC_TYPE(audio_st) == AVMEDIA_TYPE_AUDIO && AV_GET_CODEC_ATTRIBUTES(audio_st, audio_codec_ctx)->frame_size <= 1)
838 return;
839 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
840 // FFmpeg < 4.0
841 if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
842 return;
843 #else
844 if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
845 return;
846 #endif
847
848 // FLUSH VIDEO ENCODER
849 if (info.has_video)
850 for (;;) {
851
852 // Increment PTS (in frames and scaled to the codec's timebase)
853 video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
854
855 AVPacket pkt;
856 av_init_packet(&pkt);
857 pkt.data = NULL;
858 pkt.size = 0;
859
860 /* encode the image */
861 int got_packet = 0;
862 int error_code = 0;
863
864 #if IS_FFMPEG_3_2
865 // Encode video packet (latest version of FFmpeg)
866 error_code = avcodec_send_frame(video_codec_ctx, NULL);
867 got_packet = 0;
868 while (error_code >= 0) {
869 error_code = avcodec_receive_packet(video_codec_ctx, &pkt);
870 if (error_code == AVERROR(EAGAIN)|| error_code == AVERROR_EOF) {
871 got_packet = 0;
872 // Write packet
873 avcodec_flush_buffers(video_codec_ctx);
874 break;
875 }
876 av_packet_rescale_ts(&pkt, video_codec_ctx->time_base, video_st->time_base);
877 pkt.stream_index = video_st->index;
878 error_code = av_interleaved_write_frame(oc, &pkt);
879 }
880 #else // IS_FFMPEG_3_2
881
882 // Encode video packet (older than FFmpeg 3.2)
883 error_code = avcodec_encode_video2(video_codec_ctx, &pkt, NULL, &got_packet);
884
885 #endif // IS_FFMPEG_3_2
886
887 if (error_code < 0) {
888 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + av_err2string(error_code) + "]", "error_code", error_code);
889 }
890 if (!got_packet) {
891 break;
892 }
893
894 // set the timestamp
895 av_packet_rescale_ts(&pkt, video_codec_ctx->time_base, video_st->time_base);
896 pkt.stream_index = video_st->index;
897
898 // Write packet
899 error_code = av_interleaved_write_frame(oc, &pkt);
900 if (error_code < 0) {
901 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + av_err2string(error_code) + "]", "error_code", error_code);
902 }
903 }
904
905 // FLUSH AUDIO ENCODER
906 if (info.has_audio) {
907 for (;;) {
908 AVPacket pkt;
909 av_init_packet(&pkt);
910 pkt.data = NULL;
911 pkt.size = 0;
912 pkt.pts = pkt.dts = audio_timestamp;
913
914 /* encode the image */
915 int error_code = 0;
916 int got_packet = 0;
917 #if IS_FFMPEG_3_2
918 error_code = avcodec_send_frame(audio_codec_ctx, NULL);
919 #else
920 error_code = avcodec_encode_audio2(audio_codec_ctx, &pkt, NULL, &got_packet);
921 #endif
922 if (error_code < 0) {
923 ZmqLogger::Instance()->AppendDebugMethod(
924 "FFmpegWriter::flush_encoders ERROR [" + av_err2string(error_code) + "]",
925 "error_code", error_code);
926 }
927 if (!got_packet) {
928 break;
929 }
930
931 // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
932 // but it fixes lots of PTS related issues when I do this.
933 pkt.pts = pkt.dts = audio_timestamp;
934
935 // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
936 av_packet_rescale_ts(&pkt, audio_codec_ctx->time_base, audio_st->time_base);
937
938 // set stream
939 pkt.stream_index = audio_st->index;
940 pkt.flags |= AV_PKT_FLAG_KEY;
941
942 // Write packet
943 error_code = av_interleaved_write_frame(oc, &pkt);
944 if (error_code < 0) {
945 ZmqLogger::Instance()->AppendDebugMethod(
946 "FFmpegWriter::flush_encoders ERROR [" + av_err2string(error_code) + "]",
947 "error_code", error_code);
948 }
949
950 // Increment PTS by duration of packet
951 audio_timestamp += pkt.duration;
952
953 // deallocate memory for packet
954 AV_FREE_PACKET(&pkt);
955 }
956 }
957
958 }
959
960 // Close the video codec
close_video(AVFormatContext * oc,AVStream * st)961 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
962 {
963 #if USE_HW_ACCEL
964 if (hw_en_on && hw_en_supported) {
965 if (hw_device_ctx) {
966 av_buffer_unref(&hw_device_ctx);
967 hw_device_ctx = NULL;
968 }
969 }
970 #endif // USE_HW_ACCEL
971 }
972
973 // Close the audio codec
close_audio(AVFormatContext * oc,AVStream * st)974 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
975 {
976 // Clear buffers
977 delete[] samples;
978 delete[] audio_outbuf;
979 delete[] audio_encoder_buffer;
980 samples = NULL;
981 audio_outbuf = NULL;
982 audio_encoder_buffer = NULL;
983
984 // Deallocate resample buffer
985 if (avr) {
986 SWR_CLOSE(avr);
987 SWR_FREE(&avr);
988 avr = NULL;
989 }
990
991 if (avr_planar) {
992 SWR_CLOSE(avr_planar);
993 SWR_FREE(&avr_planar);
994 avr_planar = NULL;
995 }
996 }
997
998 // Close the writer
Close()999 void FFmpegWriter::Close() {
1000 // Write trailer (if needed)
1001 if (!write_trailer)
1002 WriteTrailer();
1003
1004 // Close each codec
1005 if (video_st)
1006 close_video(oc, video_st);
1007 if (audio_st)
1008 close_audio(oc, audio_st);
1009
1010 // Deallocate image scalers
1011 if (image_rescalers.size() > 0)
1012 RemoveScalers();
1013
1014 if (!(fmt->flags & AVFMT_NOFILE)) {
1015 /* close the output file */
1016 avio_close(oc->pb);
1017 }
1018
1019 // Reset frame counters
1020 video_timestamp = 0;
1021 audio_timestamp = 0;
1022
1023 // Free the context which frees the streams too
1024 avformat_free_context(oc);
1025 oc = NULL;
1026
1027 // Close writer
1028 is_open = false;
1029 prepare_streams = false;
1030 write_header = false;
1031 write_trailer = false;
1032
1033 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close");
1034 }
1035
1036 // Add an AVFrame to the cache
add_avframe(std::shared_ptr<Frame> frame,AVFrame * av_frame)1037 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame *av_frame) {
1038 // Add AVFrame to map (if it does not already exist)
1039 if (!av_frames.count(frame)) {
1040 // Add av_frame
1041 av_frames[frame] = av_frame;
1042 } else {
1043 // Do not add, and deallocate this AVFrame
1044 AV_FREE_FRAME(&av_frame);
1045 }
1046 }
1047
1048 // Add an audio output stream
add_audio_stream()1049 AVStream *FFmpegWriter::add_audio_stream() {
1050 AVCodecContext *c;
1051 AVStream *st;
1052
1053 // Find the audio codec
1054 AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1055 if (codec == NULL)
1056 throw InvalidCodec("A valid audio codec could not be found for this file.", path);
1057
1058 // Free any previous memory allocations
1059 if (audio_codec_ctx != NULL) {
1060 AV_FREE_CONTEXT(audio_codec_ctx);
1061 }
1062
1063 // Create a new audio stream
1064 AV_FORMAT_NEW_STREAM(oc, audio_codec_ctx, codec, st)
1065
1066 c->codec_id = codec->id;
1067 c->codec_type = AVMEDIA_TYPE_AUDIO;
1068
1069 // Set the sample parameters
1070 c->bit_rate = info.audio_bit_rate;
1071 c->channels = info.channels;
1072
1073 // Set valid sample rate (or throw error)
1074 if (codec->supported_samplerates) {
1075 int i;
1076 for (i = 0; codec->supported_samplerates[i] != 0; i++)
1077 if (info.sample_rate == codec->supported_samplerates[i]) {
1078 // Set the valid sample rate
1079 c->sample_rate = info.sample_rate;
1080 break;
1081 }
1082 if (codec->supported_samplerates[i] == 0)
1083 throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
1084 } else
1085 // Set sample rate
1086 c->sample_rate = info.sample_rate;
1087
1088
1089 // Set a valid number of channels (or throw error)
1090 const uint64_t channel_layout = info.channel_layout;
1091 if (codec->channel_layouts) {
1092 int i;
1093 for (i = 0; codec->channel_layouts[i] != 0; i++)
1094 if (channel_layout == codec->channel_layouts[i]) {
1095 // Set valid channel layout
1096 c->channel_layout = channel_layout;
1097 break;
1098 }
1099 if (codec->channel_layouts[i] == 0)
1100 throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1101 } else
1102 // Set valid channel layout
1103 c->channel_layout = channel_layout;
1104
1105 // Choose a valid sample_fmt
1106 if (codec->sample_fmts) {
1107 for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) {
1108 // Set sample format to 1st valid format (and then exit loop)
1109 c->sample_fmt = codec->sample_fmts[i];
1110 break;
1111 }
1112 }
1113 if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
1114 // Default if no sample formats found
1115 c->sample_fmt = AV_SAMPLE_FMT_S16;
1116 }
1117
1118 // some formats want stream headers to be separate
1119 if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1120 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1121 // FFmpeg 3.0+
1122 c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1123 #else
1124 c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1125 #endif
1126
1127 AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1128 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_audio_stream", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->channels", c->channels, "c->sample_fmt", c->sample_fmt, "c->channel_layout", c->channel_layout, "c->sample_rate", c->sample_rate);
1129
1130 return st;
1131 }
1132
1133 // Add a video output stream
add_video_stream()1134 AVStream *FFmpegWriter::add_video_stream() {
1135 AVCodecContext *c;
1136 AVStream *st;
1137
1138 // Find the video codec
1139 AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1140 if (codec == NULL)
1141 throw InvalidCodec("A valid video codec could not be found for this file.", path);
1142
1143 // Create a new video stream
1144 AV_FORMAT_NEW_STREAM(oc, video_codec_ctx, codec, st)
1145
1146 c->codec_id = codec->id;
1147 c->codec_type = AVMEDIA_TYPE_VIDEO;
1148
1149 /* Init video encoder options */
1150 if (info.video_bit_rate >= 1000
1151 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1152 && c->codec_id != AV_CODEC_ID_AV1
1153 #endif
1154 ) {
1155 c->bit_rate = info.video_bit_rate;
1156 if (info.video_bit_rate >= 1500000) {
1157 if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1158 c->qmin = 2;
1159 c->qmax = 30;
1160 }
1161 }
1162 // Here should be the setting for low fixed bitrate
1163 // Defaults are used because mpeg2 otherwise had problems
1164 } else {
1165 // Check if codec supports crf or qp
1166 switch (c->codec_id) {
1167 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1168 // FFmpeg 4.0+
1169 case AV_CODEC_ID_AV1 :
1170 // TODO: Set `crf` or `qp` according to bitrate, as bitrate is not supported by these encoders yet.
1171 if (info.video_bit_rate >= 1000) {
1172 c->bit_rate = 0;
1173 if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1174 int calculated_quality = 35;
1175 if (info.video_bit_rate < 500000) calculated_quality = 50;
1176 if (info.video_bit_rate > 5000000) calculated_quality = 10;
1177 av_opt_set_int(c->priv_data, "crf", calculated_quality, 0);
1178 info.video_bit_rate = calculated_quality;
1179 } else {
1180 int calculated_quality = 50;
1181 if (info.video_bit_rate < 500000) calculated_quality = 60;
1182 if (info.video_bit_rate > 5000000) calculated_quality = 15;
1183 av_opt_set_int(c->priv_data, "qp", calculated_quality, 0);
1184 info.video_bit_rate = calculated_quality;
1185 } // medium
1186 }
1187 if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
1188 av_opt_set_int(c->priv_data, "preset", 6, 0);
1189 av_opt_set_int(c->priv_data, "forced-idr",1,0);
1190 }
1191 else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
1192 av_opt_set_int(c->priv_data, "speed", 7, 0);
1193 av_opt_set_int(c->priv_data, "tile-rows", 2, 0);
1194 av_opt_set_int(c->priv_data, "tile-columns", 4, 0);
1195 }
1196 else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1197 // Set number of tiles to a fixed value
1198 // TODO: Allow user to chose their own number of tiles
1199 av_opt_set_int(c->priv_data, "tile-rows", 1, 0); // log2 of number of rows
1200 av_opt_set_int(c->priv_data, "tile-columns", 2, 0); // log2 of number of columns
1201 av_opt_set_int(c->priv_data, "row-mt", 1, 0); // use multiple cores
1202 av_opt_set_int(c->priv_data, "cpu-used", 3, 0); // default is 1, usable is 4
1203 }
1204 //break;
1205 #endif
1206 case AV_CODEC_ID_VP9 :
1207 case AV_CODEC_ID_HEVC :
1208 case AV_CODEC_ID_VP8 :
1209 case AV_CODEC_ID_H264 :
1210 if (info.video_bit_rate < 40) {
1211 c->qmin = 0;
1212 c->qmax = 63;
1213 } else {
1214 c->qmin = info.video_bit_rate - 5;
1215 c->qmax = 63;
1216 }
1217 break;
1218 default:
1219 // Here should be the setting for codecs that don't support crf
1220 // For now defaults are used
1221 break;
1222 }
1223 }
1224
1225 //TODO: Implement variable bitrate feature (which actually works). This implementation throws
1226 //invalid bitrate errors and rc buffer underflow errors, etc...
1227 //c->rc_min_rate = info.video_bit_rate;
1228 //c->rc_max_rate = info.video_bit_rate;
1229 //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
1230 //if ( !c->rc_initial_buffer_occupancy )
1231 // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
1232
1233 /* resolution must be a multiple of two */
1234 // TODO: require /2 height and width
1235 c->width = info.width;
1236 c->height = info.height;
1237
1238 /* time base: this is the fundamental unit of time (in seconds) in terms
1239 of which frame timestamps are represented. for fixed-fps content,
1240 timebase should be 1/framerate and timestamp increments should be
1241 identically 1. */
1242 c->time_base.num = info.video_timebase.num;
1243 c->time_base.den = info.video_timebase.den;
1244 // AVCodecContext->framerate was added in FFmpeg 2.6
1245 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 26, 0)
1246 c->framerate = av_inv_q(c->time_base);
1247 #endif
1248 st->avg_frame_rate = av_inv_q(c->time_base);
1249 st->time_base.num = info.video_timebase.num;
1250 st->time_base.den = info.video_timebase.den;
1251 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1252 #pragma GCC diagnostic push
1253 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
1254 st->codec->time_base.num = info.video_timebase.num;
1255 st->codec->time_base.den = info.video_timebase.den;
1256 #pragma GCC diagnostic pop
1257 #endif
1258
1259 c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
1260 c->max_b_frames = 10;
1261 if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
1262 /* just for testing, we also add B frames */
1263 c->max_b_frames = 2;
1264 if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
1265 /* Needed to avoid using macroblocks in which some coeffs overflow.
1266 This does not happen with normal video, it just happens here as
1267 the motion of the chroma plane does not match the luma plane. */
1268 c->mb_decision = 2;
1269 // some formats want stream headers to be separate
1270 if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1271 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1272 // FFmpeg 3.0+
1273 c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1274 #else
1275 c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1276 #endif
1277
1278 // Find all supported pixel formats for this codec
1279 const PixelFormat *supported_pixel_formats = codec->pix_fmts;
1280 while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
1281 // Assign the 1st valid pixel format (if one is missing)
1282 if (c->pix_fmt == PIX_FMT_NONE)
1283 c->pix_fmt = *supported_pixel_formats;
1284 ++supported_pixel_formats;
1285 }
1286
1287 // Codec doesn't have any pix formats?
1288 if (c->pix_fmt == PIX_FMT_NONE) {
1289 if (fmt->video_codec == AV_CODEC_ID_RAWVIDEO) {
1290 // Raw video should use RGB24
1291 c->pix_fmt = PIX_FMT_RGB24;
1292
1293 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1294 // FFmpeg < 4.0
1295 if (strcmp(fmt->name, "gif") != 0)
1296 // If not GIF format, skip the encoding process
1297 // Set raw picture flag (so we don't encode this video)
1298 oc->oformat->flags |= AVFMT_RAWPICTURE;
1299 #endif
1300 } else {
1301 // Set the default codec
1302 c->pix_fmt = PIX_FMT_YUV420P;
1303 }
1304 }
1305
1306 AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1307 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1308 // FFmpeg < 4.0
1309 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (std::string)fmt->name + " : " + (std::string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "AVFMT_RAWPICTURE", AVFMT_RAWPICTURE);
1310 #else
1311 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (std::string)fmt->name + " : " + (std::string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags);
1312 #endif
1313
1314 return st;
1315 }
1316
1317 // open audio codec
open_audio(AVFormatContext * oc,AVStream * st)1318 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) {
1319 AVCodec *codec;
1320 AV_GET_CODEC_FROM_STREAM(st, audio_codec_ctx)
1321
1322 // Set number of threads equal to number of processors (not to exceed 16)
1323 audio_codec_ctx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
1324
1325 // Find the audio encoder
1326 codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1327 if (!codec)
1328 codec = avcodec_find_encoder(audio_codec_ctx->codec_id);
1329 if (!codec)
1330 throw InvalidCodec("Could not find codec", path);
1331
1332 // Init options
1333 AVDictionary *opts = NULL;
1334 av_dict_set(&opts, "strict", "experimental", 0);
1335
1336 // Open the codec
1337 if (avcodec_open2(audio_codec_ctx, codec, &opts) < 0)
1338 throw InvalidCodec("Could not open audio codec", path);
1339 AV_COPY_PARAMS_FROM_CONTEXT(st, audio_codec_ctx);
1340
1341 // Free options
1342 av_dict_free(&opts);
1343
1344 // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
1345 // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
1346 if (audio_codec_ctx->frame_size <= 1) {
1347 // No frame size found... so calculate
1348 audio_input_frame_size = 50000 / info.channels;
1349
1350 int s = AV_FIND_DECODER_CODEC_ID(st);
1351 switch (s) {
1352 case AV_CODEC_ID_PCM_S16LE:
1353 case AV_CODEC_ID_PCM_S16BE:
1354 case AV_CODEC_ID_PCM_U16LE:
1355 case AV_CODEC_ID_PCM_U16BE:
1356 audio_input_frame_size >>= 1;
1357 break;
1358 default:
1359 break;
1360 }
1361 } else {
1362 // Set frame size based on the codec
1363 audio_input_frame_size = audio_codec_ctx->frame_size;
1364 }
1365
1366 // Set the initial frame size (since it might change during resampling)
1367 initial_audio_input_frame_size = audio_input_frame_size;
1368
1369 // Allocate array for samples
1370 samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1371
1372 // Set audio output buffer (used to store the encoded audio)
1373 audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1374 audio_outbuf = new uint8_t[audio_outbuf_size];
1375
1376 // Set audio packet encoding buffer
1377 audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1378 audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1379
1380 // Add audio metadata (if any)
1381 for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1382 av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1383 }
1384
1385 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec_ctx->thread_count", audio_codec_ctx->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE);
1386 }
1387
1388 // open video codec
open_video(AVFormatContext * oc,AVStream * st)1389 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
1390 AVCodec *codec;
1391 AV_GET_CODEC_FROM_STREAM(st, video_codec_ctx)
1392
1393 // Set number of threads equal to number of processors (not to exceed 16)
1394 video_codec_ctx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
1395
1396 #if USE_HW_ACCEL
1397 if (hw_en_on && hw_en_supported) {
1398 //char *dev_hw = NULL;
1399 char adapter[256];
1400 char *adapter_ptr = NULL;
1401 int adapter_num;
1402 // Use the hw device given in the environment variable HW_EN_DEVICE_SET or the default if not set
1403 adapter_num = openshot::Settings::Instance()->HW_EN_DEVICE_SET;
1404 std::clog << "Encoding Device Nr: " << adapter_num << "\n";
1405 if (adapter_num < 3 && adapter_num >=0) {
1406 #if defined(__unix__)
1407 snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
1408 // Maybe 127 is better because the first card would be 1?!
1409 adapter_ptr = adapter;
1410 #elif defined(_WIN32) || defined(__APPLE__)
1411 adapter_ptr = NULL;
1412 #else
1413 adapter_ptr = NULL;
1414 #endif
1415 }
1416 else {
1417 adapter_ptr = NULL; // Just to be sure
1418 }
1419 // Check if it is there and writable
1420 #if defined(__unix__)
1421 if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
1422 #elif defined(_WIN32) || defined(__APPLE__)
1423 if( adapter_ptr != NULL ) {
1424 #else
1425 if( adapter_ptr != NULL ) {
1426 #endif
1427 ZmqLogger::Instance()->AppendDebugMethod("Encode Device present using device", "adapter", adapter_num);
1428 }
1429 else {
1430 adapter_ptr = NULL; // use default
1431 ZmqLogger::Instance()->AppendDebugMethod("Encode Device not present, using default");
1432 }
1433 if (av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type,
1434 adapter_ptr, NULL, 0) < 0) {
1435 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video ERROR creating hwdevice, Codec name:", info.vcodec.c_str(), -1);
1436 throw InvalidCodec("Could not create hwdevice", path);
1437 }
1438 }
1439 #endif // USE_HW_ACCEL
1440
1441 /* find the video encoder */
1442 codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1443 if (!codec)
1444 codec = avcodec_find_encoder(AV_FIND_DECODER_CODEC_ID(st));
1445 if (!codec)
1446 throw InvalidCodec("Could not find codec", path);
1447
1448 /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1449 if (video_codec_ctx->max_b_frames && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1450 video_codec_ctx->max_b_frames = 0;
1451
1452 // Init options
1453 AVDictionary *opts = NULL;
1454 av_dict_set(&opts, "strict", "experimental", 0);
1455
1456 #if USE_HW_ACCEL
1457 if (hw_en_on && hw_en_supported) {
1458 video_codec_ctx->pix_fmt = hw_en_av_pix_fmt;
1459
1460 // for the list of possible options, see the list of codec-specific options:
1461 // e.g. ffmpeg -h encoder=h264_vaapi or ffmpeg -h encoder=hevc_vaapi
1462 // and "man ffmpeg-codecs"
1463
1464 // For VAAPI, it is safer to explicitly set rc_mode instead of relying on auto-selection
1465 // which is ffmpeg version-specific.
1466 if (hw_en_av_pix_fmt == AV_PIX_FMT_VAAPI) {
1467 int64_t qp;
1468 if (av_opt_get_int(video_codec_ctx->priv_data, "qp", 0, &qp) != 0 || qp == 0) {
1469 // unless "qp" was set for CQP, switch to VBR RC mode
1470 av_opt_set(video_codec_ctx->priv_data, "rc_mode", "VBR", 0);
1471
1472 // In the current state (ffmpeg-4.2-4 libva-mesa-driver-19.1.5-1) to use VBR,
1473 // one has to specify both bit_rate and maxrate, otherwise a small low quality file is generated on Intel iGPU).
1474 video_codec_ctx->rc_max_rate = video_codec_ctx->bit_rate;
1475 }
1476 }
1477
1478 switch (video_codec_ctx->codec_id) {
1479 case AV_CODEC_ID_H264:
1480 video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames
1481 video_codec_ctx->profile = FF_PROFILE_H264_BASELINE | FF_PROFILE_H264_CONSTRAINED;
1482 av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0);
1483 av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0);
1484 av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN);
1485 break;
1486 case AV_CODEC_ID_HEVC:
1487 // tested to work with defaults
1488 break;
1489 case AV_CODEC_ID_VP9:
1490 // tested to work with defaults
1491 break;
1492 default:
1493 ZmqLogger::Instance()->AppendDebugMethod("No codec-specific options defined for this codec. HW encoding may fail",
1494 "codec_id", video_codec_ctx->codec_id);
1495 break;
1496 }
1497
1498 // set hw_frames_ctx for encoder's AVCodecContext
1499 int err;
1500 if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0) {
1501 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context",
1502 "width", info.width, "height", info.height, av_err2string(err), -1);
1503 }
1504 }
1505 #endif // USE_HW_ACCEL
1506
1507 /* open the codec */
1508 if (avcodec_open2(video_codec_ctx, codec, &opts) < 0)
1509 throw InvalidCodec("Could not open video codec", path);
1510 AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx);
1511
1512 // Free options
1513 av_dict_free(&opts);
1514
1515 // Add video metadata (if any)
1516 for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1517 av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1518 }
1519
1520 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video", "video_codec_ctx->thread_count", video_codec_ctx->thread_count);
1521
1522 }
1523
1524 // write all queued frames' audio to the video file
1525 void FFmpegWriter::write_audio_packets(bool is_final) {
1526 // Init audio buffers / variables
1527 int total_frame_samples = 0;
1528 int frame_position = 0;
1529 int channels_in_frame = 0;
1530 int sample_rate_in_frame = 0;
1531 int samples_in_frame = 0;
1532 ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1533
1534 // Create a new array (to hold all S16 audio samples, for the current queued frames
1535 unsigned int all_queued_samples_size = sizeof(int16_t) * (queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE);
1536 int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size);
1537 int16_t *all_resampled_samples = NULL;
1538 int16_t *final_samples_planar = NULL;
1539 int16_t *final_samples = NULL;
1540
1541 // Loop through each queued audio frame
1542 while (!queued_audio_frames.empty()) {
1543 // Get front frame (from the queue)
1544 std::shared_ptr<Frame> frame = queued_audio_frames.front();
1545
1546 // Get the audio details from this frame
1547 sample_rate_in_frame = frame->SampleRate();
1548 samples_in_frame = frame->GetAudioSamplesCount();
1549 channels_in_frame = frame->GetAudioChannelsCount();
1550 channel_layout_in_frame = frame->ChannelsLayout();
1551
1552 // Get audio sample array
1553 float *frame_samples_float = NULL;
1554 // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1555 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
1556
1557 // Calculate total samples
1558 total_frame_samples = samples_in_frame * channels_in_frame;
1559
1560 // Translate audio sample values back to 16 bit integers with saturation
1561 const int16_t max16 = 32767;
1562 const int16_t min16 = -32768;
1563 for (int s = 0; s < total_frame_samples; s++, frame_position++) {
1564 float valF = frame_samples_float[s] * (1 << 15);
1565 int16_t conv;
1566 if (valF > max16) {
1567 conv = max16;
1568 } else if (valF < min16) {
1569 conv = min16;
1570 } else {
1571 conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
1572 }
1573
1574 // Copy into buffer
1575 all_queued_samples[frame_position] = conv;
1576 }
1577
1578 // Deallocate float array
1579 delete[] frame_samples_float;
1580
1581 // Remove front item
1582 queued_audio_frames.pop_front();
1583
1584 } // end while
1585
1586
1587 // Update total samples (since we've combined all queued frames)
1588 total_frame_samples = frame_position;
1589 int remaining_frame_samples = total_frame_samples;
1590 int samples_position = 0;
1591
1592
1593 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "is_final", is_final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO);
1594
1595 // Keep track of the original sample format
1596 AVSampleFormat output_sample_fmt = audio_codec_ctx->sample_fmt;
1597
1598 AVFrame *audio_frame = NULL;
1599 if (!is_final) {
1600 // Create input frame (and allocate arrays)
1601 audio_frame = AV_ALLOCATE_FRAME();
1602 AV_RESET_FRAME(audio_frame);
1603 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1604
1605 // Fill input frame with sample data
1606 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples, all_queued_samples_size, 0);
1607 if (error_code < 0) {
1608 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + av_err2string(error_code) + "]", "error_code", error_code);
1609 }
1610
1611 // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1612 switch (audio_codec_ctx->sample_fmt) {
1613 case AV_SAMPLE_FMT_FLTP: {
1614 output_sample_fmt = AV_SAMPLE_FMT_FLT;
1615 break;
1616 }
1617 case AV_SAMPLE_FMT_S32P: {
1618 output_sample_fmt = AV_SAMPLE_FMT_S32;
1619 break;
1620 }
1621 case AV_SAMPLE_FMT_S16P: {
1622 output_sample_fmt = AV_SAMPLE_FMT_S16;
1623 break;
1624 }
1625 case AV_SAMPLE_FMT_U8P: {
1626 output_sample_fmt = AV_SAMPLE_FMT_U8;
1627 break;
1628 }
1629 default: {
1630 // This is only here to silence unused-enum warnings
1631 break;
1632 }
1633 }
1634
1635 // Update total samples & input frame size (due to bigger or smaller data types)
1636 total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1637 total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1638
1639 // Create output frame (and allocate arrays)
1640 AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1641 AV_RESET_FRAME(audio_converted);
1642 audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1643 av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1644
1645 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (1st resampling)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", output_sample_fmt, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
1646
1647 // setup resample context
1648 if (!avr) {
1649 avr = SWR_ALLOC();
1650 av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1651 av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1652 av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1653 av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1654 av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1655 av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1656 av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1657 av_opt_set_int(avr, "out_channels", info.channels, 0);
1658 SWR_INIT(avr);
1659 }
1660 // Convert audio samples
1661 int nb_samples = SWR_CONVERT(
1662 avr, // audio resample context
1663 audio_converted->data, // output data pointers
1664 audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1665 audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1666 audio_frame->data, // input data pointers
1667 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1668 audio_frame->nb_samples // number of input samples to convert
1669 );
1670
1671 // Set remaining samples
1672 remaining_frame_samples = total_frame_samples;
1673
1674 // Create a new array (to hold all resampled S16 audio samples)
1675 all_resampled_samples = (int16_t *) av_malloc(
1676 sizeof(int16_t) * nb_samples * info.channels
1677 * (av_get_bytes_per_sample(output_sample_fmt) /
1678 av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1679 );
1680
1681 // Copy audio samples over original samples
1682 memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1683
1684 // Remove converted audio
1685 av_freep(&(audio_frame->data[0]));
1686 AV_FREE_FRAME(&audio_frame);
1687 av_freep(&audio_converted->data[0]);
1688 AV_FREE_FRAME(&audio_converted);
1689 all_queued_samples = NULL; // this array cleared with above call
1690
1691 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples);
1692 }
1693
1694 // Loop until no more samples
1695 while (remaining_frame_samples > 0 || is_final) {
1696 // Get remaining samples needed for this packet
1697 int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1698
1699 // Determine how many samples we need
1700 int diff = 0;
1701 if (remaining_frame_samples >= remaining_packet_samples) {
1702 diff = remaining_packet_samples;
1703 } else {
1704 diff = remaining_frame_samples;
1705 }
1706
1707 // Copy frame samples into the packet samples array
1708 if (!is_final)
1709 //TODO: Make this more sane
1710 memcpy(
1711 samples + (audio_input_position
1712 * (av_get_bytes_per_sample(output_sample_fmt) /
1713 av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1714 ),
1715 all_resampled_samples + samples_position,
1716 diff * av_get_bytes_per_sample(output_sample_fmt)
1717 );
1718
1719 // Increment counters
1720 audio_input_position += diff;
1721 samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1722 remaining_frame_samples -= diff;
1723
1724 // Do we have enough samples to proceed?
1725 if (audio_input_position < (audio_input_frame_size * info.channels) && !is_final)
1726 // Not enough samples to encode... so wait until the next frame
1727 break;
1728
1729 // Convert to planar (if needed by audio codec)
1730 AVFrame *frame_final = AV_ALLOCATE_FRAME();
1731 AV_RESET_FRAME(frame_final);
1732 if (av_sample_fmt_is_planar(audio_codec_ctx->sample_fmt)) {
1733 ZmqLogger::Instance()->AppendDebugMethod(
1734 "FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)",
1735 "in_sample_fmt", output_sample_fmt,
1736 "out_sample_fmt", audio_codec_ctx->sample_fmt,
1737 "in_sample_rate", info.sample_rate,
1738 "out_sample_rate", info.sample_rate,
1739 "in_channels", info.channels,
1740 "out_channels", info.channels
1741 );
1742
1743 // setup resample context
1744 if (!avr_planar) {
1745 avr_planar = SWR_ALLOC();
1746 av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1747 av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1748 av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1749 av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec_ctx->sample_fmt, 0); // planar not allowed here
1750 av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1751 av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1752 av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1753 av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1754 SWR_INIT(avr_planar);
1755 }
1756
1757 // Create input frame (and allocate arrays)
1758 audio_frame = AV_ALLOCATE_FRAME();
1759 AV_RESET_FRAME(audio_frame);
1760 audio_frame->nb_samples = audio_input_position / info.channels;
1761
1762 // Create a new array
1763 final_samples_planar = (int16_t *) av_malloc(
1764 sizeof(int16_t) * audio_frame->nb_samples * info.channels
1765 * (av_get_bytes_per_sample(output_sample_fmt) /
1766 av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1767 );
1768
1769 // Copy audio into buffer for frame
1770 memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1771
1772 // Fill input frame with sample data
1773 avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt,
1774 (uint8_t *) final_samples_planar, audio_encoder_buffer_size, 0);
1775
1776 // Create output frame (and allocate arrays)
1777 frame_final->nb_samples = audio_input_frame_size;
1778 frame_final->channels = info.channels;
1779 frame_final->format = audio_codec_ctx->sample_fmt;
1780 frame_final->channel_layout = info.channel_layout;
1781 av_samples_alloc(frame_final->data, frame_final->linesize, info.channels,
1782 frame_final->nb_samples, audio_codec_ctx->sample_fmt, 0);
1783
1784 // Convert audio samples
1785 int nb_samples = SWR_CONVERT(
1786 avr_planar, // audio resample context
1787 frame_final->data, // output data pointers
1788 frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1789 frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1790 audio_frame->data, // input data pointers
1791 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1792 audio_frame->nb_samples // number of input samples to convert
1793 );
1794
1795 // Copy audio samples over original samples
1796 if (nb_samples > 0) {
1797 memcpy(samples, frame_final->data[0],
1798 nb_samples * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt) * info.channels);
1799 }
1800
1801 // deallocate AVFrame
1802 av_freep(&(audio_frame->data[0]));
1803 AV_FREE_FRAME(&audio_frame);
1804 all_queued_samples = NULL; // this array cleared with above call
1805
1806 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples);
1807
1808 } else {
1809 // Create a new array
1810 final_samples = (int16_t *) av_malloc(
1811 sizeof(int16_t) * audio_input_position
1812 * (av_get_bytes_per_sample(audio_codec_ctx->sample_fmt) /
1813 av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1814 );
1815
1816 // Copy audio into buffer for frame
1817 memcpy(final_samples, samples,
1818 audio_input_position * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt));
1819
1820 // Init the nb_samples property
1821 frame_final->nb_samples = audio_input_frame_size;
1822
1823 // Fill the final_frame AVFrame with audio (non planar)
1824 avcodec_fill_audio_frame(frame_final, audio_codec_ctx->channels,
1825 audio_codec_ctx->sample_fmt, (uint8_t *) final_samples,
1826 audio_encoder_buffer_size, 0);
1827 }
1828
1829 // Set the AVFrame's PTS
1830 frame_final->pts = audio_timestamp;
1831
1832 // Init the packet
1833 AVPacket pkt;
1834 av_init_packet(&pkt);
1835 pkt.data = audio_encoder_buffer;
1836 pkt.size = audio_encoder_buffer_size;
1837
1838 // Set the packet's PTS prior to encoding
1839 pkt.pts = pkt.dts = audio_timestamp;
1840
1841 /* encode the audio samples */
1842 int got_packet_ptr = 0;
1843
1844 #if IS_FFMPEG_3_2
1845 // Encode audio (latest version of FFmpeg)
1846 int error_code;
1847 int ret = 0;
1848 int frame_finished = 0;
1849 error_code = ret = avcodec_send_frame(audio_codec_ctx, frame_final);
1850 if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1851 avcodec_send_frame(audio_codec_ctx, NULL);
1852 }
1853 else {
1854 if (ret >= 0)
1855 pkt.size = 0;
1856 ret = avcodec_receive_packet(audio_codec_ctx, &pkt);
1857 if (ret >= 0)
1858 frame_finished = 1;
1859 if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1860 avcodec_flush_buffers(audio_codec_ctx);
1861 ret = 0;
1862 }
1863 if (ret >= 0) {
1864 ret = frame_finished;
1865 }
1866 }
1867 if (!pkt.data && !frame_finished)
1868 {
1869 ret = -1;
1870 }
1871 got_packet_ptr = ret;
1872 #else
1873 // Encode audio (older versions of FFmpeg)
1874 int error_code = avcodec_encode_audio2(audio_codec_ctx, &pkt, frame_final, &got_packet_ptr);
1875 #endif
1876 /* if zero size, it means the image was buffered */
1877 if (error_code == 0 && got_packet_ptr) {
1878
1879 // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1880 // but it fixes lots of PTS related issues when I do this.
1881 pkt.pts = pkt.dts = audio_timestamp;
1882
1883 // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1884 av_packet_rescale_ts(&pkt, audio_codec_ctx->time_base, audio_st->time_base);
1885
1886 // set stream
1887 pkt.stream_index = audio_st->index;
1888 pkt.flags |= AV_PKT_FLAG_KEY;
1889
1890 /* write the compressed frame in the media file */
1891 error_code = av_interleaved_write_frame(oc, &pkt);
1892 }
1893
1894 if (error_code < 0) {
1895 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + av_err2string(error_code) + "]", "error_code", error_code);
1896 }
1897
1898 // Increment PTS (no pkt.duration, so calculate with maths)
1899 audio_timestamp += FFMIN(audio_input_frame_size, audio_input_position);
1900
1901 // deallocate AVFrame
1902 av_freep(&(frame_final->data[0]));
1903 AV_FREE_FRAME(&frame_final);
1904
1905 // deallocate memory for packet
1906 AV_FREE_PACKET(&pkt);
1907
1908 // Reset position
1909 audio_input_position = 0;
1910 is_final = false;
1911 }
1912
1913 // Delete arrays (if needed)
1914 if (all_resampled_samples) {
1915 av_freep(&all_resampled_samples);
1916 all_resampled_samples = NULL;
1917 }
1918 if (all_queued_samples) {
1919 av_freep(&all_queued_samples);
1920 all_queued_samples = NULL;
1921 }
1922 }
1923
1924 // Allocate an AVFrame object
1925 AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer) {
1926 // Create an RGB AVFrame
1927 AVFrame *new_av_frame = NULL;
1928
1929 // Allocate an AVFrame structure
1930 new_av_frame = AV_ALLOCATE_FRAME();
1931 if (new_av_frame == NULL)
1932 throw OutOfMemory("Could not allocate AVFrame", path);
1933
1934 // Determine required buffer size and allocate buffer
1935 *buffer_size = AV_GET_IMAGE_SIZE(pix_fmt, width, height);
1936
1937 // Create buffer (if not provided)
1938 if (!new_buffer) {
1939 // New Buffer
1940 new_buffer = (uint8_t *) av_malloc(*buffer_size * sizeof(uint8_t));
1941 // Attach buffer to AVFrame
1942 AV_COPY_PICTURE_DATA(new_av_frame, new_buffer, pix_fmt, width, height);
1943 new_av_frame->width = width;
1944 new_av_frame->height = height;
1945 new_av_frame->format = pix_fmt;
1946 }
1947
1948 // return AVFrame
1949 return new_av_frame;
1950 }
1951
1952 // process video frame
1953 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame) {
1954 // Determine the height & width of the source image
1955 int source_image_width = frame->GetWidth();
1956 int source_image_height = frame->GetHeight();
1957
1958 // Do nothing if size is 1x1 (i.e. no image in this frame)
1959 if (source_image_height == 1 && source_image_width == 1)
1960 return;
1961
1962 // Init rescalers (if not initialized yet)
1963 if (image_rescalers.size() == 0)
1964 InitScalers(source_image_width, source_image_height);
1965
1966 // Get a unique rescaler (for this thread)
1967 SwsContext *scaler = image_rescalers[rescaler_position];
1968 rescaler_position++;
1969 if (rescaler_position == num_of_rescalers)
1970 rescaler_position = 0;
1971
1972 // Allocate an RGB frame & final output frame
1973 int bytes_source = 0;
1974 int bytes_final = 0;
1975 AVFrame *frame_source = NULL;
1976 const uchar *pixels = NULL;
1977
1978 // Get a list of pixels from source image
1979 pixels = frame->GetPixels();
1980
1981 // Init AVFrame for source image & final (converted image)
1982 frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t *) pixels);
1983 #if IS_FFMPEG_3_2
1984 AVFrame *frame_final;
1985 #if USE_HW_ACCEL
1986 if (hw_en_on && hw_en_supported) {
1987 frame_final = allocate_avframe(AV_PIX_FMT_NV12, info.width, info.height, &bytes_final, NULL);
1988 } else
1989 #endif // USE_HW_ACCEL
1990 {
1991 frame_final = allocate_avframe(
1992 (AVPixelFormat)(video_st->codecpar->format),
1993 info.width, info.height, &bytes_final, NULL
1994 );
1995 }
1996 #else
1997 AVFrame *frame_final = allocate_avframe(video_codec_ctx->pix_fmt, info.width, info.height, &bytes_final, NULL);
1998 #endif // IS_FFMPEG_3_2
1999
2000 // Fill with data
2001 AV_COPY_PICTURE_DATA(frame_source, (uint8_t *) pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
2002 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final);
2003
2004 // Resize & convert pixel format
2005 sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
2006 source_image_height, frame_final->data, frame_final->linesize);
2007
2008 // Add resized AVFrame to av_frames map
2009 add_avframe(frame, frame_final);
2010
2011 // Deallocate memory
2012 AV_FREE_FRAME(&frame_source);
2013 }
2014
2015 // write video frame
2016 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *frame_final) {
2017 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
2018 // FFmpeg 4.0+
2019 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet",
2020 "frame->number", frame->number, "oc->oformat->flags", oc->oformat->flags);
2021
2022 if (AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO) {
2023 #else
2024 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet",
2025 "frame->number", frame->number,
2026 "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE);
2027
2028 if (oc->oformat->flags & AVFMT_RAWPICTURE) {
2029 #endif
2030 // Raw video case.
2031 AVPacket pkt;
2032 av_init_packet(&pkt);
2033
2034 pkt.flags |= AV_PKT_FLAG_KEY;
2035 pkt.stream_index = video_st->index;
2036 pkt.data = (uint8_t *) frame_final->data;
2037 pkt.size = sizeof(AVPicture);
2038
2039 // Set PTS (in frames and scaled to the codec's timebase)
2040 pkt.pts = video_timestamp;
2041
2042 /* write the compressed frame in the media file */
2043 int error_code = av_interleaved_write_frame(oc, &pkt);
2044 if (error_code < 0) {
2045 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + av_err2string(error_code) + "]", "error_code", error_code);
2046 return false;
2047 }
2048
2049 // Deallocate packet
2050 AV_FREE_PACKET(&pkt);
2051
2052 } else
2053 {
2054
2055 AVPacket pkt;
2056 av_init_packet(&pkt);
2057 pkt.data = NULL;
2058 pkt.size = 0;
2059 pkt.pts = pkt.dts = AV_NOPTS_VALUE;
2060
2061 // Assign the initial AVFrame PTS from the frame counter
2062 frame_final->pts = video_timestamp;
2063 #if USE_HW_ACCEL
2064 if (hw_en_on && hw_en_supported) {
2065 if (!(hw_frame = av_frame_alloc())) {
2066 std::clog << "Error code: av_hwframe_alloc\n";
2067 }
2068 if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) {
2069 std::clog << "Error code: av_hwframe_get_buffer\n";
2070 }
2071 if (!hw_frame->hw_frames_ctx) {
2072 std::clog << "Error hw_frames_ctx.\n";
2073 }
2074 hw_frame->format = AV_PIX_FMT_NV12;
2075 if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) {
2076 std::clog << "Error while transferring frame data to surface.\n";
2077 }
2078 av_frame_copy_props(hw_frame, frame_final);
2079 }
2080 #endif // USE_HW_ACCEL
2081 /* encode the image */
2082 int got_packet_ptr = 0;
2083 int error_code = 0;
2084 #if IS_FFMPEG_3_2
2085 // Write video packet (latest version of FFmpeg)
2086 int ret;
2087
2088 #if USE_HW_ACCEL
2089 if (hw_en_on && hw_en_supported) {
2090 ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!!
2091 } else
2092 #endif // USE_HW_ACCEL
2093 {
2094 ret = avcodec_send_frame(video_codec_ctx, frame_final);
2095 }
2096 error_code = ret;
2097 if (ret < 0 ) {
2098 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet (Frame not sent)");
2099 if (ret == AVERROR(EAGAIN) ) {
2100 std::clog << "Frame EAGAIN\n";
2101 }
2102 if (ret == AVERROR_EOF ) {
2103 std::clog << "Frame AVERROR_EOF\n";
2104 }
2105 avcodec_send_frame(video_codec_ctx, NULL);
2106 }
2107 else {
2108 while (ret >= 0) {
2109 ret = avcodec_receive_packet(video_codec_ctx, &pkt);
2110
2111 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
2112 avcodec_flush_buffers(video_codec_ctx);
2113 got_packet_ptr = 0;
2114 break;
2115 }
2116 if (ret == 0) {
2117 got_packet_ptr = 1;
2118 break;
2119 }
2120 }
2121 }
2122 #else
2123 // Write video packet (older than FFmpeg 3.2)
2124 error_code = avcodec_encode_video2(video_codec_ctx, &pkt, frame_final, &got_packet_ptr);
2125 if (error_code != 0) {
2126 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + av_err2string(error_code) + "]", "error_code", error_code);
2127 }
2128 if (got_packet_ptr == 0) {
2129 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet (Frame gotpacket error)");
2130 }
2131 #endif // IS_FFMPEG_3_2
2132
2133 /* if zero size, it means the image was buffered */
2134 if (error_code == 0 && got_packet_ptr) {
2135 // set the timestamp
2136 av_packet_rescale_ts(&pkt, video_codec_ctx->time_base, video_st->time_base);
2137 pkt.stream_index = video_st->index;
2138
2139 /* write the compressed frame in the media file */
2140 int result = av_interleaved_write_frame(oc, &pkt);
2141 if (result < 0) {
2142 ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + av_err2string(result) + "]", "result", result);
2143 return false;
2144 }
2145 }
2146
2147 // Deallocate packet
2148 AV_FREE_PACKET(&pkt);
2149 #if USE_HW_ACCEL
2150 if (hw_en_on && hw_en_supported) {
2151 if (hw_frame) {
2152 av_frame_free(&hw_frame);
2153 hw_frame = NULL;
2154 }
2155 }
2156 #endif // USE_HW_ACCEL
2157 }
2158
2159 // Increment PTS (in frames and scaled to the codec's timebase)
2160 video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
2161
2162 // Success
2163 return true;
2164 }
2165
2166 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
2167 void FFmpegWriter::OutputStreamInfo() {
2168 // output debug info
2169 av_dump_format(oc, 0, path.c_str(), 1);
2170 }
2171
2172 // Init a collection of software rescalers (thread safe)
2173 void FFmpegWriter::InitScalers(int source_width, int source_height) {
2174 int scale_mode = SWS_FAST_BILINEAR;
2175 if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
2176 scale_mode = SWS_BICUBIC;
2177 }
2178
2179 // Init software rescalers vector (many of them, one for each thread)
2180 for (int x = 0; x < num_of_rescalers; x++) {
2181 // Init the software scaler from FFMpeg
2182 #if USE_HW_ACCEL
2183 if (hw_en_on && hw_en_supported) {
2184 img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA,
2185 info.width, info.height, AV_PIX_FMT_NV12, scale_mode, NULL, NULL, NULL);
2186 } else
2187 #endif // USE_HW_ACCEL
2188 {
2189 img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA,
2190 info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(video_st, video_st->codec),
2191 scale_mode, NULL, NULL, NULL);
2192 }
2193
2194 // Add rescaler to vector
2195 image_rescalers.push_back(img_convert_ctx);
2196 }
2197 }
2198
2199 // Set audio resample options
2200 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
2201 original_sample_rate = sample_rate;
2202 original_channels = channels;
2203 }
2204
2205 // Remove & deallocate all software scalers
2206 void FFmpegWriter::RemoveScalers() {
2207 // Close all rescalers
2208 for (int x = 0; x < num_of_rescalers; x++)
2209 sws_freeContext(image_rescalers[x]);
2210
2211 // Clear vector
2212 image_rescalers.clear();
2213 }
2214