1 /*
2 * GPAC - Multimedia Framework C SDK
3 *
4 * Authors: Jean Le Feuvre
5 * Copyright (c) Telecom ParisTech 2017-2018
6 * All rights reserved
7 *
8 * This file is part of GPAC / ffmpeg decode filter
9 *
10 * GPAC is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU Lesser General Public License as published by
12 * the Free Software Foundation; either version 2, or (at your option)
13 * any later version.
14 *
15 * GPAC is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with this library; see the file COPYING. If not, write to
22 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 *
24 */
25
26 #include <gpac/setup.h>
27
28 #ifdef GPAC_HAS_FFMPEG
29
30 #include "ff_common.h"
31
32 #include <libswscale/swscale.h>
33
34 #define FF_CHECK_PROP(_name, _ffname, _type) if (ctx->_name != ctx->decoder->_ffname) { \
35 gf_filter_pid_set_property(ctx->out_pid, _type, &PROP_UINT( (u32) ctx->decoder->_ffname ) ); \
36 ctx->_name = (u32) ctx->decoder->_ffname; \
37 } \
38
39 #define FF_CHECK_PROPL(_name, _ffname, _type) if (ctx->_name != ctx->decoder->_ffname) { \
40 gf_filter_pid_set_property(ctx->out_pid, _type, &PROP_LONGUINT( (u32) ctx->decoder->_ffname ) ); \
41 ctx->_name = (u32) ctx->decoder->_ffname; \
42 } \
43
44 #define FF_CHECK_PROP_VAL(_name, _val, _type) if (ctx->_name != _val) { \
45 gf_filter_pid_set_property(ctx->out_pid, _type, &PROP_UINT( _val ) ); \
46 ctx->_name = _val; \
47 } \
48
49 static GF_Err ffdec_configure_pid(GF_Filter *filter, GF_FilterPid *pid, Bool is_remove);
50
51
52 typedef struct _gf_ffdec_ctx
53 {
54 //internal data
55 Bool initialized;
56
57 Bool owns_context;
58 AVCodecContext *decoder;
59 //decode options
60 AVDictionary *options;
61
62 Bool reconfig_pending;
63
64 GF_FilterPid *in_pid, *out_pid;
65 //media type
66 u32 type;
67 //CRC32 of extra_data
68 u32 extra_data_crc;
69
70 GF_Err (*process)(GF_Filter *filter, struct _gf_ffdec_ctx *ctx);
71
72 u32 flush_done;
73
74 //for now we don't share the data
75 AVFrame *frame;
76 //audio state
77 u32 channels, sample_rate, sample_fmt, bytes_per_sample;
78 u64 channel_layout;
79 u32 frame_start;
80 u32 nb_samples_already_in_frame;
81
82 //video state
83 u32 width, height, pixel_fmt, stride, stride_uv;
84 GF_Fraction sar;
85 struct SwsContext *sws_ctx;
86
87 GF_List *src_packets;
88
89 Bool drop_non_refs;
90 } GF_FFDecodeCtx;
91
ffdec_initialize(GF_Filter * filter)92 static GF_Err ffdec_initialize(GF_Filter *filter)
93 {
94 GF_FFDecodeCtx *ctx = (GF_FFDecodeCtx *) gf_filter_get_udta(filter);
95 ctx->initialized = GF_TRUE;
96 ctx->src_packets = gf_list_new();
97
98 ffmpeg_setup_logs(GF_LOG_CODEC);
99 return GF_OK;
100 }
101
ffdec_finalize(GF_Filter * filter)102 static void ffdec_finalize(GF_Filter *filter)
103 {
104 GF_FFDecodeCtx *ctx = (GF_FFDecodeCtx *) gf_filter_get_udta(filter);
105
106 if (ctx->options) av_dict_free(&ctx->options);
107 if (ctx->frame) av_frame_free(&ctx->frame);
108 if (ctx->sws_ctx) sws_freeContext(ctx->sws_ctx);
109
110 while (gf_list_count(ctx->src_packets)) {
111 GF_FilterPacket *pck = gf_list_pop_back(ctx->src_packets);
112 gf_filter_pck_unref(pck);
113 }
114 gf_list_del(ctx->src_packets);
115
116 if (ctx->owns_context && ctx->decoder) {
117 if (ctx->decoder->extradata) gf_free(ctx->decoder->extradata);
118 avcodec_close(ctx->decoder);
119 }
120 return;
121 }
122
ffdec_process_video(GF_Filter * filter,struct _gf_ffdec_ctx * ctx)123 static GF_Err ffdec_process_video(GF_Filter *filter, struct _gf_ffdec_ctx *ctx)
124 {
125 AVPacket pkt;
126 AVFrame *frame;
127 AVPicture pict;
128 Bool is_eos=GF_FALSE;
129 s32 res;
130 s32 gotpic;
131 const char *data = NULL;
132 Bool seek_flag = GF_FALSE;
133 u32 i, count;
134 u32 size=0, pix_fmt, outsize, pix_out, stride, stride_uv, uv_height, nb_planes;
135 u8 *out_buffer;
136 GF_FilterPacket *pck_src;
137 GF_FilterPacket *dst_pck;
138 GF_FilterPacket *pck = gf_filter_pid_get_packet(ctx->in_pid);
139
140 if (ctx->reconfig_pending) {
141 pck = NULL;
142 } else if (!pck) {
143 is_eos = gf_filter_pid_is_eos(ctx->in_pid);
144 if (!is_eos) return GF_OK;
145 }
146
147 if (pck && ctx->drop_non_refs && !gf_filter_pck_get_sap(pck)) {
148 gf_filter_pid_drop_packet(ctx->in_pid);
149 return GF_OK;
150 }
151
152 frame = ctx->frame;
153
154 av_init_packet(&pkt);
155
156 if (pck) {
157 data = gf_filter_pck_get_data(pck, &size);
158
159 if (!size) {
160 gf_filter_pid_drop_packet(ctx->in_pid);
161 return GF_OK;
162 }
163
164 pck_src = pck;
165 gf_filter_pck_ref_props(&pck_src);
166 if (pck_src) gf_list_add(ctx->src_packets, pck_src);
167
168 //seems ffmpeg is not properly handling the decoding after a flush, we close and reopen the codec
169 if (ctx->flush_done) {
170 const AVCodec *codec = ctx->decoder->codec;
171 avcodec_close(ctx->decoder);
172 avcodec_open2(ctx->decoder, codec, NULL );
173 ctx->flush_done = GF_FALSE;
174 }
175
176 pkt.dts = gf_filter_pck_get_dts(pck);
177 pkt.pts = gf_filter_pck_get_cts(pck);
178 pkt.duration = gf_filter_pck_get_duration(pck);
179 if (gf_filter_pck_get_sap(pck)>0)
180 pkt.flags = AV_PKT_FLAG_KEY;
181 }
182 pkt.data = (uint8_t*)data;
183 pkt.size = size;
184
185 /*TOCHECK: for AVC bitstreams after ISMA decryption, in case (as we do) the decryption DRM tool
186 doesn't put back nalu size, we have to do it ourselves, but we can't modify input data...*/
187
188 gotpic=0;
189 res = avcodec_decode_video2(ctx->decoder, frame, &gotpic, &pkt);
190 if (pck) gf_filter_pid_drop_packet(ctx->in_pid);
191
192 if (!gotpic) {
193 if (is_eos) {
194 ctx->flush_done = GF_TRUE;
195 gf_filter_pid_set_eos(ctx->out_pid);
196 return GF_EOS;
197 }
198 if (ctx->reconfig_pending) {
199 if (ctx->decoder->extradata) gf_free(ctx->decoder->extradata);
200 ctx->decoder->extradata = NULL;
201 avcodec_close(ctx->decoder);
202 ctx->decoder = NULL;
203 ctx->reconfig_pending = GF_FALSE;
204 //these properties are checked after decode, when we reconfigure we copy props from input to output
205 //so we need to make sure we retrigger pid config even if these did not change
206 ctx->pixel_fmt = 0;
207 ctx->width = 0;
208 ctx->height = 0;
209 ctx->stride = 0;
210 ctx->stride_uv = 0;
211 ctx->sar.num = ctx->sar.den = 0;
212 while (gf_list_count(ctx->src_packets)) {
213 GF_FilterPacket *ref_pck = gf_list_pop_back(ctx->src_packets);
214 gf_filter_pck_unref(ref_pck);
215 }
216 GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[FFDec] PID %s reconfigure pending and all frames flushed, reconfguring\n", gf_filter_pid_get_name(ctx->in_pid) ));
217 return ffdec_configure_pid(filter, ctx->in_pid, GF_FALSE);
218 }
219 }
220
221 if (res < 0) {
222 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s failed to decode frame PTS "LLU": %s\n", gf_filter_pid_get_name(ctx->in_pid), pkt.pts, av_err2str(res) ));
223 return GF_NON_COMPLIANT_BITSTREAM;
224 }
225 if (!gotpic) return GF_OK;
226
227 pix_fmt = ffmpeg_pixfmt_to_gpac(ctx->decoder->pix_fmt);
228 if (!pix_fmt) pix_fmt = GF_PIXEL_RGB;
229
230 //update all props
231 FF_CHECK_PROP_VAL(pixel_fmt, pix_fmt, GF_PROP_PID_PIXFMT)
232 FF_CHECK_PROP(width, width, GF_PROP_PID_WIDTH)
233 FF_CHECK_PROP(height, height, GF_PROP_PID_HEIGHT)
234
235 stride = stride_uv = uv_height = nb_planes = 0;
236 if (! gf_pixel_get_size_info(pix_fmt, ctx->width, ctx->height, &outsize, &stride, &stride_uv, &nb_planes, &uv_height) ) {
237 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s failed to query pixelformat size infon", gf_filter_pid_get_name(ctx->in_pid) ));
238 return GF_NOT_SUPPORTED;
239 }
240
241 FF_CHECK_PROP_VAL(stride, stride, GF_PROP_PID_STRIDE)
242 FF_CHECK_PROP_VAL(stride_uv, stride_uv, GF_PROP_PID_STRIDE_UV)
243 if (ctx->sar.num * ctx->decoder->sample_aspect_ratio.den != ctx->sar.den * ctx->decoder->sample_aspect_ratio.num) {
244 ctx->sar.num = ctx->decoder->sample_aspect_ratio.num;
245 ctx->sar.den = ctx->decoder->sample_aspect_ratio.den;
246
247 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_SAR, &PROP_FRAC( ctx->sar ) );
248 }
249
250 memset(&pict, 0, sizeof(pict));
251
252
253 pck_src = NULL;
254 count = gf_list_count(ctx->src_packets);
255 for (i=0; i<count; i++) {
256 pck_src = gf_list_get(ctx->src_packets, i);
257 if (gf_filter_pck_get_cts(pck_src) == frame->pkt_pts) break;
258 pck_src = NULL;
259 }
260
261 seek_flag = GF_FALSE;
262 if (pck_src) {
263 seek_flag = gf_filter_pck_get_seek_flag(pck_src);
264 }
265 //this was a seek frame, do not dispatch
266 if (seek_flag) {
267 if (pck_src) {
268 gf_list_del_item(ctx->src_packets, pck_src);
269 gf_filter_pck_unref(pck_src);
270 }
271 return GF_OK;
272 }
273
274 dst_pck = gf_filter_pck_new_alloc(ctx->out_pid, outsize, &out_buffer);
275
276 if (pck_src) {
277 if (dst_pck) gf_filter_pck_merge_properties(pck_src, dst_pck);
278 gf_list_del_item(ctx->src_packets, pck_src);
279 gf_filter_pck_unref(pck_src);
280 } else {
281 if (dst_pck) gf_filter_pck_set_sap(dst_pck, GF_FILTER_SAP_1);
282 }
283 if (!dst_pck) return GF_OUT_OF_MEM;
284
285
286 //TODO: cleanup, we should not convert pixel format in the decoder but through filters !
287 switch (ctx->pixel_fmt) {
288 case GF_PIXEL_RGB:
289 pict.data[0] = (uint8_t *)out_buffer;
290 pict.linesize[0] = 3*ctx->width;
291 pix_out = AV_PIX_FMT_RGB24;
292 break;
293 case GF_PIXEL_RGBA:
294 pict.data[0] = (uint8_t *)out_buffer;
295 pict.linesize[0] = 4*ctx->width;
296 pix_out = AV_PIX_FMT_RGBA;
297 break;
298 case GF_PIXEL_YUV:
299 case GF_PIXEL_YUV_10:
300 pict.data[0] = (uint8_t *)out_buffer;
301 pict.data[1] = (uint8_t *)out_buffer + ctx->stride * ctx->height;
302 pict.data[2] = (uint8_t *)pict.data[1] + ctx->stride_uv * uv_height;
303 pict.linesize[0] = ctx->stride;
304 pict.linesize[1] = pict.linesize[2] = ctx->stride_uv;
305 if (ctx->pixel_fmt == GF_PIXEL_YUV_10)
306 pix_out = AV_PIX_FMT_YUV420P10LE;
307 else
308 pix_out = AV_PIX_FMT_YUV420P;
309 break;
310
311 case GF_PIXEL_YUV422:
312 case GF_PIXEL_YUV422_10:
313 pict.data[0] = (uint8_t *)out_buffer;
314 pict.data[1] = (uint8_t *)out_buffer + ctx->stride * ctx->height;
315 pict.data[2] = (uint8_t *)pict.data[1] + ctx->stride_uv * ctx->height;
316 pict.linesize[0] = ctx->stride;
317 pict.linesize[1] = pict.linesize[2] = ctx->stride_uv;
318 if (ctx->pixel_fmt == GF_PIXEL_YUV422_10)
319 pix_out = AV_PIX_FMT_YUV422P10LE;
320 else
321 pix_out = AV_PIX_FMT_YUV422P;
322 break;
323
324 case GF_PIXEL_YUV444:
325 case GF_PIXEL_YUV444_10:
326 pict.data[0] = (uint8_t *)out_buffer;
327 pict.data[1] = (uint8_t *)out_buffer + ctx->stride * ctx->height;
328 pict.data[2] = (uint8_t *)out_buffer + 2*ctx->stride * ctx->height;
329 pict.linesize[0] = pict.linesize[1] = pict.linesize[2] = ctx->stride;
330 if (ctx->pixel_fmt == GF_PIXEL_YUV444_10)
331 pix_out = AV_PIX_FMT_YUV444P10LE;
332 else
333 pix_out = AV_PIX_FMT_YUV444P;
334 break;
335
336 default:
337 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] Unsupported pixel format %s, patch welcome\n", av_get_pix_fmt_name(ctx->decoder->pix_fmt) ));
338
339 gf_filter_pck_discard(dst_pck);
340
341 return GF_NOT_SUPPORTED;
342 }
343
344 ctx->sws_ctx = sws_getCachedContext(ctx->sws_ctx,
345 ctx->decoder->width, ctx->decoder->height, ctx->decoder->pix_fmt,
346 ctx->width, ctx->height, pix_out, SWS_BICUBIC, NULL, NULL, NULL);
347 if (ctx->sws_ctx) {
348 sws_scale(ctx->sws_ctx, (const uint8_t * const*)frame->data, frame->linesize, 0, ctx->height, pict.data, pict.linesize);
349 }
350
351 gf_filter_pck_set_seek_flag(dst_pck, GF_FALSE);
352
353 if (frame->interlaced_frame)
354 gf_filter_pck_set_interlaced(dst_pck, frame->top_field_first ? 2 : 1);
355
356 gf_filter_pck_send(dst_pck);
357 return GF_OK;
358 }
359
360
ffdec_process_audio(GF_Filter * filter,struct _gf_ffdec_ctx * ctx)361 static GF_Err ffdec_process_audio(GF_Filter *filter, struct _gf_ffdec_ctx *ctx)
362 {
363 AVPacket pkt;
364 s32 gotpic;
365 s32 len, in_size, i;
366 u32 output_size;
367 Bool is_eos=GF_FALSE;
368 u8 *data;
369 AVFrame *frame;
370 GF_FilterPacket *dst_pck, *src_pck;
371 GF_FilterPacket *pck = gf_filter_pid_get_packet(ctx->in_pid);
372
373 if (ctx->reconfig_pending) {
374 pck = NULL;
375 } else if (!pck) {
376 is_eos = gf_filter_pid_is_eos(ctx->in_pid);
377 if (!is_eos) return GF_OK;
378 }
379
380 av_init_packet(&pkt);
381 if (pck) pkt.data = (uint8_t *) gf_filter_pck_get_data(pck, &in_size);
382
383 if (pck) {
384 src_pck = pck;
385 gf_filter_pck_ref_props(&src_pck);
386 if (src_pck) gf_list_add(ctx->src_packets, src_pck);
387
388 if (!pkt.data) {
389 gf_filter_pid_drop_packet(ctx->in_pid);
390 return GF_OK;
391 }
392
393 pkt.pts = gf_filter_pck_get_cts(pck);
394 pkt.dts = gf_filter_pck_get_dts(pck);
395
396 pkt.size = in_size;
397 if ((s32) ctx->frame_start > pkt.size) ctx->frame_start = 0;
398 //seek to last byte consumed by the previous decode4()
399 else if (ctx->frame_start) {
400 pkt.data += ctx->frame_start;
401 pkt.size -= ctx->frame_start;
402 }
403 pkt.duration = gf_filter_pck_get_duration(pck);
404 if (gf_filter_pck_get_sap(pck)>0)
405 pkt.flags = AV_PKT_FLAG_KEY;
406
407 } else {
408 pkt.size = 0;
409 }
410
411 frame = ctx->frame;
412 len = avcodec_decode_audio4(ctx->decoder, frame, &gotpic, &pkt);
413
414 //this will handle eos as well
415 if ((len<0) || !gotpic) {
416 ctx->frame_start = 0;
417 if (pck) gf_filter_pid_drop_packet(ctx->in_pid);
418 if (pkt.size && (len<0)) {
419 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s failed to decode frame PTS "LLU": %s\n", gf_filter_pid_get_name(ctx->in_pid), pkt.pts, av_err2str(len) ));
420 return GF_NON_COMPLIANT_BITSTREAM;
421 }
422 if (is_eos) {
423 gf_filter_pid_set_eos(ctx->out_pid);
424 return GF_EOS;
425 }
426 if (ctx->reconfig_pending) {
427 if (ctx->decoder->extradata) gf_free(ctx->decoder->extradata);
428 ctx->decoder->extradata = NULL;
429 avcodec_close(ctx->decoder);
430 ctx->decoder = NULL;
431 ctx->reconfig_pending = GF_FALSE;
432 //these properties are checked after decode, when we reconfigure we copy props from input to output
433 //so we need to make sure we retrigger pid config even if these did not change
434 ctx->sample_fmt = 0;
435 ctx->sample_rate = 0;
436 ctx->channels = 0;
437 ctx->channel_layout = 0;
438 GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[FFDec] PID %s reconfigure pending and all frames flushed, reconfguring\n", gf_filter_pid_get_name(ctx->in_pid) ));
439 return ffdec_configure_pid(filter, ctx->in_pid, GF_FALSE);
440 }
441 return GF_OK;
442 }
443
444 FF_CHECK_PROP(channels, channels, GF_PROP_PID_NUM_CHANNELS)
445 FF_CHECK_PROPL(channel_layout, channel_layout, GF_PROP_PID_CHANNEL_LAYOUT)
446 FF_CHECK_PROP(sample_rate, sample_rate, GF_PROP_PID_SAMPLE_RATE)
447
448 output_size = frame->nb_samples*ctx->channels*ctx->bytes_per_sample;
449 dst_pck = gf_filter_pck_new_alloc(ctx->out_pid, output_size, &data);
450
451 switch (frame->format) {
452 case AV_SAMPLE_FMT_U8P:
453 case AV_SAMPLE_FMT_S16P:
454 case AV_SAMPLE_FMT_S32P:
455 case AV_SAMPLE_FMT_FLTP:
456 case AV_SAMPLE_FMT_DBLP:
457 for (i=0; (u32) i< ctx->channels; i++) {
458 char *inputChannel = frame->extended_data[i];
459 memcpy(data, inputChannel, ctx->bytes_per_sample * frame->nb_samples);
460 data += ctx->bytes_per_sample * frame->nb_samples;
461 }
462 break;
463 default:
464 memcpy(data, ctx->frame->data[0], ctx->bytes_per_sample * frame->nb_samples * ctx->channels);
465 break;
466 }
467
468 //we don't follow the same approach as in video, we assume the codec works with one in one out
469 //and use the first entry in src packets to match in order to copy the properties
470 //a nicer approach would be to count delay frames (number of frames used to initialize)
471 //and backmerge properties from the last packet in to the last-nb_init_frames
472 src_pck = gf_list_get(ctx->src_packets, 0);
473
474 if (src_pck) {
475 gf_filter_pck_merge_properties(src_pck, dst_pck);
476 gf_list_rem(ctx->src_packets, 0);
477 gf_filter_pck_unref(src_pck);
478 }
479
480 if (frame->pkt_pts != AV_NOPTS_VALUE) {
481 u64 pts = frame->pkt_pts;
482 u32 timescale = gf_filter_pck_get_timescale(pck);
483 if (ctx->nb_samples_already_in_frame) {
484 if (ctx->sample_rate == timescale) {
485 pts += ctx->nb_samples_already_in_frame;
486 }
487 }
488 gf_filter_pck_set_cts(dst_pck, pts);
489 }
490 if (frame->pkt_dts != AV_NOPTS_VALUE) {
491 gf_filter_pck_set_dts(dst_pck, frame->pkt_dts);
492 }
493
494
495 gf_filter_pck_send(dst_pck);
496
497 ctx->frame_start += len;
498 //done with this input packet
499 if (in_size <= (s32) ctx->frame_start) {
500 frame->nb_samples = 0;
501 ctx->frame_start = 0;
502 ctx->nb_samples_already_in_frame = 0;
503 gf_filter_pid_drop_packet(ctx->in_pid);
504 return GF_OK;
505 }
506 //still some data to decode in packet, don't drop it
507 //todo: check if frame->pkt_pts or frame->pts is updated by ffmpeg, otherwise do it ourselves !
508 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] Code not yet tested - frame PTS was "LLU" - nb samples dec %d\n", frame->pkt_pts, frame->nb_samples));
509 ctx->nb_samples_already_in_frame += frame->nb_samples;
510 frame->nb_samples = 0;
511
512 return ffdec_process_audio(filter, ctx);
513 }
514
515 #ifdef FF_SUB_SUPPORT
ffdec_process_subtitle(GF_Filter * filter,struct _gf_ffdec_ctx * ctx)516 static GF_Err ffdec_process_subtitle(GF_Filter *filter, struct _gf_ffdec_ctx *ctx)
517 {
518 AVPacket pkt;
519 AVSubtitle subs;
520 s32 gotpic;
521 s32 len, in_size;
522 Bool is_eos=GF_FALSE;
523
524 GF_FilterPacket *pck = gf_filter_pid_get_packet(ctx->in_pid);
525
526 if (!pck) {
527 is_eos = gf_filter_pid_is_eos(ctx->in_pid);
528 if (!is_eos) return GF_OK;
529 }
530 av_init_packet(&pkt);
531 if (pck) pkt.data = (uint8_t *) gf_filter_pck_get_data(pck, &in_size);
532
533 if (!is_eos) {
534 u64 dts;
535 pkt.pts = gf_filter_pck_get_cts(pck);
536
537 //copy over SAP and duration in dts
538 dts = gf_filter_pck_get_sap(pck);
539 dts <<= 32;
540 dts |= gf_filter_pck_get_duration(pck);
541 pkt.dts = dts;
542
543 pkt.size = in_size;
544 if (ctx->frame_start > pkt.size) ctx->frame_start = 0;
545 //seek to last byte consumed by the previous decode4()
546 else if (ctx->frame_start) {
547 pkt.data += ctx->frame_start;
548 pkt.size -= ctx->frame_start;
549 }
550 } else {
551 pkt.size = 0;
552 }
553
554 memset(&subs, 0, sizeof(AVSubtitle));
555 len = avcodec_decode_subtitle2(ctx->decoder, &subs, &gotpic, &pkt);
556
557 //this will handle eos as well
558 if ((len<0) || !gotpic) {
559 ctx->frame_start = 0;
560 if (pck) gf_filter_pid_drop_packet(ctx->in_pid);
561 if (len<0) {
562 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s failed to decode frame PTS "LLU": %s\n", gf_filter_pid_get_name(ctx->in_pid), pkt.pts, av_err2str(len) ));
563 return GF_NON_COMPLIANT_BITSTREAM;
564 }
565 if (is_eos) {
566 gf_filter_pid_set_eos(ctx->out_pid);
567 return GF_EOS;
568 }
569 return GF_OK;
570 }
571 //TODO - do we want to remap to TX3G/other and handle the rendering some place else, or do we do the rendering here ?
572
573
574 avsubtitle_free(&subs);
575 if (pck) gf_filter_pid_drop_packet(ctx->in_pid);
576 return GF_OK;
577 }
578 #endif
579
ffdec_process(GF_Filter * filter)580 static GF_Err ffdec_process(GF_Filter *filter)
581 {
582 GF_FFDecodeCtx *ctx = (GF_FFDecodeCtx *) gf_filter_get_udta(filter);
583 if (gf_filter_pid_would_block(ctx->out_pid))
584 return GF_OK;
585 return ctx->process(filter, ctx);
586 }
587
ffdec_configure_pid(GF_Filter * filter,GF_FilterPid * pid,Bool is_remove)588 static GF_Err ffdec_configure_pid(GF_Filter *filter, GF_FilterPid *pid, Bool is_remove)
589 {
590 s32 res;
591 u32 type=0, gpac_codecid=0;
592 const GF_PropertyValue *prop;
593 GF_FFDecodeCtx *ctx = (GF_FFDecodeCtx *) gf_filter_get_udta(filter);
594
595 //disconnect of src pid (not yet supported)
596 if (is_remove) {
597 //one in one out, this is simple
598 if (ctx->out_pid) gf_filter_pid_remove(ctx->out_pid);
599 return GF_OK;
600 }
601
602 //check our PID: streamtype and codecid
603 prop = gf_filter_pid_get_property(pid, GF_PROP_PID_STREAM_TYPE);
604 if (!prop) return GF_NOT_SUPPORTED;
605
606 type = prop->value.uint;
607 switch (type) {
608 case GF_STREAM_AUDIO:
609 case GF_STREAM_VISUAL:
610 #ifdef FF_SUB_SUPPORT
611 case GF_STREAM_TEXT:
612 #endif
613 break;
614 default:
615 return GF_NOT_SUPPORTED;
616 }
617 prop = gf_filter_pid_get_property(pid, GF_PROP_PID_CODECID);
618 if (!prop) return GF_NOT_SUPPORTED;
619 gpac_codecid = prop->value.uint;
620 if (gpac_codecid==GF_CODECID_RAW)
621 return GF_NOT_SUPPORTED;
622
623
624 //initial config or update
625 if (!ctx->in_pid || (ctx->in_pid==pid)) {
626 ctx->in_pid = pid;
627 if (!ctx->type) ctx->type = type;
628 else if (ctx->type != type) {
629 return GF_NOT_SUPPORTED;
630 }
631 } else {
632 //only one input pid in ctx
633 if (ctx->in_pid) return GF_REQUIRES_NEW_INSTANCE;
634 }
635
636 prop = gf_filter_pid_get_property(pid, GF_PROP_PID_WIDTH);
637 ctx->width = prop ? prop->value.uint : 320;
638 prop = gf_filter_pid_get_property(pid, GF_PROP_PID_HEIGHT);
639 ctx->height = prop ? prop->value.uint : 240;
640
641
642 if (gpac_codecid == GF_CODECID_FFMPEG) {
643 AVCodec *codec=NULL;
644 prop = gf_filter_pid_get_property(pid, GF_FFMPEG_DECODER_CONFIG);
645 if (!prop || !prop->value.ptr) {
646 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s codec context not exposed by demuxer !\n", gf_filter_pid_get_name(pid) ));
647 return GF_SERVICE_ERROR;
648 }
649 ctx->decoder = prop->value.ptr;
650 codec = avcodec_find_decoder(ctx->decoder->codec_id);
651 if (!codec) return GF_NOT_SUPPORTED;
652
653 res = avcodec_open2(ctx->decoder, codec, NULL );
654 if (res < 0) {
655 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s failed to open codec context: %s\n", gf_filter_pid_get_name(pid), av_err2str(res) ));
656 return GF_NON_COMPLIANT_BITSTREAM;
657 }
658 }
659 //we reconfigure the stream
660 else {
661 AVCodec *codec=NULL;
662 u32 codec_id, ff_codectag=0;
663 if (ctx->decoder) {
664 codec_id = ffmpeg_codecid_from_gpac(gpac_codecid, NULL);
665 //same codec, same config, don't reinit
666 if (ctx->decoder->codec->id == codec_id) {
667 u32 cfg_crc=0;
668 prop = gf_filter_pid_get_property(pid, GF_PROP_PID_DECODER_CONFIG);
669 if (prop && prop->value.data.ptr && prop->value.data.size) {
670 cfg_crc = gf_crc_32(prop->value.data.ptr, prop->value.data.size);
671 }
672 if (cfg_crc == ctx->extra_data_crc) {
673 goto reuse_codec_context;
674 }
675 }
676
677 //we could further optimize by detecting we have the same codecid and injecting the extradata
678 //but this is not 100% reliable, and will require parsing AVC/HEVC config
679 //since this seems to work properly with decoder close/open, we keep it as is
680 ctx->reconfig_pending = GF_TRUE;
681 GF_LOG(GF_LOG_INFO, GF_LOG_CODEC, ("[FFDec] PID %s reconfigure detected, flushing frame\n", gf_filter_pid_get_name(pid) ));
682 return GF_OK;
683 }
684
685 codec_id = ffmpeg_codecid_from_gpac(gpac_codecid, &ff_codectag);
686 if (codec_id) codec = avcodec_find_decoder(codec_id);
687 if (!codec) {
688 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] No decoder found for codec %s\n", gf_codecid_name(gpac_codecid) ));
689 return GF_NOT_SUPPORTED;
690 }
691
692 ctx->decoder = avcodec_alloc_context3(NULL);
693 if (! ctx->decoder) return GF_OUT_OF_MEM;
694 ctx->owns_context = GF_TRUE;
695 if (ff_codectag)
696 ctx->decoder->codec_tag = ff_codectag;
697
698 ffmpeg_set_enc_dec_flags(ctx->options, ctx->decoder);
699
700 //we may have a dsi here!
701 prop = gf_filter_pid_get_property(pid, GF_PROP_PID_DECODER_CONFIG);
702 if (prop && prop->value.data.ptr && prop->value.data.size) {
703 //looks loke ffmpeg wants the fLaC keyword
704 if (gpac_codecid==GF_CODECID_FLAC) {
705 ctx->decoder->extradata_size = prop->value.data.size+4;
706 ctx->decoder->extradata = gf_malloc(sizeof(char) * prop->value.data.size+4);
707 memcpy(ctx->decoder->extradata, "fLaC", 4);
708 memcpy(ctx->decoder->extradata+4, prop->value.data.ptr, prop->value.data.size);
709 } else {
710 ctx->decoder->extradata_size = prop->value.data.size;
711 ctx->decoder->extradata = gf_malloc(sizeof(char) * prop->value.data.size);
712 memcpy(ctx->decoder->extradata, prop->value.data.ptr, prop->value.data.size);
713 }
714 ctx->extra_data_crc = gf_crc_32(prop->value.data.ptr, prop->value.data.size);
715 }
716
717 res = avcodec_open2(ctx->decoder, codec, NULL );
718 if (res < 0) {
719 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] PID %s failed to open codec context: %s\n", gf_filter_pid_get_name(pid), av_err2str(res) ));
720 return GF_NON_COMPLIANT_BITSTREAM;
721 }
722 }
723
724 ffmpeg_report_unused_options(filter, ctx->options);
725
726 //we're good to go, declare our output pid
727 ctx->in_pid = pid;
728 if (!ctx->out_pid) {
729 char szCodecName[1000];
730 ctx->out_pid = gf_filter_pid_new(filter);
731
732 //to change once we implement on-the-fly codec change
733 sprintf(szCodecName, "ffdec:%s", ctx->decoder->codec->name ? ctx->decoder->codec->name : "unknown");
734 gf_filter_set_name(filter, szCodecName);
735 gf_filter_pid_set_framing_mode(ctx->in_pid, GF_TRUE);
736 }
737
738 reuse_codec_context:
739 //copy props it at init config or at reconfig
740 if (ctx->out_pid) {
741 gf_filter_pid_copy_properties(ctx->out_pid, ctx->in_pid);
742 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_CODECID, &PROP_UINT(GF_CODECID_RAW) );
743 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_DECODER_CONFIG, NULL );
744 }
745
746 if (type==GF_STREAM_VISUAL) {
747 u32 pix_fmt;
748 ctx->process = ffdec_process_video;
749 //for some streams, we don't have w/h/pixfmt after opening the decoder
750 //to make sure we are not confusing potential filters expecting them, init to default values
751 if (ctx->decoder->pix_fmt>=0) {
752 pix_fmt = ffmpeg_pixfmt_to_gpac(ctx->decoder->pix_fmt);
753 if (!pix_fmt) {
754 GF_LOG(GF_LOG_WARNING, GF_LOG_CODEC, ("[FFDec] Unsupported pixel format %d, defaulting to RGB\n", pix_fmt));
755 pix_fmt = GF_PIXEL_RGB;
756 }
757 FF_CHECK_PROP_VAL(pixel_fmt, pix_fmt, GF_PROP_PID_PIXFMT)
758 } else {
759 ctx->pixel_fmt = GF_PIXEL_YUV;
760 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_PIXFMT, &PROP_UINT( ctx->pixel_fmt) );
761 }
762
763 if (ctx->decoder->width) {
764 FF_CHECK_PROP(width, width, GF_PROP_PID_WIDTH)
765 } else {
766 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_WIDTH, &PROP_UINT( ctx->width) );
767 }
768 if (ctx->decoder->height) {
769 FF_CHECK_PROP(height, height, GF_PROP_PID_HEIGHT)
770 } else {
771 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_HEIGHT, &PROP_UINT( ctx->height) );
772 }
773 if (ctx->decoder->sample_aspect_ratio.num && ctx->decoder->sample_aspect_ratio.den) {
774 ctx->sar.num = ctx->decoder->sample_aspect_ratio.num;
775 ctx->sar.den = ctx->decoder->sample_aspect_ratio.den;
776 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_SAR, &PROP_FRAC( ctx->sar) );
777 }
778 if (!ctx->frame)
779 ctx->frame = av_frame_alloc();
780
781 if (ctx->pixel_fmt) {
782 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_PIXFMT, &PROP_UINT( ctx->pixel_fmt) );
783 }
784
785 } else if (type==GF_STREAM_AUDIO) {
786 ctx->process = ffdec_process_audio;
787 ctx->sample_fmt = ffmpeg_audio_fmt_to_gpac(ctx->decoder->sample_fmt);
788 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_AUDIO_FORMAT, &PROP_UINT(ctx->sample_fmt) );
789 ctx->bytes_per_sample = gf_audio_fmt_bit_depth(ctx->sample_fmt) / 8;
790
791 //override PID props with what decoder gives us
792 if (ctx->decoder->channels) {
793 FF_CHECK_PROP(channels, channels, GF_PROP_PID_NUM_CHANNELS)
794 }
795 if (ctx->decoder->channel_layout) {
796 u64 ch_lay = ffmpeg_channel_layout_to_gpac(ctx->decoder->channel_layout);
797 if (ctx->channel_layout != ch_lay) {
798 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_CHANNEL_LAYOUT, &PROP_LONGUINT(ch_lay ) );
799 ctx->channel_layout = ch_lay;
800 }
801 }
802 if (ctx->decoder->sample_rate) {
803 FF_CHECK_PROP(sample_rate, sample_rate, GF_PROP_PID_SAMPLE_RATE)
804 }
805 if (!ctx->frame)
806 ctx->frame = av_frame_alloc();
807
808 if (ctx->sample_fmt) {
809 gf_filter_pid_set_property(ctx->out_pid, GF_PROP_PID_AUDIO_FORMAT, &PROP_UINT( ctx->sample_fmt) );
810 }
811
812 } else {
813 #ifdef FF_SUB_SUPPORT
814 ctx->process = ffdec_process_subtitle;
815 #endif
816 }
817 return GF_OK;
818 }
819
820
ffdec_update_arg(GF_Filter * filter,const char * arg_name,const GF_PropertyValue * arg_val)821 static GF_Err ffdec_update_arg(GF_Filter *filter, const char *arg_name, const GF_PropertyValue *arg_val)
822 {
823 s32 res;
824 GF_FFDecodeCtx *ctx = gf_filter_get_udta(filter);
825
826 //initial parsing of arguments
827 if (!ctx->initialized) {
828 switch (arg_val->type) {
829 case GF_PROP_STRING:
830 res = av_dict_set(&ctx->options, arg_name, arg_val->value.string, 0);
831 if (res<0) {
832 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] Failed to set option %s:%s\n", arg_name, arg_val ));
833 }
834 break;
835 default:
836 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[FFDec] Failed to set option %s:%s, unrecognized type %d\n", arg_name, arg_val, arg_val->type ));
837 return GF_NOT_SUPPORTED;
838 }
839 return GF_OK;
840 }
841 //updates of arguments, not supported for ffmpeg decoders
842 return GF_NOT_SUPPORTED;
843 }
844
ffdec_process_event(GF_Filter * filter,const GF_FilterEvent * evt)845 static Bool ffdec_process_event(GF_Filter *filter, const GF_FilterEvent *evt)
846 {
847 GF_FFDecodeCtx *ctx = (GF_FFDecodeCtx *) gf_filter_get_udta(filter);
848
849 if ((evt->base.type==GF_FEVT_PLAY) || (evt->base.type==GF_FEVT_SET_SPEED) || (evt->base.type==GF_FEVT_RESUME)) {
850 ctx->drop_non_refs = evt->play.drop_non_ref;
851 }
852 return GF_FALSE;
853 }
854
855 static const GF_FilterCapability FFDecodeCaps[] =
856 {
857 CAP_UINT(GF_CAPS_INPUT_OUTPUT, GF_PROP_PID_STREAM_TYPE, GF_STREAM_VISUAL),
858 CAP_BOOL(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_UNFRAMED, GF_TRUE),
859 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_RAW),
860 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_NONE),
861 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_SVC),
862 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_LHVC),
863 CAP_UINT(GF_CAPS_OUTPUT, GF_PROP_PID_CODECID, GF_CODECID_RAW),
864 CAP_BOOL(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_TILE_BASE, GF_TRUE),
865 {0},
866 CAP_UINT(GF_CAPS_INPUT_OUTPUT, GF_PROP_PID_STREAM_TYPE, GF_STREAM_AUDIO),
867 CAP_BOOL(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_UNFRAMED, GF_TRUE),
868 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_NONE),
869 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_RAW),
870 CAP_UINT(GF_CAPS_OUTPUT, GF_PROP_PID_CODECID, GF_CODECID_RAW),
871
872 #ifdef FF_SUB_SUPPORT
873 {0},
874 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_STREAM_TYPE, GF_STREAM_TEXT),
875 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_TEXT_MPEG4),
876 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_TX3G),
877 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_WEBVTT),
878 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_SUBS_XML),
879 CAP_UINT(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_CODECID, GF_CODECID_SIMPLE_TEXT),
880 CAP_UINT(GF_CAPS_OUTPUT, GF_PROP_PID_STREAM_TYPE, GF_STREAM_TEXT),
881 CAP_UINT(GF_CAPS_OUTPUT, GF_PROP_PID_CODECID, GF_CODECID_RAW),
882 #endif
883
884 };
885
886 GF_FilterRegister FFDecodeRegister = {
887 .name = "ffdec",
888 .version = LIBAVCODEC_IDENT,
889 GF_FS_SET_DESCRIPTION("FFMPEG decoder")
890 GF_FS_SET_HELP("See FFMPEG documentation (https://ffmpeg.org/documentation.html) for more details")
891 .private_size = sizeof(GF_FFDecodeCtx),
892 SETCAPS(FFDecodeCaps),
893 .initialize = ffdec_initialize,
894 .finalize = ffdec_finalize,
895 .configure_pid = ffdec_configure_pid,
896 .process = ffdec_process,
897 .update_arg = ffdec_update_arg,
898 .process_event = ffdec_process_event,
899 .flags = GF_FS_REG_META,
900 //use middle priorty, so that hardware decs/other native impl in gpac can take over if needed
901 //don't use lowest one since we use this for scalable codecs
902 .priority = 128
903
904 };
905
906
907 static const GF_FilterArgs FFDecodeArgs[] =
908 {
909 { "*", -1, "any possible options defined for AVCodecContext and sub-classes. See `gpac -hx ffdec` and `gpac -hx ffdec:*`", GF_PROP_STRING, NULL, NULL, GF_FS_ARG_META},
910 {0}
911 };
912
ffdec_register(GF_FilterSession * session)913 const GF_FilterRegister *ffdec_register(GF_FilterSession *session)
914 {
915 ffmpeg_build_register(session, &FFDecodeRegister, FFDecodeArgs, 1, FF_REG_TYPE_DECODE);
916 return &FFDecodeRegister;
917 }
918
919 #else
920 #include <gpac/filters.h>
ffdec_register(GF_FilterSession * session)921 const GF_FilterRegister *ffdec_register(GF_FilterSession *session)
922 {
923 return NULL;
924 }
925 #endif
926
927