1 /*
2 * GPAC - Multimedia Framework C SDK
3 *
4 * Authors: Jean Le Feuvre
5 * Copyright (c) Telecom ParisTech 2000-2018
6 * All rights reserved
7 *
8 * This file is part of GPAC / VideoToolBox decoder filter
9 *
10 * GPAC is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU Lesser General Public License as published by
12 * the Free Software Foundation; either version 2, or (at your option)
13 * any later version.
14 *
15 * GPAC is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with this library; see the file COPYING. If not, write to
22 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 *
24 */
25
26 //do not include math.h we would have a conflict with Fixed ... we're lucky we don't need maths routines here
27 #define _GF_MATH_H_
28
29 #include <gpac/thread.h>
30
31 #if !defined(GPAC_DISABLE_AV_PARSERS) && ( defined(GPAC_CONFIG_DARWIN) || defined(GPAC_CONFIG_IOS) )
32
33 #include <stdint.h>
34
35 #define Picture QuickdrawPicture
36 #include <VideoToolbox/VideoToolbox.h>
37 #undef Picture
38
39 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
40 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
41 #endif
42
43 #include <gpac/maths.h>
44 #include <gpac/internal/media_dev.h>
45 #include <gpac/constants.h>
46 #include <gpac/filters.h>
47
48 #include "../../src/compositor/gl_inc.h"
49
50
51 #ifdef GPAC_CONFIG_IOS
52 #define VTB_GL_TEXTURE
53
54 #define GF_CVGLTextureREF CVOpenGLESTextureRef
55 #define GF_CVGLTextureCacheREF CVOpenGLESTextureCacheRef
56 #define GF_kCVPixelBufferOpenGLCompatibilityKey kCVPixelBufferOpenGLESCompatibilityKey
57 #define GF_CVOpenGLTextureCacheFlush CVOpenGLESTextureCacheFlush
58 #define GF_CVOpenGLTextureGetTarget CVOpenGLESTextureGetTarget
59 #define GF_CVOpenGLTextureGetName CVOpenGLESTextureGetName
60
61 #else
62
63 //not working yet, not sure why
64 //#define VTB_GL_TEXTURE
65
66 #include <CoreVideo/CVOpenGLTexture.h>
67
68 #define GF_CVGLTextureREF CVOpenGLTextureRef
69 #define GF_CVGLTextureCacheREF CVOpenGLTextureCacheRef
70 #define GF_kCVPixelBufferOpenGLCompatibilityKey kCVPixelBufferOpenGLCompatibilityKey
71 #define GF_CVOpenGLTextureCacheFlush CVOpenGLTextureCacheFlush
72 #define GF_CVOpenGLTextureGetTarget CVOpenGLTextureGetTarget
73 #define GF_CVOpenGLTextureGetName CVOpenGLTextureGetName
74
75
76 #endif
77
78 #ifndef GPAC_DISABLE_AV_PARSERS
79
80 typedef struct
81 {
82 //opts
83 u32 reorder, ofmt;
84 Bool no_copy;
85 Bool disable_hw;
86
87 //internal
88 // GF_FilterPid *ipid;
89 GF_List *streams;
90 GF_FilterPid *opid;
91 u32 width, height, stride;
92 GF_Fraction pixel_ar;
93 u32 pix_fmt;
94 u32 out_size;
95 u32 cfg_crc;
96 u32 codecid;
97 Bool is_hardware;
98
99 GF_Err last_error;
100
101 int vtb_type;
102 VTDecompressionSessionRef vtb_session;
103 CMFormatDescriptionRef fmt_desc;
104
105 GF_List *frames, *frames_res;
106 GF_FilterPacket *cur_pck;
107
108 u8 chroma_format, luma_bit_depth, chroma_bit_depth;
109 Bool frame_size_changed;
110 Bool reorder_detected;
111 Bool drop_non_refs;
112
113 volatile u32 decoded_frames_pending;
114 u32 reorder_probe;
115 Bool reconfig_needed;
116 u64 last_cts_out;
117 u32 last_timescale_out;
118
119 //MPEG-1/2 specific
120 Bool init_mpeg12;
121
122 //MPEG-4 specific
123 Bool skip_mpeg4_vosh;
124 char *vosh;
125 u32 vosh_size;
126
127 //NAL-based specific
128 GF_BitStream *nal_bs;
129 GF_BitStream *ps_bs;
130
131 GF_BitStream *nalu_rewrite_bs;
132 u8 *nalu_buffer;
133 u32 nalu_buffer_alloc;
134
135 Bool is_avc;
136 Bool is_annex_b;
137
138 u32 nalu_size_length;
139
140 GF_List *SPSs, *PPSs, *VPSs;
141 s32 active_sps, active_pps, active_vps;
142 u32 active_sps_crc, active_pps_crc, active_vps_crc;
143
144 AVCState avc;
145 Bool check_h264_isma;
146
147 HEVCState hevc;
148 Bool is_hevc;
149
150 Bool profile_supported, can_reconfig;
151 u32 nb_consecutive_errors;
152 //openGL output
153 #ifdef VTB_GL_TEXTURE
154 Bool use_gl_textures;
155 GF_CVGLTextureCacheREF cache_texture;
156 #endif
157 void *gl_context;
158
159 struct __vtb_frame_ifce *last_frame_sent;
160 } GF_VTBDecCtx;
161
162
163 typedef struct __vtb_frame_ifce
164 {
165 GF_FilterFrameInterface frame_ifce;
166
167 Bool locked;
168 CVPixelBufferRef frame;
169 GF_VTBDecCtx *ctx;
170 GF_FilterPacket *pck_src;
171 //openGL mode
172 #ifdef VTB_GL_TEXTURE
173 GF_CVGLTextureREF y, u, v;
174 #endif
175 } GF_VTBHWFrame;
176
177 static void vtbdec_delete_decoder(GF_VTBDecCtx *ctx);
178 static GF_Err vtbdec_flush_frame(GF_Filter *filter, GF_VTBDecCtx *ctx);
179
vtbdec_on_frame(void * opaque,void * sourceFrameRefCon,OSStatus status,VTDecodeInfoFlags flags,CVImageBufferRef image,CMTime pts,CMTime duration)180 static void vtbdec_on_frame(void *opaque, void *sourceFrameRefCon, OSStatus status, VTDecodeInfoFlags flags, CVImageBufferRef image, CMTime pts, CMTime duration)
181 {
182 GF_VTBDecCtx *ctx = (GF_VTBDecCtx *)opaque;
183 GF_VTBHWFrame *frame;
184 u32 i, count, timescale;
185 u64 cts, dts;
186 assert(ctx->cur_pck);
187
188 if (!image) {
189 if (status != kCVReturnSuccess) {
190 ctx->last_error = GF_NON_COMPLIANT_BITSTREAM;
191 ctx->nb_consecutive_errors++;
192 //if we can reconfigure and this is a SAP, reconfig if too many errors or first frame after reconfig
193 if (ctx->can_reconfig && gf_filter_pck_get_sap(ctx->cur_pck)
194 && (!ctx->profile_supported || (ctx->nb_consecutive_errors>10))
195 ) {
196 ctx->last_error = GF_PROFILE_NOT_SUPPORTED;
197 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Decode error - status %d - trying filter chain reload\n", status));
198 } else {
199 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Decode error - status %d\n", status));
200 }
201 return;
202 }
203 GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[VTB] No output buffer\n"));
204 return;
205 }
206 if (gf_filter_pck_get_seek_flag(ctx->cur_pck) ) {
207 GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[VTB] Frame marked as seek, not dispatching - status %d\n", status));
208 return;
209 }
210
211 if (ctx->reorder_probe) {
212 ctx->reorder_probe--;
213 }
214
215 ctx->profile_supported = GF_TRUE;
216 ctx->nb_consecutive_errors=0;
217 frame = gf_list_pop_back(ctx->frames_res);
218 if (!frame) {
219 GF_SAFEALLOC(frame, GF_VTBHWFrame);
220 if (!frame) return;
221 } else {
222 memset(frame, 0, sizeof(GF_VTBHWFrame));
223 }
224
225 assert( gf_filter_pck_get_seek_flag(ctx->cur_pck) == 0 );
226
227 frame->frame_ifce.user_data = frame;
228 frame->frame = CVPixelBufferRetain(image);
229 frame->pck_src = ctx->cur_pck;
230 gf_filter_pck_ref_props(&frame->pck_src);
231
232 frame->ctx = ctx;
233 cts = gf_filter_pck_get_cts(frame->pck_src);
234 dts = gf_filter_pck_get_dts(frame->pck_src);
235 timescale = gf_filter_pck_get_timescale(frame->pck_src);
236
237 GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[VTB] Decoded frame DTS "LLU" CTS "LLU" timescale %d\n", dts, cts, timescale));
238
239 if (!ctx->last_timescale_out)
240 ctx->last_timescale_out = gf_filter_pck_get_timescale(frame->pck_src);
241
242 count = gf_list_count(ctx->frames);
243 for (i=0; i<count; i++) {
244 GF_VTBHWFrame *aframe = gf_list_get(ctx->frames, i);
245 Bool insert = GF_FALSE;
246 u64 acts, adts, atimescale;
247 s64 diff;
248
249 acts = gf_filter_pck_get_cts(aframe->pck_src);
250 adts = gf_filter_pck_get_dts(aframe->pck_src);
251 atimescale = gf_filter_pck_get_timescale(aframe->pck_src);
252
253 if (adts > dts) {
254 ctx->reorder_probe=0;
255 ctx->reorder_detected=GF_FALSE;
256 break;
257 }
258 if ((timescale == atimescale) && (ctx->last_timescale_out == timescale)) {
259 diff = (s64) acts - (s64) cts;
260 if ((diff>0) && (cts > ctx->last_cts_out) ) {
261 insert = GF_TRUE;
262 }
263 } else {
264 diff = (s64) (acts * timescale) - (s64) (cts * atimescale);
265 if ((diff>0) && (ctx->last_timescale_out * cts > timescale * ctx->last_cts_out) ) {
266 insert = GF_TRUE;
267 }
268 }
269 if (insert) {
270 gf_list_insert(ctx->frames, frame, i);
271 ctx->reorder_detected = GF_TRUE;
272 return;
273 }
274 }
275 gf_list_add(ctx->frames, frame);
276 }
277
vtbdec_create_buffer_attributes(GF_VTBDecCtx * ctx,OSType pix_fmt)278 static CFDictionaryRef vtbdec_create_buffer_attributes(GF_VTBDecCtx *ctx, OSType pix_fmt)
279 {
280 CFMutableDictionaryRef buffer_attributes;
281 CFMutableDictionaryRef surf_props;
282 CFNumberRef w;
283 CFNumberRef h;
284 CFNumberRef pixel_fmt;
285
286 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &ctx->width);
287 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &ctx->height);
288 pixel_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
289
290 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault, 4, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
291 surf_props = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
292
293 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
294 CFRelease(w);
295 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
296 CFRelease(h);
297 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, pixel_fmt);
298 CFRelease(pixel_fmt);
299
300 #ifdef VTB_GL_TEXTURE
301 if (ctx->use_gl_textures)
302 CFDictionarySetValue(buffer_attributes, GF_kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
303 #endif
304
305 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, surf_props);
306 CFRelease(surf_props);
307
308 return buffer_attributes;
309 }
310
vtbdec_init_decoder(GF_Filter * filter,GF_VTBDecCtx * ctx)311 static GF_Err vtbdec_init_decoder(GF_Filter *filter, GF_VTBDecCtx *ctx)
312 {
313 CFMutableDictionaryRef dec_dsi, dec_type;
314 CFMutableDictionaryRef dsi;
315 VTDecompressionOutputCallbackRecord cbacks;
316 CFDictionaryRef buffer_attribs;
317 OSStatus status;
318 OSType kColorSpace;
319 const GF_PropertyValue *p;
320 CFDataRef data = NULL;
321 u8 *dsi_data=NULL;
322 u32 dsi_data_size=0;
323 u32 w, h;
324 GF_FilterPid *pid;
325 w = h = 0;
326
327 dec_dsi = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
328
329 if (ctx->ofmt==1) {
330 kColorSpace = kCVPixelFormatType_420YpCbCr8Planar;
331 ctx->pix_fmt = GF_PIXEL_YUV;
332 } else if (ctx->ofmt==2) {
333 kColorSpace = kCVPixelFormatType_24RGB;
334 ctx->pix_fmt = GF_PIXEL_RGB;
335 } else {
336 kColorSpace = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
337 ctx->pix_fmt = GF_PIXEL_NV12;
338 }
339
340 ctx->reorder_probe = ctx->reorder;
341 ctx->reorder_detected = GF_FALSE;
342 pid = gf_list_get(ctx->streams, 0);
343 p = gf_filter_pid_get_property(pid, GF_PROP_PID_WIDTH);
344 if (p) w = p->value.uint;
345 p = gf_filter_pid_get_property(pid, GF_PROP_PID_HEIGHT);
346 if (p) h = p->value.uint;
347
348
349 p = gf_filter_pid_get_property(pid, GF_PROP_PID_DECODER_CONFIG);
350
351 switch (ctx->codecid) {
352 case GF_CODECID_AVC:
353 if (gf_list_count(ctx->SPSs) && gf_list_count(ctx->PPSs)) {
354 s32 idx;
355 u32 i;
356 GF_AVCConfig *cfg;
357 GF_AVCConfigSlot *sps = NULL;
358 GF_AVCConfigSlot *pps = NULL;
359
360 for (i=0; i<gf_list_count(ctx->SPSs); i++) {
361 sps = gf_list_get(ctx->SPSs, i);
362 if (ctx->active_sps<0) ctx->active_sps = sps->id;
363
364 if (sps->id==ctx->active_sps) {
365 ctx->active_sps_crc = gf_crc_32(sps->data, sps->size);
366 break;
367 }
368 sps = NULL;
369 }
370 if (!sps) return GF_NON_COMPLIANT_BITSTREAM;
371 for (i=0; i<gf_list_count(ctx->PPSs); i++) {
372 pps = gf_list_get(ctx->PPSs, i);
373 if (ctx->active_pps<0) ctx->active_pps = pps->id;
374
375 if (pps->id==ctx->active_pps) {
376 ctx->active_pps_crc = gf_crc_32(pps->data, pps->size);
377 break;
378 }
379 pps = NULL;
380 }
381 if (!pps) return GF_NON_COMPLIANT_BITSTREAM;
382 ctx->reconfig_needed = GF_FALSE;
383
384 ctx->vtb_type = kCMVideoCodecType_H264;
385
386 if (gf_media_avc_read_sps(sps->data, sps->size, &ctx->avc, 0, NULL)<0)
387 return GF_NON_COMPLIANT_BITSTREAM;
388 if (gf_media_avc_read_pps(pps->data, pps->size, &ctx->avc)<0)
389 return GF_NON_COMPLIANT_BITSTREAM;
390
391 idx = ctx->active_sps;
392 ctx->width = ctx->avc.sps[idx].width;
393 ctx->height = ctx->avc.sps[idx].height;
394 if (ctx->avc.sps[idx].vui.par_num && ctx->avc.sps[idx].vui.par_den) {
395 ctx->pixel_ar.num = ctx->avc.sps[idx].vui.par_num;
396 ctx->pixel_ar.den = ctx->avc.sps[idx].vui.par_den;
397 } else {
398 ctx->pixel_ar.num = ctx->pixel_ar.den = 1;
399 }
400 ctx->chroma_format = ctx->avc.sps[idx].chroma_format;
401 ctx->luma_bit_depth = 8 + ctx->avc.sps[idx].luma_bit_depth_m8;
402 ctx->chroma_bit_depth = 8 + ctx->avc.sps[idx].chroma_bit_depth_m8;
403
404 switch (ctx->chroma_format) {
405 case 2:
406 #ifndef GPAC_CONFIG_IOS
407 //422 decoding doesn't seem supported ...
408 if (ctx->luma_bit_depth>8) {
409 kColorSpace = kCVPixelFormatType_422YpCbCr10;
410 ctx->pix_fmt = GF_PIXEL_YUV422_10;
411 } else
412 #endif
413 {
414 kColorSpace = kCVPixelFormatType_422YpCbCr8;
415 ctx->pix_fmt = GF_PIXEL_YUV422;
416 }
417 break;
418 case 3:
419 #ifndef GPAC_CONFIG_IOS
420 if (ctx->luma_bit_depth>8) {
421 kColorSpace = kCVPixelFormatType_444YpCbCr10;
422 ctx->pix_fmt = GF_PIXEL_YUV444_10;
423 } else
424 #endif
425 {
426 kColorSpace = kCVPixelFormatType_444YpCbCr8;
427 ctx->pix_fmt = GF_PIXEL_YUV444;
428 }
429 break;
430 default:
431 #if !defined(GPAC_CONFIG_IOS) && defined(AVAILABLE_MAC_OS_X_VERSION_10_13_AND_LATER)
432 if (ctx->luma_bit_depth>8) {
433 kColorSpace = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange;
434 ctx->pix_fmt = GF_PIXEL_NV12_10;
435 }
436 #endif
437 break;
438 }
439 //always rewrite with current sps and pps
440 cfg = gf_odf_avc_cfg_new();
441 cfg->configurationVersion = 1;
442 cfg->profile_compatibility = ctx->avc.sps[idx].prof_compat;
443 cfg->AVCProfileIndication = ctx->avc.sps[idx].profile_idc;
444 cfg->AVCLevelIndication = ctx->avc.sps[idx].level_idc;
445 cfg->chroma_format = ctx->avc.sps[idx].chroma_format;
446 cfg->luma_bit_depth = 8 + ctx->avc.sps[idx].luma_bit_depth_m8;
447 cfg->chroma_bit_depth = 8 + ctx->avc.sps[idx].chroma_bit_depth_m8;
448 cfg->nal_unit_size = 4;
449
450 //we send only the active SPS and PPS, otherwise vtb complains !!
451 gf_list_add(cfg->sequenceParameterSets, sps);
452 gf_list_add(cfg->pictureParameterSets, pps);
453 gf_odf_avc_cfg_write(cfg, &dsi_data, &dsi_data_size);
454 gf_list_reset(cfg->sequenceParameterSets);
455 gf_list_reset(cfg->pictureParameterSets);
456 gf_odf_avc_cfg_del((cfg));
457
458 dsi = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
459 data = CFDataCreate(kCFAllocatorDefault, (const UInt8*)dsi_data, dsi_data_size);
460 if (data) {
461 CFDictionarySetValue(dsi, CFSTR("avcC"), data);
462 CFDictionarySetValue(dec_dsi, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, dsi);
463 CFRelease(data);
464 }
465 CFRelease(dsi);
466
467 gf_free(dsi_data);
468 }
469 break;
470
471 case GF_CODECID_HEVC:
472 if (gf_list_count(ctx->SPSs) && gf_list_count(ctx->PPSs) && gf_list_count(ctx->VPSs)) {
473 s32 idx;
474 u32 i;
475 GF_HEVCConfig *cfg;
476 GF_HEVCParamArray *vpsa = NULL;
477 GF_HEVCParamArray *spsa = NULL;
478 GF_HEVCParamArray *ppsa = NULL;
479 GF_AVCConfigSlot *vps = NULL;
480 GF_AVCConfigSlot *sps = NULL;
481 GF_AVCConfigSlot *pps = NULL;
482
483 for (i=0; i<gf_list_count(ctx->VPSs); i++) {
484 vps = gf_list_get(ctx->VPSs, i);
485 if (ctx->active_vps<0) ctx->active_vps = vps->id;
486
487 if (vps->id==ctx->active_vps) break;
488 vps = NULL;
489 }
490 if (!vps) return GF_NON_COMPLIANT_BITSTREAM;
491
492 for (i=0; i<gf_list_count(ctx->SPSs); i++) {
493 sps = gf_list_get(ctx->SPSs, i);
494 if (ctx->active_sps<0) ctx->active_sps = sps->id;
495
496 if (sps->id==ctx->active_sps) break;
497 sps = NULL;
498 }
499 if (!sps) return GF_NON_COMPLIANT_BITSTREAM;
500 for (i=0; i<gf_list_count(ctx->PPSs); i++) {
501 pps = gf_list_get(ctx->PPSs, i);
502 if (ctx->active_pps<0) ctx->active_pps = pps->id;
503
504 if (pps->id==ctx->active_pps) break;
505 pps = NULL;
506 }
507 if (!pps) return GF_NON_COMPLIANT_BITSTREAM;
508 ctx->reconfig_needed = GF_FALSE;
509
510 ctx->vtb_type = kCMVideoCodecType_HEVC;
511
512 idx = ctx->active_sps;
513 ctx->width = ctx->hevc.sps[idx].width;
514 ctx->height = ctx->hevc.sps[idx].height;
515 if (ctx->hevc.sps[idx].aspect_ratio_info_present_flag && ctx->hevc.sps[idx].sar_width && ctx->hevc.sps[idx].sar_height) {
516 ctx->pixel_ar.num = ctx->hevc.sps[idx].sar_width;
517 ctx->pixel_ar.den = ctx->hevc.sps[idx].sar_height;
518 } else {
519 ctx->pixel_ar.num = ctx->pixel_ar.den = 1;
520 }
521 ctx->chroma_format = ctx->hevc.sps[idx].chroma_format_idc;
522 ctx->luma_bit_depth = ctx->hevc.sps[idx].bit_depth_luma;
523 ctx->chroma_bit_depth = ctx->hevc.sps[idx].bit_depth_chroma;
524
525 switch (ctx->chroma_format) {
526 case 2:
527 //422 decoding doesn't seem supported ...
528 if (ctx->luma_bit_depth>8) {
529 kColorSpace = kCVPixelFormatType_422YpCbCr10;
530 ctx->pix_fmt = GF_PIXEL_YUV422_10;
531 } else {
532 kColorSpace = kCVPixelFormatType_422YpCbCr8;
533 ctx->pix_fmt = GF_PIXEL_YUV422;
534 }
535 break;
536 case 3:
537 if (ctx->luma_bit_depth>8) {
538 kColorSpace = kCVPixelFormatType_444YpCbCr10;
539 ctx->pix_fmt = GF_PIXEL_YUV444_10;
540 } else {
541 kColorSpace = kCVPixelFormatType_444YpCbCr8;
542 ctx->pix_fmt = GF_PIXEL_YUV444;
543 }
544 break;
545 default:
546 if (ctx->luma_bit_depth>8) {
547 kColorSpace = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
548 ctx->pix_fmt = GF_PIXEL_NV12;
549 }
550 break;
551 }
552 //always rewrite with current sps and pps
553 cfg = gf_odf_hevc_cfg_new();
554 cfg->configurationVersion = 1;
555 cfg->profile_space = ctx->hevc.sps[idx].ptl.profile_space;
556 cfg->tier_flag = ctx->hevc.sps[idx].ptl.tier_flag;
557 cfg->profile_idc = ctx->hevc.sps[idx].ptl.profile_idc;
558 cfg->general_profile_compatibility_flags = ctx->hevc.sps[idx].ptl.profile_compatibility_flag;
559 cfg->progressive_source_flag = ctx->hevc.sps[idx].ptl.general_progressive_source_flag;
560 cfg->interlaced_source_flag = ctx->hevc.sps[idx].ptl.general_interlaced_source_flag;
561 cfg->non_packed_constraint_flag = ctx->hevc.sps[idx].ptl.general_non_packed_constraint_flag;
562 cfg->frame_only_constraint_flag = ctx->hevc.sps[idx].ptl.general_frame_only_constraint_flag;
563
564 cfg->constraint_indicator_flags = ctx->hevc.sps[idx].ptl.general_reserved_44bits;
565 cfg->level_idc = ctx->hevc.sps[idx].ptl.level_idc;
566
567 cfg->luma_bit_depth = ctx->hevc.sps[idx].bit_depth_luma;
568 cfg->chroma_bit_depth = ctx->hevc.sps[idx].bit_depth_chroma;
569 cfg->chromaFormat = ctx->hevc.sps[idx].chroma_format_idc;
570 cfg->complete_representation = GF_TRUE;
571
572 cfg->nal_unit_size = 4;
573
574 GF_SAFEALLOC(vpsa, GF_HEVCParamArray);
575 if (!vpsa) return GF_OUT_OF_MEM;
576 vpsa->array_completeness = 1;
577 vpsa->type = GF_HEVC_NALU_VID_PARAM;
578 vpsa->nalus = gf_list_new();
579 gf_list_add(vpsa->nalus, vps);
580 gf_list_add(cfg->param_array, vpsa);
581
582 GF_SAFEALLOC(spsa, GF_HEVCParamArray);
583 if (!spsa) return GF_OUT_OF_MEM;
584 spsa->array_completeness = 1;
585 spsa->type = GF_HEVC_NALU_SEQ_PARAM;
586 spsa->nalus = gf_list_new();
587 gf_list_add(spsa->nalus, sps);
588 gf_list_add(cfg->param_array, spsa);
589
590 GF_SAFEALLOC(ppsa, GF_HEVCParamArray);
591 if (!ppsa) return GF_OUT_OF_MEM;
592 ppsa->array_completeness = 1;
593 ppsa->type = GF_HEVC_NALU_PIC_PARAM;
594 //we send all PPS
595 ppsa->nalus = ctx->PPSs;
596
597 gf_list_add(cfg->param_array, ppsa);
598
599 gf_odf_hevc_cfg_write(cfg, &dsi_data, &dsi_data_size);
600 gf_list_reset(vpsa->nalus);
601 gf_list_reset(spsa->nalus);
602 ppsa->nalus = NULL;
603 gf_odf_hevc_cfg_del(cfg);
604
605 dsi = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
606 data = CFDataCreate(kCFAllocatorDefault, (const UInt8*)dsi_data, dsi_data_size);
607 if (data) {
608 CFDictionarySetValue(dsi, CFSTR("hvcC"), data);
609 CFDictionarySetValue(dec_dsi, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, dsi);
610 CFRelease(data);
611 }
612 CFRelease(dsi);
613
614 gf_free(dsi_data);
615 }
616 break;
617
618 case GF_CODECID_MPEG2_SIMPLE:
619 case GF_CODECID_MPEG2_MAIN:
620 case GF_CODECID_MPEG2_SNR:
621 case GF_CODECID_MPEG2_SPATIAL:
622 case GF_CODECID_MPEG2_HIGH:
623 case GF_CODECID_MPEG2_422:
624
625 ctx->vtb_type = kCMVideoCodecType_MPEG2Video;
626 if (!ctx->width || !ctx->height) {
627 ctx->init_mpeg12 = GF_TRUE;
628 return GF_OK;
629 }
630 ctx->init_mpeg12 = GF_FALSE;
631 ctx->reconfig_needed = GF_FALSE;
632 break;
633
634 case GF_CODECID_MPEG1:
635 ctx->vtb_type = kCMVideoCodecType_MPEG1Video;
636 if (!ctx->width || !ctx->height) {
637 ctx->init_mpeg12 = GF_TRUE;
638 return GF_OK;
639 }
640 ctx->init_mpeg12 = GF_FALSE;
641 ctx->reconfig_needed = GF_FALSE;
642 break;
643 case GF_CODECID_MPEG4_PART2 :
644 {
645 char *vosh = NULL;
646 u32 vosh_size = 0;
647 ctx->vtb_type = kCMVideoCodecType_MPEG4Video;
648
649 if (!p || !p->value.data.ptr) {
650 vosh = ctx->vosh;
651 vosh_size = ctx->vosh_size;
652 } else {
653 vosh = p->value.data.ptr;
654 vosh_size = p->value.data.size;
655 }
656 ctx->reconfig_needed = GF_FALSE;
657
658 if (vosh) {
659 GF_M4VDecSpecInfo vcfg;
660 GF_BitStream *bs;
661 GF_ESD *esd;
662
663 gf_m4v_get_config(vosh, vosh_size, &vcfg);
664 ctx->width = vcfg.width;
665 ctx->height = vcfg.height;
666 esd = gf_odf_desc_esd_new(2);
667 esd->decoderConfig->decoderSpecificInfo->data = vosh;
668 esd->decoderConfig->decoderSpecificInfo->dataLength = vosh_size;
669 bs = gf_bs_new(NULL, 0, GF_BITSTREAM_WRITE);
670 gf_bs_write_u32(bs, 0);
671 gf_odf_desc_write_bs((GF_Descriptor *) esd, bs);
672 gf_bs_get_content(bs, &dsi_data, &dsi_data_size);
673 gf_bs_del(bs);
674 esd->decoderConfig->decoderSpecificInfo->data = NULL;
675 esd->decoderConfig->decoderSpecificInfo->dataLength = 0;
676 gf_odf_desc_del((GF_Descriptor*)esd);
677
678 dsi = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
679 data = CFDataCreate(kCFAllocatorDefault, (const UInt8*) dsi_data, dsi_data_size);
680 gf_free(dsi_data);
681
682 if (data) {
683 CFDictionarySetValue(dsi, CFSTR("esds"), data);
684 CFDictionarySetValue(dec_dsi, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, dsi);
685 CFRelease(data);
686 }
687 CFRelease(dsi);
688
689 ctx->skip_mpeg4_vosh = GF_FALSE;
690 } else {
691 ctx->skip_mpeg4_vosh = GF_TRUE;
692 return GF_OK;
693 }
694 break;
695 }
696 case GF_CODECID_H263:
697 case GF_CODECID_S263:
698 ctx->reorder_probe = 0;
699 ctx->reconfig_needed = GF_FALSE;
700 if (w && h) {
701 ctx->width = w;
702 ctx->height = h;
703 ctx->vtb_type = kCMVideoCodecType_H263;
704 break;
705 }
706 break;
707
708 default :
709 ctx->reconfig_needed = GF_FALSE;
710 return GF_NOT_SUPPORTED;
711 }
712 //not yet ready
713 if (! ctx->width || !ctx->height) return GF_OK;
714
715 /*status = */CMVideoFormatDescriptionCreate(kCFAllocatorDefault, ctx->vtb_type, ctx->width, ctx->height, dec_dsi, &ctx->fmt_desc);
716
717 if (!ctx->fmt_desc) {
718 if (dec_dsi) CFRelease(dec_dsi);
719 return GF_NON_COMPLIANT_BITSTREAM;
720 }
721 buffer_attribs = vtbdec_create_buffer_attributes(ctx, kColorSpace);
722
723 cbacks.decompressionOutputCallback = vtbdec_on_frame;
724 cbacks.decompressionOutputRefCon = ctx;
725
726 status = 1;
727 if (!ctx->disable_hw) {
728 dec_type = CFDictionaryCreateMutable(kCFAllocatorDefault, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
729 CFDictionarySetValue(dec_type, kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, kCFBooleanTrue);
730 ctx->is_hardware = GF_TRUE;
731
732 status = VTDecompressionSessionCreate(NULL, ctx->fmt_desc, dec_type, buffer_attribs, &cbacks, &ctx->vtb_session);
733
734 if (dec_type)
735 CFRelease(dec_type);
736 }
737
738 //if HW decoder not available or disabled , try soft one
739 if (status) {
740 status = VTDecompressionSessionCreate(NULL, ctx->fmt_desc, NULL, buffer_attribs, &cbacks, &ctx->vtb_session);
741 ctx->is_hardware = GF_FALSE;
742 }
743
744 if (dec_dsi)
745 CFRelease(dec_dsi);
746 if (buffer_attribs)
747 CFRelease(buffer_attribs);
748
749 switch (status) {
750 case kVTVideoDecoderNotAvailableNowErr:
751 case kVTVideoDecoderUnsupportedDataFormatErr:
752 return GF_NOT_SUPPORTED;
753 case kVTVideoDecoderMalfunctionErr:
754 return GF_IO_ERR;
755 case kVTVideoDecoderBadDataErr :
756 return GF_NOT_SUPPORTED;
757
758 case kVTPixelTransferNotSupportedErr:
759 case kVTCouldNotFindVideoDecoderErr:
760 return GF_NOT_SUPPORTED;
761 case 0:
762 break;
763 default:
764 return GF_SERVICE_ERROR;
765 }
766
767 //good to go !
768 ctx->stride = ctx->width;
769 if (ctx->pix_fmt == GF_PIXEL_YUV422) {
770 ctx->out_size = ctx->width*ctx->height*2;
771 } else if (ctx->pix_fmt == GF_PIXEL_YUV444) {
772 ctx->out_size = ctx->width*ctx->height*3;
773 } else if (ctx->pix_fmt == GF_PIXEL_RGB) {
774 ctx->out_size = ctx->width*ctx->height*3;
775 ctx->stride *= 3;
776 } else {
777 // (ctx->pix_fmt == GF_PIXEL_YUV)
778 ctx->out_size = ctx->width*ctx->height*3/2;
779 }
780 if (ctx->luma_bit_depth>8) {
781 ctx->out_size *= 2;
782 }
783 ctx->frame_size_changed = GF_TRUE;
784 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_WIDTH, &PROP_UINT(ctx->width) );
785 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_HEIGHT, &PROP_UINT(ctx->height) );
786 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_STRIDE, &PROP_UINT(ctx->stride) );
787 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_PAR, &PROP_FRAC(ctx->pixel_ar) );
788 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_PIXFMT, &PROP_UINT(ctx->pix_fmt) );
789 ctx->profile_supported = GF_FALSE;
790 ctx->can_reconfig = !gf_opts_get_bool("core", "no-reassign");
791
792 switch (ctx->vtb_type) {
793 case kCMVideoCodecType_H264:
794 gf_filter_set_name(filter, ctx->is_hardware ? "VTB:Hardware:AVC|H264" : "VTB:Software:AVC|H264");
795 break;
796 case kCMVideoCodecType_MPEG2Video:
797 gf_filter_set_name(filter, ctx->is_hardware ? "VTB:Hardware:MPEG2" : "VTB:Software:MPEG2");
798 break;
799 case kCMVideoCodecType_MPEG4Video:
800 gf_filter_set_name(filter, ctx->is_hardware ? "VTB:Hardware:MPEG4P2" : "VTB:Software:MPEG4P2");
801 break;
802 case kCMVideoCodecType_H263:
803 gf_filter_set_name(filter, ctx->is_hardware ? "VTB:Hardware:H263" : "VTB:Software:H263");
804 break;
805 case kCMVideoCodecType_MPEG1Video:
806 gf_filter_set_name(filter, ctx->is_hardware ? "VTB:Hardware:MPEG1" : "VTB:Software:MPEG1");
807 break;
808 default:
809 break;
810 }
811 return GF_OK;
812 }
813
vtbdec_register_param_sets(GF_VTBDecCtx * ctx,char * data,u32 size,Bool is_sps,u8 hevc_nal_type)814 static void vtbdec_register_param_sets(GF_VTBDecCtx *ctx, char *data, u32 size, Bool is_sps, u8 hevc_nal_type)
815 {
816 Bool add = GF_TRUE;
817 u32 i, count;
818 s32 ps_id;
819 GF_List *dest = NULL;
820
821 if (!ctx->ps_bs) ctx->ps_bs = gf_bs_new(data, size, GF_BITSTREAM_READ);
822 else gf_bs_reassign_buffer(ctx->ps_bs, data, size);
823
824 if (hevc_nal_type) {
825 if (hevc_nal_type==GF_HEVC_NALU_SEQ_PARAM) {
826 dest = ctx->SPSs;
827 ps_id = gf_media_hevc_read_sps_bs(ctx->ps_bs, &ctx->hevc);
828 if (ps_id<0) return;
829 }
830 else if (hevc_nal_type==GF_HEVC_NALU_PIC_PARAM) {
831 dest = ctx->PPSs;
832 ps_id = gf_media_hevc_read_pps_bs(ctx->ps_bs, &ctx->hevc);
833 if (ps_id<0) return;
834 }
835 else if (hevc_nal_type==GF_HEVC_NALU_VID_PARAM) {
836 dest = ctx->VPSs;
837 ps_id = gf_media_hevc_read_vps_bs(ctx->ps_bs, &ctx->hevc);
838 if (ps_id<0) return;
839 }
840
841 } else {
842 dest = is_sps ? ctx->SPSs : ctx->PPSs;
843
844 if (is_sps) {
845 ps_id = gf_media_avc_read_sps_bs(ctx->ps_bs, &ctx->avc, 0, NULL);
846 if (ps_id<0) return;
847 } else {
848 ps_id = gf_media_avc_read_pps_bs(ctx->ps_bs, &ctx->avc);
849 if (ps_id<0) return;
850 }
851 }
852
853 count = gf_list_count(dest);
854 for (i=0; i<count; i++) {
855 GF_AVCConfigSlot *a_slc = gf_list_get(dest, i);
856 if (a_slc->id != ps_id) continue;
857 //not same size or different content but same ID, remove old xPS
858 if ((a_slc->size != size) || memcmp(a_slc->data, data, size) ) {
859 gf_free(a_slc->data);
860 gf_free(a_slc);
861 gf_list_rem(dest, i);
862 break;
863 } else {
864 add = GF_FALSE;
865 }
866 break;
867 }
868 if (add) {
869 GF_AVCConfigSlot *slc;
870 GF_SAFEALLOC(slc, GF_AVCConfigSlot);
871 if (!slc) return;
872 slc->data = gf_malloc(size);
873 if (!slc->data) {
874 gf_free(slc);
875 return;
876 }
877 memcpy(slc->data, data, size);
878 slc->size = size;
879 slc->id = ps_id;
880 slc->crc = gf_crc_32(data, size);
881 gf_list_add(dest, slc);
882 }
883 }
884
vtbdec_purge_param_sets(GF_VTBDecCtx * ctx,Bool is_sps,s32 idx)885 static u32 vtbdec_purge_param_sets(GF_VTBDecCtx *ctx, Bool is_sps, s32 idx)
886 {
887 u32 i, j, count, crc_res = 0;
888 GF_List *dest = is_sps ? ctx->SPSs : ctx->PPSs;
889
890 //remove all xPS sharing the same ID, use only the last occurence
891 count = gf_list_count(dest);
892 for (i=0; i<count; i++) {
893 GF_AVCConfigSlot *slc = gf_list_get(dest, i);
894 if (slc->id != idx) continue;
895 crc_res = slc->crc;
896
897 for (j=i+1; j<count; j++) {
898 GF_AVCConfigSlot *a_slc = gf_list_get(dest, j);
899 if (a_slc->id != slc->id) continue;
900 //not same size or different content but same ID, remove old xPS
901 if ((slc->size != a_slc->size) || memcmp(a_slc->data, slc->data, a_slc->size) ) {
902 crc_res = a_slc->crc;
903 gf_free(slc->data);
904 gf_free(slc);
905 gf_list_rem(dest, i);
906 i--;
907 count--;
908 break;
909 }
910 }
911 }
912 return crc_res;
913 }
914
vtbdec_del_param_list(GF_List * list)915 static void vtbdec_del_param_list(GF_List *list)
916 {
917 while (gf_list_count(list)) {
918 GF_AVCConfigSlot *slc = gf_list_get(list, 0);
919 gf_free(slc->data);
920 gf_free(slc);
921 gf_list_rem(list, 0);
922 }
923 gf_list_del(list);
924 }
925
vtbdec_configure_pid(GF_Filter * filter,GF_FilterPid * pid,Bool is_remove)926 static GF_Err vtbdec_configure_pid(GF_Filter *filter, GF_FilterPid *pid, Bool is_remove)
927 {
928 const GF_PropertyValue *p, *dsi;
929 u32 codecid, dsi_crc;
930 GF_Err e;
931 GF_FilterPid *base_pid = NULL;
932 GF_VTBDecCtx *ctx = gf_filter_get_udta(filter);
933
934 if (is_remove) {
935 if (ctx->opid) gf_filter_pid_remove(ctx->opid);
936 ctx->opid = NULL;
937 gf_list_del_item(ctx->streams, pid);
938 return GF_OK;
939 }
940 if (! gf_filter_pid_check_caps(pid)) {
941 while (gf_list_count(ctx->frames)) {
942 vtbdec_flush_frame(filter, ctx);
943 }
944 return GF_NOT_SUPPORTED;
945 }
946 p = gf_filter_pid_get_property(pid, GF_PROP_PID_CODECID);
947 if (!p) {
948 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTBDec] Missing codecid, cannot initialize\n"));
949 return GF_NOT_SUPPORTED;
950 }
951 codecid = p->value.uint;
952
953 base_pid = gf_list_get(ctx->streams, 0);
954 p = gf_filter_pid_get_property(pid, GF_PROP_PID_DEPENDENCY_ID);
955 if (!p && base_pid && (base_pid != pid)) return GF_REQUIRES_NEW_INSTANCE;
956 else if (p) {
957 u32 i;
958 u32 base_idx_plus_one = 0;
959
960 if (ctx->codecid != GF_CODECID_HEVC) return GF_REQUIRES_NEW_INSTANCE;
961
962 for (i=0; i<gf_list_count(ctx->streams); i++) {
963 GF_FilterPid *ipid = gf_list_get(ctx->streams, i);
964 const GF_PropertyValue *p_dep;
965 if (ipid==pid) continue;
966
967 p_dep = gf_filter_pid_get_property(ipid, GF_PROP_PID_ID);
968 if (p_dep && p_dep->value.uint == p->value.uint) {
969 base_idx_plus_one = i+1;
970 break;
971 }
972 }
973 if (!base_idx_plus_one) return GF_REQUIRES_NEW_INSTANCE;
974
975 //no support for L-HEVC
976 if (codecid != GF_CODECID_HEVC) return GF_NOT_SUPPORTED;
977 if (gf_list_find(ctx->streams, pid) < 0) {
978 gf_list_insert(ctx->streams, pid, base_idx_plus_one);
979 }
980 //no configure for temporal enhancements
981 p = gf_filter_pid_get_property(pid, GF_PROP_PID_FPS);
982 if (ctx->opid && p) gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_FPS, p);
983 return GF_OK;
984 }
985
986
987 dsi = gf_filter_pid_get_property(pid, GF_PROP_PID_DECODER_CONFIG);
988 dsi_crc = dsi ? gf_crc_32(dsi->value.data.ptr, dsi->value.data.size) : 0;
989 if ((codecid==ctx->codecid) && (dsi_crc == ctx->cfg_crc) && ctx->width && ctx->height) {
990 gf_filter_pid_copy_properties(ctx->opid, pid);
991 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_CODECID, &PROP_UINT(GF_CODECID_RAW) );
992 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_DECODER_CONFIG, NULL);
993 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_DECODER_CONFIG_ENHANCEMENT, NULL);
994 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_WIDTH, &PROP_UINT(ctx->width) );
995 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_HEIGHT, &PROP_UINT(ctx->height) );
996 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_STRIDE, &PROP_UINT(ctx->stride) );
997 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_PAR, &PROP_FRAC(ctx->pixel_ar) );
998 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_PIXFMT, &PROP_UINT(ctx->pix_fmt) );
999 return GF_OK;
1000 }
1001 //need a reset !
1002 if (ctx->vtb_session) {
1003 //flush all pending frames and mark reconfigure as pending
1004 ctx->reconfig_needed = GF_TRUE;
1005 while (gf_list_count(ctx->frames)) {
1006 vtbdec_flush_frame(filter, ctx);
1007 }
1008 }
1009 if (gf_list_find(ctx->streams, pid) < 0) {
1010 gf_list_insert(ctx->streams, pid, 0);
1011 }
1012 ctx->cfg_crc = dsi_crc;
1013 ctx->codecid = codecid;
1014 gf_filter_set_max_extra_input_pids(filter, (codecid==GF_CODECID_HEVC) ? 5 : 0);
1015
1016 if (!ctx->opid) {
1017 ctx->opid = gf_filter_pid_new(filter);
1018 gf_filter_pid_set_framing_mode(pid, GF_TRUE);
1019 }
1020
1021 //copy properties at init or reconfig
1022 gf_filter_pid_copy_properties(ctx->opid, pid);
1023 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_CODECID, &PROP_UINT(GF_CODECID_RAW) );
1024 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_DECODER_CONFIG, NULL);
1025 gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_DECODER_CONFIG_ENHANCEMENT, NULL);
1026
1027 ctx->nalu_size_length = 0;
1028 ctx->is_annex_b = GF_FALSE;
1029
1030 //check AVC config
1031 if (codecid==GF_CODECID_AVC) {
1032
1033 if (ctx->SPSs) vtbdec_del_param_list(ctx->SPSs);
1034 ctx->SPSs = gf_list_new();
1035 if (ctx->PPSs) vtbdec_del_param_list(ctx->PPSs);
1036 ctx->PPSs = gf_list_new();
1037
1038 ctx->is_avc = GF_TRUE;
1039 ctx->check_h264_isma = GF_TRUE;
1040
1041 ctx->avc.sps_active_idx = ctx->avc.pps_active_idx = -1;
1042 ctx->active_sps = ctx->active_pps = -1;
1043 ctx->active_sps_crc = ctx->active_pps_crc = 0;
1044
1045 if (!dsi || !dsi->value.data.ptr) {
1046 ctx->is_annex_b = GF_TRUE;
1047 ctx->width=ctx->height=128;
1048 ctx->out_size = ctx->width*ctx->height*3/2;
1049 ctx->pix_fmt = GF_PIXEL_YUV;
1050 return GF_OK;
1051 } else {
1052 u32 i;
1053 GF_AVCConfigSlot *slc;
1054 GF_AVCConfig *cfg = gf_odf_avc_cfg_read(dsi->value.data.ptr, dsi->value.data.size);
1055 for (i=0; i<gf_list_count(cfg->sequenceParameterSets); i++) {
1056 slc = gf_list_get(cfg->sequenceParameterSets, i);
1057 slc->id = -1;
1058 vtbdec_register_param_sets(ctx, slc->data, slc->size, GF_TRUE, 0);
1059 }
1060
1061 for (i=0; i<gf_list_count(cfg->pictureParameterSets); i++) {
1062 slc = gf_list_get(cfg->pictureParameterSets, i);
1063 slc->id = -1;
1064 vtbdec_register_param_sets(ctx, slc->data, slc->size, GF_FALSE, 0);
1065 }
1066
1067 slc = gf_list_get(ctx->SPSs, 0);
1068 if (slc) {
1069 ctx->active_sps = slc->id;
1070 ctx->active_sps_crc = gf_crc_32(slc->data, slc->size);
1071 }
1072
1073 slc = gf_list_get(ctx->PPSs, 0);
1074 if (slc) {
1075 ctx->active_pps = slc->id;
1076 ctx->active_pps_crc = gf_crc_32(slc->data, slc->size);
1077 }
1078
1079 ctx->nalu_size_length = cfg->nal_unit_size;
1080 if (gf_list_count(ctx->SPSs) && gf_list_count(ctx->PPSs) && !ctx->reconfig_needed ) {
1081 e = vtbdec_init_decoder(filter, ctx);
1082 } else {
1083 e = GF_OK;
1084 }
1085 gf_odf_avc_cfg_del(cfg);
1086 return e;
1087 }
1088 }
1089
1090 //check HEVC config
1091 if (codecid==GF_CODECID_HEVC) {
1092 if (ctx->SPSs) vtbdec_del_param_list(ctx->SPSs);
1093 ctx->SPSs = gf_list_new();
1094 if (ctx->SPSs) vtbdec_del_param_list(ctx->PPSs);
1095 ctx->PPSs = gf_list_new();
1096 if (ctx->SPSs) vtbdec_del_param_list(ctx->VPSs);
1097 ctx->VPSs = gf_list_new();
1098 ctx->is_hevc = GF_TRUE;
1099
1100 ctx->hevc.sps_active_idx = -1;
1101 ctx->active_sps = ctx->active_pps = ctx->active_vps = -1;
1102
1103 if (!dsi || !dsi->value.data.ptr) {
1104 ctx->is_annex_b = GF_TRUE;
1105 ctx->width=ctx->height=128;
1106 ctx->out_size = ctx->width*ctx->height*3/2;
1107 ctx->pix_fmt = GF_PIXEL_YUV;
1108 return GF_OK;
1109 } else {
1110 u32 i, j;
1111 GF_AVCConfigSlot *slc;
1112 GF_HEVCConfig *cfg = gf_odf_hevc_cfg_read(dsi->value.data.ptr, dsi->value.data.size, GF_FALSE);
1113
1114 for (i=0; i<gf_list_count(cfg->param_array); i++) {
1115 GF_HEVCParamArray *pa = gf_list_get(cfg->param_array, i);
1116
1117
1118 for (j=0; j<gf_list_count(pa->nalus); j++) {
1119 slc = gf_list_get(pa->nalus, j);
1120 slc->id = -1;
1121
1122 vtbdec_register_param_sets(ctx, slc->data, slc->size, GF_FALSE, pa->type);
1123 }
1124 }
1125
1126 slc = gf_list_get(ctx->SPSs, 0);
1127 if (slc) ctx->active_sps = slc->id;
1128
1129 slc = gf_list_get(ctx->PPSs, 0);
1130 if (slc) ctx->active_pps = slc->id;
1131
1132 slc = gf_list_get(ctx->VPSs, 0);
1133 if (slc) ctx->active_vps = slc->id;
1134
1135 ctx->nalu_size_length = cfg->nal_unit_size;
1136 if (gf_list_count(ctx->SPSs) && gf_list_count(ctx->PPSs) && gf_list_count(ctx->VPSs) && !ctx->reconfig_needed) {
1137 e = vtbdec_init_decoder(filter, ctx);
1138 } else {
1139 e = GF_OK;
1140 }
1141 gf_odf_hevc_cfg_del(cfg);
1142 return e;
1143 }
1144 }
1145
1146 if (ctx->vtb_session) {
1147 assert(ctx->reconfig_needed);
1148 return GF_OK;
1149 }
1150
1151 //check VOSH config
1152 if (codecid==GF_CODECID_MPEG4_PART2) {
1153 if (!dsi || !dsi->value.data.ptr) {
1154 ctx->width=ctx->height=128;
1155 ctx->out_size = ctx->width*ctx->height*3/2;
1156 ctx->pix_fmt = GF_PIXEL_YUV;
1157 } else {
1158 return vtbdec_init_decoder(filter, ctx);
1159 }
1160 }
1161
1162 return vtbdec_init_decoder(filter, ctx);
1163 }
1164
1165
vtbdec_delete_decoder(GF_VTBDecCtx * ctx)1166 static void vtbdec_delete_decoder(GF_VTBDecCtx *ctx)
1167 {
1168 if (ctx->fmt_desc) {
1169 CFRelease(ctx->fmt_desc);
1170 ctx->fmt_desc = NULL;
1171 }
1172 if (ctx->vtb_session) {
1173 VTDecompressionSessionInvalidate(ctx->vtb_session);
1174 ctx->vtb_session=NULL;
1175 }
1176 vtbdec_del_param_list(ctx->SPSs);
1177 ctx->SPSs = NULL;
1178 vtbdec_del_param_list(ctx->PPSs);
1179 ctx->PPSs = NULL;
1180 vtbdec_del_param_list(ctx->VPSs);
1181 ctx->VPSs = NULL;
1182 }
1183
vtbdec_parse_nal_units(GF_Filter * filter,GF_VTBDecCtx * ctx,char * inBuffer,u32 inBufferLength,char ** out_buffer,u32 * out_size)1184 static GF_Err vtbdec_parse_nal_units(GF_Filter *filter, GF_VTBDecCtx *ctx, char *inBuffer, u32 inBufferLength, char **out_buffer, u32 *out_size)
1185 {
1186 u32 i, sc_size=0;
1187 char *ptr = inBuffer;
1188 u32 nal_size;
1189 GF_Err e = GF_OK;
1190 Bool reassign_bs = GF_TRUE;
1191 Bool check_reconfig = GF_FALSE;
1192
1193 if (out_buffer) {
1194 *out_buffer = NULL;
1195 *out_size = 0;
1196 }
1197
1198 if (!ctx->nalu_size_length) {
1199 nal_size = gf_media_nalu_next_start_code((u8 *) inBuffer, inBufferLength, &sc_size);
1200 if (!sc_size) return GF_NON_COMPLIANT_BITSTREAM;
1201 ptr += nal_size + sc_size;
1202 assert(inBufferLength >= nal_size + sc_size);
1203 inBufferLength -= nal_size + sc_size;
1204 }
1205
1206 while (inBufferLength) {
1207 Bool add_nal = GF_TRUE;
1208 u8 nal_type, nal_hdr;
1209
1210 if (ctx->nalu_size_length) {
1211 nal_size = 0;
1212 for (i=0; i<ctx->nalu_size_length; i++) {
1213 nal_size = (nal_size<<8) + ((u8) ptr[i]);
1214 }
1215
1216 if (nal_size > inBufferLength) {
1217 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error parsing NAL: size indicated %d but %d bytes only in payload\n", nal_size, inBufferLength));
1218 break;
1219 }
1220 ptr += ctx->nalu_size_length;
1221 } else {
1222 nal_size = gf_media_nalu_next_start_code((const u8 *) ptr, inBufferLength, &sc_size);
1223 }
1224
1225 if (nal_size==0) {
1226 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error parsing NAL: size 0 shall never happen\n", nal_size));
1227
1228 if (ctx->nalu_size_length) {
1229 if (inBufferLength < ctx->nalu_size_length) break;
1230 inBufferLength -= ctx->nalu_size_length;
1231 } else {
1232 if (!sc_size || (inBufferLength < sc_size)) break;
1233 inBufferLength -= sc_size;
1234 ptr += sc_size;
1235 }
1236 continue;
1237 }
1238
1239 if (ctx->is_avc) {
1240 if (!ctx->nal_bs) ctx->nal_bs = gf_bs_new(ptr, nal_size, GF_BITSTREAM_READ);
1241 else gf_bs_reassign_buffer(ctx->nal_bs, ptr, nal_size);
1242
1243 nal_hdr = ptr[0];
1244 nal_type = nal_hdr & 0x1F;
1245 switch (nal_type) {
1246 case GF_AVC_NALU_SEQ_PARAM:
1247 vtbdec_register_param_sets(ctx, ptr, nal_size, GF_TRUE, 0);
1248 add_nal = GF_FALSE;
1249 break;
1250 case GF_AVC_NALU_PIC_PARAM:
1251 vtbdec_register_param_sets(ctx, ptr, nal_size, GF_FALSE, 0);
1252 add_nal = GF_FALSE;
1253 break;
1254 case GF_AVC_NALU_ACCESS_UNIT:
1255 case GF_AVC_NALU_END_OF_SEQ:
1256 case GF_AVC_NALU_END_OF_STREAM:
1257 case GF_AVC_NALU_FILLER_DATA:
1258 add_nal = GF_FALSE;
1259 break;
1260 default:
1261 break;
1262 }
1263
1264 gf_media_avc_parse_nalu(ctx->nal_bs, &ctx->avc);
1265
1266 if ((nal_type<=GF_AVC_NALU_IDR_SLICE) && ctx->avc.s_info.sps) {
1267 if (ctx->avc.sps_active_idx != ctx->active_sps) {
1268 ctx->reconfig_needed = 1;
1269 ctx->active_sps = ctx->avc.sps_active_idx;
1270 ctx->active_pps = ctx->avc.s_info.pps->id;
1271 return GF_OK;
1272 }
1273 }
1274 } else if (ctx->is_hevc) {
1275 u8 temporal_id, ayer_id;
1276
1277 if (!ctx->nal_bs) ctx->nal_bs = gf_bs_new(ptr, nal_size, GF_BITSTREAM_READ);
1278 else gf_bs_reassign_buffer(ctx->nal_bs, ptr, nal_size);
1279
1280 s32 res = gf_media_hevc_parse_nalu_bs(ctx->nal_bs, &ctx->hevc, &nal_type, &temporal_id, &ayer_id);
1281 if (res>=0) {
1282 switch (nal_type) {
1283 case GF_HEVC_NALU_VID_PARAM:
1284 case GF_HEVC_NALU_SEQ_PARAM:
1285 case GF_HEVC_NALU_PIC_PARAM:
1286 vtbdec_register_param_sets(ctx, ptr, nal_size, GF_FALSE, nal_type);
1287 add_nal = GF_FALSE;
1288 break;
1289 case GF_HEVC_NALU_ACCESS_UNIT:
1290 case GF_HEVC_NALU_END_OF_SEQ:
1291 case GF_HEVC_NALU_END_OF_STREAM:
1292 case GF_HEVC_NALU_FILLER_DATA:
1293 add_nal = GF_FALSE;
1294 break;
1295 default:
1296 break;
1297 }
1298
1299 if ((nal_type<=GF_HEVC_NALU_SLICE_CRA) && ctx->hevc.s_info.sps) {
1300 if (ctx->hevc.sps_active_idx != ctx->active_sps) {
1301 ctx->reconfig_needed = 1;
1302 ctx->active_sps = ctx->hevc.sps_active_idx;
1303 ctx->active_pps = ctx->hevc.s_info.pps->id;
1304 ctx->active_vps = ctx->hevc.s_info.sps->vps_id;
1305 return GF_OK;
1306 }
1307 }
1308 }
1309 }
1310
1311 //if sps and pps are ready, init decoder
1312 if (!ctx->vtb_session && gf_list_count(ctx->SPSs) && gf_list_count(ctx->PPSs) ) {
1313 e = vtbdec_init_decoder(filter, ctx);
1314 if (e) {
1315 return e;
1316 }
1317 }
1318
1319 if (!out_buffer) add_nal = GF_FALSE;
1320 else if (add_nal && !ctx->vtb_session) add_nal = GF_FALSE;
1321
1322 if (add_nal) {
1323 if (reassign_bs) {
1324 if (!ctx->nalu_rewrite_bs) ctx->nalu_rewrite_bs = gf_bs_new(NULL, 0, GF_BITSTREAM_WRITE);
1325 else {
1326 gf_bs_reassign_buffer(ctx->nalu_rewrite_bs, ctx->nalu_buffer, ctx->nalu_buffer_alloc);
1327 //detach from context until we get the output of the bistream
1328 ctx->nalu_buffer = NULL;
1329 ctx->nalu_buffer_alloc = 0;
1330 }
1331 reassign_bs = GF_FALSE;
1332 }
1333
1334 gf_bs_write_u32(ctx->nalu_rewrite_bs, nal_size);
1335 gf_bs_write_data(ctx->nalu_rewrite_bs, ptr, nal_size);
1336 }
1337
1338 ptr += nal_size;
1339 if (ctx->nalu_size_length) {
1340 if (inBufferLength < nal_size + ctx->nalu_size_length) break;
1341 inBufferLength -= nal_size + ctx->nalu_size_length;
1342 } else {
1343 if (!sc_size || (inBufferLength < nal_size + sc_size)) break;
1344 inBufferLength -= nal_size + sc_size;
1345 ptr += sc_size;
1346 }
1347 }
1348
1349 if (check_reconfig && ctx->avc.s_info.pps ) {
1350 u32 sps_crc, pps_crc;
1351 sps_crc = vtbdec_purge_param_sets(ctx, GF_TRUE, ctx->avc.s_info.pps->sps_id);
1352 pps_crc = vtbdec_purge_param_sets(ctx, GF_FALSE, ctx->avc.s_info.pps->id);
1353
1354 if ((sps_crc != ctx->active_sps_crc) || (pps_crc != ctx->active_pps_crc) ) {
1355 ctx->reconfig_needed = 1;
1356 ctx->active_sps = ctx->avc.s_info.pps->sps_id;
1357 ctx->active_pps = ctx->avc.s_info.pps->id;
1358 ctx->active_sps_crc = sps_crc;
1359 ctx->active_pps_crc = pps_crc;
1360 }
1361 }
1362
1363 if (!reassign_bs) {
1364 //get output without truncating the allocated buffer, repass the buffer at the next AU
1365 gf_bs_get_content_no_truncate(ctx->nalu_rewrite_bs, &ctx->nalu_buffer, out_size, &ctx->nalu_buffer_alloc);
1366 *out_buffer = ctx->nalu_buffer;
1367 }
1368 return e;
1369 }
1370
1371
1372 static GF_Err vtbdec_send_output_frame(GF_Filter *filter, GF_VTBDecCtx *ctx);
1373
vtbdec_flush_frame(GF_Filter * filter,GF_VTBDecCtx * ctx)1374 static GF_Err vtbdec_flush_frame(GF_Filter *filter, GF_VTBDecCtx *ctx)
1375 {
1376 GF_VTBHWFrame *vtbframe;
1377 OSStatus status;
1378 OSType type;
1379
1380 if (ctx->no_copy) return vtbdec_send_output_frame(filter, ctx);
1381
1382 vtbframe = gf_list_pop_front(ctx->frames);
1383 GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[VTB] Outputing frame DTS "LLU" CTS "LLU" timescale %d\n", gf_filter_pck_get_dts(vtbframe->pck_src), gf_filter_pck_get_cts(vtbframe->pck_src), gf_filter_pck_get_timescale(vtbframe->pck_src)));
1384
1385
1386 status = CVPixelBufferLockBaseAddress(vtbframe->frame, kCVPixelBufferLock_ReadOnly);
1387 if (status != kCVReturnSuccess) {
1388 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locking frame data\n"));
1389 gf_list_add(ctx->frames_res, vtbframe);
1390 return GF_IO_ERR;
1391 }
1392
1393 type = CVPixelBufferGetPixelFormatType(vtbframe->frame);
1394
1395 if ((type==kCVPixelFormatType_420YpCbCr8Planar)
1396 || (type==kCVPixelFormatType_420YpCbCr8PlanarFullRange)
1397 || (type==kCVPixelFormatType_422YpCbCr8_yuvs)
1398 || (type==kCVPixelFormatType_444YpCbCr8)
1399 || (type=='444v')
1400 || (type==kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)
1401 || (type==kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
1402 ) {
1403 u32 i, j, nb_planes = (u32) CVPixelBufferGetPlaneCount(vtbframe->frame);
1404 u8 *dst;
1405 u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(vtbframe->frame, 0);
1406
1407 GF_FilterPacket *dst_pck = gf_filter_pck_new_alloc(ctx->opid, ctx->out_size, &dst);
1408
1409 //TOCHECK - for now the 3 planes are consecutive in VideoToolbox
1410 if (stride==ctx->width) {
1411 char *data = CVPixelBufferGetBaseAddressOfPlane(vtbframe->frame, 0);
1412 memcpy(dst, data, sizeof(char)*ctx->out_size);
1413 } else {
1414 for (i=0; i<nb_planes; i++) {
1415 char *data = CVPixelBufferGetBaseAddressOfPlane(vtbframe->frame, i);
1416 u32 stride = (u32) CVPixelBufferGetBytesPerRowOfPlane(vtbframe->frame, i);
1417 u32 w, h = (u32) CVPixelBufferGetHeightOfPlane(vtbframe->frame, i);
1418 w = ctx->width;
1419 if (i) {
1420 switch (ctx->pix_fmt) {
1421 case GF_PIXEL_YUV444:
1422 break;
1423 case GF_PIXEL_YUV422:
1424 case GF_PIXEL_YUV:
1425 w /= 2;
1426 break;
1427 }
1428 }
1429 if (stride != w) {
1430 for (j=0; j<h; j++) {
1431 memcpy(dst, data, sizeof(char)*w);
1432 dst += w;
1433 data += stride;
1434 }
1435 } else {
1436 memcpy(dst, data, sizeof(char)*h*stride);
1437 dst += sizeof(char)*h*stride;
1438 }
1439 }
1440 }
1441
1442 gf_filter_pck_merge_properties(vtbframe->pck_src, dst_pck);
1443 ctx->last_cts_out = gf_filter_pck_get_cts(vtbframe->pck_src);
1444 ctx->last_timescale_out = gf_filter_pck_get_timescale(vtbframe->pck_src);
1445 gf_filter_pck_unref(vtbframe->pck_src);
1446 vtbframe->pck_src = NULL;
1447 gf_filter_pck_send(dst_pck);
1448 }
1449 CVPixelBufferUnlockBaseAddress(vtbframe->frame, kCVPixelBufferLock_ReadOnly);
1450 gf_list_add(ctx->frames_res, vtbframe);
1451 return GF_OK;
1452 }
vtbdec_process(GF_Filter * filter)1453 static GF_Err vtbdec_process(GF_Filter *filter)
1454 {
1455 OSStatus status;
1456 CMSampleBufferRef sample = NULL;
1457 CMBlockBufferRef block_buffer = NULL;
1458 char *in_data=NULL;
1459 u32 in_data_size;
1460 char *in_buffer;
1461 u32 in_buffer_size, frames_count, i, count, nb_eos;
1462 u64 min_dts;
1463 GF_Err e;
1464 GF_VTBDecCtx *ctx = gf_filter_get_udta(filter);
1465 GF_FilterPacket *pck;
1466 GF_FilterPid *ref_pid = NULL;
1467
1468 //figure out min DTS
1469 count = gf_list_count(ctx->streams);
1470 nb_eos = 0;
1471 min_dts = 0;
1472 for (i=0; i<count; i++) {
1473 u64 dts;
1474 GF_FilterPid *pid = gf_list_get(ctx->streams, i);
1475
1476 pck = gf_filter_pid_get_packet(pid);
1477 if (!pck) {
1478 if (gf_filter_pid_is_eos(pid)) {
1479 nb_eos++;
1480 continue;
1481 } else {
1482 return GF_OK;
1483 }
1484 }
1485 dts = gf_filter_pck_get_dts(pck);
1486 dts *= 1000;
1487 dts /= gf_filter_pck_get_timescale(pck);
1488 if (!min_dts || (min_dts>dts)) {
1489 min_dts = dts;
1490 ref_pid = pid;
1491 }
1492 }
1493
1494 if (nb_eos==count) {
1495 while (gf_list_count(ctx->frames)) {
1496 vtbdec_flush_frame(filter, ctx);
1497 }
1498 gf_filter_pid_set_eos(ctx->opid);
1499 return GF_EOS;
1500 }
1501 assert(ref_pid);
1502 pck = gf_filter_pid_get_packet(ref_pid);
1503 assert(pck);
1504
1505 if (ctx->drop_non_refs && !gf_filter_pck_get_sap(pck)) {
1506 gf_filter_pid_drop_packet(ref_pid);
1507 return GF_OK;
1508 }
1509
1510 in_buffer = (char *) gf_filter_pck_get_data(pck, &in_buffer_size);
1511
1512 //discard empty packets
1513 if (!in_buffer || !in_buffer_size) {
1514 gf_filter_pid_drop_packet(ref_pid);
1515 //if inbuffer is null this is a hardware frame, should never happen
1516 return in_buffer ? GF_OK : GF_NOT_SUPPORTED;
1517 }
1518
1519 if (ctx->skip_mpeg4_vosh) {
1520 GF_M4VDecSpecInfo dsi;
1521 dsi.width = dsi.height = 0;
1522 e = gf_m4v_get_config(in_buffer, in_buffer_size, &dsi);
1523 //found a vosh - remove it from payload, init decoder if needed
1524 if ((e==GF_OK) && dsi.width && dsi.height) {
1525 if (!ctx->vtb_session) {
1526 ctx->vosh = in_buffer;
1527 ctx->vosh_size = dsi.next_object_start;
1528 e = vtbdec_init_decoder(filter, ctx);
1529 if (e) {
1530 gf_filter_pid_drop_packet(ref_pid);
1531 return e;
1532 }
1533
1534 //enfoce removal for all frames
1535 ctx->skip_mpeg4_vosh = GF_TRUE;
1536 }
1537 ctx->vosh_size = dsi.next_object_start;
1538 } else if (!ctx->vtb_session) {
1539 gf_filter_pid_drop_packet(ref_pid);
1540 return GF_OK;
1541 }
1542 }
1543
1544 if (ctx->init_mpeg12) {
1545 GF_M4VDecSpecInfo dsi;
1546 dsi.width = dsi.height = 0;
1547
1548 e = gf_mpegv12_get_config(in_buffer, in_buffer_size, &dsi);
1549 if ((e==GF_OK) && dsi.width && dsi.height) {
1550 ctx->width = dsi.width;
1551 ctx->height = dsi.height;
1552 ctx->pixel_ar.num = dsi.par_num;
1553 ctx->pixel_ar.den = dsi.par_den;
1554
1555 e = vtbdec_init_decoder(filter, ctx);
1556 if (e) {
1557 gf_filter_pid_drop_packet(ref_pid);
1558 return e;
1559 }
1560 }
1561
1562 if (!ctx->vtb_session) {
1563 gf_filter_pid_drop_packet(ref_pid);
1564 return GF_OK;
1565 }
1566 }
1567
1568 if (ctx->check_h264_isma) {
1569 if (in_buffer && !in_buffer[0] && !in_buffer[1] && !in_buffer[2] && (in_buffer[3]==0x01)) {
1570 ctx->check_h264_isma=GF_FALSE;
1571 ctx->nalu_size_length=0;
1572 ctx->is_annex_b=GF_TRUE;
1573 }
1574 }
1575
1576 //Always parse AVC data , remove SPS/PPS/... and reconfig if needed
1577 if (ctx->is_annex_b || ctx->nalu_size_length) {
1578
1579 e = vtbdec_parse_nal_units(filter, ctx, in_buffer, in_buffer_size, &in_data, &in_data_size);
1580 if (e) {
1581 gf_filter_pid_drop_packet(ref_pid);
1582 return e;
1583 }
1584 } else if (ctx->vosh_size) {
1585 in_data = in_buffer + ctx->vosh_size;
1586 in_data_size = in_buffer_size - ctx->vosh_size;
1587 ctx->vosh_size = 0;
1588 } else {
1589 in_data = in_buffer;
1590 in_data_size = in_buffer_size;
1591 }
1592
1593 if (ctx->reconfig_needed) {
1594 //flush all pending frames
1595 while (gf_list_count(ctx->frames)) {
1596 vtbdec_flush_frame(filter, ctx);
1597 }
1598 //waiting for last frame to be discarded
1599 if (ctx->no_copy && ctx->decoded_frames_pending) {
1600 ctx->last_frame_sent->frame_ifce.flags = GF_FRAME_IFCE_BLOCKING;
1601 gf_filter_ask_rt_reschedule(filter, 0);
1602 return GF_OK;
1603 }
1604 if (ctx->fmt_desc) {
1605 CFRelease(ctx->fmt_desc);
1606 ctx->fmt_desc = NULL;
1607 }
1608 if (ctx->vtb_session) {
1609 VTDecompressionSessionInvalidate(ctx->vtb_session);
1610 ctx->vtb_session=NULL;
1611 }
1612 vtbdec_init_decoder(filter, ctx);
1613 return GF_OK;
1614 }
1615 if (!ctx->vtb_session) {
1616 gf_filter_pid_drop_packet(ref_pid);
1617 return GF_OK;
1618 }
1619
1620
1621 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, in_data, in_data_size, kCFAllocatorNull, NULL, 0, in_data_size, 0, &block_buffer);
1622
1623 if (status || (block_buffer == NULL) ) {
1624 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Failed to allocate block buffer of %d bytes\n", in_data_size));
1625 gf_filter_pid_drop_packet(ref_pid);
1626 return GF_IO_ERR;
1627 }
1628
1629 status = CMSampleBufferCreate(kCFAllocatorDefault, block_buffer, TRUE, NULL, NULL, ctx->fmt_desc, 1, 0, NULL, 0, NULL, &sample);
1630
1631 if (status || (sample==NULL)) {
1632 if (block_buffer)
1633 CFRelease(block_buffer);
1634
1635 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Failed to create sample buffer for %d bytes\n", in_data_size));
1636 gf_filter_pid_drop_packet(ref_pid);
1637 return GF_IO_ERR;
1638 }
1639 GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[VTB] Decoding frame DTS "LLU" ms\n", min_dts));
1640 ctx->cur_pck = pck;
1641 ctx->last_error = GF_OK;
1642 status = VTDecompressionSessionDecodeFrame(ctx->vtb_session, sample, 0, NULL, 0);
1643 if (!status)
1644 status = VTDecompressionSessionWaitForAsynchronousFrames(ctx->vtb_session);
1645
1646
1647 CFRelease(block_buffer);
1648 CFRelease(sample);
1649
1650 //profile not supported, request codec change - do not discard input frame
1651 if (ctx->last_error==GF_PROFILE_NOT_SUPPORTED) {
1652 ctx->cur_pck = NULL;
1653 return ctx->last_error;
1654 }
1655
1656 gf_filter_pid_drop_packet(ref_pid);
1657 ctx->cur_pck = NULL;
1658
1659 if (ctx->last_error) return ctx->last_error;
1660
1661 if (status)
1662 return GF_NON_COMPLIANT_BITSTREAM;
1663
1664 frames_count = gf_list_count(ctx->frames);
1665 if (!frames_count) {
1666 return ctx->last_error;
1667 }
1668 //probing for reordering, or reordering is on but not enough frames: wait before we dispatch
1669 if (ctx->reorder_probe) {
1670 return GF_OK;
1671 }
1672
1673 if (ctx->reorder_detected && (frames_count<ctx->reorder) )
1674 return GF_OK;
1675
1676 return vtbdec_flush_frame(filter, ctx);
1677 }
1678
vtbframe_release(GF_Filter * filter,GF_FilterPid * pid,GF_FilterPacket * pck)1679 void vtbframe_release(GF_Filter *filter, GF_FilterPid *pid, GF_FilterPacket *pck)
1680 {
1681 GF_FilterFrameInterface *frame = gf_filter_pck_get_frame_interface(pck);
1682 GF_VTBHWFrame *f = (GF_VTBHWFrame *)frame->user_data;
1683 if (f->locked) {
1684 CVPixelBufferUnlockBaseAddress(f->frame, kCVPixelBufferLock_ReadOnly);
1685 }
1686 #ifdef VTB_GL_TEXTURE
1687 if (f->y) CVBufferRelease(f->y);
1688 if (f->u) CVBufferRelease(f->u);
1689 if (f->v) CVBufferRelease(f->v);
1690 if (f->ctx->cache_texture)
1691 GF_CVOpenGLTextureCacheFlush(f->ctx->cache_texture, 0);
1692 #endif
1693
1694 if (f->frame) {
1695 CVPixelBufferRelease(f->frame);
1696 f->frame = NULL;
1697 }
1698
1699 safe_int_dec(&f->ctx->decoded_frames_pending);
1700 gf_list_add(f->ctx->frames_res, f);
1701 }
1702
vtbframe_get_plane(GF_FilterFrameInterface * frame,u32 plane_idx,const u8 ** outPlane,u32 * outStride)1703 GF_Err vtbframe_get_plane(GF_FilterFrameInterface *frame, u32 plane_idx, const u8 **outPlane, u32 *outStride)
1704 {
1705 OSStatus status;
1706 GF_Err e;
1707 GF_VTBHWFrame *f = (GF_VTBHWFrame *)frame->user_data;
1708 if (! outPlane || !outStride) return GF_BAD_PARAM;
1709 *outPlane = NULL;
1710 assert(f->frame);
1711 if (!f->locked) {
1712 status = CVPixelBufferLockBaseAddress(f->frame, kCVPixelBufferLock_ReadOnly);
1713 if (status != kCVReturnSuccess) {
1714 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locking frame data\n"));
1715 return GF_IO_ERR;
1716 }
1717 f->locked = GF_TRUE;
1718 }
1719 e = GF_OK;
1720
1721 if (CVPixelBufferIsPlanar(f->frame)) {
1722 *outPlane = CVPixelBufferGetBaseAddressOfPlane(f->frame, plane_idx);
1723 if (*outPlane)
1724 *outStride = (u32) CVPixelBufferGetBytesPerRowOfPlane(f->frame, plane_idx);
1725 else
1726 e = GF_EOS;
1727 } else if (plane_idx==0) {
1728 *outStride = (u32) CVPixelBufferGetBytesPerRow(f->frame);
1729 *outPlane = CVPixelBufferGetBaseAddress(f->frame);
1730 } else {
1731 e = GF_BAD_PARAM;
1732 }
1733 return e;
1734 }
1735
1736 #ifdef VTB_GL_TEXTURE
1737
1738 /*Define codec matrix*/
1739 typedef struct __matrix GF_CodecMatrix;
1740
1741 #ifdef GPAC_CONFIG_IOS
1742 void *myGetGLContext();
1743 #else
1744
1745 #include <OpenGL/CGLCurrent.h>
myGetGLContext()1746 void *myGetGLContext()
1747 {
1748 return CGLGetCurrentContext();
1749 }
1750 #endif
1751
1752
vtbframe_get_gl_texture(GF_FilterFrameInterface * frame,u32 plane_idx,u32 * gl_tex_format,u32 * gl_tex_id,GF_CodecMatrix * texcoordmatrix)1753 GF_Err vtbframe_get_gl_texture(GF_FilterFrameInterface *frame, u32 plane_idx, u32 *gl_tex_format, u32 *gl_tex_id, GF_CodecMatrix * texcoordmatrix)
1754 {
1755 OSStatus status;
1756 GLenum target_fmt;
1757 u32 w, h;
1758 GF_CVGLTextureREF *outTexture=NULL;
1759 GF_VTBHWFrame *f = (GF_VTBHWFrame *)frame->user_data;
1760 if (! gl_tex_format || !gl_tex_id) return GF_BAD_PARAM;
1761 *gl_tex_format = 0;
1762 *gl_tex_id = 0;
1763
1764 if (!f->ctx->gl_context) {
1765 f->ctx->gl_context = myGetGLContext();
1766 if (!f->ctx->gl_context) {
1767 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locating current GL context\n"));
1768 return GF_IO_ERR;
1769 }
1770 }
1771 if (! f->ctx->decoded_frames_pending) return GF_IO_ERR;
1772
1773 if (!f->ctx->cache_texture) {
1774 #ifdef GPAC_CONFIG_IOS
1775 status = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, f->ctx->gl_context, NULL, &f->ctx->cache_texture);
1776 #else
1777 status = CVOpenGLTextureCacheCreate(kCFAllocatorDefault, NULL, f->ctx->gl_context, CGLGetPixelFormat(f->ctx->gl_context), NULL, &f->ctx->cache_texture);
1778 #endif
1779 if (status != kCVReturnSuccess) {
1780 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error creating cache texture\n"));
1781 return GF_IO_ERR;
1782 }
1783 }
1784
1785 if (!f->locked) {
1786 status = CVPixelBufferLockBaseAddress(f->frame, kCVPixelBufferLock_ReadOnly);
1787 if (status != kCVReturnSuccess) {
1788 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error locking frame data\n"));
1789 return GF_IO_ERR;
1790 }
1791 f->locked = GF_TRUE;
1792 }
1793
1794 if (CVPixelBufferIsPlanar(f->frame)) {
1795 w = (u32) CVPixelBufferGetPlaneCount(f->frame);
1796 if (plane_idx >= (u32) CVPixelBufferGetPlaneCount(f->frame)) {
1797 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Wrong plane index\n"));
1798 return GF_BAD_PARAM;
1799 }
1800 } else if (plane_idx!=0) {
1801 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Wrong plane index %d on interleaved format\n", plane_idx));
1802 return GF_BAD_PARAM;
1803 }
1804
1805 target_fmt = GL_LUMINANCE;
1806 w = f->ctx->width;
1807 h = f->ctx->height;
1808 if (plane_idx) {
1809 w /= 2;
1810 h /= 2;
1811 target_fmt = GL_LUMINANCE_ALPHA;
1812 }
1813 if (plane_idx==0) {
1814 outTexture = &f->y;
1815 }
1816 else if (plane_idx==1) {
1817 outTexture = &f->u;
1818 }
1819 //don't create texture if already done !
1820 if ( *outTexture == NULL) {
1821 #ifdef GPAC_CONFIG_IOS
1822 status = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, f->ctx->cache_texture, f->frame, NULL, GL_TEXTURE_2D, target_fmt, w, h, target_fmt, GL_UNSIGNED_BYTE, plane_idx, outTexture);
1823 #else
1824 status = CVOpenGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, f->ctx->cache_texture, f->frame, NULL, outTexture);
1825 #endif
1826
1827 if (status != kCVReturnSuccess) {
1828 GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[VTB] Error creating cache texture for plane %d\n", plane_idx));
1829 return GF_IO_ERR;
1830 }
1831 }
1832 *gl_tex_format = GF_CVOpenGLTextureGetTarget(*outTexture);
1833 *gl_tex_id = GF_CVOpenGLTextureGetName(*outTexture);
1834
1835 return GF_OK;
1836 }
1837 #endif
1838
vtbdec_send_output_frame(GF_Filter * filter,GF_VTBDecCtx * ctx)1839 static GF_Err vtbdec_send_output_frame(GF_Filter *filter, GF_VTBDecCtx *ctx)
1840 {
1841 GF_VTBHWFrame *vtb_frame;
1842 GF_FilterPacket *dst_pck;
1843
1844 vtb_frame = gf_list_pop_front(ctx->frames);
1845 if (!vtb_frame) return GF_BAD_PARAM;
1846
1847 GF_LOG(GF_LOG_DEBUG, GF_LOG_CODEC, ("[VTB] Outputing frame DTS "LLU" CTS "LLU" timescale %d\n", gf_filter_pck_get_dts(vtb_frame->pck_src), gf_filter_pck_get_cts(vtb_frame->pck_src), gf_filter_pck_get_timescale(vtb_frame->pck_src)));
1848
1849 vtb_frame->frame_ifce.user_data = vtb_frame;
1850 vtb_frame->frame_ifce.get_plane = vtbframe_get_plane;
1851 #ifdef VTB_GL_TEXTURE
1852 if (ctx->use_gl_textures)
1853 vtb_frame->frame_ifce.get_gl_texture = vtbframe_get_gl_texture;
1854 #endif
1855
1856 if (ctx->reconfig_needed)
1857 vtb_frame->frame_ifce.flags = GF_FRAME_IFCE_BLOCKING;
1858
1859 safe_int_inc(&ctx->decoded_frames_pending);
1860
1861 dst_pck = gf_filter_pck_new_frame_interface(ctx->opid, &vtb_frame->frame_ifce, vtbframe_release);
1862
1863 gf_filter_pck_merge_properties(vtb_frame->pck_src, dst_pck);
1864
1865 ctx->last_cts_out = gf_filter_pck_get_cts(vtb_frame->pck_src);
1866 ctx->last_timescale_out = gf_filter_pck_get_timescale(vtb_frame->pck_src);
1867 gf_filter_pck_unref(vtb_frame->pck_src);
1868 vtb_frame->pck_src = NULL;
1869 ctx->last_frame_sent = vtb_frame;
1870 gf_filter_pck_send(dst_pck);
1871 return GF_OK;
1872 }
1873
1874
1875 #endif
1876
vtbdec_process_event(GF_Filter * filter,const GF_FilterEvent * evt)1877 static Bool vtbdec_process_event(GF_Filter *filter, const GF_FilterEvent *evt)
1878 {
1879 GF_VTBDecCtx *ctx = (GF_VTBDecCtx *) gf_filter_get_udta(filter);
1880 if (evt->base.type==GF_FEVT_PLAY) {
1881 while (gf_list_count(ctx->frames) ) {
1882 GF_VTBHWFrame *f = gf_list_pop_back(ctx->frames);
1883 if (f->pck_src) gf_filter_pck_unref(f->pck_src);
1884 f->pck_src = NULL;
1885 gf_list_add(ctx->frames_res, f);
1886 }
1887 ctx->drop_non_refs = evt->play.drop_non_ref;
1888 }
1889 else if ((evt->base.type==GF_FEVT_SET_SPEED) || (evt->base.type==GF_FEVT_RESUME)) {
1890 ctx->drop_non_refs = evt->play.drop_non_ref;
1891 }
1892 return GF_FALSE;
1893 }
1894
vtbdec_initialize(GF_Filter * filter)1895 static GF_Err vtbdec_initialize(GF_Filter *filter)
1896 {
1897 GF_VTBDecCtx *ctx = (GF_VTBDecCtx *) gf_filter_get_udta(filter);
1898 #ifdef VTB_GL_TEXTURE
1899 if (ctx->no_copy)
1900 ctx->use_gl_textures = GF_TRUE;
1901 #endif
1902
1903 ctx->frames_res = gf_list_new();
1904 ctx->frames = gf_list_new();
1905 ctx->streams = gf_list_new();
1906 return GF_OK;
1907 }
1908
vtbdec_finalize(GF_Filter * filter)1909 static void vtbdec_finalize(GF_Filter *filter)
1910 {
1911 GF_VTBDecCtx *ctx = (GF_VTBDecCtx *) gf_filter_get_udta(filter);
1912 vtbdec_delete_decoder(ctx);
1913
1914 #ifdef VTB_GL_TEXTURE
1915 if (ctx->cache_texture) {
1916 CFRelease(ctx->cache_texture);
1917 }
1918 #endif
1919
1920 if (ctx->frames) {
1921 while (gf_list_count(ctx->frames) ) {
1922 GF_VTBHWFrame *f = gf_list_pop_back(ctx->frames);
1923 if (f->pck_src) gf_filter_pck_unref(f->pck_src);
1924 gf_free(f);
1925 }
1926 gf_list_del(ctx->frames);
1927 }
1928
1929 if (ctx->frames_res) {
1930 while (gf_list_count(ctx->frames_res) ) {
1931 GF_VTBHWFrame *f = gf_list_pop_back(ctx->frames_res);
1932 if (f->pck_src) gf_filter_pck_unref(f->pck_src);
1933 gf_free(f);
1934 }
1935 gf_list_del(ctx->frames_res);
1936 }
1937 gf_list_del(ctx->streams);
1938
1939 if (ctx->nal_bs) gf_bs_del(ctx->nal_bs);
1940 if (ctx->ps_bs) gf_bs_del(ctx->ps_bs);
1941 if (ctx->nalu_rewrite_bs) gf_bs_del(ctx->nalu_rewrite_bs);
1942 if (ctx->nalu_buffer) gf_free(ctx->nalu_buffer);
1943 }
1944
1945
1946 static const GF_FilterCapability VTBDecCaps[] =
1947 {
1948 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_STREAM_TYPE, GF_STREAM_VISUAL),
1949 CAP_BOOL(GF_CAPS_INPUT_EXCLUDED, GF_PROP_PID_UNFRAMED, GF_TRUE),
1950 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG4_PART2),
1951 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_AVC),
1952 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_HEVC),
1953 CAP_BOOL(GF_CAPS_INPUT_EXCLUDED,GF_PROP_PID_TILE_BASE, GF_TRUE),
1954
1955 #ifndef GPAC_CONFIG_IOS
1956 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG2_SIMPLE),
1957 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG2_MAIN),
1958 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG2_SNR),
1959 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG2_SPATIAL),
1960 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG2_HIGH),
1961 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_MPEG2_422),
1962 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_H263),
1963 CAP_UINT(GF_CAPS_INPUT,GF_PROP_PID_CODECID, GF_CODECID_S263),
1964 #endif
1965 CAP_UINT(GF_CAPS_OUTPUT, GF_PROP_PID_STREAM_TYPE, GF_STREAM_VISUAL),
1966 CAP_UINT(GF_CAPS_OUTPUT, GF_PROP_PID_CODECID, GF_CODECID_RAW),
1967 };
1968
1969 #define OFFS(_n) #_n, offsetof(GF_VTBDecCtx, _n)
1970
1971 static const GF_FilterArgs VTBDecArgs[] =
1972 {
1973 { OFFS(reorder), "number of frames to wait for temporal re-ordering", GF_PROP_UINT, "6", NULL, GF_FS_ARG_HINT_ADVANCED},
1974 { OFFS(no_copy), "dispatch VTB frames into filter chain (no copy)", GF_PROP_BOOL, "true", NULL, GF_FS_ARG_HINT_ADVANCED},
1975 { OFFS(ofmt), "set default pixel format for decoded video. If not matched default to nv12", GF_PROP_PIXFMT, "nv12", NULL, GF_FS_ARG_HINT_ADVANCED},
1976 { OFFS(disable_hw), "disable hardware decoding", GF_PROP_BOOL, "false", NULL, 0},
1977 {}
1978 };
1979
1980 GF_FilterRegister GF_VTBDecCtxRegister = {
1981 .name = "vtbdec",
1982 GF_FS_SET_DESCRIPTION("VideoToolBox decoder")
1983 GF_FS_SET_HELP("This filter decodes MPEG-2, H263, AVC|H264 and HEVC streams through VideoToolBox. It allows GPU frame dispatch or direct frame copy.")
1984 .private_size = sizeof(GF_VTBDecCtx),
1985 .args = VTBDecArgs,
1986 .priority = 1,
1987 SETCAPS(VTBDecCaps),
1988 .initialize = vtbdec_initialize,
1989 .finalize = vtbdec_finalize,
1990 .configure_pid = vtbdec_configure_pid,
1991 .process = vtbdec_process,
1992 .max_extra_pids = 5,
1993 .process_event = vtbdec_process_event,
1994 };
1995
1996 #else
1997 #undef _GF_MATH_H_
1998 #include <gpac/maths.h>
1999 #include <gpac/filters.h>
2000
2001 #endif // !defined(GPAC_DISABLE_AV_PARSERS) && ( defined(GPAC_CONFIG_DARWIN) || defined(GPAC_CONFIG_IOS) )
2002
vtbdec_register(GF_FilterSession * session)2003 const GF_FilterRegister *vtbdec_register(GF_FilterSession *session)
2004 {
2005 #if !defined(GPAC_DISABLE_AV_PARSERS) && ( defined(GPAC_CONFIG_DARWIN) || defined(GPAC_CONFIG_IOS) )
2006 return &GF_VTBDecCtxRegister;
2007 #else
2008 return NULL;
2009 #endif
2010 }
2011