1 /* GStreamer H.264 Parser
2 * Copyright (C) <2010> Collabora ltd
3 * Copyright (C) <2010> Nokia Corporation
4 * Copyright (C) <2011> Intel Corporation
5 *
6 * Copyright (C) <2010> Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>
7 * Copyright (C) <2011> Thibault Saunier <thibault.saunier@collabora.com>
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
18 *
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
23 */
24
25 #ifdef HAVE_CONFIG_H
26 # include "config.h"
27 #endif
28
29 #include <gst/base/base.h>
30 #include <gst/pbutils/pbutils.h>
31 #include <gst/video/video.h>
32 #include "gsth264parse.h"
33
34 #include <string.h>
35
36 GST_DEBUG_CATEGORY (h264_parse_debug);
37 #define GST_CAT_DEFAULT h264_parse_debug
38
39 #define DEFAULT_CONFIG_INTERVAL (0)
40
41 enum
42 {
43 PROP_0,
44 PROP_CONFIG_INTERVAL
45 };
46
47 enum
48 {
49 GST_H264_PARSE_FORMAT_NONE,
50 GST_H264_PARSE_FORMAT_AVC,
51 GST_H264_PARSE_FORMAT_BYTE,
52 GST_H264_PARSE_FORMAT_AVC3
53 };
54
55 enum
56 {
57 GST_H264_PARSE_ALIGN_NONE = 0,
58 GST_H264_PARSE_ALIGN_NAL,
59 GST_H264_PARSE_ALIGN_AU
60 };
61
62 enum
63 {
64 GST_H264_PARSE_STATE_GOT_SPS = 1 << 0,
65 GST_H264_PARSE_STATE_GOT_PPS = 1 << 1,
66 GST_H264_PARSE_STATE_GOT_SLICE = 1 << 2,
67
68 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS = (GST_H264_PARSE_STATE_GOT_SPS |
69 GST_H264_PARSE_STATE_GOT_PPS),
70 GST_H264_PARSE_STATE_VALID_PICTURE =
71 (GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS |
72 GST_H264_PARSE_STATE_GOT_SLICE)
73 };
74
75 #define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
76 (((parse)->state & (expected_state)) == (expected_state))
77
78 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
79 GST_PAD_SINK,
80 GST_PAD_ALWAYS,
81 GST_STATIC_CAPS ("video/x-h264"));
82
83 static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
84 GST_PAD_SRC,
85 GST_PAD_ALWAYS,
86 GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, "
87 "stream-format=(string) { avc, avc3, byte-stream }, "
88 "alignment=(string) { au, nal }"));
89
90 #define parent_class gst_h264_parse_parent_class
91 G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE);
92
93 static void gst_h264_parse_finalize (GObject * object);
94
95 static gboolean gst_h264_parse_start (GstBaseParse * parse);
96 static gboolean gst_h264_parse_stop (GstBaseParse * parse);
97 static GstFlowReturn gst_h264_parse_handle_frame (GstBaseParse * parse,
98 GstBaseParseFrame * frame, gint * skipsize);
99 static GstFlowReturn gst_h264_parse_parse_frame (GstBaseParse * parse,
100 GstBaseParseFrame * frame);
101 static GstFlowReturn gst_h264_parse_pre_push_frame (GstBaseParse * parse,
102 GstBaseParseFrame * frame);
103
104 static void gst_h264_parse_set_property (GObject * object, guint prop_id,
105 const GValue * value, GParamSpec * pspec);
106 static void gst_h264_parse_get_property (GObject * object, guint prop_id,
107 GValue * value, GParamSpec * pspec);
108
109 static gboolean gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps);
110 static GstCaps *gst_h264_parse_get_caps (GstBaseParse * parse,
111 GstCaps * filter);
112 static gboolean gst_h264_parse_event (GstBaseParse * parse, GstEvent * event);
113 static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
114 GstEvent * event);
115 static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
116 GstCaps * caps);
117
118 static void
gst_h264_parse_class_init(GstH264ParseClass * klass)119 gst_h264_parse_class_init (GstH264ParseClass * klass)
120 {
121 GObjectClass *gobject_class = (GObjectClass *) klass;
122 GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
123 GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
124
125 GST_DEBUG_CATEGORY_INIT (h264_parse_debug, "h264parse", 0, "h264 parser");
126
127 gobject_class->finalize = gst_h264_parse_finalize;
128 gobject_class->set_property = gst_h264_parse_set_property;
129 gobject_class->get_property = gst_h264_parse_get_property;
130
131 g_object_class_install_property (gobject_class, PROP_CONFIG_INTERVAL,
132 g_param_spec_int ("config-interval",
133 "SPS PPS Send Interval",
134 "Send SPS and PPS Insertion Interval in seconds (sprop parameter sets "
135 "will be multiplexed in the data stream when detected.) "
136 "(0 = disabled, -1 = send with every IDR frame)",
137 -1, 3600, DEFAULT_CONFIG_INTERVAL,
138 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
139
140 /* Override BaseParse vfuncs */
141 parse_class->start = GST_DEBUG_FUNCPTR (gst_h264_parse_start);
142 parse_class->stop = GST_DEBUG_FUNCPTR (gst_h264_parse_stop);
143 parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_h264_parse_handle_frame);
144 parse_class->pre_push_frame =
145 GST_DEBUG_FUNCPTR (gst_h264_parse_pre_push_frame);
146 parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_set_caps);
147 parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_get_caps);
148 parse_class->sink_event = GST_DEBUG_FUNCPTR (gst_h264_parse_event);
149 parse_class->src_event = GST_DEBUG_FUNCPTR (gst_h264_parse_src_event);
150
151 gst_element_class_add_static_pad_template (gstelement_class, &srctemplate);
152 gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate);
153
154 gst_element_class_set_static_metadata (gstelement_class, "H.264 parser",
155 "Codec/Parser/Converter/Video",
156 "Parses H.264 streams",
157 "Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>");
158 }
159
160 static void
gst_h264_parse_init(GstH264Parse * h264parse)161 gst_h264_parse_init (GstH264Parse * h264parse)
162 {
163 h264parse->frame_out = gst_adapter_new ();
164 gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (h264parse), FALSE);
165 GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (h264parse));
166 GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (h264parse));
167
168 h264parse->aud_needed = TRUE;
169 h264parse->aud_insert = TRUE;
170 }
171
172
173 static void
gst_h264_parse_finalize(GObject * object)174 gst_h264_parse_finalize (GObject * object)
175 {
176 GstH264Parse *h264parse = GST_H264_PARSE (object);
177
178 g_object_unref (h264parse->frame_out);
179
180 G_OBJECT_CLASS (parent_class)->finalize (object);
181 }
182
183 static void
gst_h264_parse_reset_frame(GstH264Parse * h264parse)184 gst_h264_parse_reset_frame (GstH264Parse * h264parse)
185 {
186 GST_DEBUG_OBJECT (h264parse, "reset frame");
187
188 /* done parsing; reset state */
189 h264parse->current_off = -1;
190
191 h264parse->picture_start = FALSE;
192 h264parse->update_caps = FALSE;
193 h264parse->idr_pos = -1;
194 h264parse->sei_pos = -1;
195 h264parse->keyframe = FALSE;
196 h264parse->header = FALSE;
197 h264parse->frame_start = FALSE;
198 h264parse->aud_insert = TRUE;
199 h264parse->have_sps_in_frame = FALSE;
200 h264parse->have_pps_in_frame = FALSE;
201 gst_adapter_clear (h264parse->frame_out);
202 }
203
204 static void
gst_h264_parse_reset_stream_info(GstH264Parse * h264parse)205 gst_h264_parse_reset_stream_info (GstH264Parse * h264parse)
206 {
207 gint i;
208
209 h264parse->width = 0;
210 h264parse->height = 0;
211 h264parse->fps_num = 0;
212 h264parse->fps_den = 0;
213 h264parse->upstream_par_n = -1;
214 h264parse->upstream_par_d = -1;
215 h264parse->parsed_par_n = 0;
216 h264parse->parsed_par_d = 0;
217 h264parse->have_pps = FALSE;
218 h264parse->have_sps = FALSE;
219
220 h264parse->multiview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
221 h264parse->multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
222 h264parse->first_in_bundle = TRUE;
223
224 h264parse->align = GST_H264_PARSE_ALIGN_NONE;
225 h264parse->format = GST_H264_PARSE_FORMAT_NONE;
226
227 h264parse->transform = FALSE;
228 h264parse->nal_length_size = 4;
229 h264parse->packetized = FALSE;
230 h264parse->push_codec = FALSE;
231
232 gst_buffer_replace (&h264parse->codec_data, NULL);
233 gst_buffer_replace (&h264parse->codec_data_in, NULL);
234
235 gst_h264_parse_reset_frame (h264parse);
236
237 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++)
238 gst_buffer_replace (&h264parse->sps_nals[i], NULL);
239 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++)
240 gst_buffer_replace (&h264parse->pps_nals[i], NULL);
241 }
242
243 static void
gst_h264_parse_reset(GstH264Parse * h264parse)244 gst_h264_parse_reset (GstH264Parse * h264parse)
245 {
246 h264parse->last_report = GST_CLOCK_TIME_NONE;
247
248 h264parse->dts = GST_CLOCK_TIME_NONE;
249 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
250 h264parse->do_ts = TRUE;
251
252 h264parse->sent_codec_tag = FALSE;
253
254 h264parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
255 gst_event_replace (&h264parse->force_key_unit_event, NULL);
256
257 h264parse->discont = FALSE;
258
259 gst_h264_parse_reset_stream_info (h264parse);
260 }
261
262 static gboolean
gst_h264_parse_start(GstBaseParse * parse)263 gst_h264_parse_start (GstBaseParse * parse)
264 {
265 GstH264Parse *h264parse = GST_H264_PARSE (parse);
266
267 GST_DEBUG_OBJECT (parse, "start");
268 gst_h264_parse_reset (h264parse);
269
270 h264parse->nalparser = gst_h264_nal_parser_new ();
271
272 h264parse->state = 0;
273 h264parse->dts = GST_CLOCK_TIME_NONE;
274 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
275 h264parse->sei_pic_struct_pres_flag = FALSE;
276 h264parse->sei_pic_struct = 0;
277 h264parse->field_pic_flag = 0;
278
279 gst_base_parse_set_min_frame_size (parse, 6);
280
281 return TRUE;
282 }
283
284 static gboolean
gst_h264_parse_stop(GstBaseParse * parse)285 gst_h264_parse_stop (GstBaseParse * parse)
286 {
287 GstH264Parse *h264parse = GST_H264_PARSE (parse);
288
289 GST_DEBUG_OBJECT (parse, "stop");
290 gst_h264_parse_reset (h264parse);
291
292 gst_h264_nal_parser_free (h264parse->nalparser);
293
294 return TRUE;
295 }
296
297 static const gchar *
gst_h264_parse_get_string(GstH264Parse * parse,gboolean format,gint code)298 gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code)
299 {
300 if (format) {
301 switch (code) {
302 case GST_H264_PARSE_FORMAT_AVC:
303 return "avc";
304 case GST_H264_PARSE_FORMAT_BYTE:
305 return "byte-stream";
306 case GST_H264_PARSE_FORMAT_AVC3:
307 return "avc3";
308 default:
309 return "none";
310 }
311 } else {
312 switch (code) {
313 case GST_H264_PARSE_ALIGN_NAL:
314 return "nal";
315 case GST_H264_PARSE_ALIGN_AU:
316 return "au";
317 default:
318 return "none";
319 }
320 }
321 }
322
323 static void
gst_h264_parse_format_from_caps(GstCaps * caps,guint * format,guint * align)324 gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align)
325 {
326 if (format)
327 *format = GST_H264_PARSE_FORMAT_NONE;
328
329 if (align)
330 *align = GST_H264_PARSE_ALIGN_NONE;
331
332 g_return_if_fail (gst_caps_is_fixed (caps));
333
334 GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);
335
336 if (caps && gst_caps_get_size (caps) > 0) {
337 GstStructure *s = gst_caps_get_structure (caps, 0);
338 const gchar *str = NULL;
339
340 if (format) {
341 if ((str = gst_structure_get_string (s, "stream-format"))) {
342 if (strcmp (str, "avc") == 0)
343 *format = GST_H264_PARSE_FORMAT_AVC;
344 else if (strcmp (str, "byte-stream") == 0)
345 *format = GST_H264_PARSE_FORMAT_BYTE;
346 else if (strcmp (str, "avc3") == 0)
347 *format = GST_H264_PARSE_FORMAT_AVC3;
348 }
349 }
350
351 if (align) {
352 if ((str = gst_structure_get_string (s, "alignment"))) {
353 if (strcmp (str, "au") == 0)
354 *align = GST_H264_PARSE_ALIGN_AU;
355 else if (strcmp (str, "nal") == 0)
356 *align = GST_H264_PARSE_ALIGN_NAL;
357 }
358 }
359 }
360 }
361
362 /* check downstream caps to configure format and alignment */
363 static void
gst_h264_parse_negotiate(GstH264Parse * h264parse,gint in_format,GstCaps * in_caps)364 gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format,
365 GstCaps * in_caps)
366 {
367 GstCaps *caps;
368 guint format = h264parse->format;
369 guint align = h264parse->align;
370
371 g_return_if_fail ((in_caps == NULL) || gst_caps_is_fixed (in_caps));
372
373 caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
374 GST_DEBUG_OBJECT (h264parse, "allowed caps: %" GST_PTR_FORMAT, caps);
375
376 /* concentrate on leading structure, since decodebin parser
377 * capsfilter always includes parser template caps */
378 if (caps) {
379 caps = gst_caps_truncate (caps);
380 GST_DEBUG_OBJECT (h264parse, "negotiating with caps: %" GST_PTR_FORMAT,
381 caps);
382 }
383
384 h264parse->can_passthrough = FALSE;
385
386 if (in_caps && caps) {
387 if (gst_caps_can_intersect (in_caps, caps)) {
388 GST_DEBUG_OBJECT (h264parse, "downstream accepts upstream caps");
389 gst_h264_parse_format_from_caps (in_caps, &format, &align);
390 gst_caps_unref (caps);
391 caps = NULL;
392 h264parse->can_passthrough = TRUE;
393 }
394 }
395
396 /* FIXME We could fail the negotiation immediatly if caps are empty */
397 if (caps && !gst_caps_is_empty (caps)) {
398 /* fixate to avoid ambiguity with lists when parsing */
399 caps = gst_caps_fixate (caps);
400 gst_h264_parse_format_from_caps (caps, &format, &align);
401 }
402
403 /* default */
404 if (!format)
405 format = GST_H264_PARSE_FORMAT_BYTE;
406 if (!align)
407 align = GST_H264_PARSE_ALIGN_AU;
408
409 GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s",
410 gst_h264_parse_get_string (h264parse, TRUE, format),
411 gst_h264_parse_get_string (h264parse, FALSE, align));
412
413 h264parse->format = format;
414 h264parse->align = align;
415
416 h264parse->transform = in_format != h264parse->format ||
417 align == GST_H264_PARSE_ALIGN_AU;
418
419 if (caps)
420 gst_caps_unref (caps);
421 }
422
423 static GstBuffer *
gst_h264_parse_wrap_nal(GstH264Parse * h264parse,guint format,guint8 * data,guint size)424 gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data,
425 guint size)
426 {
427 GstBuffer *buf;
428 guint nl = h264parse->nal_length_size;
429 guint32 tmp;
430
431 GST_DEBUG_OBJECT (h264parse, "nal length %d", size);
432
433 buf = gst_buffer_new_allocate (NULL, 4 + size, NULL);
434 if (format == GST_H264_PARSE_FORMAT_AVC
435 || format == GST_H264_PARSE_FORMAT_AVC3) {
436 tmp = GUINT32_TO_BE (size << (32 - 8 * nl));
437 } else {
438 /* HACK: nl should always be 4 here, otherwise this won't work.
439 * There are legit cases where nl in avc stream is 2, but byte-stream
440 * SC is still always 4 bytes. */
441 nl = 4;
442 tmp = GUINT32_TO_BE (1);
443 }
444
445 gst_buffer_fill (buf, 0, &tmp, sizeof (guint32));
446 gst_buffer_fill (buf, nl, data, size);
447 gst_buffer_set_size (buf, size + nl);
448
449 return buf;
450 }
451
452 static void
gst_h264_parser_store_nal(GstH264Parse * h264parse,guint id,GstH264NalUnitType naltype,GstH264NalUnit * nalu)453 gst_h264_parser_store_nal (GstH264Parse * h264parse, guint id,
454 GstH264NalUnitType naltype, GstH264NalUnit * nalu)
455 {
456 GstBuffer *buf, **store;
457 guint size = nalu->size, store_size;
458
459 if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_SUBSET_SPS) {
460 store_size = GST_H264_MAX_SPS_COUNT;
461 store = h264parse->sps_nals;
462 GST_DEBUG_OBJECT (h264parse, "storing sps %u", id);
463 } else if (naltype == GST_H264_NAL_PPS) {
464 store_size = GST_H264_MAX_PPS_COUNT;
465 store = h264parse->pps_nals;
466 GST_DEBUG_OBJECT (h264parse, "storing pps %u", id);
467 } else
468 return;
469
470 if (id >= store_size) {
471 GST_DEBUG_OBJECT (h264parse, "unable to store nal, id out-of-range %d", id);
472 return;
473 }
474
475 buf = gst_buffer_new_allocate (NULL, size, NULL);
476 gst_buffer_fill (buf, 0, nalu->data + nalu->offset, size);
477
478 /* Indicate that buffer contain a header needed for decoding */
479 if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_PPS)
480 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);
481
482 if (store[id])
483 gst_buffer_unref (store[id]);
484
485 store[id] = buf;
486 }
487
488 #ifndef GST_DISABLE_GST_DEBUG
489 static const gchar *nal_names[] = {
490 "Unknown",
491 "Slice",
492 "Slice DPA",
493 "Slice DPB",
494 "Slice DPC",
495 "Slice IDR",
496 "SEI",
497 "SPS",
498 "PPS",
499 "AU delimiter",
500 "Sequence End",
501 "Stream End",
502 "Filler Data",
503 "SPS extension",
504 "Prefix",
505 "SPS Subset",
506 "Depth Parameter Set",
507 "Reserved", "Reserved",
508 "Slice Aux Unpartitioned",
509 "Slice Extension",
510 "Slice Depth/3D-AVC Extension"
511 };
512
513 static const gchar *
_nal_name(GstH264NalUnitType nal_type)514 _nal_name (GstH264NalUnitType nal_type)
515 {
516 if (nal_type <= GST_H264_NAL_SLICE_DEPTH)
517 return nal_names[nal_type];
518 return "Invalid";
519 }
520 #endif
521
522 static void
gst_h264_parse_process_sei_user_data(GstH264Parse * h264parse,GstH264RegisteredUserData * rud)523 gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse,
524 GstH264RegisteredUserData * rud)
525 {
526 guint16 provider_code;
527 guint32 atsc_user_id;
528 GstByteReader br;
529
530 if (rud->country_code != 0xB5)
531 return;
532
533 if (rud->data == NULL || rud->size < 2)
534 return;
535
536 gst_byte_reader_init (&br, rud->data, rud->size);
537
538 provider_code = gst_byte_reader_get_uint16_be_unchecked (&br);
539
540 /* There is also 0x29 / 47 for DirecTV, but we don't handle that for now.
541 * https://en.wikipedia.org/wiki/CEA-708#Picture_User_Data */
542 if (provider_code != 0x0031)
543 return;
544
545 /* ANSI/SCTE 128-2010a section 8.1.2 */
546 if (!gst_byte_reader_get_uint32_be (&br, &atsc_user_id))
547 return;
548
549 atsc_user_id = GUINT32_FROM_BE (atsc_user_id);
550 switch (atsc_user_id) {
551 case GST_MAKE_FOURCC ('G', 'A', '9', '4'):{
552 guint8 user_data_type_code, m;
553
554 /* 8.2.1 ATSC1_data() Semantics, Table 15 */
555 if (!gst_byte_reader_get_uint8 (&br, &user_data_type_code))
556 return;
557
558 switch (user_data_type_code) {
559 case 0x03:{
560 guint8 process_cc_data_flag;
561 guint8 cc_count, em_data, b;
562 guint i;
563
564 if (!gst_byte_reader_get_uint8 (&br, &b) || (b & 0x20) != 0)
565 break;
566
567 if (!gst_byte_reader_get_uint8 (&br, &em_data) || em_data != 0xff)
568 break;
569
570 process_cc_data_flag = (b & 0x40) != 0;
571 cc_count = b & 0x1f;
572
573 if (cc_count * 3 > gst_byte_reader_get_remaining (&br))
574 break;
575
576 if (!process_cc_data_flag || cc_count == 0) {
577 gst_byte_reader_skip_unchecked (&br, cc_count * 3);
578 break;
579 }
580
581 /* Shouldn't really happen so let's not go out of our way to handle it */
582 if (h264parse->closedcaptions_size > 0) {
583 GST_WARNING_OBJECT (h264parse, "unused pending closed captions!");
584 GST_MEMDUMP_OBJECT (h264parse, "unused captions being dropped",
585 h264parse->closedcaptions, h264parse->closedcaptions_size);
586 }
587
588 for (i = 0; i < cc_count * 3; i++) {
589 h264parse->closedcaptions[i] =
590 gst_byte_reader_get_uint8_unchecked (&br);
591 }
592 h264parse->closedcaptions_size = cc_count * 3;
593 h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_CEA708_RAW;
594
595 GST_LOG_OBJECT (h264parse, "Extracted closed captions");
596 break;
597 }
598 default:
599 GST_LOG ("Unhandled atsc1 user data type %d", atsc_user_id);
600 break;
601 }
602 if (!gst_byte_reader_get_uint8 (&br, &m) || m != 0xff)
603 GST_WARNING_OBJECT (h264parse, "Marker bits mismatch after ATSC1_data");
604 break;
605 }
606 default:
607 break;
608 }
609 }
610
611 static void
gst_h264_parse_process_sei(GstH264Parse * h264parse,GstH264NalUnit * nalu)612 gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
613 {
614 GstH264SEIMessage sei;
615 GstH264NalParser *nalparser = h264parse->nalparser;
616 GstH264ParserResult pres;
617 GArray *messages;
618 guint i;
619
620 pres = gst_h264_parser_parse_sei (nalparser, nalu, &messages);
621 if (pres != GST_H264_PARSER_OK)
622 GST_WARNING_OBJECT (h264parse, "failed to parse one or more SEI message");
623
624 /* Even if pres != GST_H264_PARSER_OK, some message could have been parsed and
625 * stored in messages.
626 */
627 for (i = 0; i < messages->len; i++) {
628 sei = g_array_index (messages, GstH264SEIMessage, i);
629 switch (sei.payloadType) {
630 case GST_H264_SEI_PIC_TIMING:
631 {
632 guint j;
633 h264parse->sei_pic_struct_pres_flag =
634 sei.payload.pic_timing.pic_struct_present_flag;
635 h264parse->sei_cpb_removal_delay =
636 sei.payload.pic_timing.cpb_removal_delay;
637 if (h264parse->sei_pic_struct_pres_flag) {
638 h264parse->sei_pic_struct = sei.payload.pic_timing.pic_struct;
639 }
640
641 h264parse->num_clock_timestamp = 0;
642
643 for (j = 0; j < 3; j++) {
644 if (sei.payload.pic_timing.clock_timestamp_flag[j]) {
645 memcpy (&h264parse->clock_timestamp[h264parse->
646 num_clock_timestamp++],
647 &sei.payload.pic_timing.clock_timestamp[j],
648 sizeof (GstH264ClockTimestamp));
649 }
650 }
651
652 GST_LOG_OBJECT (h264parse, "pic timing updated");
653 break;
654 }
655 case GST_H264_SEI_REGISTERED_USER_DATA:
656 gst_h264_parse_process_sei_user_data (h264parse,
657 &sei.payload.registered_user_data);
658 break;
659 case GST_H264_SEI_BUF_PERIOD:
660 if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
661 h264parse->dts == GST_CLOCK_TIME_NONE)
662 h264parse->ts_trn_nb = 0;
663 else
664 h264parse->ts_trn_nb = h264parse->dts;
665
666 GST_LOG_OBJECT (h264parse,
667 "new buffering period; ts_trn_nb updated: %" GST_TIME_FORMAT,
668 GST_TIME_ARGS (h264parse->ts_trn_nb));
669 break;
670
671 /* Additional messages that are not innerly useful to the
672 * element but for debugging purposes */
673 case GST_H264_SEI_RECOVERY_POINT:
674 GST_LOG_OBJECT (h264parse, "recovery point found: %u %u %u %u",
675 sei.payload.recovery_point.recovery_frame_cnt,
676 sei.payload.recovery_point.exact_match_flag,
677 sei.payload.recovery_point.broken_link_flag,
678 sei.payload.recovery_point.changing_slice_group_idc);
679 h264parse->keyframe = TRUE;
680 break;
681
682 /* Additional messages that are not innerly useful to the
683 * element but for debugging purposes */
684 case GST_H264_SEI_STEREO_VIDEO_INFO:{
685 GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
686 GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
687
688 GST_LOG_OBJECT (h264parse, "Stereo video information %u %u %u %u %u %u",
689 sei.payload.stereo_video_info.field_views_flag,
690 sei.payload.stereo_video_info.top_field_is_left_view_flag,
691 sei.payload.stereo_video_info.current_frame_is_left_view_flag,
692 sei.payload.stereo_video_info.next_frame_is_second_view_flag,
693 sei.payload.stereo_video_info.left_view_self_contained_flag,
694 sei.payload.stereo_video_info.right_view_self_contained_flag);
695
696 if (sei.payload.stereo_video_info.field_views_flag) {
697 mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
698 if (!sei.payload.stereo_video_info.top_field_is_left_view_flag)
699 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
700 } else {
701 mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
702 if (sei.payload.stereo_video_info.next_frame_is_second_view_flag) {
703 /* Mark current frame as first in bundle */
704 h264parse->first_in_bundle = TRUE;
705 if (!sei.payload.stereo_video_info.current_frame_is_left_view_flag)
706 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
707 }
708 }
709 if (mview_mode != h264parse->multiview_mode ||
710 mview_flags != h264parse->multiview_flags) {
711 h264parse->multiview_mode = mview_mode;
712 h264parse->multiview_flags = mview_flags;
713 /* output caps need to be changed */
714 gst_h264_parse_update_src_caps (h264parse, NULL);
715 }
716 break;
717 }
718 case GST_H264_SEI_FRAME_PACKING:{
719 GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
720 GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
721
722 GST_LOG_OBJECT (h264parse,
723 "frame packing arrangement message: id %u cancelled %u "
724 "type %u quincunx %u content_interpretation %d flip %u "
725 "right_first %u field_views %u is_frame0 %u",
726 sei.payload.frame_packing.frame_packing_id,
727 sei.payload.frame_packing.frame_packing_cancel_flag,
728 sei.payload.frame_packing.frame_packing_type,
729 sei.payload.frame_packing.quincunx_sampling_flag,
730 sei.payload.frame_packing.content_interpretation_type,
731 sei.payload.frame_packing.spatial_flipping_flag,
732 sei.payload.frame_packing.frame0_flipped_flag,
733 sei.payload.frame_packing.field_views_flag,
734 sei.payload.frame_packing.current_frame_is_frame0_flag);
735
736 /* Only IDs from 0->255 and 512->2^31-1 are valid. Ignore others */
737 if ((sei.payload.frame_packing.frame_packing_id >= 256 &&
738 sei.payload.frame_packing.frame_packing_id < 512) ||
739 (sei.payload.frame_packing.frame_packing_id >= (1U << 31)))
740 break; /* ignore */
741
742 if (!sei.payload.frame_packing.frame_packing_cancel_flag) {
743 /* Cancel flag sets things back to no-info */
744
745 if (sei.payload.frame_packing.content_interpretation_type == 2)
746 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
747
748 switch (sei.payload.frame_packing.frame_packing_type) {
749 case 0:
750 mview_mode = GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD;
751 break;
752 case 1:
753 mview_mode = GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED;
754 break;
755 case 2:
756 mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
757 break;
758 case 3:
759 if (sei.payload.frame_packing.quincunx_sampling_flag)
760 mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX;
761 else
762 mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
763 if (sei.payload.frame_packing.spatial_flipping_flag) {
764 /* One of the views is flopped. */
765 if (sei.payload.frame_packing.frame0_flipped_flag !=
766 ! !(mview_flags &
767 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
768 /* the left view is flopped */
769 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED;
770 else
771 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED;
772 }
773 break;
774 case 4:
775 mview_mode = GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM;
776 if (sei.payload.frame_packing.spatial_flipping_flag) {
777 /* One of the views is flipped, */
778 if (sei.payload.frame_packing.frame0_flipped_flag !=
779 ! !(mview_flags &
780 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
781 /* the left view is flipped */
782 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED;
783 else
784 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED;
785 }
786 break;
787 case 5:
788 if (sei.payload.frame_packing.content_interpretation_type == 0)
789 mview_mode = GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME;
790 else
791 mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
792 break;
793 default:
794 GST_DEBUG_OBJECT (h264parse, "Invalid frame packing type %u",
795 sei.payload.frame_packing.frame_packing_type);
796 break;
797 }
798 }
799
800 if (mview_mode != h264parse->multiview_mode ||
801 mview_flags != h264parse->multiview_flags) {
802 h264parse->multiview_mode = mview_mode;
803 h264parse->multiview_flags = mview_flags;
804 /* output caps need to be changed */
805 gst_h264_parse_update_src_caps (h264parse, NULL);
806 }
807 break;
808 }
809 }
810 }
811 g_array_free (messages, TRUE);
812 }
813
814 /* caller guarantees 2 bytes of nal payload */
815 static gboolean
gst_h264_parse_process_nal(GstH264Parse * h264parse,GstH264NalUnit * nalu)816 gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
817 {
818 guint nal_type;
819 GstH264PPS pps = { 0, };
820 GstH264SPS sps = { 0, };
821 GstH264NalParser *nalparser = h264parse->nalparser;
822 GstH264ParserResult pres;
823
824 /* nothing to do for broken input */
825 if (G_UNLIKELY (nalu->size < 2)) {
826 GST_DEBUG_OBJECT (h264parse, "not processing nal size %u", nalu->size);
827 return TRUE;
828 }
829
830 /* we have a peek as well */
831 nal_type = nalu->type;
832
833 GST_DEBUG_OBJECT (h264parse, "processing nal of type %u %s, size %u",
834 nal_type, _nal_name (nal_type), nalu->size);
835
836 switch (nal_type) {
837 case GST_H264_NAL_SUBSET_SPS:
838 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
839 return FALSE;
840 pres = gst_h264_parser_parse_subset_sps (nalparser, nalu, &sps, TRUE);
841 goto process_sps;
842
843 case GST_H264_NAL_SPS:
844 /* reset state, everything else is obsolete */
845 h264parse->state = 0;
846 pres = gst_h264_parser_parse_sps (nalparser, nalu, &sps, TRUE);
847
848 process_sps:
849 /* arranged for a fallback sps.id, so use that one and only warn */
850 if (pres != GST_H264_PARSER_OK) {
851 GST_WARNING_OBJECT (h264parse, "failed to parse SPS:");
852 return FALSE;
853 }
854
855 GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
856 h264parse->update_caps = TRUE;
857 h264parse->have_sps = TRUE;
858 h264parse->have_sps_in_frame = TRUE;
859 if (h264parse->push_codec && h264parse->have_pps) {
860 /* SPS and PPS found in stream before the first pre_push_frame, no need
861 * to forcibly push at start */
862 GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
863 h264parse->push_codec = FALSE;
864 h264parse->have_sps = FALSE;
865 h264parse->have_pps = FALSE;
866 }
867
868 gst_h264_parser_store_nal (h264parse, sps.id, nal_type, nalu);
869 gst_h264_sps_clear (&sps);
870 h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
871 h264parse->header |= TRUE;
872 break;
873 case GST_H264_NAL_PPS:
874 /* expected state: got-sps */
875 h264parse->state &= GST_H264_PARSE_STATE_GOT_SPS;
876 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
877 return FALSE;
878
879 pres = gst_h264_parser_parse_pps (nalparser, nalu, &pps);
880 /* arranged for a fallback pps.id, so use that one and only warn */
881 if (pres != GST_H264_PARSER_OK) {
882 GST_WARNING_OBJECT (h264parse, "failed to parse PPS:");
883 if (pres != GST_H264_PARSER_BROKEN_LINK)
884 return FALSE;
885 }
886
887 /* parameters might have changed, force caps check */
888 if (!h264parse->have_pps) {
889 GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
890 h264parse->update_caps = TRUE;
891 }
892 h264parse->have_pps = TRUE;
893 h264parse->have_pps_in_frame = TRUE;
894 if (h264parse->push_codec && h264parse->have_sps) {
895 /* SPS and PPS found in stream before the first pre_push_frame, no need
896 * to forcibly push at start */
897 GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
898 h264parse->push_codec = FALSE;
899 h264parse->have_sps = FALSE;
900 h264parse->have_pps = FALSE;
901 }
902
903 gst_h264_parser_store_nal (h264parse, pps.id, nal_type, nalu);
904 gst_h264_pps_clear (&pps);
905 h264parse->state |= GST_H264_PARSE_STATE_GOT_PPS;
906 h264parse->header |= TRUE;
907 break;
908 case GST_H264_NAL_SEI:
909 /* expected state: got-sps */
910 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
911 return FALSE;
912
913 h264parse->header |= TRUE;
914 gst_h264_parse_process_sei (h264parse, nalu);
915 /* mark SEI pos */
916 if (h264parse->sei_pos == -1) {
917 if (h264parse->transform)
918 h264parse->sei_pos = gst_adapter_available (h264parse->frame_out);
919 else
920 h264parse->sei_pos = nalu->sc_offset;
921 GST_DEBUG_OBJECT (h264parse, "marking SEI in frame at offset %d",
922 h264parse->sei_pos);
923 }
924 break;
925
926 case GST_H264_NAL_SLICE:
927 case GST_H264_NAL_SLICE_DPA:
928 case GST_H264_NAL_SLICE_DPB:
929 case GST_H264_NAL_SLICE_DPC:
930 case GST_H264_NAL_SLICE_IDR:
931 case GST_H264_NAL_SLICE_EXT:
932 /* expected state: got-sps|got-pps (valid picture headers) */
933 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
934 if (!GST_H264_PARSE_STATE_VALID (h264parse,
935 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
936 return FALSE;
937
938 /* don't need to parse the whole slice (header) here */
939 if (*(nalu->data + nalu->offset + nalu->header_bytes) & 0x80) {
940 /* means first_mb_in_slice == 0 */
941 /* real frame data */
942 GST_DEBUG_OBJECT (h264parse, "first_mb_in_slice = 0");
943 h264parse->frame_start = TRUE;
944 }
945 GST_DEBUG_OBJECT (h264parse, "frame start: %i", h264parse->frame_start);
946 if (nal_type == GST_H264_NAL_SLICE_EXT && !GST_H264_IS_MVC_NALU (nalu))
947 break;
948 {
949 GstH264SliceHdr slice;
950
951 pres = gst_h264_parser_parse_slice_hdr (nalparser, nalu, &slice,
952 FALSE, FALSE);
953 GST_DEBUG_OBJECT (h264parse,
954 "parse result %d, first MB: %u, slice type: %u",
955 pres, slice.first_mb_in_slice, slice.type);
956 if (pres == GST_H264_PARSER_OK) {
957 if (GST_H264_IS_I_SLICE (&slice) || GST_H264_IS_SI_SLICE (&slice))
958 h264parse->keyframe |= TRUE;
959
960 h264parse->state |= GST_H264_PARSE_STATE_GOT_SLICE;
961 h264parse->field_pic_flag = slice.field_pic_flag;
962 }
963 }
964 if (G_LIKELY (nal_type != GST_H264_NAL_SLICE_IDR &&
965 !h264parse->push_codec))
966 break;
967 /* if we need to sneak codec NALs into the stream,
968 * this is a good place, so fake it as IDR
969 * (which should be at start anyway) */
970 /* mark where config needs to go if interval expired */
971 /* mind replacement buffer if applicable */
972 if (h264parse->idr_pos == -1) {
973 if (h264parse->transform)
974 h264parse->idr_pos = gst_adapter_available (h264parse->frame_out);
975 else
976 h264parse->idr_pos = nalu->sc_offset;
977 GST_DEBUG_OBJECT (h264parse, "marking IDR in frame at offset %d",
978 h264parse->idr_pos);
979 }
980 /* if SEI preceeds (faked) IDR, then we have to insert config there */
981 if (h264parse->sei_pos >= 0 && h264parse->idr_pos > h264parse->sei_pos) {
982 h264parse->idr_pos = h264parse->sei_pos;
983 GST_DEBUG_OBJECT (h264parse, "moved IDR mark to SEI position %d",
984 h264parse->idr_pos);
985 }
986 break;
987 case GST_H264_NAL_AU_DELIMITER:
988 /* Just accumulate AU Delimiter, whether it's before SPS or not */
989 pres = gst_h264_parser_parse_nal (nalparser, nalu);
990 if (pres != GST_H264_PARSER_OK)
991 return FALSE;
992 h264parse->aud_insert = FALSE;
993 break;
994 default:
995 /* drop anything before the initial SPS */
996 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
997 return FALSE;
998
999 pres = gst_h264_parser_parse_nal (nalparser, nalu);
1000 if (pres != GST_H264_PARSER_OK)
1001 return FALSE;
1002 break;
1003 }
1004
1005 /* if AVC output needed, collect properly prefixed nal in adapter,
1006 * and use that to replace outgoing buffer data later on */
1007 if (h264parse->transform) {
1008 GstBuffer *buf;
1009
1010 GST_LOG_OBJECT (h264parse, "collecting NAL in AVC frame");
1011 buf = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
1012 nalu->data + nalu->offset, nalu->size);
1013 gst_adapter_push (h264parse->frame_out, buf);
1014 }
1015 return TRUE;
1016 }
1017
1018 /* caller guarantees at least 2 bytes of nal payload for each nal
1019 * returns TRUE if next_nal indicates that nal terminates an AU */
1020 static inline gboolean
gst_h264_parse_collect_nal(GstH264Parse * h264parse,const guint8 * data,guint size,GstH264NalUnit * nalu)1021 gst_h264_parse_collect_nal (GstH264Parse * h264parse, const guint8 * data,
1022 guint size, GstH264NalUnit * nalu)
1023 {
1024 gboolean complete;
1025 GstH264ParserResult parse_res;
1026 GstH264NalUnitType nal_type = nalu->type;
1027 GstH264NalUnit nnalu;
1028
1029 GST_DEBUG_OBJECT (h264parse, "parsing collected nal");
1030 parse_res = gst_h264_parser_identify_nalu_unchecked (h264parse->nalparser,
1031 data, nalu->offset + nalu->size, size, &nnalu);
1032
1033 if (parse_res != GST_H264_PARSER_OK)
1034 return FALSE;
1035
1036 /* determine if AU complete */
1037 GST_LOG_OBJECT (h264parse, "nal type: %d %s", nal_type, _nal_name (nal_type));
1038 /* coded slice NAL starts a picture,
1039 * i.e. other types become aggregated in front of it */
1040 h264parse->picture_start |= (nal_type == GST_H264_NAL_SLICE ||
1041 nal_type == GST_H264_NAL_SLICE_DPA || nal_type == GST_H264_NAL_SLICE_IDR);
1042
1043 /* consider a coded slices (IDR or not) to start a picture,
1044 * (so ending the previous one) if first_mb_in_slice == 0
1045 * (non-0 is part of previous one) */
1046 /* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
1047 * but in practice it works in sane cases, needs not much parsing,
1048 * and also works with broken frame_num in NAL
1049 * (where spec-wise would fail) */
1050 nal_type = nnalu.type;
1051 complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
1052 nal_type <= GST_H264_NAL_AU_DELIMITER) ||
1053 (nal_type >= 14 && nal_type <= 18));
1054
1055 GST_LOG_OBJECT (h264parse, "next nal type: %d %s", nal_type,
1056 _nal_name (nal_type));
1057 complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
1058 || nal_type == GST_H264_NAL_SLICE_DPA
1059 || nal_type == GST_H264_NAL_SLICE_IDR) &&
1060 /* first_mb_in_slice == 0 considered start of frame */
1061 (nnalu.data[nnalu.offset + nnalu.header_bytes] & 0x80);
1062
1063 GST_LOG_OBJECT (h264parse, "au complete: %d", complete);
1064
1065 return complete;
1066 }
1067
1068 static guint8 au_delim[6] = {
1069 0x00, 0x00, 0x00, 0x01, /* nal prefix */
1070 0x09, /* nal unit type = access unit delimiter */
1071 0xf0 /* allow any slice type */
1072 };
1073
1074 static GstFlowReturn
gst_h264_parse_handle_frame_packetized(GstBaseParse * parse,GstBaseParseFrame * frame)1075 gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
1076 GstBaseParseFrame * frame)
1077 {
1078 GstH264Parse *h264parse = GST_H264_PARSE (parse);
1079 GstBuffer *buffer = frame->buffer;
1080 GstFlowReturn ret = GST_FLOW_OK;
1081 GstH264ParserResult parse_res;
1082 GstH264NalUnit nalu;
1083 const guint nl = h264parse->nal_length_size;
1084 GstMapInfo map;
1085 gint left;
1086
1087 if (nl < 1 || nl > 4) {
1088 GST_DEBUG_OBJECT (h264parse, "insufficient data to split input");
1089 return GST_FLOW_NOT_NEGOTIATED;
1090 }
1091
1092 /* need to save buffer from invalidation upon _finish_frame */
1093 if (h264parse->split_packetized)
1094 buffer = gst_buffer_copy (frame->buffer);
1095
1096 gst_buffer_map (buffer, &map, GST_MAP_READ);
1097
1098 left = map.size;
1099
1100 GST_LOG_OBJECT (h264parse,
1101 "processing packet buffer of size %" G_GSIZE_FORMAT, map.size);
1102
1103 parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
1104 map.data, 0, map.size, nl, &nalu);
1105
1106 while (parse_res == GST_H264_PARSER_OK) {
1107 GST_DEBUG_OBJECT (h264parse, "AVC nal offset %d", nalu.offset + nalu.size);
1108
1109 /* either way, have a look at it */
1110 gst_h264_parse_process_nal (h264parse, &nalu);
1111
1112 /* dispatch per NALU if needed */
1113 if (h264parse->split_packetized) {
1114 GstBaseParseFrame tmp_frame;
1115
1116 gst_base_parse_frame_init (&tmp_frame);
1117 tmp_frame.flags |= frame->flags;
1118 tmp_frame.offset = frame->offset;
1119 tmp_frame.overhead = frame->overhead;
1120 tmp_frame.buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
1121 nalu.offset, nalu.size);
1122
1123 /* note we don't need to come up with a sub-buffer, since
1124 * subsequent code only considers input buffer's metadata.
1125 * Real data is either taken from input by baseclass or
1126 * a replacement output buffer is provided anyway. */
1127 gst_h264_parse_parse_frame (parse, &tmp_frame);
1128 ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size);
1129 left -= nl + nalu.size;
1130 }
1131
1132 parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
1133 map.data, nalu.offset + nalu.size, map.size, nl, &nalu);
1134 }
1135
1136 gst_buffer_unmap (buffer, &map);
1137
1138 if (!h264parse->split_packetized) {
1139 gst_h264_parse_parse_frame (parse, frame);
1140 ret = gst_base_parse_finish_frame (parse, frame, map.size);
1141 } else {
1142 gst_buffer_unref (buffer);
1143 if (G_UNLIKELY (left)) {
1144 /* should not be happening for nice AVC */
1145 GST_WARNING_OBJECT (parse, "skipping leftover AVC data %d", left);
1146 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
1147 ret = gst_base_parse_finish_frame (parse, frame, map.size);
1148 }
1149 }
1150
1151 if (parse_res == GST_H264_PARSER_NO_NAL_END ||
1152 parse_res == GST_H264_PARSER_BROKEN_DATA) {
1153
1154 if (h264parse->split_packetized) {
1155 GST_ELEMENT_ERROR (h264parse, STREAM, FAILED, (NULL),
1156 ("invalid AVC input data"));
1157
1158 return GST_FLOW_ERROR;
1159 } else {
1160 /* do not meddle to much in this case */
1161 GST_DEBUG_OBJECT (h264parse, "parsing packet failed");
1162 }
1163 }
1164
1165 return ret;
1166 }
1167
1168 static GstFlowReturn
gst_h264_parse_handle_frame(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize)1169 gst_h264_parse_handle_frame (GstBaseParse * parse,
1170 GstBaseParseFrame * frame, gint * skipsize)
1171 {
1172 GstH264Parse *h264parse = GST_H264_PARSE (parse);
1173 GstBuffer *buffer = frame->buffer;
1174 GstMapInfo map;
1175 guint8 *data;
1176 gsize size;
1177 gint current_off = 0;
1178 gboolean drain, nonext;
1179 GstH264NalParser *nalparser = h264parse->nalparser;
1180 GstH264NalUnit nalu;
1181 GstH264ParserResult pres;
1182 gint framesize;
1183 GstFlowReturn ret;
1184 gboolean au_complete;
1185
1186 if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
1187 GST_BUFFER_FLAG_DISCONT))) {
1188 h264parse->discont = TRUE;
1189 }
1190
1191 /* delegate in packetized case, no skipping should be needed */
1192 if (h264parse->packetized)
1193 return gst_h264_parse_handle_frame_packetized (parse, frame);
1194
1195 gst_buffer_map (buffer, &map, GST_MAP_READ);
1196 data = map.data;
1197 size = map.size;
1198
1199 /* expect at least 3 bytes startcode == sc, and 2 bytes NALU payload */
1200 if (G_UNLIKELY (size < 5)) {
1201 gst_buffer_unmap (buffer, &map);
1202 *skipsize = 1;
1203 return GST_FLOW_OK;
1204 }
1205
1206 /* need to configure aggregation */
1207 if (G_UNLIKELY (h264parse->format == GST_H264_PARSE_FORMAT_NONE))
1208 gst_h264_parse_negotiate (h264parse, GST_H264_PARSE_FORMAT_BYTE, NULL);
1209
1210 /* avoid stale cached parsing state */
1211 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1212 GST_LOG_OBJECT (h264parse, "parsing new frame");
1213 gst_h264_parse_reset_frame (h264parse);
1214 } else {
1215 GST_LOG_OBJECT (h264parse, "resuming frame parsing");
1216 }
1217
1218 /* Always consume the entire input buffer when in_align == ALIGN_AU */
1219 drain = GST_BASE_PARSE_DRAINING (parse)
1220 || h264parse->in_align == GST_H264_PARSE_ALIGN_AU;
1221 nonext = FALSE;
1222
1223 current_off = h264parse->current_off;
1224 if (current_off < 0)
1225 current_off = 0;
1226 g_assert (current_off < size);
1227 GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
1228
1229 /* check for initial skip */
1230 if (h264parse->current_off == -1) {
1231 pres =
1232 gst_h264_parser_identify_nalu_unchecked (nalparser, data, current_off,
1233 size, &nalu);
1234 switch (pres) {
1235 case GST_H264_PARSER_OK:
1236 if (nalu.sc_offset > 0) {
1237 int i;
1238 gboolean is_filler_data = TRUE;
1239 /* Handle filler data */
1240 for (i = 0; i < nalu.sc_offset; i++) {
1241 if (data[i] != 0x00) {
1242 is_filler_data = FALSE;
1243 break;
1244 }
1245 }
1246 if (is_filler_data) {
1247 GST_DEBUG_OBJECT (parse, "Dropping filler data %d", nalu.sc_offset);
1248 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
1249 gst_buffer_unmap (buffer, &map);
1250 ret = gst_base_parse_finish_frame (parse, frame, nalu.sc_offset);
1251 goto drop;
1252 }
1253 *skipsize = nalu.sc_offset;
1254 goto skip;
1255 }
1256 break;
1257 case GST_H264_PARSER_NO_NAL:
1258 *skipsize = size - 3;
1259 goto skip;
1260 break;
1261 default:
1262 /* should not really occur either */
1263 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1264 ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
1265 goto invalid_stream;
1266 }
1267 }
1268
1269 while (TRUE) {
1270 pres =
1271 gst_h264_parser_identify_nalu (nalparser, data, current_off, size,
1272 &nalu);
1273
1274 switch (pres) {
1275 case GST_H264_PARSER_OK:
1276 GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
1277 nalu.offset, nalu.size);
1278 break;
1279 case GST_H264_PARSER_NO_NAL_END:
1280 GST_DEBUG_OBJECT (h264parse, "not a complete nal found at offset %u",
1281 nalu.offset);
1282 /* if draining, accept it as complete nal */
1283 if (drain) {
1284 nonext = TRUE;
1285 nalu.size = size - nalu.offset;
1286 GST_DEBUG_OBJECT (h264parse, "draining, accepting with size %u",
1287 nalu.size);
1288 /* if it's not too short at least */
1289 if (nalu.size < 2)
1290 goto broken;
1291 break;
1292 }
1293 /* otherwise need more */
1294 goto more;
1295 case GST_H264_PARSER_BROKEN_LINK:
1296 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1297 ("Error parsing H.264 stream"),
1298 ("The link to structure needed for the parsing couldn't be found"));
1299 goto invalid_stream;
1300 case GST_H264_PARSER_ERROR:
1301 /* should not really occur either */
1302 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1303 ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
1304 goto invalid_stream;
1305 case GST_H264_PARSER_NO_NAL:
1306 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1307 ("Error parsing H.264 stream"), ("No H.264 NAL unit found"));
1308 goto invalid_stream;
1309 case GST_H264_PARSER_BROKEN_DATA:
1310 GST_WARNING_OBJECT (h264parse, "input stream is corrupt; "
1311 "it contains a NAL unit of length %u", nalu.size);
1312 broken:
1313 /* broken nal at start -> arrange to skip it,
1314 * otherwise have it terminate current au
1315 * (and so it will be skipped on next frame round) */
1316 if (current_off == 0) {
1317 GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
1318 *skipsize = nalu.offset;
1319 h264parse->aud_needed = TRUE;
1320 goto skip;
1321 } else {
1322 GST_DEBUG_OBJECT (h264parse, "terminating au");
1323 nalu.size = 0;
1324 nalu.offset = nalu.sc_offset;
1325 goto end;
1326 }
1327 break;
1328 default:
1329 g_assert_not_reached ();
1330 break;
1331 }
1332
1333 GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
1334 data, nalu.offset, nalu.size);
1335
1336 if (!nonext) {
1337 if (nalu.offset + nalu.size + 4 + 2 > size) {
1338 GST_DEBUG_OBJECT (h264parse, "not enough data for next NALU");
1339 if (drain) {
1340 GST_DEBUG_OBJECT (h264parse, "but draining anyway");
1341 nonext = TRUE;
1342 } else {
1343 goto more;
1344 }
1345 }
1346 }
1347
1348 if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
1349 GST_WARNING_OBJECT (h264parse,
1350 "broken/invalid nal Type: %d %s, Size: %u will be dropped",
1351 nalu.type, _nal_name (nalu.type), nalu.size);
1352 *skipsize = nalu.size;
1353 h264parse->aud_needed = TRUE;
1354 goto skip;
1355 }
1356
1357 /* Judge whether or not to insert AU Delimiter in case of byte-stream
1358 * If we're in the middle of au, we don't need to insert aud.
1359 * Otherwise, we honor the result in gst_h264_parse_process_nal.
1360 * Note that this should be done until draining if it's happening.
1361 */
1362 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL && !h264parse->aud_needed)
1363 h264parse->aud_insert = FALSE;
1364
1365 if (nonext)
1366 break;
1367
1368 /* if no next nal, we know it's complete here */
1369 au_complete = gst_h264_parse_collect_nal (h264parse, data, size, &nalu);
1370
1371 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
1372 h264parse->aud_needed = au_complete;
1373 break;
1374 }
1375
1376 if (au_complete)
1377 break;
1378
1379 GST_DEBUG_OBJECT (h264parse, "Looking for more");
1380 current_off = nalu.offset + nalu.size;
1381 }
1382
1383 end:
1384 framesize = nalu.offset + nalu.size;
1385
1386 gst_buffer_unmap (buffer, &map);
1387
1388 gst_h264_parse_parse_frame (parse, frame);
1389
1390 return gst_base_parse_finish_frame (parse, frame, framesize);
1391
1392 more:
1393 *skipsize = 0;
1394
1395 /* Restart parsing from here next time */
1396 if (current_off > 0)
1397 h264parse->current_off = current_off;
1398
1399 /* Fall-through. */
1400 out:
1401 gst_buffer_unmap (buffer, &map);
1402 return GST_FLOW_OK;
1403
1404 drop:
1405 GST_DEBUG_OBJECT (h264parse, "Dropped data");
1406 return ret;
1407
1408 skip:
1409 GST_DEBUG_OBJECT (h264parse, "skipping %d", *skipsize);
1410 /* If we are collecting access units, we need to preserve the initial
1411 * config headers (SPS, PPS et al.) and only reset the frame if another
1412 * slice NAL was received. This means that broken pictures are discarded */
1413 if (h264parse->align != GST_H264_PARSE_ALIGN_AU ||
1414 !(h264parse->state & GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS) ||
1415 (h264parse->state & GST_H264_PARSE_STATE_GOT_SLICE))
1416 gst_h264_parse_reset_frame (h264parse);
1417 goto out;
1418
1419 invalid_stream:
1420 gst_buffer_unmap (buffer, &map);
1421 return GST_FLOW_ERROR;
1422 }
1423
1424 /* byte together avc codec data based on collected pps and sps so far */
1425 static GstBuffer *
gst_h264_parse_make_codec_data(GstH264Parse * h264parse)1426 gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
1427 {
1428 GstBuffer *buf, *nal;
1429 gint i, sps_size = 0, pps_size = 0, num_sps = 0, num_pps = 0;
1430 guint8 profile_idc = 0, profile_comp = 0, level_idc = 0;
1431 gboolean found = FALSE;
1432 GstMapInfo map;
1433 guint8 *data;
1434 gint nl;
1435
1436 /* only nal payload in stored nals */
1437
1438 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1439 if ((nal = h264parse->sps_nals[i])) {
1440 gsize size = gst_buffer_get_size (nal);
1441 num_sps++;
1442 /* size bytes also count */
1443 sps_size += size + 2;
1444 if (size >= 4) {
1445 guint8 tmp[3];
1446 found = TRUE;
1447 gst_buffer_extract (nal, 1, tmp, 3);
1448 profile_idc = tmp[0];
1449 profile_comp = tmp[1];
1450 level_idc = tmp[2];
1451 }
1452 }
1453 }
1454 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1455 if ((nal = h264parse->pps_nals[i])) {
1456 num_pps++;
1457 /* size bytes also count */
1458 pps_size += gst_buffer_get_size (nal) + 2;
1459 }
1460 }
1461
1462 /* AVC3 has SPS/PPS inside the stream, not in the codec_data */
1463 if (h264parse->format == GST_H264_PARSE_FORMAT_AVC3) {
1464 num_sps = sps_size = 0;
1465 num_pps = pps_size = 0;
1466 }
1467
1468 GST_DEBUG_OBJECT (h264parse,
1469 "constructing codec_data: num_sps=%d, num_pps=%d", num_sps, num_pps);
1470
1471 if (!found || (0 == num_pps
1472 && GST_H264_PARSE_FORMAT_AVC3 != h264parse->format))
1473 return NULL;
1474
1475 buf = gst_buffer_new_allocate (NULL, 5 + 1 + sps_size + 1 + pps_size, NULL);
1476 gst_buffer_map (buf, &map, GST_MAP_WRITE);
1477 data = map.data;
1478 nl = h264parse->nal_length_size;
1479
1480 data[0] = 1; /* AVC Decoder Configuration Record ver. 1 */
1481 data[1] = profile_idc; /* profile_idc */
1482 data[2] = profile_comp; /* profile_compability */
1483 data[3] = level_idc; /* level_idc */
1484 data[4] = 0xfc | (nl - 1); /* nal_length_size_minus1 */
1485 data[5] = 0xe0 | num_sps; /* number of SPSs */
1486
1487 data += 6;
1488 if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
1489 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1490 if ((nal = h264parse->sps_nals[i])) {
1491 gsize nal_size = gst_buffer_get_size (nal);
1492 GST_WRITE_UINT16_BE (data, nal_size);
1493 gst_buffer_extract (nal, 0, data + 2, nal_size);
1494 data += 2 + nal_size;
1495 }
1496 }
1497 }
1498
1499 data[0] = num_pps;
1500 data++;
1501 if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
1502 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1503 if ((nal = h264parse->pps_nals[i])) {
1504 gsize nal_size = gst_buffer_get_size (nal);
1505 GST_WRITE_UINT16_BE (data, nal_size);
1506 gst_buffer_extract (nal, 0, data + 2, nal_size);
1507 data += 2 + nal_size;
1508 }
1509 }
1510 }
1511
1512 gst_buffer_unmap (buf, &map);
1513
1514 return buf;
1515 }
1516
1517 static void
gst_h264_parse_get_par(GstH264Parse * h264parse,gint * num,gint * den)1518 gst_h264_parse_get_par (GstH264Parse * h264parse, gint * num, gint * den)
1519 {
1520 if (h264parse->upstream_par_n != -1 && h264parse->upstream_par_d != -1) {
1521 *num = h264parse->upstream_par_n;
1522 *den = h264parse->upstream_par_d;
1523 } else {
1524 *num = h264parse->parsed_par_n;
1525 *den = h264parse->parsed_par_d;
1526 }
1527 }
1528
1529 static GstCaps *
get_compatible_profile_caps(GstH264SPS * sps)1530 get_compatible_profile_caps (GstH264SPS * sps)
1531 {
1532 GstCaps *caps = NULL;
1533 const gchar **profiles = NULL;
1534 gint i;
1535 GValue compat_profiles = G_VALUE_INIT;
1536 g_value_init (&compat_profiles, GST_TYPE_LIST);
1537
1538 switch (sps->profile_idc) {
1539 case GST_H264_PROFILE_EXTENDED:
1540 if (sps->constraint_set0_flag) { /* A.2.1 */
1541 if (sps->constraint_set1_flag) {
1542 static const gchar *profile_array[] =
1543 { "constrained-baseline", "baseline", "main", "high",
1544 "high-10", "high-4:2:2", "high-4:4:4", NULL
1545 };
1546 profiles = profile_array;
1547 } else {
1548 static const gchar *profile_array[] = { "baseline", NULL };
1549 profiles = profile_array;
1550 }
1551 } else if (sps->constraint_set1_flag) { /* A.2.2 */
1552 static const gchar *profile_array[] =
1553 { "main", "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1554 profiles = profile_array;
1555 }
1556 break;
1557 case GST_H264_PROFILE_BASELINE:
1558 if (sps->constraint_set1_flag) { /* A.2.1 */
1559 static const gchar *profile_array[] =
1560 { "baseline", "main", "high", "high-10", "high-4:2:2",
1561 "high-4:4:4", NULL
1562 };
1563 profiles = profile_array;
1564 } else {
1565 static const gchar *profile_array[] = { "extended", NULL };
1566 profiles = profile_array;
1567 }
1568 break;
1569 case GST_H264_PROFILE_MAIN:
1570 {
1571 static const gchar *profile_array[] =
1572 { "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1573 profiles = profile_array;
1574 }
1575 break;
1576 case GST_H264_PROFILE_HIGH:
1577 if (sps->constraint_set1_flag) {
1578 static const gchar *profile_array[] =
1579 { "main", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1580 profiles = profile_array;
1581 } else {
1582 static const gchar *profile_array[] =
1583 { "high-10", "high-4:2:2", "high-4:4:4", NULL };
1584 profiles = profile_array;
1585 }
1586 break;
1587 case GST_H264_PROFILE_HIGH10:
1588 if (sps->constraint_set1_flag) {
1589 static const gchar *profile_array[] =
1590 { "main", "high", "high-4:2:2", "high-4:4:4", NULL };
1591 profiles = profile_array;
1592 } else {
1593 if (sps->constraint_set3_flag) { /* A.2.8 */
1594 static const gchar *profile_array[] =
1595 { "high-10", "high-4:2:2", "high-4:4:4", "high-4:2:2-intra",
1596 "high-4:4:4-intra", NULL
1597 };
1598 profiles = profile_array;
1599 } else {
1600 static const gchar *profile_array[] =
1601 { "high-4:2:2", "high-4:4:4", NULL };
1602 profiles = profile_array;
1603 }
1604 }
1605 break;
1606 case GST_H264_PROFILE_HIGH_422:
1607 if (sps->constraint_set1_flag) {
1608 static const gchar *profile_array[] =
1609 { "main", "high", "high-10", "high-4:4:4", NULL };
1610 profiles = profile_array;
1611 } else {
1612 if (sps->constraint_set3_flag) { /* A.2.9 */
1613 static const gchar *profile_array[] =
1614 { "high-4:2:2", "high-4:4:4", "high-4:4:4-intra", NULL };
1615 profiles = profile_array;
1616 } else {
1617 static const gchar *profile_array[] = { "high-4:4:4", NULL };
1618 profiles = profile_array;
1619 }
1620 }
1621 break;
1622 case GST_H264_PROFILE_HIGH_444:
1623 if (sps->constraint_set1_flag) {
1624 static const gchar *profile_array[] =
1625 { "main", "high", "high-10", "high-4:2:2", NULL };
1626 profiles = profile_array;
1627 } else if (sps->constraint_set3_flag) { /* A.2.10 */
1628 static const gchar *profile_array[] = { "high-4:4:4", NULL };
1629 profiles = profile_array;
1630 }
1631 break;
1632 case GST_H264_PROFILE_MULTIVIEW_HIGH:
1633 if (sps->extension_type == GST_H264_NAL_EXTENSION_MVC
1634 && sps->extension.mvc.num_views_minus1 == 1) {
1635 static const gchar *profile_array[] =
1636 { "stereo-high", "multiview-high", NULL };
1637 profiles = profile_array;
1638 } else {
1639 static const gchar *profile_array[] = { "multiview-high", NULL };
1640 profiles = profile_array;
1641 }
1642 break;
1643 default:
1644 break;
1645 }
1646
1647 if (profiles) {
1648 GValue value = G_VALUE_INIT;
1649 caps = gst_caps_new_empty_simple ("video/x-h264");
1650 for (i = 0; profiles[i]; i++) {
1651 g_value_init (&value, G_TYPE_STRING);
1652 g_value_set_string (&value, profiles[i]);
1653 gst_value_list_append_value (&compat_profiles, &value);
1654 g_value_unset (&value);
1655 }
1656 gst_caps_set_value (caps, "profile", &compat_profiles);
1657 g_value_unset (&compat_profiles);
1658 }
1659
1660 return caps;
1661 }
1662
1663 /* if downstream didn't support the exact profile indicated in sps header,
1664 * check for the compatible profiles also */
1665 static void
ensure_caps_profile(GstH264Parse * h264parse,GstCaps * caps,GstH264SPS * sps)1666 ensure_caps_profile (GstH264Parse * h264parse, GstCaps * caps, GstH264SPS * sps)
1667 {
1668 GstCaps *peer_caps, *compat_caps;
1669
1670 peer_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
1671 if (!peer_caps || !gst_caps_can_intersect (caps, peer_caps)) {
1672 GstCaps *filter_caps = gst_caps_new_empty_simple ("video/x-h264");
1673
1674 if (peer_caps)
1675 gst_caps_unref (peer_caps);
1676 peer_caps =
1677 gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (h264parse),
1678 filter_caps);
1679
1680 gst_caps_unref (filter_caps);
1681 }
1682
1683 if (peer_caps && !gst_caps_can_intersect (caps, peer_caps)) {
1684 GstStructure *structure;
1685
1686 compat_caps = get_compatible_profile_caps (sps);
1687 if (compat_caps != NULL) {
1688 GstCaps *res_caps = NULL;
1689
1690 res_caps = gst_caps_intersect (peer_caps, compat_caps);
1691
1692 if (res_caps && !gst_caps_is_empty (res_caps)) {
1693 const gchar *profile_str = NULL;
1694
1695 res_caps = gst_caps_fixate (res_caps);
1696 structure = gst_caps_get_structure (res_caps, 0);
1697 profile_str = gst_structure_get_string (structure, "profile");
1698 if (profile_str) {
1699 gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile_str,
1700 NULL);
1701 GST_DEBUG_OBJECT (h264parse,
1702 "Setting compatible profile %s to the caps", profile_str);
1703 }
1704 }
1705 if (res_caps)
1706 gst_caps_unref (res_caps);
1707 gst_caps_unref (compat_caps);
1708 }
1709 }
1710 if (peer_caps)
1711 gst_caps_unref (peer_caps);
1712 }
1713
1714 static const gchar *
digit_to_string(guint digit)1715 digit_to_string (guint digit)
1716 {
1717 static const char itoa[][2] = {
1718 "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
1719 };
1720
1721 if (G_LIKELY (digit < 10))
1722 return itoa[digit];
1723 else
1724 return NULL;
1725 }
1726
1727 static const gchar *
get_profile_string(GstH264SPS * sps)1728 get_profile_string (GstH264SPS * sps)
1729 {
1730 const gchar *profile = NULL;
1731
1732 switch (sps->profile_idc) {
1733 case 66:
1734 if (sps->constraint_set1_flag)
1735 profile = "constrained-baseline";
1736 else
1737 profile = "baseline";
1738 break;
1739 case 77:
1740 profile = "main";
1741 break;
1742 case 88:
1743 profile = "extended";
1744 break;
1745 case 100:
1746 if (sps->constraint_set4_flag) {
1747 if (sps->constraint_set5_flag)
1748 profile = "constrained-high";
1749 else
1750 profile = "progressive-high";
1751 } else
1752 profile = "high";
1753 break;
1754 case 110:
1755 if (sps->constraint_set3_flag)
1756 profile = "high-10-intra";
1757 else if (sps->constraint_set4_flag)
1758 profile = "progressive-high-10";
1759 else
1760 profile = "high-10";
1761 break;
1762 case 122:
1763 if (sps->constraint_set3_flag)
1764 profile = "high-4:2:2-intra";
1765 else
1766 profile = "high-4:2:2";
1767 break;
1768 case 244:
1769 if (sps->constraint_set3_flag)
1770 profile = "high-4:4:4-intra";
1771 else
1772 profile = "high-4:4:4";
1773 break;
1774 case 44:
1775 profile = "cavlc-4:4:4-intra";
1776 break;
1777 case 118:
1778 profile = "multiview-high";
1779 break;
1780 case 128:
1781 profile = "stereo-high";
1782 break;
1783 case 83:
1784 if (sps->constraint_set5_flag)
1785 profile = "scalable-constrained-baseline";
1786 else
1787 profile = "scalable-baseline";
1788 break;
1789 case 86:
1790 if (sps->constraint_set3_flag)
1791 profile = "scalable-high-intra";
1792 else if (sps->constraint_set5_flag)
1793 profile = "scalable-constrained-high";
1794 else
1795 profile = "scalable-high";
1796 break;
1797 default:
1798 return NULL;
1799 }
1800
1801 return profile;
1802 }
1803
1804 static const gchar *
get_level_string(GstH264SPS * sps)1805 get_level_string (GstH264SPS * sps)
1806 {
1807 if (sps->level_idc == 0)
1808 return NULL;
1809 else if ((sps->level_idc == 11 && sps->constraint_set3_flag)
1810 || sps->level_idc == 9)
1811 return "1b";
1812 else if (sps->level_idc % 10 == 0)
1813 return digit_to_string (sps->level_idc / 10);
1814 else {
1815 switch (sps->level_idc) {
1816 case 11:
1817 return "1.1";
1818 case 12:
1819 return "1.2";
1820 case 13:
1821 return "1.3";
1822 case 21:
1823 return "2.1";
1824 case 22:
1825 return "2.2";
1826 case 31:
1827 return "3.1";
1828 case 32:
1829 return "3.2";
1830 case 41:
1831 return "4.1";
1832 case 42:
1833 return "4.2";
1834 case 51:
1835 return "5.1";
1836 case 52:
1837 return "5.2";
1838 default:
1839 return NULL;
1840 }
1841 }
1842 }
1843
1844 static void
gst_h264_parse_update_src_caps(GstH264Parse * h264parse,GstCaps * caps)1845 gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
1846 {
1847 GstH264SPS *sps;
1848 GstCaps *sink_caps, *src_caps;
1849 gboolean modified = FALSE;
1850 GstBuffer *buf = NULL;
1851 GstStructure *s = NULL;
1852
1853 if (G_UNLIKELY (!gst_pad_has_current_caps (GST_BASE_PARSE_SRC_PAD
1854 (h264parse))))
1855 modified = TRUE;
1856 else if (G_UNLIKELY (!h264parse->update_caps))
1857 return;
1858
1859 /* if this is being called from the first _setcaps call, caps on the sinkpad
1860 * aren't set yet and so they need to be passed as an argument */
1861 if (caps)
1862 sink_caps = gst_caps_ref (caps);
1863 else
1864 sink_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (h264parse));
1865
1866 /* carry over input caps as much as possible; override with our own stuff */
1867 if (!sink_caps)
1868 sink_caps = gst_caps_new_empty_simple ("video/x-h264");
1869 else
1870 s = gst_caps_get_structure (sink_caps, 0);
1871
1872 sps = h264parse->nalparser->last_sps;
1873 GST_DEBUG_OBJECT (h264parse, "sps: %p", sps);
1874
1875 /* only codec-data for nice-and-clean au aligned packetized avc format */
1876 if ((h264parse->format == GST_H264_PARSE_FORMAT_AVC
1877 || h264parse->format == GST_H264_PARSE_FORMAT_AVC3)
1878 && h264parse->align == GST_H264_PARSE_ALIGN_AU) {
1879 buf = gst_h264_parse_make_codec_data (h264parse);
1880 if (buf && h264parse->codec_data) {
1881 GstMapInfo map;
1882
1883 gst_buffer_map (buf, &map, GST_MAP_READ);
1884 if (map.size != gst_buffer_get_size (h264parse->codec_data) ||
1885 gst_buffer_memcmp (h264parse->codec_data, 0, map.data, map.size))
1886 modified = TRUE;
1887
1888 gst_buffer_unmap (buf, &map);
1889 } else {
1890 if (!buf && h264parse->codec_data_in)
1891 buf = gst_buffer_ref (h264parse->codec_data_in);
1892 modified = TRUE;
1893 }
1894 }
1895
1896 caps = NULL;
1897 if (G_UNLIKELY (!sps)) {
1898 caps = gst_caps_copy (sink_caps);
1899 } else {
1900 gint crop_width, crop_height;
1901 gint fps_num, fps_den;
1902 gint par_n, par_d;
1903
1904 if (sps->frame_cropping_flag) {
1905 crop_width = sps->crop_rect_width;
1906 crop_height = sps->crop_rect_height;
1907 } else {
1908 crop_width = sps->width;
1909 crop_height = sps->height;
1910 }
1911
1912 if (G_UNLIKELY (h264parse->width != crop_width ||
1913 h264parse->height != crop_height)) {
1914 GST_INFO_OBJECT (h264parse, "resolution changed %dx%d",
1915 crop_width, crop_height);
1916 h264parse->width = crop_width;
1917 h264parse->height = crop_height;
1918 modified = TRUE;
1919 }
1920
1921 /* 0/1 is set as the default in the codec parser, we will set
1922 * it in case we have no info */
1923 gst_h264_video_calculate_framerate (sps, h264parse->field_pic_flag,
1924 h264parse->sei_pic_struct, &fps_num, &fps_den);
1925 if (G_UNLIKELY (h264parse->fps_num != fps_num
1926 || h264parse->fps_den != fps_den)) {
1927 GST_DEBUG_OBJECT (h264parse, "framerate changed %d/%d", fps_num, fps_den);
1928 h264parse->fps_num = fps_num;
1929 h264parse->fps_den = fps_den;
1930 modified = TRUE;
1931 }
1932
1933 if (sps->vui_parameters.aspect_ratio_info_present_flag) {
1934 if (G_UNLIKELY ((h264parse->parsed_par_n != sps->vui_parameters.par_n)
1935 || (h264parse->parsed_par_d != sps->vui_parameters.par_d))) {
1936 h264parse->parsed_par_n = sps->vui_parameters.par_n;
1937 h264parse->parsed_par_d = sps->vui_parameters.par_d;
1938 GST_INFO_OBJECT (h264parse, "pixel aspect ratio has been changed %d/%d",
1939 h264parse->parsed_par_n, h264parse->parsed_par_d);
1940 }
1941 }
1942
1943 if (G_UNLIKELY (modified || h264parse->update_caps)) {
1944 GstVideoInterlaceMode imode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
1945 gint width, height;
1946 GstClockTime latency;
1947
1948 const gchar *caps_mview_mode = NULL;
1949 GstVideoMultiviewMode mview_mode = h264parse->multiview_mode;
1950 GstVideoMultiviewFlags mview_flags = h264parse->multiview_flags;
1951 const gchar *chroma_format = NULL;
1952 guint bit_depth_chroma;
1953
1954 fps_num = h264parse->fps_num;
1955 fps_den = h264parse->fps_den;
1956
1957 caps = gst_caps_copy (sink_caps);
1958
1959 /* sps should give this but upstream overrides */
1960 if (s && gst_structure_has_field (s, "width"))
1961 gst_structure_get_int (s, "width", &width);
1962 else
1963 width = h264parse->width;
1964
1965 if (s && gst_structure_has_field (s, "height"))
1966 gst_structure_get_int (s, "height", &height);
1967 else
1968 height = h264parse->height;
1969
1970 if (s == NULL ||
1971 !gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n,
1972 &par_d)) {
1973 gst_h264_parse_get_par (h264parse, &par_n, &par_d);
1974 if (par_n != 0 && par_d != 0) {
1975 GST_INFO_OBJECT (h264parse, "PAR %d/%d", par_n, par_d);
1976 gst_caps_set_simple (caps, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1977 par_n, par_d, NULL);
1978 } else {
1979 /* Assume par_n/par_d of 1/1 for calcs below, but don't set into caps */
1980 par_n = par_d = 1;
1981 }
1982 }
1983
1984 /* Pass through or set output stereo/multiview config */
1985 if (s && gst_structure_has_field (s, "multiview-mode")) {
1986 caps_mview_mode = gst_structure_get_string (s, "multiview-mode");
1987 gst_structure_get_flagset (s, "multiview-flags",
1988 (guint *) & mview_flags, NULL);
1989 } else if (mview_mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1990 if (gst_video_multiview_guess_half_aspect (mview_mode,
1991 width, height, par_n, par_d)) {
1992 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT;
1993 }
1994
1995 caps_mview_mode = gst_video_multiview_mode_to_caps_string (mview_mode);
1996 gst_caps_set_simple (caps, "multiview-mode", G_TYPE_STRING,
1997 caps_mview_mode, "multiview-flags",
1998 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
1999 GST_FLAG_SET_MASK_EXACT, NULL);
2000 }
2001
2002 gst_caps_set_simple (caps, "width", G_TYPE_INT, width,
2003 "height", G_TYPE_INT, height, NULL);
2004
2005 /* upstream overrides */
2006 if (s && gst_structure_has_field (s, "framerate")) {
2007 gst_structure_get_fraction (s, "framerate", &fps_num, &fps_den);
2008 }
2009
2010 /* but not necessarily or reliably this */
2011 if (fps_den > 0) {
2012 GstStructure *s2;
2013 gst_caps_set_simple (caps, "framerate",
2014 GST_TYPE_FRACTION, fps_num, fps_den, NULL);
2015 s2 = gst_caps_get_structure (caps, 0);
2016 gst_structure_get_fraction (s2, "framerate", &h264parse->parsed_fps_n,
2017 &h264parse->parsed_fps_d);
2018 gst_base_parse_set_frame_rate (GST_BASE_PARSE (h264parse), fps_num,
2019 fps_den, 0, 0);
2020 if (fps_num > 0) {
2021 latency = gst_util_uint64_scale (GST_SECOND, fps_den, fps_num);
2022 gst_base_parse_set_latency (GST_BASE_PARSE (h264parse), latency,
2023 latency);
2024 }
2025
2026 }
2027 if (sps->frame_mbs_only_flag == 0)
2028 imode = GST_VIDEO_INTERLACE_MODE_MIXED;
2029
2030 if (s && !gst_structure_has_field (s, "interlace-mode"))
2031 gst_caps_set_simple (caps, "interlace-mode", G_TYPE_STRING,
2032 gst_video_interlace_mode_to_string (imode), NULL);
2033
2034 bit_depth_chroma = sps->bit_depth_chroma_minus8 + 8;
2035
2036 switch (sps->chroma_format_idc) {
2037 case 0:
2038 chroma_format = "4:0:0";
2039 bit_depth_chroma = 0;
2040 break;
2041 case 1:
2042 chroma_format = "4:2:0";
2043 break;
2044 case 2:
2045 chroma_format = "4:2:2";
2046 break;
2047 case 3:
2048 chroma_format = "4:4:4";
2049 break;
2050 default:
2051 break;
2052 }
2053
2054 if (chroma_format)
2055 gst_caps_set_simple (caps,
2056 "chroma-format", G_TYPE_STRING, chroma_format,
2057 "bit-depth-luma", G_TYPE_UINT, sps->bit_depth_luma_minus8 + 8,
2058 "bit-depth-chroma", G_TYPE_UINT, bit_depth_chroma, NULL);
2059 }
2060 }
2061
2062 if (caps) {
2063 gst_caps_set_simple (caps, "parsed", G_TYPE_BOOLEAN, TRUE,
2064 "stream-format", G_TYPE_STRING,
2065 gst_h264_parse_get_string (h264parse, TRUE, h264parse->format),
2066 "alignment", G_TYPE_STRING,
2067 gst_h264_parse_get_string (h264parse, FALSE, h264parse->align), NULL);
2068
2069 /* set profile and level in caps */
2070 if (sps) {
2071 const gchar *profile, *level;
2072
2073 profile = get_profile_string (sps);
2074 if (profile != NULL)
2075 gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile, NULL);
2076
2077 level = get_level_string (sps);
2078 if (level != NULL)
2079 gst_caps_set_simple (caps, "level", G_TYPE_STRING, level, NULL);
2080
2081 /* relax the profile constraint to find a suitable decoder */
2082 ensure_caps_profile (h264parse, caps, sps);
2083 }
2084
2085 src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
2086
2087 if (src_caps) {
2088 /* use codec data from old caps for comparison; we don't want to resend caps
2089 if everything is same except codec data; */
2090 if (gst_structure_has_field (gst_caps_get_structure (src_caps, 0),
2091 "codec_data")) {
2092 gst_caps_set_value (caps, "codec_data",
2093 gst_structure_get_value (gst_caps_get_structure (src_caps, 0),
2094 "codec_data"));
2095 } else if (!buf) {
2096 GstStructure *s;
2097 /* remove any left-over codec-data hanging around */
2098 s = gst_caps_get_structure (caps, 0);
2099 gst_structure_remove_field (s, "codec_data");
2100 }
2101 }
2102
2103 if (!(src_caps && gst_caps_is_strictly_equal (src_caps, caps))) {
2104 /* update codec data to new value */
2105 if (buf) {
2106 gst_caps_set_simple (caps, "codec_data", GST_TYPE_BUFFER, buf, NULL);
2107 gst_buffer_replace (&h264parse->codec_data, buf);
2108 gst_buffer_unref (buf);
2109 buf = NULL;
2110 } else {
2111 GstStructure *s;
2112 /* remove any left-over codec-data hanging around */
2113 s = gst_caps_get_structure (caps, 0);
2114 gst_structure_remove_field (s, "codec_data");
2115 gst_buffer_replace (&h264parse->codec_data, NULL);
2116 }
2117
2118 gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (h264parse), caps);
2119 }
2120
2121 if (src_caps)
2122 gst_caps_unref (src_caps);
2123 gst_caps_unref (caps);
2124 }
2125
2126 gst_caps_unref (sink_caps);
2127 if (buf)
2128 gst_buffer_unref (buf);
2129 }
2130
2131 static void
gst_h264_parse_get_timestamp(GstH264Parse * h264parse,GstClockTime * out_ts,GstClockTime * out_dur,gboolean frame)2132 gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
2133 GstClockTime * out_ts, GstClockTime * out_dur, gboolean frame)
2134 {
2135 GstH264SPS *sps = h264parse->nalparser->last_sps;
2136 GstClockTime upstream;
2137 gint duration = 1;
2138
2139 g_return_if_fail (out_dur != NULL);
2140 g_return_if_fail (out_ts != NULL);
2141
2142 upstream = *out_ts;
2143 GST_LOG_OBJECT (h264parse, "Upstream ts %" GST_TIME_FORMAT,
2144 GST_TIME_ARGS (upstream));
2145
2146 if (!frame) {
2147 GST_LOG_OBJECT (h264parse, "no frame data -> 0 duration");
2148 *out_dur = 0;
2149 goto exit;
2150 } else {
2151 *out_ts = upstream;
2152 }
2153
2154 if (!sps) {
2155 GST_DEBUG_OBJECT (h264parse, "referred SPS invalid");
2156 goto exit;
2157 } else if (!sps->vui_parameters_present_flag) {
2158 GST_DEBUG_OBJECT (h264parse,
2159 "unable to compute timestamp: VUI not present");
2160 goto exit;
2161 } else if (!sps->vui_parameters.timing_info_present_flag) {
2162 GST_DEBUG_OBJECT (h264parse,
2163 "unable to compute timestamp: timing info not present");
2164 goto exit;
2165 } else if (sps->vui_parameters.time_scale == 0) {
2166 GST_DEBUG_OBJECT (h264parse,
2167 "unable to compute timestamp: time_scale = 0 "
2168 "(this is forbidden in spec; bitstream probably contains error)");
2169 goto exit;
2170 }
2171
2172 if (h264parse->sei_pic_struct_pres_flag &&
2173 h264parse->sei_pic_struct != (guint8) - 1) {
2174 /* Note that when h264parse->sei_pic_struct == -1 (unspecified), there
2175 * are ways to infer its value. This is related to computing the
2176 * TopFieldOrderCnt and BottomFieldOrderCnt, which looks
2177 * complicated and thus not implemented for the time being. Yet
2178 * the value we have here is correct for many applications
2179 */
2180 switch (h264parse->sei_pic_struct) {
2181 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
2182 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
2183 duration = 1;
2184 break;
2185 case GST_H264_SEI_PIC_STRUCT_FRAME:
2186 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2187 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2188 duration = 2;
2189 break;
2190 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2191 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2192 duration = 3;
2193 break;
2194 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2195 duration = 4;
2196 break;
2197 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2198 duration = 6;
2199 break;
2200 default:
2201 GST_DEBUG_OBJECT (h264parse,
2202 "h264parse->sei_pic_struct of unknown value %d. Not parsed",
2203 h264parse->sei_pic_struct);
2204 break;
2205 }
2206 } else {
2207 duration = h264parse->field_pic_flag ? 1 : 2;
2208 }
2209
2210 GST_LOG_OBJECT (h264parse, "frame tick duration %d", duration);
2211
2212 /*
2213 * h264parse.264 C.1.2 Timing of coded picture removal (equivalent to DTS):
2214 * Tr,n(0) = initial_cpb_removal_delay[ SchedSelIdx ] / 90000
2215 * Tr,n(n) = Tr,n(nb) + Tc * cpb_removal_delay(n)
2216 * where
2217 * Tc = num_units_in_tick / time_scale
2218 */
2219
2220 if (h264parse->ts_trn_nb != GST_CLOCK_TIME_NONE) {
2221 GST_LOG_OBJECT (h264parse, "buffering based ts");
2222 /* buffering period is present */
2223 if (upstream != GST_CLOCK_TIME_NONE) {
2224 /* If upstream timestamp is valid, we respect it and adjust current
2225 * reference point */
2226 h264parse->ts_trn_nb = upstream -
2227 (GstClockTime) gst_util_uint64_scale
2228 (h264parse->sei_cpb_removal_delay * GST_SECOND,
2229 sps->vui_parameters.num_units_in_tick,
2230 sps->vui_parameters.time_scale);
2231 } else {
2232 /* If no upstream timestamp is given, we write in new timestamp */
2233 upstream = h264parse->dts = h264parse->ts_trn_nb +
2234 (GstClockTime) gst_util_uint64_scale
2235 (h264parse->sei_cpb_removal_delay * GST_SECOND,
2236 sps->vui_parameters.num_units_in_tick,
2237 sps->vui_parameters.time_scale);
2238 }
2239 } else {
2240 GstClockTime dur;
2241
2242 GST_LOG_OBJECT (h264parse, "duration based ts");
2243 /* naive method: no removal delay specified
2244 * track upstream timestamp and provide best guess frame duration */
2245 dur = gst_util_uint64_scale (duration * GST_SECOND,
2246 sps->vui_parameters.num_units_in_tick, sps->vui_parameters.time_scale);
2247 /* sanity check */
2248 if (dur < GST_MSECOND) {
2249 GST_DEBUG_OBJECT (h264parse, "discarding dur %" GST_TIME_FORMAT,
2250 GST_TIME_ARGS (dur));
2251 } else {
2252 *out_dur = dur;
2253 }
2254 }
2255
2256 exit:
2257 if (GST_CLOCK_TIME_IS_VALID (upstream))
2258 *out_ts = h264parse->dts = upstream;
2259
2260 if (GST_CLOCK_TIME_IS_VALID (*out_dur) &&
2261 GST_CLOCK_TIME_IS_VALID (h264parse->dts))
2262 h264parse->dts += *out_dur;
2263 }
2264
2265 static GstFlowReturn
gst_h264_parse_parse_frame(GstBaseParse * parse,GstBaseParseFrame * frame)2266 gst_h264_parse_parse_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
2267 {
2268 GstH264Parse *h264parse;
2269 GstBuffer *buffer;
2270 guint av;
2271
2272 h264parse = GST_H264_PARSE (parse);
2273 buffer = frame->buffer;
2274
2275 gst_h264_parse_update_src_caps (h264parse, NULL);
2276
2277 /* don't mess with timestamps if provided by upstream,
2278 * particularly since our ts not that good they handle seeking etc */
2279 if (h264parse->do_ts)
2280 gst_h264_parse_get_timestamp (h264parse,
2281 &GST_BUFFER_TIMESTAMP (buffer), &GST_BUFFER_DURATION (buffer),
2282 h264parse->frame_start);
2283
2284 if (h264parse->keyframe)
2285 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
2286 else
2287 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
2288
2289 if (h264parse->header)
2290 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_HEADER);
2291 else
2292 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_HEADER);
2293
2294 if (h264parse->discont) {
2295 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
2296 h264parse->discont = FALSE;
2297 }
2298
2299 /* replace with transformed AVC output if applicable */
2300 av = gst_adapter_available (h264parse->frame_out);
2301 if (av) {
2302 GstBuffer *buf;
2303
2304 buf = gst_adapter_take_buffer (h264parse->frame_out, av);
2305 gst_buffer_copy_into (buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
2306 gst_buffer_replace (&frame->out_buffer, buf);
2307 gst_buffer_unref (buf);
2308 }
2309
2310 return GST_FLOW_OK;
2311 }
2312
2313 /* sends a codec NAL downstream, decorating and transforming as needed.
2314 * No ownership is taken of @nal */
2315 static GstFlowReturn
gst_h264_parse_push_codec_buffer(GstH264Parse * h264parse,GstBuffer * nal,GstClockTime ts)2316 gst_h264_parse_push_codec_buffer (GstH264Parse * h264parse,
2317 GstBuffer * nal, GstClockTime ts)
2318 {
2319 GstMapInfo map;
2320
2321 gst_buffer_map (nal, &map, GST_MAP_READ);
2322 nal = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
2323 map.data, map.size);
2324 gst_buffer_unmap (nal, &map);
2325
2326 GST_BUFFER_TIMESTAMP (nal) = ts;
2327 GST_BUFFER_DURATION (nal) = 0;
2328
2329 return gst_pad_push (GST_BASE_PARSE_SRC_PAD (h264parse), nal);
2330 }
2331
2332 static GstEvent *
check_pending_key_unit_event(GstEvent * pending_event,GstSegment * segment,GstClockTime timestamp,guint flags,GstClockTime pending_key_unit_ts)2333 check_pending_key_unit_event (GstEvent * pending_event,
2334 GstSegment * segment, GstClockTime timestamp, guint flags,
2335 GstClockTime pending_key_unit_ts)
2336 {
2337 GstClockTime running_time, stream_time;
2338 gboolean all_headers;
2339 guint count;
2340 GstEvent *event = NULL;
2341
2342 g_return_val_if_fail (segment != NULL, NULL);
2343
2344 if (pending_event == NULL)
2345 goto out;
2346
2347 if (GST_CLOCK_TIME_IS_VALID (pending_key_unit_ts) &&
2348 timestamp == GST_CLOCK_TIME_NONE)
2349 goto out;
2350
2351 running_time = gst_segment_to_running_time (segment,
2352 GST_FORMAT_TIME, timestamp);
2353
2354 GST_INFO ("now %" GST_TIME_FORMAT " wanted %" GST_TIME_FORMAT,
2355 GST_TIME_ARGS (running_time), GST_TIME_ARGS (pending_key_unit_ts));
2356 if (GST_CLOCK_TIME_IS_VALID (pending_key_unit_ts) &&
2357 running_time < pending_key_unit_ts)
2358 goto out;
2359
2360 if (flags & GST_BUFFER_FLAG_DELTA_UNIT) {
2361 GST_DEBUG ("pending force key unit, waiting for keyframe");
2362 goto out;
2363 }
2364
2365 stream_time = gst_segment_to_stream_time (segment,
2366 GST_FORMAT_TIME, timestamp);
2367
2368 if (!gst_video_event_parse_upstream_force_key_unit (pending_event,
2369 NULL, &all_headers, &count)) {
2370 gst_video_event_parse_downstream_force_key_unit (pending_event, NULL,
2371 NULL, NULL, &all_headers, &count);
2372 }
2373
2374 event =
2375 gst_video_event_new_downstream_force_key_unit (timestamp, stream_time,
2376 running_time, all_headers, count);
2377 gst_event_set_seqnum (event, gst_event_get_seqnum (pending_event));
2378
2379 out:
2380 return event;
2381 }
2382
2383 static void
gst_h264_parse_prepare_key_unit(GstH264Parse * parse,GstEvent * event)2384 gst_h264_parse_prepare_key_unit (GstH264Parse * parse, GstEvent * event)
2385 {
2386 GstClockTime running_time;
2387 guint count;
2388 #ifndef GST_DISABLE_GST_DEBUG
2389 gboolean have_sps, have_pps;
2390 gint i;
2391 #endif
2392
2393 parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
2394 gst_event_replace (&parse->force_key_unit_event, NULL);
2395
2396 gst_video_event_parse_downstream_force_key_unit (event,
2397 NULL, NULL, &running_time, NULL, &count);
2398
2399 GST_INFO_OBJECT (parse, "pushing downstream force-key-unit event %d "
2400 "%" GST_TIME_FORMAT " count %d", gst_event_get_seqnum (event),
2401 GST_TIME_ARGS (running_time), count);
2402 gst_pad_push_event (GST_BASE_PARSE_SRC_PAD (parse), event);
2403
2404 #ifndef GST_DISABLE_GST_DEBUG
2405 have_sps = have_pps = FALSE;
2406 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2407 if (parse->sps_nals[i] != NULL) {
2408 have_sps = TRUE;
2409 break;
2410 }
2411 }
2412 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2413 if (parse->pps_nals[i] != NULL) {
2414 have_pps = TRUE;
2415 break;
2416 }
2417 }
2418
2419 GST_INFO_OBJECT (parse, "preparing key unit, have sps %d have pps %d",
2420 have_sps, have_pps);
2421 #endif
2422
2423 /* set push_codec to TRUE so that pre_push_frame sends SPS/PPS again */
2424 parse->push_codec = TRUE;
2425 }
2426
2427 static gboolean
gst_h264_parse_handle_sps_pps_nals(GstH264Parse * h264parse,GstBuffer * buffer,GstBaseParseFrame * frame)2428 gst_h264_parse_handle_sps_pps_nals (GstH264Parse * h264parse,
2429 GstBuffer * buffer, GstBaseParseFrame * frame)
2430 {
2431 GstBuffer *codec_nal;
2432 gint i;
2433 gboolean send_done = FALSE;
2434 GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
2435
2436 if (h264parse->have_sps_in_frame && h264parse->have_pps_in_frame) {
2437 GST_DEBUG_OBJECT (h264parse, "SPS/PPS exist in frame, will not insert");
2438 return TRUE;
2439 }
2440
2441 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
2442 /* send separate config NAL buffers */
2443 GST_DEBUG_OBJECT (h264parse, "- sending SPS/PPS");
2444 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2445 if ((codec_nal = h264parse->sps_nals[i])) {
2446 GST_DEBUG_OBJECT (h264parse, "sending SPS nal");
2447 gst_h264_parse_push_codec_buffer (h264parse, codec_nal, timestamp);
2448 send_done = TRUE;
2449 }
2450 }
2451 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2452 if ((codec_nal = h264parse->pps_nals[i])) {
2453 GST_DEBUG_OBJECT (h264parse, "sending PPS nal");
2454 gst_h264_parse_push_codec_buffer (h264parse, codec_nal, timestamp);
2455 send_done = TRUE;
2456 }
2457 }
2458 } else {
2459 /* insert config NALs into AU */
2460 GstByteWriter bw;
2461 GstBuffer *new_buf;
2462 const gboolean bs = h264parse->format == GST_H264_PARSE_FORMAT_BYTE;
2463 const gint nls = 4 - h264parse->nal_length_size;
2464 gboolean ok;
2465
2466 gst_byte_writer_init_with_size (&bw, gst_buffer_get_size (buffer), FALSE);
2467 ok = gst_byte_writer_put_buffer (&bw, buffer, 0, h264parse->idr_pos);
2468 GST_DEBUG_OBJECT (h264parse, "- inserting SPS/PPS");
2469 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2470 if ((codec_nal = h264parse->sps_nals[i])) {
2471 gsize nal_size = gst_buffer_get_size (codec_nal);
2472 GST_DEBUG_OBJECT (h264parse, "inserting SPS nal");
2473 if (bs) {
2474 ok &= gst_byte_writer_put_uint32_be (&bw, 1);
2475 } else {
2476 ok &= gst_byte_writer_put_uint32_be (&bw, (nal_size << (nls * 8)));
2477 ok &= gst_byte_writer_set_pos (&bw,
2478 gst_byte_writer_get_pos (&bw) - nls);
2479 }
2480
2481 ok &= gst_byte_writer_put_buffer (&bw, codec_nal, 0, nal_size);
2482 send_done = TRUE;
2483 }
2484 }
2485 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2486 if ((codec_nal = h264parse->pps_nals[i])) {
2487 gsize nal_size = gst_buffer_get_size (codec_nal);
2488 GST_DEBUG_OBJECT (h264parse, "inserting PPS nal");
2489 if (bs) {
2490 ok &= gst_byte_writer_put_uint32_be (&bw, 1);
2491 } else {
2492 ok &= gst_byte_writer_put_uint32_be (&bw, (nal_size << (nls * 8)));
2493 ok &= gst_byte_writer_set_pos (&bw,
2494 gst_byte_writer_get_pos (&bw) - nls);
2495 }
2496 ok &= gst_byte_writer_put_buffer (&bw, codec_nal, 0, nal_size);
2497 send_done = TRUE;
2498 }
2499 }
2500 ok &= gst_byte_writer_put_buffer (&bw, buffer, h264parse->idr_pos, -1);
2501 /* collect result and push */
2502 new_buf = gst_byte_writer_reset_and_get_buffer (&bw);
2503 gst_buffer_copy_into (new_buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
2504 /* should already be keyframe/IDR, but it may not have been,
2505 * so mark it as such to avoid being discarded by picky decoder */
2506 GST_BUFFER_FLAG_UNSET (new_buf, GST_BUFFER_FLAG_DELTA_UNIT);
2507 gst_buffer_replace (&frame->out_buffer, new_buf);
2508 gst_buffer_unref (new_buf);
2509 /* some result checking seems to make some compilers happy */
2510 if (G_UNLIKELY (!ok)) {
2511 GST_ERROR_OBJECT (h264parse, "failed to insert SPS/PPS");
2512 }
2513 }
2514
2515 return send_done;
2516 }
2517
2518 static GstFlowReturn
gst_h264_parse_pre_push_frame(GstBaseParse * parse,GstBaseParseFrame * frame)2519 gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
2520 {
2521 GstH264Parse *h264parse;
2522 GstBuffer *buffer;
2523 GstEvent *event;
2524
2525 h264parse = GST_H264_PARSE (parse);
2526
2527 if (!h264parse->sent_codec_tag) {
2528 GstTagList *taglist;
2529 GstCaps *caps;
2530
2531 /* codec tag */
2532 caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (parse));
2533 if (caps == NULL) {
2534 if (GST_PAD_IS_FLUSHING (GST_BASE_PARSE_SRC_PAD (h264parse))) {
2535 GST_INFO_OBJECT (h264parse, "Src pad is flushing");
2536 return GST_FLOW_FLUSHING;
2537 } else {
2538 GST_INFO_OBJECT (h264parse, "Src pad is not negotiated!");
2539 return GST_FLOW_NOT_NEGOTIATED;
2540 }
2541 }
2542
2543 taglist = gst_tag_list_new_empty ();
2544 gst_pb_utils_add_codec_description_to_tag_list (taglist,
2545 GST_TAG_VIDEO_CODEC, caps);
2546 gst_caps_unref (caps);
2547
2548 gst_base_parse_merge_tags (parse, taglist, GST_TAG_MERGE_REPLACE);
2549 gst_tag_list_unref (taglist);
2550
2551 /* also signals the end of first-frame processing */
2552 h264parse->sent_codec_tag = TRUE;
2553 }
2554
2555 /* In case of byte-stream, insert au delimeter by default
2556 * if it doesn't exist */
2557 if (h264parse->aud_insert && h264parse->format == GST_H264_PARSE_FORMAT_BYTE) {
2558 if (h264parse->align == GST_H264_PARSE_ALIGN_AU) {
2559 GstMemory *mem =
2560 gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, (guint8 *) au_delim,
2561 sizeof (au_delim), 0, sizeof (au_delim), NULL, NULL);
2562
2563 frame->out_buffer = gst_buffer_copy (frame->buffer);
2564 gst_buffer_prepend_memory (frame->out_buffer, mem);
2565 if (h264parse->idr_pos >= 0)
2566 h264parse->idr_pos += sizeof (au_delim);
2567
2568 buffer = frame->out_buffer;
2569 } else {
2570 GstBuffer *aud_buffer = gst_buffer_new_allocate (NULL, 2, NULL);
2571 gst_buffer_fill (aud_buffer, 0, (guint8 *) (au_delim + 4), 2);
2572
2573 buffer = frame->buffer;
2574 gst_h264_parse_push_codec_buffer (h264parse, aud_buffer,
2575 GST_BUFFER_TIMESTAMP (buffer));
2576 gst_buffer_unref (aud_buffer);
2577 }
2578 } else {
2579 buffer = frame->buffer;
2580 }
2581
2582 if (h264parse->closedcaptions_size > 0) {
2583 gst_buffer_add_video_caption_meta (buffer,
2584 h264parse->closedcaptions_type, h264parse->closedcaptions,
2585 h264parse->closedcaptions_size);
2586
2587 h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
2588 h264parse->closedcaptions_size = 0;
2589 }
2590
2591 if ((event = check_pending_key_unit_event (h264parse->force_key_unit_event,
2592 &parse->segment, GST_BUFFER_TIMESTAMP (buffer),
2593 GST_BUFFER_FLAGS (buffer), h264parse->pending_key_unit_ts))) {
2594 gst_h264_parse_prepare_key_unit (h264parse, event);
2595 }
2596
2597 /* periodic SPS/PPS sending */
2598 if (h264parse->interval > 0 || h264parse->push_codec) {
2599 GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
2600 guint64 diff;
2601 gboolean initial_frame = FALSE;
2602
2603 /* init */
2604 if (!GST_CLOCK_TIME_IS_VALID (h264parse->last_report)) {
2605 h264parse->last_report = timestamp;
2606 initial_frame = TRUE;
2607 }
2608
2609 if (h264parse->idr_pos >= 0) {
2610 GST_LOG_OBJECT (h264parse, "IDR nal at offset %d", h264parse->idr_pos);
2611
2612 if (timestamp > h264parse->last_report)
2613 diff = timestamp - h264parse->last_report;
2614 else
2615 diff = 0;
2616
2617 GST_LOG_OBJECT (h264parse,
2618 "now %" GST_TIME_FORMAT ", last SPS/PPS %" GST_TIME_FORMAT,
2619 GST_TIME_ARGS (timestamp), GST_TIME_ARGS (h264parse->last_report));
2620
2621 GST_DEBUG_OBJECT (h264parse,
2622 "interval since last SPS/PPS %" GST_TIME_FORMAT,
2623 GST_TIME_ARGS (diff));
2624
2625 if (GST_TIME_AS_SECONDS (diff) >= h264parse->interval ||
2626 initial_frame || h264parse->push_codec) {
2627 GstClockTime new_ts;
2628
2629 /* avoid overwriting a perfectly fine timestamp */
2630 new_ts = GST_CLOCK_TIME_IS_VALID (timestamp) ? timestamp :
2631 h264parse->last_report;
2632
2633 if (gst_h264_parse_handle_sps_pps_nals (h264parse, buffer, frame)) {
2634 h264parse->last_report = new_ts;
2635 }
2636 }
2637 /* we pushed whatever we had */
2638 h264parse->push_codec = FALSE;
2639 h264parse->have_sps = FALSE;
2640 h264parse->have_pps = FALSE;
2641 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
2642 }
2643 } else if (h264parse->interval == -1) {
2644 if (h264parse->idr_pos >= 0) {
2645 GST_LOG_OBJECT (h264parse, "IDR nal at offset %d", h264parse->idr_pos);
2646
2647 gst_h264_parse_handle_sps_pps_nals (h264parse, buffer, frame);
2648
2649 /* we pushed whatever we had */
2650 h264parse->push_codec = FALSE;
2651 h264parse->have_sps = FALSE;
2652 h264parse->have_pps = FALSE;
2653 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
2654 }
2655 }
2656
2657 /* Fixme: setting passthrough mode casuing multiple issues:
2658 * For nal aligned multiresoluton streams, passthrough mode make h264parse
2659 * unable to advertise the new resoultions. Also causing issues while
2660 * parsing MVC streams when it has two layers.
2661 * Disabing passthourgh mode for now */
2662 #if 0
2663 /* If SPS/PPS and a keyframe have been parsed, and we're not converting,
2664 * we might switch to passthrough mode now on the basis that we've seen
2665 * the SEI packets and know optional caps params (such as multiview).
2666 * This is an efficiency optimisation that relies on stream properties
2667 * remaining uniform in practice. */
2668 if (h264parse->can_passthrough) {
2669 if (h264parse->keyframe && h264parse->have_sps && h264parse->have_pps) {
2670 GST_LOG_OBJECT (parse, "Switching to passthrough mode");
2671 gst_base_parse_set_passthrough (parse, TRUE);
2672 }
2673 }
2674 #endif
2675
2676 {
2677 guint i = 0;
2678
2679 for (i = 0; i < h264parse->num_clock_timestamp; i++) {
2680 GstH264ClockTimestamp *tim = &h264parse->clock_timestamp[i];
2681 GstVideoTimeCodeFlags flags = 0;
2682 gint field_count = -1;
2683 guint n_frames;
2684
2685 /* Table D-1 */
2686 switch (h264parse->sei_pic_struct) {
2687 case GST_H264_SEI_PIC_STRUCT_FRAME:
2688 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
2689 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
2690 field_count = h264parse->sei_pic_struct;
2691 break;
2692 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2693 field_count = i + 1;
2694 break;
2695 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2696 field_count = 2 - i;
2697 break;
2698 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2699 field_count = i % 2 ? 2 : 1;
2700 break;
2701 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2702 field_count = i % 2 ? 1 : 2;
2703 break;
2704 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2705 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2706 field_count = 0;
2707 break;
2708 }
2709
2710 if (field_count == -1) {
2711 GST_WARNING_OBJECT (parse,
2712 "failed to determine field count for timecode");
2713 field_count = 0;
2714 }
2715
2716 /* dropping of the two lowest (value 0 and 1) n_frames
2717 * counts when seconds_value is equal to 0 and
2718 * minutes_value is not an integer multiple of 10 */
2719 if (tim->counting_type == 4)
2720 flags |= GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME;
2721
2722 if (tim->ct_type == GST_H264_CT_TYPE_INTERLACED)
2723 flags |= GST_VIDEO_TIME_CODE_FLAGS_INTERLACED;
2724
2725 n_frames =
2726 gst_util_uint64_scale_int (tim->n_frames, 1,
2727 2 - tim->nuit_field_based_flag);
2728
2729 gst_buffer_add_video_time_code_meta_full (buffer,
2730 h264parse->parsed_fps_n,
2731 h264parse->parsed_fps_d,
2732 NULL,
2733 flags,
2734 tim->hours_flag ? tim->hours_value : 0,
2735 tim->minutes_flag ? tim->minutes_value : 0,
2736 tim->seconds_flag ? tim->seconds_value : 0, n_frames, field_count);
2737 }
2738
2739 h264parse->num_clock_timestamp = 0;
2740 }
2741
2742 gst_h264_parse_reset_frame (h264parse);
2743
2744 return GST_FLOW_OK;
2745 }
2746
2747 static gboolean
gst_h264_parse_set_caps(GstBaseParse * parse,GstCaps * caps)2748 gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps)
2749 {
2750 GstH264Parse *h264parse;
2751 GstStructure *str;
2752 const GValue *codec_data_value;
2753 GstBuffer *codec_data = NULL;
2754 gsize size;
2755 guint format, align, off;
2756 GstH264NalUnit nalu;
2757 GstH264ParserResult parseres;
2758 GstCaps *old_caps;
2759
2760 h264parse = GST_H264_PARSE (parse);
2761
2762 /* reset */
2763 h264parse->push_codec = FALSE;
2764
2765 old_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (parse));
2766 if (old_caps) {
2767 if (!gst_caps_is_equal (old_caps, caps))
2768 gst_h264_parse_reset_stream_info (h264parse);
2769 gst_caps_unref (old_caps);
2770 }
2771
2772 str = gst_caps_get_structure (caps, 0);
2773
2774 /* accept upstream info if provided */
2775 gst_structure_get_int (str, "width", &h264parse->width);
2776 gst_structure_get_int (str, "height", &h264parse->height);
2777 gst_structure_get_fraction (str, "framerate", &h264parse->fps_num,
2778 &h264parse->fps_den);
2779 gst_structure_get_fraction (str, "pixel-aspect-ratio",
2780 &h264parse->upstream_par_n, &h264parse->upstream_par_d);
2781
2782 /* get upstream format and align from caps */
2783 gst_h264_parse_format_from_caps (caps, &format, &align);
2784
2785 codec_data_value = gst_structure_get_value (str, "codec_data");
2786
2787 /* fix up caps without stream-format for max. backwards compatibility */
2788 if (format == GST_H264_PARSE_FORMAT_NONE) {
2789 /* codec_data implies avc */
2790 if (codec_data_value != NULL) {
2791 GST_ERROR ("video/x-h264 caps with codec_data but no stream-format=avc");
2792 format = GST_H264_PARSE_FORMAT_AVC;
2793 } else {
2794 /* otherwise assume bytestream input */
2795 GST_ERROR ("video/x-h264 caps without codec_data or stream-format");
2796 format = GST_H264_PARSE_FORMAT_BYTE;
2797 }
2798 }
2799
2800 /* avc caps sanity checks */
2801 if (format == GST_H264_PARSE_FORMAT_AVC) {
2802 /* AVC requires codec_data, AVC3 might have one and/or SPS/PPS inline */
2803 if (codec_data_value == NULL)
2804 goto avc_caps_codec_data_missing;
2805
2806 /* AVC implies alignment=au, everything else is not allowed */
2807 if (align == GST_H264_PARSE_ALIGN_NONE)
2808 align = GST_H264_PARSE_ALIGN_AU;
2809 else if (align != GST_H264_PARSE_ALIGN_AU)
2810 goto avc_caps_wrong_alignment;
2811 }
2812
2813 /* bytestream caps sanity checks */
2814 if (format == GST_H264_PARSE_FORMAT_BYTE) {
2815 /* should have SPS/PSS in-band (and/or oob in streamheader field) */
2816 if (codec_data_value != NULL)
2817 goto bytestream_caps_with_codec_data;
2818 }
2819
2820 /* packetized video has codec_data (required for AVC, optional for AVC3) */
2821 if (codec_data_value != NULL) {
2822 GstMapInfo map;
2823 guint8 *data;
2824 guint num_sps, num_pps;
2825 #ifndef GST_DISABLE_GST_DEBUG
2826 guint profile;
2827 #endif
2828 gint i;
2829
2830 GST_DEBUG_OBJECT (h264parse, "have packetized h264");
2831 /* make note for optional split processing */
2832 h264parse->packetized = TRUE;
2833
2834 /* codec_data field should hold a buffer */
2835 if (!GST_VALUE_HOLDS_BUFFER (codec_data_value))
2836 goto avc_caps_codec_data_wrong_type;
2837
2838 codec_data = gst_value_get_buffer (codec_data_value);
2839 if (!codec_data)
2840 goto avc_caps_codec_data_missing;
2841 gst_buffer_map (codec_data, &map, GST_MAP_READ);
2842 data = map.data;
2843 size = map.size;
2844
2845 /* parse the avcC data */
2846 if (size < 7) { /* when numSPS==0 and numPPS==0, length is 7 bytes */
2847 gst_buffer_unmap (codec_data, &map);
2848 goto avcc_too_small;
2849 }
2850 /* parse the version, this must be 1 */
2851 if (data[0] != 1) {
2852 gst_buffer_unmap (codec_data, &map);
2853 goto wrong_version;
2854 }
2855 #ifndef GST_DISABLE_GST_DEBUG
2856 /* AVCProfileIndication */
2857 /* profile_compat */
2858 /* AVCLevelIndication */
2859 profile = (data[1] << 16) | (data[2] << 8) | data[3];
2860 GST_DEBUG_OBJECT (h264parse, "profile %06x", profile);
2861 #endif
2862
2863 /* 6 bits reserved | 2 bits lengthSizeMinusOne */
2864 /* this is the number of bytes in front of the NAL units to mark their
2865 * length */
2866 h264parse->nal_length_size = (data[4] & 0x03) + 1;
2867 GST_DEBUG_OBJECT (h264parse, "nal length size %u",
2868 h264parse->nal_length_size);
2869
2870 num_sps = data[5] & 0x1f;
2871 off = 6;
2872 for (i = 0; i < num_sps; i++) {
2873 parseres = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
2874 data, off, size, 2, &nalu);
2875 if (parseres != GST_H264_PARSER_OK) {
2876 gst_buffer_unmap (codec_data, &map);
2877 goto avcc_too_small;
2878 }
2879
2880 gst_h264_parse_process_nal (h264parse, &nalu);
2881 off = nalu.offset + nalu.size;
2882 }
2883
2884 if (off >= size) {
2885 gst_buffer_unmap (codec_data, &map);
2886 goto avcc_too_small;
2887 }
2888 num_pps = data[off];
2889 off++;
2890
2891 for (i = 0; i < num_pps; i++) {
2892 parseres = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
2893 data, off, size, 2, &nalu);
2894 if (parseres != GST_H264_PARSER_OK) {
2895 gst_buffer_unmap (codec_data, &map);
2896 goto avcc_too_small;
2897 }
2898
2899 gst_h264_parse_process_nal (h264parse, &nalu);
2900 off = nalu.offset + nalu.size;
2901 }
2902
2903 gst_buffer_unmap (codec_data, &map);
2904
2905 gst_buffer_replace (&h264parse->codec_data_in, codec_data);
2906
2907 /* don't confuse codec_data with inband sps/pps */
2908 h264parse->have_sps_in_frame = FALSE;
2909 h264parse->have_pps_in_frame = FALSE;
2910 } else if (format == GST_H264_PARSE_FORMAT_BYTE) {
2911 GST_DEBUG_OBJECT (h264parse, "have bytestream h264");
2912 /* nothing to pre-process */
2913 h264parse->packetized = FALSE;
2914 /* we have 4 sync bytes */
2915 h264parse->nal_length_size = 4;
2916 } else {
2917 /* probably AVC3 without codec_data field, anything to do here? */
2918 }
2919
2920 {
2921 GstCaps *in_caps;
2922
2923 /* prefer input type determined above */
2924 in_caps = gst_caps_new_simple ("video/x-h264",
2925 "parsed", G_TYPE_BOOLEAN, TRUE,
2926 "stream-format", G_TYPE_STRING,
2927 gst_h264_parse_get_string (h264parse, TRUE, format),
2928 "alignment", G_TYPE_STRING,
2929 gst_h264_parse_get_string (h264parse, FALSE, align), NULL);
2930 /* negotiate with downstream, sets ->format and ->align */
2931 gst_h264_parse_negotiate (h264parse, format, in_caps);
2932 gst_caps_unref (in_caps);
2933 }
2934
2935 if (format == h264parse->format && align == h264parse->align) {
2936 /* we did parse codec-data and might supplement src caps */
2937 gst_h264_parse_update_src_caps (h264parse, caps);
2938 } else if (format == GST_H264_PARSE_FORMAT_AVC
2939 || format == GST_H264_PARSE_FORMAT_AVC3) {
2940 /* if input != output, and input is avc, must split before anything else */
2941 /* arrange to insert codec-data in-stream if needed.
2942 * src caps are only arranged for later on */
2943 h264parse->push_codec = TRUE;
2944 h264parse->have_sps = FALSE;
2945 h264parse->have_pps = FALSE;
2946 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL)
2947 h264parse->split_packetized = TRUE;
2948 h264parse->packetized = TRUE;
2949 }
2950
2951 h264parse->in_align = align;
2952
2953 return TRUE;
2954
2955 /* ERRORS */
2956 avc_caps_codec_data_wrong_type:
2957 {
2958 GST_WARNING_OBJECT (parse, "H.264 AVC caps, codec_data field not a buffer");
2959 goto refuse_caps;
2960 }
2961 avc_caps_codec_data_missing:
2962 {
2963 GST_WARNING_OBJECT (parse, "H.264 AVC caps, but no codec_data");
2964 goto refuse_caps;
2965 }
2966 avc_caps_wrong_alignment:
2967 {
2968 GST_WARNING_OBJECT (parse, "H.264 AVC caps with NAL alignment, must be AU");
2969 goto refuse_caps;
2970 }
2971 bytestream_caps_with_codec_data:
2972 {
2973 GST_WARNING_OBJECT (parse, "H.264 bytestream caps with codec_data is not "
2974 "expected, send SPS/PPS in-band with data or in streamheader field");
2975 goto refuse_caps;
2976 }
2977 avcc_too_small:
2978 {
2979 GST_DEBUG_OBJECT (h264parse, "avcC size %" G_GSIZE_FORMAT " < 8", size);
2980 goto refuse_caps;
2981 }
2982 wrong_version:
2983 {
2984 GST_DEBUG_OBJECT (h264parse, "wrong avcC version");
2985 goto refuse_caps;
2986 }
2987 refuse_caps:
2988 {
2989 GST_WARNING_OBJECT (h264parse, "refused caps %" GST_PTR_FORMAT, caps);
2990 return FALSE;
2991 }
2992 }
2993
2994 static void
remove_fields(GstCaps * caps,gboolean all)2995 remove_fields (GstCaps * caps, gboolean all)
2996 {
2997 guint i, n;
2998
2999 n = gst_caps_get_size (caps);
3000 for (i = 0; i < n; i++) {
3001 GstStructure *s = gst_caps_get_structure (caps, i);
3002
3003 if (all) {
3004 gst_structure_remove_field (s, "alignment");
3005 gst_structure_remove_field (s, "stream-format");
3006 }
3007 gst_structure_remove_field (s, "parsed");
3008 }
3009 }
3010
3011 static GstCaps *
gst_h264_parse_get_caps(GstBaseParse * parse,GstCaps * filter)3012 gst_h264_parse_get_caps (GstBaseParse * parse, GstCaps * filter)
3013 {
3014 GstCaps *peercaps, *templ;
3015 GstCaps *res, *tmp, *pcopy;
3016
3017 templ = gst_pad_get_pad_template_caps (GST_BASE_PARSE_SINK_PAD (parse));
3018 if (filter) {
3019 GstCaps *fcopy = gst_caps_copy (filter);
3020 /* Remove the fields we convert */
3021 remove_fields (fcopy, TRUE);
3022 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), fcopy);
3023 gst_caps_unref (fcopy);
3024 } else
3025 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), NULL);
3026
3027 pcopy = gst_caps_copy (peercaps);
3028 remove_fields (pcopy, TRUE);
3029
3030 res = gst_caps_intersect_full (pcopy, templ, GST_CAPS_INTERSECT_FIRST);
3031 gst_caps_unref (pcopy);
3032 gst_caps_unref (templ);
3033
3034 if (filter) {
3035 GstCaps *tmp = gst_caps_intersect_full (res, filter,
3036 GST_CAPS_INTERSECT_FIRST);
3037 gst_caps_unref (res);
3038 res = tmp;
3039 }
3040
3041 /* Try if we can put the downstream caps first */
3042 pcopy = gst_caps_copy (peercaps);
3043 remove_fields (pcopy, FALSE);
3044 tmp = gst_caps_intersect_full (pcopy, res, GST_CAPS_INTERSECT_FIRST);
3045 gst_caps_unref (pcopy);
3046 if (!gst_caps_is_empty (tmp))
3047 res = gst_caps_merge (tmp, res);
3048 else
3049 gst_caps_unref (tmp);
3050
3051 gst_caps_unref (peercaps);
3052 return res;
3053 }
3054
3055 static gboolean
gst_h264_parse_event(GstBaseParse * parse,GstEvent * event)3056 gst_h264_parse_event (GstBaseParse * parse, GstEvent * event)
3057 {
3058 gboolean res;
3059 GstH264Parse *h264parse = GST_H264_PARSE (parse);
3060
3061 switch (GST_EVENT_TYPE (event)) {
3062 case GST_EVENT_CUSTOM_DOWNSTREAM:
3063 {
3064 GstClockTime timestamp, stream_time, running_time;
3065 gboolean all_headers;
3066 guint count;
3067
3068 if (gst_video_event_is_force_key_unit (event)) {
3069 gst_video_event_parse_downstream_force_key_unit (event,
3070 ×tamp, &stream_time, &running_time, &all_headers, &count);
3071
3072 GST_INFO_OBJECT (h264parse,
3073 "received downstream force key unit event, "
3074 "seqnum %d running_time %" GST_TIME_FORMAT
3075 " all_headers %d count %d", gst_event_get_seqnum (event),
3076 GST_TIME_ARGS (running_time), all_headers, count);
3077 if (h264parse->force_key_unit_event) {
3078 GST_INFO_OBJECT (h264parse, "ignoring force key unit event "
3079 "as one is already queued");
3080 } else {
3081 h264parse->pending_key_unit_ts = running_time;
3082 gst_event_replace (&h264parse->force_key_unit_event, event);
3083 }
3084 gst_event_unref (event);
3085 res = TRUE;
3086 } else {
3087 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3088 break;
3089 }
3090 break;
3091 }
3092 case GST_EVENT_FLUSH_STOP:
3093 h264parse->dts = GST_CLOCK_TIME_NONE;
3094 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
3095 h264parse->push_codec = TRUE;
3096
3097 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3098 break;
3099 case GST_EVENT_SEGMENT:
3100 {
3101 const GstSegment *segment;
3102
3103 gst_event_parse_segment (event, &segment);
3104 /* don't try to mess with more subtle cases (e.g. seek) */
3105 if (segment->format == GST_FORMAT_TIME &&
3106 (segment->start != 0 || segment->rate != 1.0
3107 || segment->applied_rate != 1.0))
3108 h264parse->do_ts = FALSE;
3109
3110 h264parse->last_report = GST_CLOCK_TIME_NONE;
3111
3112 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3113 break;
3114 }
3115 default:
3116 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3117 break;
3118 }
3119 return res;
3120 }
3121
3122 static gboolean
gst_h264_parse_src_event(GstBaseParse * parse,GstEvent * event)3123 gst_h264_parse_src_event (GstBaseParse * parse, GstEvent * event)
3124 {
3125 gboolean res;
3126 GstH264Parse *h264parse = GST_H264_PARSE (parse);
3127
3128 switch (GST_EVENT_TYPE (event)) {
3129 case GST_EVENT_CUSTOM_UPSTREAM:
3130 {
3131 GstClockTime running_time;
3132 gboolean all_headers;
3133 guint count;
3134
3135 if (gst_video_event_is_force_key_unit (event)) {
3136 gst_video_event_parse_upstream_force_key_unit (event,
3137 &running_time, &all_headers, &count);
3138
3139 GST_INFO_OBJECT (h264parse, "received upstream force-key-unit event, "
3140 "seqnum %d running_time %" GST_TIME_FORMAT
3141 " all_headers %d count %d", gst_event_get_seqnum (event),
3142 GST_TIME_ARGS (running_time), all_headers, count);
3143
3144 if (all_headers) {
3145 h264parse->pending_key_unit_ts = running_time;
3146 gst_event_replace (&h264parse->force_key_unit_event, event);
3147 }
3148 }
3149 res = GST_BASE_PARSE_CLASS (parent_class)->src_event (parse, event);
3150 break;
3151 }
3152 default:
3153 res = GST_BASE_PARSE_CLASS (parent_class)->src_event (parse, event);
3154 break;
3155 }
3156
3157 return res;
3158 }
3159
3160 static void
gst_h264_parse_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)3161 gst_h264_parse_set_property (GObject * object, guint prop_id,
3162 const GValue * value, GParamSpec * pspec)
3163 {
3164 GstH264Parse *parse;
3165
3166 parse = GST_H264_PARSE (object);
3167
3168 switch (prop_id) {
3169 case PROP_CONFIG_INTERVAL:
3170 parse->interval = g_value_get_int (value);
3171 break;
3172 default:
3173 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
3174 break;
3175 }
3176 }
3177
3178 static void
gst_h264_parse_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)3179 gst_h264_parse_get_property (GObject * object, guint prop_id,
3180 GValue * value, GParamSpec * pspec)
3181 {
3182 GstH264Parse *parse;
3183
3184 parse = GST_H264_PARSE (object);
3185
3186 switch (prop_id) {
3187 case PROP_CONFIG_INTERVAL:
3188 g_value_set_int (value, parse->interval);
3189 break;
3190 default:
3191 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
3192 break;
3193 }
3194 }
3195