1 /* GStreamer
2  * Copyright (C) 2008 Axis Communications <dev-gstreamer@axis.com>
3  * @author Bjorn Ostby <bjorn.ostby@axis.com>
4  * @author Peter Kjellerstedt <peter.kjellerstedt@axis.com>
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Library General Public
8  * License as published by the Free Software Foundation; either
9  * version 2 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Library General Public License for more details.
15  *
16  * You should have received a copy of the GNU Library General Public
17  * License along with this library; if not, write to the
18  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19  * Boston, MA 02110-1301, USA.
20  */
21 
22 /**
23  * SECTION:element-rtpjpegpay
24  *
25  * Payload encode JPEG pictures into RTP packets according to RFC 2435.
26  * For detailed information see: http://www.rfc-editor.org/rfc/rfc2435.txt
27  *
28  * The payloader takes a JPEG picture, scans the header for quantization
29  * tables (if needed) and constructs the RTP packet header followed by
30  * the actual JPEG entropy scan.
31  *
32  * The payloader assumes that correct width and height is found in the caps.
33  */
34 
35 #ifdef HAVE_CONFIG_H
36 #  include "config.h"
37 #endif
38 
39 #include <string.h>
40 #include <gst/rtp/gstrtpbuffer.h>
41 #include <gst/video/video.h>
42 
43 #include "gstrtpjpegpay.h"
44 #include "gstrtputils.h"
45 
46 static GstStaticPadTemplate gst_rtp_jpeg_pay_sink_template =
47     GST_STATIC_PAD_TEMPLATE ("sink",
48     GST_PAD_SINK,
49     GST_PAD_ALWAYS,
50     GST_STATIC_CAPS ("image/jpeg; " "video/x-jpeg")
51     );
52 
53 static GstStaticPadTemplate gst_rtp_jpeg_pay_src_template =
54     GST_STATIC_PAD_TEMPLATE ("src",
55     GST_PAD_SRC,
56     GST_PAD_ALWAYS,
57     GST_STATIC_CAPS ("application/x-rtp, "
58         "  media = (string) \"video\", "
59         "  payload = (int) " GST_RTP_PAYLOAD_JPEG_STRING ", "
60         "  clock-rate = (int) 90000,   "
61         "  encoding-name = (string) \"JPEG\", "
62         "  width = (int) [ 1, 65536 ], " "  height = (int) [ 1, 65536 ]; "
63         " application/x-rtp, "
64         "  media = (string) \"video\", "
65         "  payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", "
66         "  clock-rate = (int) 90000,   "
67         "  encoding-name = (string) \"JPEG\", "
68         "  width = (int) [ 1, 65536 ], " "  height = (int) [ 1, 65536 ]")
69     );
70 
71 GST_DEBUG_CATEGORY_STATIC (rtpjpegpay_debug);
72 #define GST_CAT_DEFAULT (rtpjpegpay_debug)
73 
74 /*
75  * QUANT_PREFIX_LEN:
76  *
77  * Prefix length in the header before the quantization tables:
78  * Two size bytes and one byte for precision
79  */
80 #define QUANT_PREFIX_LEN     3
81 
82 
83 typedef enum _RtpJpegMarker RtpJpegMarker;
84 
85 /*
86  * RtpJpegMarker:
87  * @JPEG_MARKER: Prefix for JPEG marker
88  * @JPEG_MARKER_SOI: Start of Image marker
89  * @JPEG_MARKER_JFIF: JFIF marker
90  * @JPEG_MARKER_CMT: Comment marker
91  * @JPEG_MARKER_DQT: Define Quantization Table marker
92  * @JPEG_MARKER_SOF: Start of Frame marker
93  * @JPEG_MARKER_DHT: Define Huffman Table marker
94  * @JPEG_MARKER_SOS: Start of Scan marker
95  * @JPEG_MARKER_EOI: End of Image marker
96  * @JPEG_MARKER_DRI: Define Restart Interval marker
97  * @JPEG_MARKER_H264: H264 marker
98  *
99  * Identifers for markers in JPEG header
100  */
101 enum _RtpJpegMarker
102 {
103   JPEG_MARKER = 0xFF,
104   JPEG_MARKER_SOI = 0xD8,
105   JPEG_MARKER_JFIF = 0xE0,
106   JPEG_MARKER_CMT = 0xFE,
107   JPEG_MARKER_DQT = 0xDB,
108   JPEG_MARKER_SOF = 0xC0,
109   JPEG_MARKER_DHT = 0xC4,
110   JPEG_MARKER_JPG = 0xC8,
111   JPEG_MARKER_SOS = 0xDA,
112   JPEG_MARKER_EOI = 0xD9,
113   JPEG_MARKER_DRI = 0xDD,
114   JPEG_MARKER_APP0 = 0xE0,
115   JPEG_MARKER_H264 = 0xE4,      /* APP4 */
116   JPEG_MARKER_APP15 = 0xEF,
117   JPEG_MARKER_JPG0 = 0xF0,
118   JPEG_MARKER_JPG13 = 0xFD
119 };
120 
121 #define DEFAULT_JPEG_QUANT    255
122 
123 #define DEFAULT_JPEG_QUALITY  255
124 #define DEFAULT_JPEG_TYPE     1
125 
126 enum
127 {
128   PROP_0,
129   PROP_JPEG_QUALITY,
130   PROP_JPEG_TYPE
131 };
132 
133 enum
134 {
135   Q_TABLE_0 = 0,
136   Q_TABLE_1,
137   Q_TABLE_MAX                   /* only support for two tables at the moment */
138 };
139 
140 typedef struct _RtpJpegHeader RtpJpegHeader;
141 
142 /*
143  * RtpJpegHeader:
144  * @type_spec: type specific
145  * @offset: fragment offset
146  * @type: type field
147  * @q: quantization table for this frame
148  * @width: width of image in 8-pixel multiples
149  * @height: height of image in 8-pixel multiples
150  *
151  * 0                   1                   2                   3
152  * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
153  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
154  * | Type-specific |              Fragment Offset                  |
155  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
156  * |      Type     |       Q       |     Width     |     Height    |
157  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
158  */
159 struct _RtpJpegHeader
160 {
161   guint type_spec:8;
162   guint offset:24;
163   guint8 type;
164   guint8 q;
165   guint8 width;
166   guint8 height;
167 };
168 
169 /*
170  * RtpQuantHeader
171  * @mbz: must be zero
172  * @precision: specify size of quantization tables
173  * @length: length of quantization data
174  *
175  * 0                   1                   2                   3
176  * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
177  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
178  * |      MBZ      |   Precision   |             Length            |
179  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
180  * |                    Quantization Table Data                    |
181  * |                              ...                              |
182  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
183  */
184 typedef struct
185 {
186   guint8 mbz;
187   guint8 precision;
188   guint16 length;
189 } RtpQuantHeader;
190 
191 typedef struct
192 {
193   guint8 size;
194   const guint8 *data;
195 } RtpQuantTable;
196 
197 /*
198  * RtpRestartMarkerHeader:
199  * @restartInterval: number of MCUs that appear between restart markers
200  * @restartFirstLastCount: a combination of the first packet mark in the chunk
201  *                         last packet mark in the chunk and the position of the
202  *                         first restart interval in the current "chunk"
203  *
204  *    0                   1                   2                   3
205  *   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
206  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
207  *  |       Restart Interval        |F|L|       Restart Count       |
208  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
209  *
210  *  The restart marker header is implemented according to the following
211  *  methodology specified in section 3.1.7 of rfc2435.txt.
212  *
213  *  "If the restart intervals in a frame are not guaranteed to be aligned
214  *  with packet boundaries, the F (first) and L (last) bits MUST be set
215  *  to 1 and the Restart Count MUST be set to 0x3FFF.  This indicates
216  *  that a receiver MUST reassemble the entire frame before decoding it."
217  *
218  */
219 
220 typedef struct
221 {
222   guint16 restart_interval;
223   guint16 restart_count;
224 } RtpRestartMarkerHeader;
225 
226 typedef struct
227 {
228   guint8 id;
229   guint8 samp;
230   guint8 qt;
231 } CompInfo;
232 
233 /* FIXME: restart marker header currently unsupported */
234 
235 static void gst_rtp_jpeg_pay_set_property (GObject * object, guint prop_id,
236     const GValue * value, GParamSpec * pspec);
237 
238 static void gst_rtp_jpeg_pay_get_property (GObject * object, guint prop_id,
239     GValue * value, GParamSpec * pspec);
240 
241 static gboolean gst_rtp_jpeg_pay_setcaps (GstRTPBasePayload * basepayload,
242     GstCaps * caps);
243 
244 static GstFlowReturn gst_rtp_jpeg_pay_handle_buffer (GstRTPBasePayload * pad,
245     GstBuffer * buffer);
246 
247 #define gst_rtp_jpeg_pay_parent_class parent_class
248 G_DEFINE_TYPE (GstRtpJPEGPay, gst_rtp_jpeg_pay, GST_TYPE_RTP_BASE_PAYLOAD);
249 
250 static void
gst_rtp_jpeg_pay_class_init(GstRtpJPEGPayClass * klass)251 gst_rtp_jpeg_pay_class_init (GstRtpJPEGPayClass * klass)
252 {
253   GObjectClass *gobject_class;
254   GstElementClass *gstelement_class;
255   GstRTPBasePayloadClass *gstrtpbasepayload_class;
256 
257   gobject_class = (GObjectClass *) klass;
258   gstelement_class = (GstElementClass *) klass;
259   gstrtpbasepayload_class = (GstRTPBasePayloadClass *) klass;
260 
261   gobject_class->set_property = gst_rtp_jpeg_pay_set_property;
262   gobject_class->get_property = gst_rtp_jpeg_pay_get_property;
263 
264   gst_element_class_add_static_pad_template (gstelement_class,
265       &gst_rtp_jpeg_pay_src_template);
266   gst_element_class_add_static_pad_template (gstelement_class,
267       &gst_rtp_jpeg_pay_sink_template);
268 
269   gst_element_class_set_static_metadata (gstelement_class, "RTP JPEG payloader",
270       "Codec/Payloader/Network/RTP",
271       "Payload-encodes JPEG pictures into RTP packets (RFC 2435)",
272       "Axis Communications <dev-gstreamer@axis.com>");
273 
274   gstrtpbasepayload_class->set_caps = gst_rtp_jpeg_pay_setcaps;
275   gstrtpbasepayload_class->handle_buffer = gst_rtp_jpeg_pay_handle_buffer;
276 
277   g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_JPEG_QUALITY,
278       g_param_spec_int ("quality", "Quality",
279           "Quality factor on JPEG data (unused)", 0, 255, 255,
280           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
281 
282   g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_JPEG_TYPE,
283       g_param_spec_int ("type", "Type",
284           "Default JPEG Type, overwritten by SOF when present", 0, 255,
285           DEFAULT_JPEG_TYPE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
286 
287   GST_DEBUG_CATEGORY_INIT (rtpjpegpay_debug, "rtpjpegpay", 0,
288       "Motion JPEG RTP Payloader");
289 }
290 
291 static void
gst_rtp_jpeg_pay_init(GstRtpJPEGPay * pay)292 gst_rtp_jpeg_pay_init (GstRtpJPEGPay * pay)
293 {
294   pay->quality = DEFAULT_JPEG_QUALITY;
295   pay->quant = DEFAULT_JPEG_QUANT;
296   pay->type = DEFAULT_JPEG_TYPE;
297   pay->width = -1;
298   pay->height = -1;
299 
300   GST_RTP_BASE_PAYLOAD_PT (pay) = GST_RTP_PAYLOAD_JPEG;
301 }
302 
303 static gboolean
gst_rtp_jpeg_pay_setcaps(GstRTPBasePayload * basepayload,GstCaps * caps)304 gst_rtp_jpeg_pay_setcaps (GstRTPBasePayload * basepayload, GstCaps * caps)
305 {
306   GstStructure *caps_structure = gst_caps_get_structure (caps, 0);
307   GstRtpJPEGPay *pay;
308   gboolean res;
309   gint width = -1, height = -1;
310   gint num = 0, denom;
311   gchar *rate = NULL;
312   gchar *dim = NULL;
313 
314   pay = GST_RTP_JPEG_PAY (basepayload);
315 
316   /* these properties are mandatory, but they might be adjusted by the SOF, if there
317    * is one. */
318   if (!gst_structure_get_int (caps_structure, "height", &height) || height <= 0) {
319     goto invalid_dimension;
320   }
321 
322   if (!gst_structure_get_int (caps_structure, "width", &width) || width <= 0) {
323     goto invalid_dimension;
324   }
325 
326   if (gst_structure_get_fraction (caps_structure, "framerate", &num, &denom) &&
327       (num < 0 || denom <= 0)) {
328     goto invalid_framerate;
329   }
330 
331   if (height > 2040 || width > 2040) {
332     pay->height = 0;
333     pay->width = 0;
334   } else {
335     pay->height = GST_ROUND_UP_8 (height) / 8;
336     pay->width = GST_ROUND_UP_8 (width) / 8;
337   }
338 
339   gst_rtp_base_payload_set_options (basepayload, "video",
340       basepayload->pt != GST_RTP_PAYLOAD_JPEG, "JPEG", 90000);
341 
342   if (num > 0) {
343     gdouble framerate;
344     gst_util_fraction_to_double (num, denom, &framerate);
345     rate = g_strdup_printf ("%f", framerate);
346   }
347 
348   if (pay->width == 0) {
349     GST_DEBUG_OBJECT (pay,
350         "width or height are greater than 2040, adding x-dimensions to caps");
351     dim = g_strdup_printf ("%d,%d", width, height);
352   }
353 
354   if (rate != NULL && dim != NULL) {
355     res = gst_rtp_base_payload_set_outcaps (basepayload, "a-framerate",
356         G_TYPE_STRING, rate, "x-dimensions", G_TYPE_STRING, dim, NULL);
357   } else if (rate != NULL && dim == NULL) {
358     res = gst_rtp_base_payload_set_outcaps (basepayload, "a-framerate",
359         G_TYPE_STRING, rate, NULL);
360   } else if (rate == NULL && dim != NULL) {
361     res = gst_rtp_base_payload_set_outcaps (basepayload, "x-dimensions",
362         G_TYPE_STRING, dim, NULL);
363   } else {
364     res = gst_rtp_base_payload_set_outcaps (basepayload, NULL);
365   }
366 
367   g_free (dim);
368   g_free (rate);
369 
370   return res;
371 
372   /* ERRORS */
373 invalid_dimension:
374   {
375     GST_ERROR_OBJECT (pay, "Invalid width/height from caps");
376     return FALSE;
377   }
378 invalid_framerate:
379   {
380     GST_ERROR_OBJECT (pay, "Invalid framerate from caps");
381     return FALSE;
382   }
383 }
384 
385 static guint
gst_rtp_jpeg_pay_header_size(const guint8 * data,guint offset)386 gst_rtp_jpeg_pay_header_size (const guint8 * data, guint offset)
387 {
388   return data[offset] << 8 | data[offset + 1];
389 }
390 
391 static guint
gst_rtp_jpeg_pay_read_quant_table(const guint8 * data,guint size,guint offset,RtpQuantTable tables[])392 gst_rtp_jpeg_pay_read_quant_table (const guint8 * data, guint size,
393     guint offset, RtpQuantTable tables[])
394 {
395   guint quant_size, tab_size;
396   guint8 prec;
397   guint8 id;
398 
399   if (offset + 2 > size)
400     goto too_small;
401 
402   quant_size = gst_rtp_jpeg_pay_header_size (data, offset);
403   if (quant_size < 2)
404     goto small_quant_size;
405 
406   /* clamp to available data */
407   if (offset + quant_size > size)
408     quant_size = size - offset;
409 
410   offset += 2;
411   quant_size -= 2;
412 
413   while (quant_size > 0) {
414     /* not enough to read the id */
415     if (offset + 1 > size)
416       break;
417 
418     id = data[offset] & 0x0f;
419     if (id == 15)
420       /* invalid id received - corrupt data */
421       goto invalid_id;
422 
423     prec = (data[offset] & 0xf0) >> 4;
424     if (prec)
425       tab_size = 128;
426     else
427       tab_size = 64;
428 
429     /* there is not enough for the table */
430     if (quant_size < tab_size + 1)
431       goto no_table;
432 
433     GST_LOG ("read quant table %d, tab_size %d, prec %02x", id, tab_size, prec);
434 
435     tables[id].size = tab_size;
436     tables[id].data = &data[offset + 1];
437 
438     tab_size += 1;
439     quant_size -= tab_size;
440     offset += tab_size;
441   }
442 done:
443   return offset + quant_size;
444 
445   /* ERRORS */
446 too_small:
447   {
448     GST_WARNING ("not enough data");
449     return size;
450   }
451 small_quant_size:
452   {
453     GST_WARNING ("quant_size too small (%u < 2)", quant_size);
454     return size;
455   }
456 invalid_id:
457   {
458     GST_WARNING ("invalid id");
459     goto done;
460   }
461 no_table:
462   {
463     GST_WARNING ("not enough data for table (%u < %u)", quant_size,
464         tab_size + 1);
465     goto done;
466   }
467 }
468 
469 static gboolean
gst_rtp_jpeg_pay_read_sof(GstRtpJPEGPay * pay,const guint8 * data,guint size,guint * offset,CompInfo info[],RtpQuantTable tables[],gulong tables_elements)470 gst_rtp_jpeg_pay_read_sof (GstRtpJPEGPay * pay, const guint8 * data,
471     guint size, guint * offset, CompInfo info[], RtpQuantTable tables[],
472     gulong tables_elements)
473 {
474   guint sof_size, off;
475   guint width, height, infolen;
476   CompInfo elem;
477   gint i, j;
478 
479   off = *offset;
480 
481   /* we need at least 17 bytes for the SOF */
482   if (off + 17 > size)
483     goto wrong_size;
484 
485   sof_size = gst_rtp_jpeg_pay_header_size (data, off);
486   if (sof_size < 17)
487     goto wrong_length;
488 
489   *offset += sof_size;
490 
491   /* skip size */
492   off += 2;
493 
494   /* precision should be 8 */
495   if (data[off++] != 8)
496     goto bad_precision;
497 
498   /* read dimensions */
499   height = data[off] << 8 | data[off + 1];
500   width = data[off + 2] << 8 | data[off + 3];
501   off += 4;
502 
503   GST_LOG_OBJECT (pay, "got dimensions %ux%u", height, width);
504 
505   if (height == 0) {
506     goto invalid_dimension;
507   }
508   if (height > 2040) {
509     height = 0;
510   }
511   if (width == 0) {
512     goto invalid_dimension;
513   }
514   if (width > 2040) {
515     width = 0;
516   }
517 
518   if (height == 0 || width == 0) {
519     pay->height = 0;
520     pay->width = 0;
521   } else {
522     pay->height = GST_ROUND_UP_8 (height) / 8;
523     pay->width = GST_ROUND_UP_8 (width) / 8;
524   }
525 
526   /* we only support 3 components */
527   if (data[off++] != 3)
528     goto bad_components;
529 
530   infolen = 0;
531   for (i = 0; i < 3; i++) {
532     elem.id = data[off++];
533     elem.samp = data[off++];
534     elem.qt = data[off++];
535     GST_LOG_OBJECT (pay, "got comp %d, samp %02x, qt %d", elem.id, elem.samp,
536         elem.qt);
537     /* insertion sort from the last element to the first */
538     for (j = infolen; j > 1; j--) {
539       if (G_LIKELY (info[j - 1].id < elem.id))
540         break;
541       info[j] = info[j - 1];
542     }
543     info[j] = elem;
544     infolen++;
545   }
546 
547   /* see that the components are supported */
548   if (info[0].samp == 0x21)
549     pay->type = 0;
550   else if (info[0].samp == 0x22)
551     pay->type = 1;
552   else
553     goto invalid_comp;
554 
555   if (!(info[1].samp == 0x11))
556     goto invalid_comp;
557 
558   if (!(info[2].samp == 0x11))
559     goto invalid_comp;
560 
561   return TRUE;
562 
563   /* ERRORS */
564 wrong_size:
565   {
566     GST_ELEMENT_WARNING (pay, STREAM, FORMAT,
567         ("Wrong size %u (needed %u).", size, off + 17), (NULL));
568     return FALSE;
569   }
570 wrong_length:
571   {
572     GST_ELEMENT_WARNING (pay, STREAM, FORMAT,
573         ("Wrong SOF length %u.", sof_size), (NULL));
574     return FALSE;
575   }
576 bad_precision:
577   {
578     GST_ELEMENT_WARNING (pay, STREAM, FORMAT,
579         ("Wrong precision, expecting 8."), (NULL));
580     return FALSE;
581   }
582 invalid_dimension:
583   {
584     GST_ELEMENT_WARNING (pay, STREAM, FORMAT,
585         ("Wrong dimension, size %ux%u", width, height), (NULL));
586     return FALSE;
587   }
588 bad_components:
589   {
590     GST_ELEMENT_WARNING (pay, STREAM, FORMAT,
591         ("Wrong number of components"), (NULL));
592     return FALSE;
593   }
594 invalid_comp:
595   {
596     GST_ELEMENT_WARNING (pay, STREAM, FORMAT, ("Invalid component"), (NULL));
597     return FALSE;
598   }
599 }
600 
601 static gboolean
gst_rtp_jpeg_pay_read_dri(GstRtpJPEGPay * pay,const guint8 * data,guint size,guint * offset,RtpRestartMarkerHeader * dri)602 gst_rtp_jpeg_pay_read_dri (GstRtpJPEGPay * pay, const guint8 * data,
603     guint size, guint * offset, RtpRestartMarkerHeader * dri)
604 {
605   guint dri_size, off;
606 
607   off = *offset;
608 
609   /* we need at least 4 bytes for the DRI */
610   if (off + 4 > size)
611     goto wrong_size;
612 
613   dri_size = gst_rtp_jpeg_pay_header_size (data, off);
614   if (dri_size < 4)
615     goto wrong_length;
616 
617   *offset += dri_size;
618   off += 2;
619 
620   dri->restart_interval = g_htons ((data[off] << 8) | (data[off + 1]));
621   dri->restart_count = g_htons (0xFFFF);
622 
623   return dri->restart_interval > 0;
624 
625 wrong_size:
626   {
627     GST_WARNING ("not enough data for DRI");
628     *offset = size;
629     return FALSE;
630   }
631 wrong_length:
632   {
633     GST_WARNING ("DRI size too small (%u)", dri_size);
634     *offset += dri_size;
635     return FALSE;
636   }
637 }
638 
639 static RtpJpegMarker
gst_rtp_jpeg_pay_scan_marker(const guint8 * data,guint size,guint * offset)640 gst_rtp_jpeg_pay_scan_marker (const guint8 * data, guint size, guint * offset)
641 {
642   while ((data[(*offset)++] != JPEG_MARKER) && ((*offset) < size));
643 
644   if (G_UNLIKELY ((*offset) >= size)) {
645     GST_LOG ("found EOI marker");
646     return JPEG_MARKER_EOI;
647   } else {
648     guint8 marker;
649 
650     marker = data[*offset];
651     GST_LOG ("found 0x%02x marker at offset %u", marker, *offset);
652     (*offset)++;
653     return marker;
654   }
655 }
656 
657 #define RTP_HEADER_LEN 12
658 
659 static GstFlowReturn
gst_rtp_jpeg_pay_handle_buffer(GstRTPBasePayload * basepayload,GstBuffer * buffer)660 gst_rtp_jpeg_pay_handle_buffer (GstRTPBasePayload * basepayload,
661     GstBuffer * buffer)
662 {
663   GstRtpJPEGPay *pay;
664   GstClockTime timestamp;
665   GstFlowReturn ret = GST_FLOW_ERROR;
666   RtpJpegHeader jpeg_header;
667   RtpQuantHeader quant_header;
668   RtpRestartMarkerHeader restart_marker_header;
669   RtpQuantTable tables[15] = { {0, NULL}, };
670   CompInfo info[3] = { {0,}, };
671   guint quant_data_size;
672   GstMapInfo map;
673   guint8 *data;
674   gsize size;
675   guint mtu, max_payload_size;
676   guint bytes_left;
677   guint jpeg_header_size = 0;
678   guint offset;
679   gboolean frame_done;
680   gboolean sos_found, sof_found, dqt_found, dri_found;
681   gint i;
682   GstBufferList *list = NULL;
683   gboolean discont;
684 
685   pay = GST_RTP_JPEG_PAY (basepayload);
686   mtu = GST_RTP_BASE_PAYLOAD_MTU (pay);
687 
688   gst_buffer_map (buffer, &map, GST_MAP_READ);
689   data = map.data;
690   size = map.size;
691   timestamp = GST_BUFFER_PTS (buffer);
692   offset = 0;
693   discont = GST_BUFFER_IS_DISCONT (buffer);
694 
695   GST_LOG_OBJECT (pay, "got buffer size %" G_GSIZE_FORMAT
696       " , timestamp %" GST_TIME_FORMAT, size, GST_TIME_ARGS (timestamp));
697 
698   /* parse the jpeg header for 'start of scan' and read quant tables if needed */
699   sos_found = FALSE;
700   dqt_found = FALSE;
701   sof_found = FALSE;
702   dri_found = FALSE;
703 
704   while (!sos_found && (offset < size)) {
705     gint marker;
706 
707     GST_LOG_OBJECT (pay, "checking from offset %u", offset);
708     switch ((marker = gst_rtp_jpeg_pay_scan_marker (data, size, &offset))) {
709       case JPEG_MARKER_JFIF:
710       case JPEG_MARKER_CMT:
711       case JPEG_MARKER_DHT:
712       case JPEG_MARKER_H264:
713         GST_LOG_OBJECT (pay, "skipping marker");
714         offset += gst_rtp_jpeg_pay_header_size (data, offset);
715         break;
716       case JPEG_MARKER_SOF:
717         if (!gst_rtp_jpeg_pay_read_sof (pay, data, size, &offset, info, tables,
718                 G_N_ELEMENTS (tables)))
719           goto invalid_format;
720         sof_found = TRUE;
721         break;
722       case JPEG_MARKER_DQT:
723         GST_LOG ("DQT found");
724         offset = gst_rtp_jpeg_pay_read_quant_table (data, size, offset, tables);
725         dqt_found = TRUE;
726         break;
727       case JPEG_MARKER_SOS:
728         sos_found = TRUE;
729         GST_LOG_OBJECT (pay, "SOS found");
730         jpeg_header_size = offset + gst_rtp_jpeg_pay_header_size (data, offset);
731         break;
732       case JPEG_MARKER_EOI:
733         GST_WARNING_OBJECT (pay, "EOI reached before SOS!");
734         break;
735       case JPEG_MARKER_SOI:
736         GST_LOG_OBJECT (pay, "SOI found");
737         break;
738       case JPEG_MARKER_DRI:
739         GST_LOG_OBJECT (pay, "DRI found");
740         if (gst_rtp_jpeg_pay_read_dri (pay, data, size, &offset,
741                 &restart_marker_header))
742           dri_found = TRUE;
743         break;
744       default:
745         if (marker == JPEG_MARKER_JPG ||
746             (marker >= JPEG_MARKER_JPG0 && marker <= JPEG_MARKER_JPG13) ||
747             (marker >= JPEG_MARKER_APP0 && marker <= JPEG_MARKER_APP15)) {
748           GST_LOG_OBJECT (pay, "skipping marker");
749           offset += gst_rtp_jpeg_pay_header_size (data, offset);
750         } else {
751           GST_FIXME_OBJECT (pay, "unhandled marker 0x%02x", marker);
752         }
753         break;
754     }
755   }
756   if (!dqt_found || !sof_found)
757     goto unsupported_jpeg;
758 
759   /* by now we should either have negotiated the width/height or the SOF header
760    * should have filled us in */
761   if (pay->width < 0 || pay->height < 0) {
762     goto no_dimension;
763   }
764 
765   GST_LOG_OBJECT (pay, "header size %u", jpeg_header_size);
766 
767   size -= jpeg_header_size;
768   data += jpeg_header_size;
769   offset = 0;
770 
771   if (dri_found)
772     pay->type += 64;
773 
774   /* prepare stuff for the jpeg header */
775   jpeg_header.type_spec = 0;
776   jpeg_header.type = pay->type;
777   jpeg_header.q = pay->quant;
778   jpeg_header.width = pay->width;
779   jpeg_header.height = pay->height;
780 
781   /* collect the quant headers sizes */
782   quant_header.mbz = 0;
783   quant_header.precision = 0;
784   quant_header.length = 0;
785   quant_data_size = 0;
786 
787   if (pay->quant > 127) {
788     /* for the Y and U component, look up the quant table and its size. quant
789      * tables for U and V should be the same */
790     for (i = 0; i < 2; i++) {
791       guint qsize;
792       guint qt;
793 
794       qt = info[i].qt;
795       if (qt >= G_N_ELEMENTS (tables))
796         goto invalid_quant;
797 
798       qsize = tables[qt].size;
799       if (qsize == 0)
800         goto invalid_quant;
801 
802       quant_header.precision |= (qsize == 64 ? 0 : (1 << i));
803       quant_data_size += qsize;
804     }
805     quant_header.length = g_htons (quant_data_size);
806     quant_data_size += sizeof (quant_header);
807   }
808 
809   GST_LOG_OBJECT (pay, "quant_data size %u", quant_data_size);
810 
811   bytes_left = sizeof (jpeg_header) + quant_data_size + size;
812 
813   if (dri_found)
814     bytes_left += sizeof (restart_marker_header);
815 
816   max_payload_size = mtu - (RTP_HEADER_LEN + sizeof (jpeg_header));
817   list = gst_buffer_list_new_sized ((bytes_left / max_payload_size) + 1);
818 
819   frame_done = FALSE;
820   do {
821     GstBuffer *outbuf;
822     guint8 *payload;
823     guint payload_size;
824     guint header_size;
825     GstBuffer *paybuf;
826     GstRTPBuffer rtp = { NULL };
827     guint rtp_header_size = gst_rtp_buffer_calc_header_len (0);
828 
829     /* The available room is the packet MTU, minus the RTP header length. */
830     payload_size =
831         (bytes_left < (mtu - rtp_header_size) ? bytes_left :
832         (mtu - rtp_header_size));
833 
834     header_size = sizeof (jpeg_header) + quant_data_size;
835     if (dri_found)
836       header_size += sizeof (restart_marker_header);
837 
838     outbuf = gst_rtp_buffer_new_allocate (header_size, 0, 0);
839 
840     gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp);
841 
842     if (payload_size == bytes_left) {
843       GST_LOG_OBJECT (pay, "last packet of frame");
844       frame_done = TRUE;
845       gst_rtp_buffer_set_marker (&rtp, 1);
846     }
847 
848     payload = gst_rtp_buffer_get_payload (&rtp);
849 
850     /* update offset */
851 #if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
852     jpeg_header.offset = ((offset & 0x0000FF) << 16) |
853         ((offset & 0xFF0000) >> 16) | (offset & 0x00FF00);
854 #else
855     jpeg_header.offset = offset;
856 #endif
857     memcpy (payload, &jpeg_header, sizeof (jpeg_header));
858     payload += sizeof (jpeg_header);
859     payload_size -= sizeof (jpeg_header);
860 
861     if (dri_found) {
862       memcpy (payload, &restart_marker_header, sizeof (restart_marker_header));
863       payload += sizeof (restart_marker_header);
864       payload_size -= sizeof (restart_marker_header);
865     }
866 
867     /* only send quant table with first packet */
868     if (G_UNLIKELY (quant_data_size > 0)) {
869       memcpy (payload, &quant_header, sizeof (quant_header));
870       payload += sizeof (quant_header);
871 
872       /* copy the quant tables for luma and chrominance */
873       for (i = 0; i < 2; i++) {
874         guint qsize;
875         guint qt;
876 
877         qt = info[i].qt;
878         qsize = tables[qt].size;
879         memcpy (payload, tables[qt].data, qsize);
880 
881         GST_LOG_OBJECT (pay, "component %d using quant %d, size %d", i, qt,
882             qsize);
883 
884         payload += qsize;
885       }
886       payload_size -= quant_data_size;
887       bytes_left -= quant_data_size;
888       quant_data_size = 0;
889     }
890     GST_LOG_OBJECT (pay, "sending payload size %d", payload_size);
891     gst_rtp_buffer_unmap (&rtp);
892 
893     /* create a new buf to hold the payload */
894     paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
895         jpeg_header_size + offset, payload_size);
896 
897     /* join memory parts */
898     gst_rtp_copy_video_meta (pay, outbuf, paybuf);
899     outbuf = gst_buffer_append (outbuf, paybuf);
900 
901     GST_BUFFER_PTS (outbuf) = timestamp;
902 
903     if (discont) {
904       GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DISCONT);
905       /* Only the first outputted buffer has the DISCONT flag */
906       discont = FALSE;
907     }
908 
909     /* and add to list */
910     gst_buffer_list_insert (list, -1, outbuf);
911 
912     bytes_left -= payload_size;
913     offset += payload_size;
914     data += payload_size;
915   }
916   while (!frame_done);
917 
918   /* push the whole buffer list at once */
919   ret = gst_rtp_base_payload_push_list (basepayload, list);
920 
921   gst_buffer_unmap (buffer, &map);
922   gst_buffer_unref (buffer);
923 
924   return ret;
925 
926   /* ERRORS */
927 unsupported_jpeg:
928   {
929     GST_ELEMENT_WARNING (pay, STREAM, FORMAT, ("Unsupported JPEG"), (NULL));
930     gst_buffer_unmap (buffer, &map);
931     gst_buffer_unref (buffer);
932     return GST_FLOW_OK;
933   }
934 no_dimension:
935   {
936     GST_ELEMENT_WARNING (pay, STREAM, FORMAT, ("No size given"), (NULL));
937     gst_buffer_unmap (buffer, &map);
938     gst_buffer_unref (buffer);
939     return GST_FLOW_OK;
940   }
941 invalid_format:
942   {
943     /* error was posted */
944     gst_buffer_unmap (buffer, &map);
945     gst_buffer_unref (buffer);
946     return GST_FLOW_OK;
947   }
948 invalid_quant:
949   {
950     GST_ELEMENT_WARNING (pay, STREAM, FORMAT, ("Invalid quant tables"), (NULL));
951     gst_buffer_unmap (buffer, &map);
952     gst_buffer_unref (buffer);
953     return GST_FLOW_OK;
954   }
955 }
956 
957 static void
gst_rtp_jpeg_pay_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)958 gst_rtp_jpeg_pay_set_property (GObject * object, guint prop_id,
959     const GValue * value, GParamSpec * pspec)
960 {
961   GstRtpJPEGPay *rtpjpegpay;
962 
963   rtpjpegpay = GST_RTP_JPEG_PAY (object);
964 
965   switch (prop_id) {
966     case PROP_JPEG_QUALITY:
967       rtpjpegpay->quality = g_value_get_int (value);
968       GST_DEBUG_OBJECT (object, "quality = %d", rtpjpegpay->quality);
969       break;
970     case PROP_JPEG_TYPE:
971       rtpjpegpay->type = g_value_get_int (value);
972       GST_DEBUG_OBJECT (object, "type = %d", rtpjpegpay->type);
973       break;
974     default:
975       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
976       break;
977   }
978 }
979 
980 static void
gst_rtp_jpeg_pay_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)981 gst_rtp_jpeg_pay_get_property (GObject * object, guint prop_id,
982     GValue * value, GParamSpec * pspec)
983 {
984   GstRtpJPEGPay *rtpjpegpay;
985 
986   rtpjpegpay = GST_RTP_JPEG_PAY (object);
987 
988   switch (prop_id) {
989     case PROP_JPEG_QUALITY:
990       g_value_set_int (value, rtpjpegpay->quality);
991       break;
992     case PROP_JPEG_TYPE:
993       g_value_set_int (value, rtpjpegpay->type);
994       break;
995     default:
996       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
997       break;
998   }
999 }
1000 
1001 gboolean
gst_rtp_jpeg_pay_plugin_init(GstPlugin * plugin)1002 gst_rtp_jpeg_pay_plugin_init (GstPlugin * plugin)
1003 {
1004   return gst_element_register (plugin, "rtpjpegpay", GST_RANK_SECONDARY,
1005       GST_TYPE_RTP_JPEG_PAY);
1006 }
1007