1 /*
2 mediastreamer2 library - modular sound and video processing and streaming
3 Copyright (C) 2006-2010 Belledonne Communications SARL (simon.morlat@linphone.org)
4
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License
7 as published by the Free Software Foundation; either version 2
8 of the License, or (at your option) any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 */
19
20
21 #include "mediastreamer2/msvideo.h"
22 #if !defined(NO_FFMPEG)
23 #include "ffmpeg-priv.h"
24 #endif
25
26 #ifdef _WIN32
27 #include <malloc.h>
28 #endif
29
30 #if MS_HAS_ARM
31 #include "msvideo_neon.h"
32 #endif
33
34 #ifndef INT32_MAX
35 #define INT32_MAX 017777777777
36 #endif
37
ms_pix_fmt_to_string(MSPixFmt fmt)38 const char *ms_pix_fmt_to_string(MSPixFmt fmt){
39 switch(fmt){
40 case MS_YUV420P: return "MS_YUV420P";
41 case MS_YUYV : return "MS_YUYV";
42 case MS_RGB24: return "MS_RGB24";
43 case MS_RGB24_REV: return "MS_RGB24_REV";
44 case MS_MJPEG: return "MS_MJPEG";
45 case MS_UYVY: return "MS_UYVY";
46 case MS_YUY2: return "MS_YUY2";
47 case MS_RGBA32: return "MS_RGBA32";
48 case MS_RGB565: return "MS_RGB565";
49 case MS_H264: return "MS_H264";
50 case MS_PIX_FMT_UNKNOWN: return "MS_PIX_FMT_UNKNOWN";
51 }
52 return "bad format";
53 }
54
55 struct _mblk_video_header {
56 uint16_t w, h;
57 int pad[3];
58 };
59 typedef struct _mblk_video_header mblk_video_header;
60
ms_yuv_buf_init(YuvBuf * buf,int w,int h,int stride,uint8_t * ptr)61 void ms_yuv_buf_init(YuvBuf *buf, int w, int h, int stride, uint8_t *ptr){
62 int ysize,usize;
63 ysize=stride*(h & 0x1 ? h +1 : h);
64 usize=ysize/4;
65 buf->w=w;
66 buf->h=h;
67 buf->planes[0]=ptr;
68 buf->planes[1]=buf->planes[0]+ysize;
69 buf->planes[2]=buf->planes[1]+usize;
70 buf->planes[3]=0;
71 buf->strides[0]=stride;
72 buf->strides[1]=stride/2;
73 buf->strides[2]=buf->strides[1];
74 buf->strides[3]=0;
75 }
76
ms_yuv_buf_init_from_mblk(YuvBuf * buf,mblk_t * m)77 int ms_yuv_buf_init_from_mblk(YuvBuf *buf, mblk_t *m){
78 int w,h;
79
80 // read header
81 mblk_video_header* hdr = (mblk_video_header*)dblk_base(m->b_datap);
82 w = hdr->w;
83 h = hdr->h;
84
85 if (m->b_cont == NULL)
86 ms_yuv_buf_init(buf,w,h,w,m->b_rptr);
87 else
88 ms_yuv_buf_init(buf,w,h,w,m->b_cont->b_rptr);
89 return 0;
90 }
91
92
ms_yuv_buf_init_from_mblk_with_size(YuvBuf * buf,mblk_t * m,int w,int h)93 int ms_yuv_buf_init_from_mblk_with_size(YuvBuf *buf, mblk_t *m, int w, int h){
94 if (m->b_cont!=NULL) m=m->b_cont; /*skip potential video header */
95 ms_yuv_buf_init(buf,w,h,w,m->b_rptr);
96 return 0;
97 }
98
ms_picture_init_from_mblk_with_size(MSPicture * buf,mblk_t * m,MSPixFmt fmt,int w,int h)99 int ms_picture_init_from_mblk_with_size(MSPicture *buf, mblk_t *m, MSPixFmt fmt, int w, int h){
100 if (m->b_cont!=NULL) m=m->b_cont; /*skip potential video header */
101 switch(fmt){
102 case MS_YUV420P:
103 return ms_yuv_buf_init_from_mblk_with_size(buf,m,w,h);
104 break;
105 case MS_YUY2:
106 case MS_YUYV:
107 case MS_UYVY:
108 memset(buf,0,sizeof(*buf));
109 buf->w=w;
110 buf->h=h;
111 buf->planes[0]=m->b_rptr;
112 buf->strides[0]=w*2;
113 break;
114 case MS_RGB24:
115 case MS_RGB24_REV:
116 memset(buf,0,sizeof(*buf));
117 buf->w=w;
118 buf->h=h;
119 buf->planes[0]=m->b_rptr;
120 buf->strides[0]=w*3;
121 break;
122 default:
123 ms_fatal("FIXME: unsupported format %i",fmt);
124 return -1;
125 }
126 return 0;
127 }
128
ms_yuv_buf_alloc(YuvBuf * buf,int w,int h)129 mblk_t * ms_yuv_buf_alloc(YuvBuf *buf, int w, int h){
130 int size=(w * (h & 0x1 ? h+1 : h) *3)/2; /*swscale doesn't like odd numbers of line*/
131 const int header_size = sizeof(mblk_video_header);
132 const int padding=16;
133 mblk_t *msg=allocb(header_size + size+padding,0);
134 // write width/height in header
135 mblk_video_header* hdr = (mblk_video_header*)msg->b_wptr;
136 hdr->w = w;
137 hdr->h = h;
138 msg->b_rptr += header_size;
139 msg->b_wptr += header_size;
140 ms_yuv_buf_init(buf,w,h,w,msg->b_wptr);
141 msg->b_wptr+=size;
142 return msg;
143 }
144
ms_yuv_buf_alloc_from_buffer(int w,int h,mblk_t * buffer)145 mblk_t* ms_yuv_buf_alloc_from_buffer(int w, int h, mblk_t* buffer) {
146 const int header_size =sizeof(mblk_video_header);
147 mblk_t *msg=allocb(header_size,0);
148 // write width/height in header
149 mblk_video_header* hdr = (mblk_video_header*)msg->b_wptr;
150 hdr->w = w;
151 hdr->h = h;
152 msg->b_rptr += header_size;
153 msg->b_wptr += header_size;
154 // append real image buffer
155 msg->b_cont = buffer;
156 return msg;
157 }
158
row_copy(const uint8_t * src,uint8_t * dst,size_t width,size_t src_pix_stride,size_t dst_pix_stride)159 static void row_copy(const uint8_t *src, uint8_t *dst, size_t width, size_t src_pix_stride, size_t dst_pix_stride) {
160 if(src_pix_stride == 1 && dst_pix_stride == 1) {
161 memcpy(dst, src, width);
162 } else {
163 const uint8_t *r_ptr = src;
164 uint8_t *w_ptr = dst;
165 const uint8_t *src_end = src + width * src_pix_stride;
166 const uint8_t *dst_end = dst + width * dst_pix_stride;
167 while(r_ptr < src_end && w_ptr < dst_end) {
168 *w_ptr = *r_ptr;
169 r_ptr += src_pix_stride;
170 w_ptr += dst_pix_stride;
171 }
172 }
173 }
174
plane_copy(const uint8_t * src_plane,size_t src_row_stride,size_t src_pix_stride,const MSRect * src_roi,uint8_t * dst_plane,size_t dst_row_stride,size_t dst_pix_stride,const MSRect * dst_roi)175 static void plane_copy(const uint8_t *src_plane, size_t src_row_stride, size_t src_pix_stride, const MSRect *src_roi,
176 uint8_t *dst_plane, size_t dst_row_stride, size_t dst_pix_stride, const MSRect *dst_roi) {
177
178 const uint8_t *r_ptr = src_plane + (src_roi->y * src_row_stride + src_roi->x * src_pix_stride);
179 uint8_t *w_ptr = dst_plane + (dst_roi->y * dst_row_stride + dst_roi->x * dst_pix_stride);
180
181 if (src_row_stride == dst_row_stride && src_pix_stride == 1 && dst_pix_stride == 1 && src_roi == dst_roi) {
182 memcpy(w_ptr, r_ptr, dst_row_stride * dst_roi->h);
183 } else {
184 int i;
185 for(i=0; i<src_roi->h; i++) {
186 row_copy(r_ptr, w_ptr, src_roi->w, src_pix_stride, dst_pix_stride);
187 r_ptr += src_row_stride;
188 w_ptr += dst_row_stride;
189 }
190 }
191 }
192
ms_yuv_buf_copy(uint8_t * src_planes[],const int src_strides[],uint8_t * dst_planes[],const int dst_strides[],MSVideoSize roi)193 void ms_yuv_buf_copy(uint8_t *src_planes[], const int src_strides[],
194 uint8_t *dst_planes[], const int dst_strides[], MSVideoSize roi) {
195 MSRect roi_rect = {0, 0, roi.width, roi.height};
196 plane_copy(src_planes[0], src_strides[0], 1, &roi_rect, dst_planes[0], dst_strides[0], 1, &roi_rect);
197 roi_rect.w /= 2;
198 roi_rect.h /= 2;
199 plane_copy(src_planes[1], src_strides[1], 1, &roi_rect, dst_planes[1], dst_strides[1], 1, &roi_rect);
200 plane_copy(src_planes[2], src_strides[2], 1, &roi_rect, dst_planes[2], dst_strides[2], 1, &roi_rect);
201 }
202
ms_yuv_buf_copy_with_pix_strides(uint8_t * src_planes[],const int src_row_strides[],const int src_pix_strides[],MSRect src_roi,uint8_t * dst_planes[],const int dst_row_strides[],const int dst_pix_strides[],MSRect dst_roi)203 void ms_yuv_buf_copy_with_pix_strides(uint8_t *src_planes[], const int src_row_strides[], const int src_pix_strides[], MSRect src_roi,
204 uint8_t *dst_planes[], const int dst_row_strides[], const int dst_pix_strides[], MSRect dst_roi) {
205 plane_copy(src_planes[0], src_row_strides[0], src_pix_strides[0], &src_roi, dst_planes[0], dst_row_strides[0], dst_pix_strides[0], &dst_roi);
206
207 src_roi.x /= 2;
208 src_roi.y /= 2;
209 src_roi.w /= 2;
210 src_roi.h /= 2;
211
212 dst_roi.x /= 2;
213 dst_roi.y /= 2;
214 dst_roi.w /= 2;
215 dst_roi.h /= 2;
216
217 plane_copy(src_planes[1], src_row_strides[1], src_pix_strides[1], &src_roi, dst_planes[1], dst_row_strides[1], dst_pix_strides[1], &dst_roi);
218 plane_copy(src_planes[2], src_row_strides[2], src_pix_strides[2], &src_roi, dst_planes[2], dst_row_strides[2], dst_pix_strides[2], &dst_roi);
219 }
220
ms_yuv_buf_allocator_new(void)221 MSYuvBufAllocator *ms_yuv_buf_allocator_new(void) {
222 msgb_allocator_t *allocator = (msgb_allocator_t *)ms_new0(msgb_allocator_t, 1);
223 msgb_allocator_init(allocator);
224 return allocator;
225 }
226
ms_yuv_buf_allocator_get(MSYuvBufAllocator * obj,MSPicture * buf,int w,int h)227 mblk_t *ms_yuv_buf_allocator_get(MSYuvBufAllocator *obj, MSPicture *buf, int w, int h) {
228 int size=(w * (h & 0x1 ? h+1 : h) *3)/2; /*swscale doesn't like odd numbers of line*/
229 const int header_size = sizeof(mblk_video_header);
230 const int padding=16;
231 mblk_t *msg = msgb_allocator_alloc(obj, header_size + size+padding);
232 mblk_video_header* hdr = (mblk_video_header*)msg->b_wptr;
233 hdr->w = w;
234 hdr->h = h;
235 msg->b_rptr += header_size;
236 msg->b_wptr += header_size;
237 ms_yuv_buf_init(buf,w,h,w,msg->b_wptr);
238 msg->b_wptr+=size;
239 return msg;
240 }
241
ms_yuv_buf_allocator_free(MSYuvBufAllocator * obj)242 void ms_yuv_buf_allocator_free(MSYuvBufAllocator *obj) {
243 mblk_t *m;
244 int possibly_leaked = 0;
245 for(m = qbegin(&obj->q); !qend(&obj->q,m); m = qnext(&obj->q, m)){
246 if (dblk_ref_value(m->b_datap) > 1) possibly_leaked++;
247 }
248 msgb_allocator_uninit(obj);
249 ms_free(obj);
250 if (possibly_leaked > 0){
251 ms_warning("ms_yuv_buf_allocator_free(): leaving %i mblk_t still ref'd, possible leak.", possibly_leaked);
252 }
253 }
254
plane_horizontal_mirror(uint8_t * p,int linesize,int w,int h)255 static void plane_horizontal_mirror(uint8_t *p, int linesize, int w, int h){
256 int i,j;
257 uint8_t tmp;
258 for(j=0;j<h;++j){
259 for(i=0;i<w/2;++i){
260 const int idx_target_pixel = w-1-i;
261 tmp=p[i];
262 p[i]=p[idx_target_pixel];
263 p[idx_target_pixel]=tmp;
264 }
265 p+=linesize;
266 }
267 }
plane_central_mirror(uint8_t * p,int linesize,int w,int h)268 static void plane_central_mirror(uint8_t *p, int linesize, int w, int h){
269 int i,j;
270 uint8_t tmp;
271 uint8_t *end_of_image = p + (h-1)*linesize+w-1;
272 uint8_t *image_center=p+(h/2)*linesize + w/2;
273 for(j=0;j<h/2;++j){
274 for(i=0;i<w && p<image_center;++i){
275 tmp=*p;
276 *p=*end_of_image;
277 *end_of_image=tmp;
278 ++p;
279 --end_of_image;
280 }
281 p+=linesize-w;
282 end_of_image-=linesize-w;
283 }
284 }
plane_vertical_mirror(uint8_t * p,int linesize,int w,int h)285 static void plane_vertical_mirror(uint8_t *p, int linesize, int w, int h){
286 int j;
287 uint8_t *tmp=alloca(w*sizeof(int));
288 uint8_t *bottom_line = p + (h-1)*linesize;
289 for(j=0;j<h/2;++j){
290 memcpy(tmp, p, w);
291 memcpy(p, bottom_line, w);
292 memcpy(bottom_line, tmp, w);
293 p+=linesize;
294 bottom_line-=linesize;
295 }
296 }
297
plane_mirror(MSMirrorType type,uint8_t * p,int linesize,int w,int h)298 static void plane_mirror(MSMirrorType type, uint8_t *p, int linesize, int w, int h){
299 switch (type){
300 case MS_HORIZONTAL_MIRROR:
301 plane_horizontal_mirror(p,linesize,w,h);
302 break;
303 case MS_VERTICAL_MIRROR:
304 plane_vertical_mirror(p,linesize,w,h);
305 break;
306 case MS_CENTRAL_MIRROR:
307 plane_central_mirror(p,linesize,w,h);
308 break;
309 case MS_NO_MIRROR:
310 break;
311 }
312 }
313
314 /*in place horizontal mirroring*/
ms_yuv_buf_mirror(YuvBuf * buf)315 void ms_yuv_buf_mirror(YuvBuf *buf){
316 ms_yuv_buf_mirrors(buf, MS_HORIZONTAL_MIRROR);
317 }
318
319 /*in place mirroring*/
ms_yuv_buf_mirrors(YuvBuf * buf,MSMirrorType type)320 void ms_yuv_buf_mirrors(YuvBuf *buf, MSMirrorType type){
321 plane_mirror(type, buf->planes[0],buf->strides[0],buf->w,buf->h);
322 plane_mirror(type, buf->planes[1],buf->strides[1],buf->w/2,buf->h/2);
323 plane_mirror(type, buf->planes[2],buf->strides[2],buf->w/2,buf->h/2);
324 }
325
326 #ifndef MAKEFOURCC
327 #define MAKEFOURCC(a,b,c,d) ((d)<<24 | (c)<<16 | (b)<<8 | (a))
328 #endif
329
ms_fourcc_to_pix_fmt(uint32_t fourcc)330 MSPixFmt ms_fourcc_to_pix_fmt(uint32_t fourcc){
331 MSPixFmt ret;
332 switch (fourcc){
333 case MAKEFOURCC('I','4','2','0'):
334 ret=MS_YUV420P;
335 break;
336 case MAKEFOURCC('Y','U','Y','2'):
337 ret=MS_YUY2;
338 break;
339 case MAKEFOURCC('Y','U','Y','V'):
340 ret=MS_YUYV;
341 break;
342 case MAKEFOURCC('U','Y','V','Y'):
343 ret=MS_UYVY;
344 break;
345 case MAKEFOURCC('M','J','P','G'):
346 ret=MS_MJPEG;
347 break;
348 case 0: /*BI_RGB on windows*/
349 ret=MS_RGB24;
350 break;
351 default:
352 ret=MS_PIX_FMT_UNKNOWN;
353 }
354 return ret;
355 }
356
rgb24_mirror(uint8_t * buf,int w,int h,int linesize)357 void rgb24_mirror(uint8_t *buf, int w, int h, int linesize){
358 int i,j;
359 int r,g,b;
360 int end=w*3;
361 for(i=0;i<h;++i){
362 for(j=0;j<end/2;j+=3){
363 r=buf[j];
364 g=buf[j+1];
365 b=buf[j+2];
366 buf[j]=buf[end-j-3];
367 buf[j+1]=buf[end-j-2];
368 buf[j+2]=buf[end-j-1];
369 buf[end-j-3]=r;
370 buf[end-j-2]=g;
371 buf[end-j-1]=b;
372 }
373 buf+=linesize;
374 }
375 }
376
rgb24_revert(uint8_t * buf,int w,int h,int linesize)377 void rgb24_revert(uint8_t *buf, int w, int h, int linesize){
378 uint8_t *p,*pe;
379 int i,j;
380 uint8_t *end=buf+((h-1)*linesize);
381 uint8_t exch;
382 p=buf;
383 pe=end-1;
384 for(i=0;i<h/2;++i){
385 for(j=0;j<w*3;++j){
386 exch=p[i];
387 p[i]=pe[-i];
388 pe[-i]=exch;
389 }
390 p+=linesize;
391 pe-=linesize;
392 }
393 }
394
rgb24_copy_revert(uint8_t * dstbuf,int dstlsz,const uint8_t * srcbuf,int srclsz,MSVideoSize roi)395 void rgb24_copy_revert(uint8_t *dstbuf, int dstlsz,
396 const uint8_t *srcbuf, int srclsz, MSVideoSize roi){
397 int i,j;
398 const uint8_t *psrc;
399 uint8_t *pdst;
400 psrc=srcbuf;
401 pdst=dstbuf+(dstlsz*(roi.height-1));
402 for(i=0;i<roi.height;++i){
403 for(j=0;j<roi.width*3;++j){
404 pdst[(roi.width*3)-1-j]=psrc[j];
405 }
406 pdst-=dstlsz;
407 psrc+=srclsz;
408 }
409 }
410
411 static MSVideoSize _ordered_vsizes[]={
412 {MS_VIDEO_SIZE_QCIF_W,MS_VIDEO_SIZE_QCIF_H},
413 {MS_VIDEO_SIZE_QVGA_W,MS_VIDEO_SIZE_QVGA_H},
414 {MS_VIDEO_SIZE_CIF_W,MS_VIDEO_SIZE_CIF_H},
415 {MS_VIDEO_SIZE_VGA_W,MS_VIDEO_SIZE_VGA_H},
416 {MS_VIDEO_SIZE_4CIF_W,MS_VIDEO_SIZE_4CIF_H},
417 {MS_VIDEO_SIZE_720P_W,MS_VIDEO_SIZE_720P_H},
418 {0,0}
419 };
420
ms_video_size_get_just_lower_than(MSVideoSize vs)421 MSVideoSize ms_video_size_get_just_lower_than(MSVideoSize vs){
422 MSVideoSize *p;
423 MSVideoSize ret;
424 ret.width=0;
425 ret.height=0;
426 for(p=_ordered_vsizes;p->width!=0;++p){
427 if (ms_video_size_greater_than(vs,*p) && !ms_video_size_equal(vs,*p)){
428 ret=*p;
429 }else return ret;
430 }
431 return ret;
432 }
433
ms_rgb_to_yuv(const uint8_t rgb[3],uint8_t yuv[3])434 void ms_rgb_to_yuv(const uint8_t rgb[3], uint8_t yuv[3]){
435 yuv[0]=(uint8_t)(0.257*rgb[0] + 0.504*rgb[1] + 0.098*rgb[2] + 16);
436 yuv[1]=(uint8_t)(-0.148*rgb[0] - 0.291*rgb[1] + 0.439*rgb[2] + 128);
437 yuv[2]=(uint8_t)(0.439*rgb[0] - 0.368*rgb[1] - 0.071*rgb[2] + 128);
438 }
439
440 #if !defined(NO_FFMPEG)
441
442
ms_pix_fmt_to_ffmpeg(MSPixFmt fmt)443 int ms_pix_fmt_to_ffmpeg(MSPixFmt fmt){
444 switch(fmt){
445 case MS_RGBA32:
446 return AV_PIX_FMT_RGBA;
447 case MS_RGB24:
448 return AV_PIX_FMT_RGB24;
449 case MS_RGB24_REV:
450 return AV_PIX_FMT_BGR24;
451 case MS_YUV420P:
452 return AV_PIX_FMT_YUV420P;
453 case MS_YUYV:
454 return AV_PIX_FMT_YUYV422;
455 case MS_UYVY:
456 return AV_PIX_FMT_UYVY422;
457 case MS_YUY2:
458 return AV_PIX_FMT_YUYV422; /* <- same as MS_YUYV */
459 case MS_RGB565:
460 return AV_PIX_FMT_RGB565;
461 default:
462 ms_fatal("format not supported.");
463 return -1;
464 }
465 return -1;
466 }
467
ffmpeg_pix_fmt_to_ms(int fmt)468 MSPixFmt ffmpeg_pix_fmt_to_ms(int fmt){
469 switch(fmt){
470 case AV_PIX_FMT_RGB24:
471 return MS_RGB24;
472 case AV_PIX_FMT_BGR24:
473 return MS_RGB24_REV;
474 case AV_PIX_FMT_YUV420P:
475 return MS_YUV420P;
476 case AV_PIX_FMT_YUYV422:
477 return MS_YUYV; /* same as MS_YUY2 */
478 case AV_PIX_FMT_UYVY422:
479 return MS_UYVY;
480 case AV_PIX_FMT_RGBA:
481 return MS_RGBA32;
482 case AV_PIX_FMT_RGB565:
483 return MS_RGB565;
484 default:
485 ms_fatal("format not supported.");
486 return MS_YUV420P; /* default */
487 }
488 return MS_YUV420P; /* default */
489 }
490
491 struct _MSFFScalerContext{
492 struct SwsContext *ctx;
493 int src_h;
494 };
495
496 typedef struct _MSFFScalerContext MSFFScalerContext;
497
ff_create_swscale_context(int src_w,int src_h,MSPixFmt src_fmt,int dst_w,int dst_h,MSPixFmt dst_fmt,int flags)498 static MSScalerContext *ff_create_swscale_context(int src_w, int src_h, MSPixFmt src_fmt,
499 int dst_w, int dst_h, MSPixFmt dst_fmt, int flags){
500 int ff_flags=0;
501 MSFFScalerContext *ctx=ms_new0(MSFFScalerContext,1);
502 ctx->src_h=src_h;
503 #if MS_HAS_ARM
504 ff_flags|=SWS_FAST_BILINEAR;
505 #else
506 if (flags & MS_SCALER_METHOD_BILINEAR)
507 ff_flags|=SWS_BILINEAR;
508 else if (flags & MS_SCALER_METHOD_NEIGHBOUR)
509 ff_flags|=SWS_BILINEAR;
510 #endif
511 ctx->ctx=sws_getContext (src_w,src_h,ms_pix_fmt_to_ffmpeg (src_fmt),
512 dst_w,dst_h,ms_pix_fmt_to_ffmpeg (dst_fmt),ff_flags,NULL,NULL,NULL);
513 if (ctx->ctx==NULL){
514 ms_free(ctx);
515 ctx=NULL;
516 }
517 return (MSScalerContext*)ctx;
518 }
519
ff_sws_scale(MSScalerContext * ctx,uint8_t * src[],int src_strides[],uint8_t * dst[],int dst_strides[])520 static int ff_sws_scale(MSScalerContext *ctx, uint8_t *src[], int src_strides[], uint8_t *dst[], int dst_strides[]){
521 MSFFScalerContext *fctx=(MSFFScalerContext*)ctx;
522 #if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(0,9,0)
523 int err=sws_scale(fctx->ctx,(const uint8_t * const*)src,src_strides,0,fctx->src_h,dst,dst_strides);
524 #else
525 int err=sws_scale(fctx->ctx,(uint8_t **)src,src_strides,0,fctx->src_h,dst,dst_strides);
526 #endif
527 if (err<0) return -1;
528 return 0;
529 }
530
ff_sws_free(MSScalerContext * ctx)531 static void ff_sws_free(MSScalerContext *ctx){
532 MSFFScalerContext *fctx=(MSFFScalerContext*)ctx;
533 if (fctx->ctx) sws_freeContext(fctx->ctx);
534 ms_free(ctx);
535 }
536
537 MSScalerDesc ffmpeg_scaler={
538 ff_create_swscale_context,
539 ff_sws_scale,
540 ff_sws_free
541 };
542
543 #endif
544
545 #if 0
546
547 /*
548 We use openmax-dl (from ARM) to optimize some scaling routines.
549 */
550
551 #include "omxIP.h"
552
553 typedef struct AndroidScalerCtx{
554 MSFFScalerContext base;
555 OMXIPColorSpace cs;
556 OMXSize src_size;
557 OMXSize dst_size;
558 bool_t use_omx;
559 }AndroidScalerCtx;
560
561 /* for android we use ffmpeg's scaler except for YUV420P-->RGB565, for which we prefer
562 another arm neon optimized routine */
563
564 static MSScalerContext *android_create_scaler_context(int src_w, int src_h, MSPixFmt src_fmt,
565 int dst_w, int dst_h, MSPixFmt dst_fmt, int flags){
566 AndroidScalerCtx *ctx=ms_new0(AndroidScalerCtx,1);
567 if (src_fmt==MS_YUV420P && dst_fmt==MS_RGB565){
568 ctx->use_omx=TRUE;
569 ctx->cs=OMX_IP_BGR565;
570 ctx->src_size.width=src_w;
571 ctx->src_size.height=src_h;
572 ctx->dst_size.width=dst_w;
573 ctx->dst_size.height=dst_h;
574 }else{
575 unsigned int ff_flags=0;
576 ctx->base.src_h=src_h;
577 if (flags & MS_SCALER_METHOD_BILINEAR)
578 ff_flags|=SWS_BILINEAR;
579 else if (flags & MS_SCALER_METHOD_NEIGHBOUR)
580 ff_flags|=SWS_BILINEAR;
581 ctx->base.ctx=sws_getContext (src_w,src_h,ms_pix_fmt_to_ffmpeg (src_fmt),
582 dst_w,dst_h,ms_pix_fmt_to_ffmpeg (dst_fmt),ff_flags,NULL,NULL,NULL);
583 if (ctx->base.ctx==NULL){
584 ms_free(ctx);
585 ctx=NULL;
586 }
587 }
588 return (MSScalerContext *)ctx;
589 }
590
591 static int android_scaler_process(MSScalerContext *ctx, uint8_t *src[], int src_strides[], uint8_t *dst[], int dst_strides[]){
592 AndroidScalerCtx *actx=(AndroidScalerCtx*)ctx;
593 if (actx->use_omx){
594 int ret;
595 OMX_U8 *osrc[3];
596 OMX_INT osrc_strides[3];
597 OMX_INT xrr_max;
598 OMX_INT yrr_max;
599
600 osrc[0]=src[0];
601 osrc[1]=src[1];
602 osrc[2]=src[2];
603 osrc_strides[0]=src_strides[0];
604 osrc_strides[1]=src_strides[1];
605 osrc_strides[2]=src_strides[2];
606
607 xrr_max = (OMX_INT) ((( (OMX_F32) ((actx->src_size.width&~1)-1) / ((actx->dst_size.width&~1)-1))) * (1<<16) +0.5);
608 yrr_max = (OMX_INT) ((( (OMX_F32) ((actx->src_size.height&~1)-1) / ((actx->dst_size.height&~1)-1))) * (1<< 16) +0.5);
609
610 ret=omxIPCS_YCbCr420RszCscRotBGR_U8_P3C3R((const OMX_U8**)osrc,osrc_strides,actx->src_size,dst[0],dst_strides[0],actx->dst_size,actx->cs,
611 OMX_IP_BILINEAR, OMX_IP_DISABLE, xrr_max,yrr_max);
612 if (ret!=OMX_Sts_NoErr){
613 ms_error("omxIPCS_YCbCr420RszCscRotBGR_U8_P3C3R() failed : %i",ret);
614 return -1;
615 }
616 return 0;
617 }
618 return ff_sws_scale(ctx,src,src_strides,dst,dst_strides);
619 }
620
621 static void android_scaler_free(MSScalerContext *ctx){
622 ff_sws_free(ctx);
623 }
624
625 static MSScalerDesc android_scaler={
626 android_create_scaler_context,
627 android_scaler_process,
628 android_scaler_free
629 };
630
631 #endif
632
633 #ifdef __ANDROID__
634 #include "cpu-features.h"
635 #endif
636
637 #if defined(__ANDROID__) && defined(MS_HAS_ARM) && !defined(__aarch64__)
638 extern MSScalerDesc ms_android_scaler;
639 #endif
640
641 static MSScalerDesc *scaler_impl=NULL;
642
643
ms_scaler_create_context(int src_w,int src_h,MSPixFmt src_fmt,int dst_w,int dst_h,MSPixFmt dst_fmt,int flags)644 MSScalerContext *ms_scaler_create_context(int src_w, int src_h, MSPixFmt src_fmt,
645 int dst_w, int dst_h, MSPixFmt dst_fmt, int flags){
646 if (!scaler_impl){
647 #if defined(__ANDROID__) && defined(MS_HAS_ARM) && !defined(__aarch64__)
648 if (android_getCpuFamily() == ANDROID_CPU_FAMILY_ARM && (android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0){
649 scaler_impl = &ms_android_scaler;
650 }
651 #elif !defined(NO_FFMPEG)
652 scaler_impl=&ffmpeg_scaler;
653 #endif
654 }
655
656 if (scaler_impl)
657 return scaler_impl->create_context(src_w,src_h,src_fmt,dst_w,dst_h,dst_fmt, flags);
658 ms_fatal("No scaler implementation built-in, please supply one with ms_video_set_scaler_impl ()");
659 return NULL;
660 }
661
ms_scaler_process(MSScalerContext * ctx,uint8_t * src[],int src_strides[],uint8_t * dst[],int dst_strides[])662 int ms_scaler_process(MSScalerContext *ctx, uint8_t *src[], int src_strides[], uint8_t *dst[], int dst_strides[]){
663 return scaler_impl->context_process(ctx,src,src_strides,dst,dst_strides);
664 }
665
ms_scaler_context_free(MSScalerContext * ctx)666 void ms_scaler_context_free(MSScalerContext *ctx){
667 scaler_impl->context_free(ctx);
668 }
669
ms_video_set_scaler_impl(MSScalerDesc * desc)670 void ms_video_set_scaler_impl(MSScalerDesc *desc){
671 scaler_impl=desc;
672 }
673
674 /* Can rotate Y, U or V plane; use step=2 for interleaved UV planes otherwise step=1*/
rotate_plane_down_scale_by_2(int wDest,int hDest,int full_width,const uint8_t * src,uint8_t * dst,int step,bool_t clockWise,bool_t downscale)675 static void rotate_plane_down_scale_by_2(int wDest, int hDest, int full_width, const uint8_t* src, uint8_t* dst, int step, bool_t clockWise,bool_t downscale) {
676 int factor = downscale?2:1;
677 int hSrc = wDest*factor;
678 int wSrc = hDest*factor;
679 int src_stride = full_width*step*factor;
680
681 int signed_dst_stride;
682 int incr;
683 int y,x;
684
685 if (clockWise) {
686 /* ms_warning("start writing destination buffer from top right");*/
687 dst += wDest - 1;
688 incr = 1;
689 signed_dst_stride = wDest;
690 } else {
691 /* ms_warning("start writing destination buffer from top right");*/
692 dst += wDest * (hDest - 1);
693 incr = -1;
694 signed_dst_stride = -wDest;
695 }
696 for (y=0; y<hSrc; y+=factor) {
697 uint8_t* dst2 = dst;
698 for (x=0; x<step*wSrc; x+=step*factor) {
699 /* Copy a line in source buffer (left to right)
700 Clockwise: Store a column in destination buffer (top to bottom)
701 Not clockwise: Store a column in destination buffer (bottom to top)
702 */
703 *dst2 = src[x];
704 dst2 += signed_dst_stride;
705 }
706 dst -= incr;
707 src += src_stride;
708 }
709 }
710
711 #ifdef __ANDROID__
712
713 static int hasNeon = -1;
714 #elif MS_HAS_ARM_NEON
715 static int hasNeon = 1;
716 #elif MS_HAS_ARM
717 static int hasNeon = 0;
718 #endif
719
720 /* Destination and source images may have their dimensions inverted.*/
copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(MSYuvBufAllocator * allocator,const uint8_t * y,const uint8_t * cbcr,int rotation,int w,int h,int y_byte_per_row,int cbcr_byte_per_row,bool_t uFirstvSecond,bool_t down_scale)721 mblk_t *copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(MSYuvBufAllocator *allocator, const uint8_t* y, const uint8_t * cbcr, int rotation, int w, int h, int y_byte_per_row,int cbcr_byte_per_row, bool_t uFirstvSecond, bool_t down_scale) {
722 MSPicture pict;
723 int uv_w=w/2;
724 int uv_h=h/2;
725 const uint8_t* srcu;
726 uint8_t* dstu;
727 const uint8_t* srcv;
728 uint8_t* dstv;
729 int factor = down_scale?2:1;
730 mblk_t * yuv_block;
731
732 #ifdef __ANDROID__
733 if (hasNeon == -1) {
734 hasNeon = (((android_getCpuFamily() == ANDROID_CPU_FAMILY_ARM) && ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0))
735 || (android_getCpuFamily() == ANDROID_CPU_FAMILY_ARM64));
736 #ifdef __aarch64__
737 ms_warning("Warning: ARM64 NEON routines for video rotation are not yet implemented for Android: using SOFT version!");
738 #endif
739 }
740 #endif
741
742 yuv_block = ms_yuv_buf_allocator_get(allocator, &pict, w, h);
743
744 if (!uFirstvSecond) {
745 unsigned char* tmp = pict.planes[1];
746 pict.planes[1] = pict.planes[2];
747 pict.planes[2] = tmp;
748 }
749
750 if (rotation % 180 == 0) {
751 int i,j;
752 uint8_t* u_dest=pict.planes[1], *v_dest=pict.planes[2];
753
754 if (rotation == 0) {
755 #if MS_HAS_ARM
756 if (hasNeon) {
757 deinterlace_down_scale_neon(y, cbcr, pict.planes[0], u_dest, v_dest, w, h, y_byte_per_row, cbcr_byte_per_row,down_scale);
758 } else
759 #endif
760 {
761 // plain copy
762 for(i=0; i<h; i++) {
763 if (down_scale) {
764 for(j=0 ; j<w;j++) {
765 pict.planes[0][i*w+j] = y[i*2*y_byte_per_row+j*2];
766 }
767 } else {
768 memcpy(&pict.planes[0][i*w], &y[i*y_byte_per_row], w);
769 }
770 }
771 // de-interlace u/v
772 for (i=0; i<uv_h; i++) {
773 for(j=0; j<uv_w; j++) {
774 *u_dest++ = cbcr[cbcr_byte_per_row*i*factor + 2*j*factor];
775 *v_dest++ = cbcr[cbcr_byte_per_row*i*factor + 2*j*factor + 1];
776 }
777 }
778 }
779 } else {
780 #if defined(__arm__)
781 if (hasNeon) {
782 deinterlace_down_scale_and_rotate_180_neon(y, cbcr, pict.planes[0], u_dest, v_dest, w, h, y_byte_per_row, cbcr_byte_per_row,down_scale);
783 } else
784 #endif
785 {
786 // 180° y rotation
787 for(i=0; i<h; i++) {
788 for(j=0 ; j<w;j++) {
789 pict.planes[0][i*w+j] = y[(h-1-i)*y_byte_per_row*factor+(w-1-j)*factor];
790 }
791 }
792 // 180° rotation + de-interlace u/v
793 for (i=0; i<uv_h; i++) {
794 for(j=0; j<uv_w; j++) {
795 *u_dest++ = cbcr[cbcr_byte_per_row*(uv_h-1-i)*factor + 2*(uv_w-1-j)*factor];
796 *v_dest++ = cbcr[cbcr_byte_per_row*(uv_h-1-i)*factor + 2*(uv_w-1-j)*factor + 1];
797 }
798 }
799 }
800 }
801 } else {
802 bool_t clockwise = rotation == 90 ? TRUE : FALSE;
803 // Rotate Y
804 #if defined(__arm__)
805 if (hasNeon) {
806 if (clockwise) {
807 rotate_down_scale_plane_neon_clockwise(w,h,y_byte_per_row,(uint8_t*)y,pict.planes[0],down_scale);
808 } else {
809 rotate_down_scale_plane_neon_anticlockwise(w,h,y_byte_per_row,(uint8_t*)y,pict.planes[0], down_scale);
810 }
811 } else
812 #endif
813 {
814 uint8_t* dsty = pict.planes[0];
815 uint8_t* srcy = (uint8_t*) y;
816 rotate_plane_down_scale_by_2(w,h,y_byte_per_row,srcy,dsty,1, clockwise, down_scale);
817 }
818
819 #if defined(__arm__)
820 if (hasNeon) {
821 rotate_down_scale_cbcr_to_cr_cb(uv_w,uv_h, cbcr_byte_per_row/2, (uint8_t*)cbcr, pict.planes[2], pict.planes[1],clockwise,down_scale);
822 } else
823 #endif
824 {
825 // Copying U
826 srcu = cbcr;
827 dstu = pict.planes[1];
828 rotate_plane_down_scale_by_2(uv_w,uv_h,cbcr_byte_per_row/2,srcu,dstu, 2, clockwise, down_scale);
829 // Copying V
830 srcv = srcu + 1;
831 dstv = pict.planes[2];
832 rotate_plane_down_scale_by_2(uv_w,uv_h,cbcr_byte_per_row/2,srcv,dstv, 2, clockwise,down_scale);
833 }
834 }
835
836 return yuv_block;
837 }
838
copy_ycbcrbiplanar_to_true_yuv_with_rotation(MSYuvBufAllocator * allocator,const uint8_t * y,const uint8_t * cbcr,int rotation,int w,int h,int y_byte_per_row,int cbcr_byte_per_row,bool_t uFirstvSecond)839 mblk_t *copy_ycbcrbiplanar_to_true_yuv_with_rotation(MSYuvBufAllocator *allocator, const uint8_t* y, const uint8_t * cbcr, int rotation, int w, int h, int y_byte_per_row,int cbcr_byte_per_row, bool_t uFirstvSecond) {
840 return copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(allocator, y, cbcr, rotation, w, h, y_byte_per_row, cbcr_byte_per_row, uFirstvSecond, FALSE);
841 }
842
ms_video_init_framerate_controller(MSFrameRateController * ctrl,float fps)843 void ms_video_init_framerate_controller(MSFrameRateController* ctrl, float fps) {
844 ctrl->start_time = 0;
845 ctrl->th_frame_count = -1;
846 ctrl->fps = fps;
847 }
848
ms_video_capture_new_frame(MSFrameRateController * ctrl,uint64_t current_time)849 bool_t ms_video_capture_new_frame(MSFrameRateController* ctrl, uint64_t current_time) {
850 int cur_frame;
851 float elapsed;
852
853 /* init controller */
854 if (ctrl->th_frame_count==-1){
855 ctrl->start_time = current_time;
856 ctrl->th_frame_count = 0;
857 }
858
859 elapsed = ((float)(current_time - ctrl->start_time))/1000.0f;
860 cur_frame = (int)(elapsed * ctrl->fps);
861
862 if (cur_frame>=ctrl->th_frame_count){
863 ctrl->th_frame_count++;
864 return TRUE;
865 } else {
866 return FALSE;
867 }
868 }
869
ms_average_fps_init(MSAverageFPS * afps,const char * ctx)870 void ms_average_fps_init(MSAverageFPS* afps, const char* ctx) {
871 afps->last_frame_time = (uint64_t)-1;
872 afps->last_print_time = (uint64_t)-1;
873 afps->mean_inter_frame = 0;
874 afps->context = ctx;
875 if (!ctx || strstr(ctx, "%f") == 0) {
876 ms_error("Invalid MSAverageFPS context given '%s' (must be not null and must contain one occurence of '%%f'", ctx);
877 }
878 }
879
880 /*compatibility, deprecated*/
ms_video_init_average_fps(MSAverageFPS * afps,const char * ctx)881 void ms_video_init_average_fps(MSAverageFPS* afps, const char* ctx){
882 ms_average_fps_init(afps,ctx);
883 }
884
ms_average_fps_update(MSAverageFPS * afps,uint64_t current_time)885 bool_t ms_average_fps_update(MSAverageFPS* afps, uint64_t current_time) {
886 if (afps->last_frame_time!=(uint64_t)-1){
887 float frame_interval=(float)(current_time - afps->last_frame_time)/1000.0f;
888 if (afps->mean_inter_frame==0){
889 afps->mean_inter_frame=frame_interval;
890 }else{
891 afps->mean_inter_frame=(0.8f*afps->mean_inter_frame)+(0.2f*frame_interval);
892 }
893 } else {
894 afps->last_print_time = current_time;
895 }
896 afps->last_frame_time=current_time;
897
898 if ((current_time - afps->last_print_time > 5000) && afps->mean_inter_frame!=0){
899 ms_message(afps->context, 1/afps->mean_inter_frame);
900 afps->last_print_time = current_time;
901 return TRUE;
902 }
903 return FALSE;
904 }
905
906 /*compatibility, deprecated*/
ms_video_update_average_fps(MSAverageFPS * afps,uint64_t current_time)907 bool_t ms_video_update_average_fps(MSAverageFPS* afps, uint64_t current_time){
908 return ms_average_fps_update(afps,current_time);
909 }
910
ms_average_fps_get(const MSAverageFPS * afps)911 float ms_average_fps_get(const MSAverageFPS* afps){
912 return afps->mean_inter_frame!=0 ? 1.0f/afps->mean_inter_frame : 0.0f;
913 }
914
ms_video_find_best_configuration_for_bitrate(const MSVideoConfiguration * vconf_list,int bitrate,int cpu_count)915 MSVideoConfiguration ms_video_find_best_configuration_for_bitrate(const MSVideoConfiguration *vconf_list, int bitrate , int cpu_count) {
916 const MSVideoConfiguration *vconf_it = vconf_list;
917 MSVideoConfiguration best_vconf={0};
918 int max_pixels=0;
919
920 /* search for configuration that has compatible cpu count, compatible bitrate, biggest video size, and greater fps*/
921 while(TRUE) {
922 int pixels=vconf_it->vsize.width*vconf_it->vsize.height;
923 if ((cpu_count>=vconf_it->mincpu && bitrate>=vconf_it->required_bitrate) || vconf_it->required_bitrate==0){
924 if (pixels>max_pixels){
925 best_vconf=*vconf_it;
926 max_pixels=pixels;
927 }else if (pixels==max_pixels){
928 if (best_vconf.fps<vconf_it->fps){
929 best_vconf=*vconf_it;
930 }
931 }
932 }
933 if (vconf_it->required_bitrate==0) {
934 break;
935 }
936 vconf_it++;
937 }
938 best_vconf.required_bitrate=bitrate>best_vconf.bitrate_limit ? best_vconf.bitrate_limit : bitrate;
939 return best_vconf;
940 }
941
ms_video_find_best_configuration_for_size(const MSVideoConfiguration * vconf_list,MSVideoSize vsize,int cpu_count)942 MSVideoConfiguration ms_video_find_best_configuration_for_size(const MSVideoConfiguration *vconf_list, MSVideoSize vsize, int cpu_count) {
943 const MSVideoConfiguration *vconf_it = vconf_list;
944 MSVideoConfiguration best_vconf={0};
945 int min_score=INT32_MAX;
946 int ref_pixels=vsize.height*vsize.width;
947
948 /* search for configuration that is first nearest to target video size, then second has the greater fps,
949 * but any case making sure the the cpu count is sufficient*/
950 while(TRUE) {
951 int pixels=vconf_it->vsize.width*vconf_it->vsize.height;
952 int score=abs(pixels-ref_pixels);
953 if (cpu_count>=vconf_it->mincpu){
954 if (score<min_score){
955 best_vconf=*vconf_it;
956 min_score=score;
957 }else if (score==min_score){
958 if (best_vconf.fps<vconf_it->fps){
959 best_vconf=*vconf_it;
960 }
961 }
962 }
963 if (vconf_it->required_bitrate==0) {
964 break;
965 }
966 vconf_it++;
967
968 }
969 best_vconf.vsize=vsize;
970 return best_vconf;
971 }
972
ms_video_find_best_configuration_for_size_and_bitrate(const MSVideoConfiguration * vconf_list,MSVideoSize vsize,int cpu_count,int bitrate)973 MSVideoConfiguration ms_video_find_best_configuration_for_size_and_bitrate(const MSVideoConfiguration *vconf_list, MSVideoSize vsize, int cpu_count, int bitrate) {
974 const MSVideoConfiguration *vconf_it = vconf_list;
975 MSVideoConfiguration best_vconf={0};
976 int min_score=INT32_MAX;
977 int ref_pixels=vsize.height*vsize.width;
978
979 if (bitrate == 0) return ms_video_find_best_configuration_for_size(vconf_list, vsize, cpu_count);
980
981 /* search for configuration that is first nearest to target video size, then target bitrate and finally has the greater fps,
982 * but any case making sure the the cpu count is sufficient*/
983 while(TRUE) {
984 int pixels=vconf_it->vsize.width*vconf_it->vsize.height;
985 int score=abs(pixels-ref_pixels);
986 if (cpu_count>=vconf_it->mincpu){
987 if (score<min_score){
988 best_vconf=*vconf_it;
989 min_score=score;
990 }else if (score==min_score) {
991 if (bitrate <= vconf_it->bitrate_limit && bitrate >= vconf_it->required_bitrate) {
992 best_vconf=*vconf_it;
993 }
994 }
995 }
996 if (vconf_it->required_bitrate==0) {
997 break;
998 }
999 vconf_it++;
1000
1001 }
1002 best_vconf.vsize = vsize;
1003 return best_vconf;
1004 }
1005