1 /* $Id$ */
2 /*
3 * Copyright (C) 2019 Teluu Inc. (http://www.teluu.com)
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19 #include <pjmedia/vid_conf.h>
20 #include <pjmedia/clock.h>
21 #include <pjmedia/converter.h>
22 #include <pjmedia/errno.h>
23 #include <pj/array.h>
24 #include <pj/log.h>
25 #include <pj/os.h>
26
27 #if defined(PJMEDIA_HAS_VIDEO) && (PJMEDIA_HAS_VIDEO != 0)
28
29
30 #define CONF_NAME "vidconf"
31 #define CONF_SIGN PJMEDIA_SIG_VID_CONF
32
33 /* If set, conf will stop clock when there is no ports connection. However,
34 * this may cause stuck if port remove/disconnect is called from the clock
35 * callback. So better disable this for now.
36 */
37 #define AUTO_STOP_CLOCK 0
38
39 /* Maximum number of consecutive errors that will only be printed once. */
40 #define MAX_ERR_COUNT 150
41
42 /* Clockrate for video timestamp unit */
43 #define TS_CLOCK_RATE 90000
44
45 #define THIS_FILE "vid_conf.c"
46 #define TRACE_(x) PJ_LOG(5,x)
47
48
49 /*
50 * Conference bridge.
51 */
52 struct pjmedia_vid_conf
53 {
54 pjmedia_vid_conf_setting opt; /**< Settings. */
55 unsigned port_cnt; /**< Current number of ports. */
56 unsigned connect_cnt; /**< Total number of connections */
57 pj_mutex_t *mutex; /**< Conference mutex. */
58 struct vconf_port **ports; /**< Array of ports. */
59 pjmedia_clock *clock; /**< Clock. */
60 };
61
62
63 /*
64 * Rendering state: converter, layout settings, etc.
65 */
66 typedef struct render_state
67 {
68 pjmedia_format_id src_fmt_id; /**< Source format ID. */
69 pjmedia_rect_size src_frame_size; /**< Source frame size. */
70 pjmedia_rect src_rect; /**< Source region to be rendered. */
71
72 pjmedia_format_id dst_fmt_id; /**< Destination format ID. */
73 pjmedia_rect_size dst_frame_size; /**< Destination frame size. */
74 pjmedia_rect dst_rect; /**< Destination region. */
75
76 pjmedia_converter *converter; /**< Converter. */
77
78 } render_state;
79
80
81 /*
82 * Conference bridge port.
83 */
84 typedef struct vconf_port
85 {
86 pj_pool_t *pool; /**< Pool. */
87 unsigned idx; /**< Port index. */
88 pj_str_t name; /**< Port name. */
89 pjmedia_port *port; /**< Video port. */
90 pj_uint32_t ts_interval; /**< Port put/get interval. */
91 pj_timestamp ts_next; /**< Time for next put/get_frame(). */
92 void *get_buf; /**< Buffer for get_frame(). */
93 pj_size_t get_buf_size; /**< Buffer size for get_frame(). */
94 void *put_buf; /**< Buffer for put_frame(). */
95 pj_size_t put_buf_size; /**< Buffer size for put_frame(). */
96
97 unsigned listener_cnt; /**< Number of listeners. */
98 unsigned *listener_slots;/**< Array of listeners (for info). */
99
100 unsigned transmitter_cnt;/**<Number of transmitters. */
101 unsigned *transmitter_slots;/**< Array of transmitters. */
102 pj_pool_t **render_pool; /**< Array of pool for render state */
103 render_state **render_states; /**< Array of render_state (one for
104 each transmitter). */
105
106 pj_status_t last_err; /**< Last error status. */
107 unsigned last_err_cnt; /**< Last error count. */
108 } vconf_port;
109
110
111 /* Prototypes */
112 static void on_clock_tick(const pj_timestamp *ts, void *user_data);
113 static pj_status_t render_src_frame(vconf_port *src, vconf_port *sink,
114 unsigned transmitter_idx);
115 static void update_render_state(pjmedia_vid_conf *vid_conf, vconf_port *cp);
116 static void cleanup_render_state(vconf_port *cp,
117 unsigned transmitter_idx);
118
119
120 /*
121 * Initialize video conference settings with default values.
122 */
pjmedia_vid_conf_setting_default(pjmedia_vid_conf_setting * opt)123 PJ_DEF(void) pjmedia_vid_conf_setting_default(pjmedia_vid_conf_setting *opt)
124 {
125 pj_bzero(opt, sizeof(*opt));
126 opt->max_slot_cnt = 32;
127 opt->frame_rate = 60;
128 }
129
130
131 /*
132 * Create a video conference bridge.
133 */
pjmedia_vid_conf_create(pj_pool_t * pool,const pjmedia_vid_conf_setting * opt,pjmedia_vid_conf ** p_vid_conf)134 PJ_DEF(pj_status_t) pjmedia_vid_conf_create(
135 pj_pool_t *pool,
136 const pjmedia_vid_conf_setting *opt,
137 pjmedia_vid_conf **p_vid_conf)
138 {
139 pjmedia_vid_conf *vid_conf;
140 pjmedia_clock_param clock_param;
141 pj_status_t status;
142
143 PJ_ASSERT_RETURN(pool && p_vid_conf, PJ_EINVAL);
144
145 /* Allocate conf structure */
146 vid_conf = PJ_POOL_ZALLOC_T(pool, pjmedia_vid_conf);
147 PJ_ASSERT_RETURN(vid_conf, PJ_ENOMEM);
148
149 /* Init settings */
150 if (opt) {
151 vid_conf->opt = *opt;
152 } else {
153 pjmedia_vid_conf_setting_default(&vid_conf->opt);
154 }
155
156 /* Allocate ports */
157 vid_conf->ports = (vconf_port**)
158 pj_pool_zalloc(pool, vid_conf->opt.max_slot_cnt *
159 sizeof(vconf_port*));
160 PJ_ASSERT_RETURN(vid_conf->ports, PJ_ENOMEM);
161
162 /* Create mutex */
163 status = pj_mutex_create_recursive(pool, CONF_NAME, &vid_conf->mutex);
164 if (status != PJ_SUCCESS) {
165 pjmedia_vid_conf_destroy(vid_conf);
166 return status;
167 }
168
169 /* Create clock */
170 pj_bzero(&clock_param, sizeof(clock_param));
171 clock_param.clock_rate = TS_CLOCK_RATE;
172 clock_param.usec_interval = 1000000 / vid_conf->opt.frame_rate;
173 status = pjmedia_clock_create2(pool, &clock_param, 0, &on_clock_tick,
174 vid_conf, &vid_conf->clock);
175 if (status != PJ_SUCCESS) {
176 pjmedia_vid_conf_destroy(vid_conf);
177 return status;
178 }
179
180 /* Done */
181 *p_vid_conf = vid_conf;
182
183 PJ_LOG(5,(THIS_FILE, "Created video conference bridge with %d ports",
184 vid_conf->opt.max_slot_cnt));
185
186 return PJ_SUCCESS;
187 }
188
189
190 /*
191 * Destroy video conference bridge.
192 */
pjmedia_vid_conf_destroy(pjmedia_vid_conf * vid_conf)193 PJ_DEF(pj_status_t) pjmedia_vid_conf_destroy(pjmedia_vid_conf *vid_conf)
194 {
195 unsigned i;
196
197 PJ_ASSERT_RETURN(vid_conf, PJ_EINVAL);
198
199 /* Destroy clock */
200 if (vid_conf->clock) {
201 pjmedia_clock_destroy(vid_conf->clock);
202 vid_conf->clock = NULL;
203 }
204
205 /* Remove any registered ports (at least to cleanup their pool) */
206 for (i=0; i < vid_conf->opt.max_slot_cnt; ++i) {
207 pjmedia_vid_conf_remove_port(vid_conf, i);
208 }
209
210 /* Destroy mutex */
211 if (vid_conf->mutex) {
212 pj_mutex_destroy(vid_conf->mutex);
213 vid_conf->mutex = NULL;
214 }
215
216 PJ_LOG(5,(THIS_FILE, "Video conference bridge destroyed"));
217
218 return PJ_SUCCESS;
219 }
220
221
222 /*
223 * Add a media port to the video conference bridge.
224 */
pjmedia_vid_conf_add_port(pjmedia_vid_conf * vid_conf,pj_pool_t * parent_pool,pjmedia_port * port,const pj_str_t * name,void * opt,unsigned * p_slot)225 PJ_DEF(pj_status_t) pjmedia_vid_conf_add_port( pjmedia_vid_conf *vid_conf,
226 pj_pool_t *parent_pool,
227 pjmedia_port *port,
228 const pj_str_t *name,
229 void *opt,
230 unsigned *p_slot)
231 {
232 pj_pool_t *pool;
233 vconf_port *cport;
234 unsigned index;
235
236 PJ_ASSERT_RETURN(vid_conf && parent_pool && port, PJ_EINVAL);
237 PJ_ASSERT_RETURN(port->info.fmt.type==PJMEDIA_TYPE_VIDEO &&
238 port->info.fmt.detail_type==PJMEDIA_FORMAT_DETAIL_VIDEO,
239 PJ_EINVAL);
240 PJ_UNUSED_ARG(opt);
241
242 /* If name is not specified, use the port's name */
243 if (!name)
244 name = &port->info.name;
245
246 pj_mutex_lock(vid_conf->mutex);
247
248 if (vid_conf->port_cnt >= vid_conf->opt.max_slot_cnt) {
249 pj_assert(!"Too many ports");
250 pj_mutex_unlock(vid_conf->mutex);
251 return PJ_ETOOMANY;
252 }
253
254 /* Find empty port in the conference bridge. */
255 for (index=0; index < vid_conf->opt.max_slot_cnt; ++index) {
256 if (vid_conf->ports[index] == NULL)
257 break;
258 }
259 pj_assert(index != vid_conf->opt.max_slot_cnt);
260
261 /* Create pool */
262 pool = pj_pool_create(parent_pool->factory, name->ptr, 500, 500, NULL);
263 PJ_ASSERT_RETURN(pool, PJ_ENOMEM);
264
265 /* Create port. */
266 cport = PJ_POOL_ZALLOC_T(pool, vconf_port);
267 PJ_ASSERT_RETURN(cport, PJ_ENOMEM);
268
269 /* Set pool, port, index, and name */
270 cport->pool = pool;
271 cport->port = port;
272 cport->idx = index;
273 pj_strdup_with_null(pool, &cport->name, name);
274
275 /* Init put/get_frame() intervals */
276 {
277 pjmedia_ratio *fps = &port->info.fmt.det.vid.fps;
278 pj_uint32_t vconf_interval = (pj_uint32_t)
279 (TS_CLOCK_RATE * 1.0 /
280 vid_conf->opt.frame_rate);
281 cport->ts_interval = (pj_uint32_t)(TS_CLOCK_RATE * 1.0 /
282 fps->num * fps->denum);
283
284 /* Normalize the interval */
285 if (cport->ts_interval < vconf_interval) {
286 cport->ts_interval = vconf_interval;
287 PJ_LOG(3,(THIS_FILE, "Warning: frame rate of port %s is higher "
288 "than video conference bridge (%d > %d)",
289 name->ptr, (int)(fps->num/fps->denum),
290 vid_conf->opt.frame_rate));
291 }
292 }
293
294 /* Allocate buffer for put/get_frame() */
295 {
296 const pjmedia_video_format_info *vfi;
297 pjmedia_video_apply_fmt_param vafp;
298 pj_status_t status;
299
300 vfi = pjmedia_get_video_format_info(NULL, port->info.fmt.id);
301 if (!vfi) {
302 PJ_LOG(4,(THIS_FILE, "pjmedia_vid_conf_add_port(): "
303 "unrecognized format %04X",
304 port->info.fmt.id));
305 return PJMEDIA_EBADFMT;
306 }
307
308 pj_bzero(&vafp, sizeof(vafp));
309 vafp.size = port->info.fmt.det.vid.size;
310 status = (*vfi->apply_fmt)(vfi, &vafp);
311 if (status != PJ_SUCCESS) {
312 PJ_LOG(4,(THIS_FILE, "pjmedia_vid_conf_add_port(): "
313 "Failed to apply format %04X",
314 port->info.fmt.id));
315 return status;
316 }
317 if (port->put_frame) {
318 cport->put_buf_size = vafp.framebytes;
319 cport->put_buf = pj_pool_zalloc(cport->pool, cport->put_buf_size);
320 }
321 if (port->get_frame) {
322 cport->get_buf_size = vafp.framebytes;
323 cport->get_buf = pj_pool_zalloc(cport->pool, cport->get_buf_size);
324 }
325 }
326
327 /* Create listener array */
328 cport->listener_slots = (unsigned*)
329 pj_pool_zalloc(pool,
330 vid_conf->opt.max_slot_cnt *
331 sizeof(unsigned));
332 PJ_ASSERT_RETURN(cport->listener_slots, PJ_ENOMEM);
333
334 /* Create transmitter array */
335 cport->transmitter_slots = (unsigned*)
336 pj_pool_zalloc(pool,
337 vid_conf->opt.max_slot_cnt *
338 sizeof(unsigned));
339 PJ_ASSERT_RETURN(cport->transmitter_slots, PJ_ENOMEM);
340
341 /* Create pointer-to-render_state array */
342 cport->render_states = (render_state**)
343 pj_pool_zalloc(pool,
344 vid_conf->opt.max_slot_cnt *
345 sizeof(render_state*));
346 PJ_ASSERT_RETURN(cport->render_states, PJ_ENOMEM);
347
348 /* Create pointer-to-render-pool array */
349 cport->render_pool = (pj_pool_t**)
350 pj_pool_zalloc(pool,
351 vid_conf->opt.max_slot_cnt *
352 sizeof(pj_pool_t*));
353 PJ_ASSERT_RETURN(cport->render_pool, PJ_ENOMEM);
354
355 /* Register the conf port. */
356 vid_conf->ports[index] = cport;
357 vid_conf->port_cnt++;
358
359 PJ_LOG(4,(THIS_FILE,"Added port %d (%.*s)",
360 index, (int)cport->name.slen, cport->name.ptr));
361
362 pj_mutex_unlock(vid_conf->mutex);
363
364 /* Done. */
365 if (p_slot) {
366 *p_slot = index;
367 }
368
369 return PJ_SUCCESS;
370 }
371
372
373 /*
374 * Remove a media port from the video conference bridge.
375 */
pjmedia_vid_conf_remove_port(pjmedia_vid_conf * vid_conf,unsigned slot)376 PJ_DEF(pj_status_t) pjmedia_vid_conf_remove_port( pjmedia_vid_conf *vid_conf,
377 unsigned slot)
378 {
379 vconf_port *cport;
380
381 PJ_ASSERT_RETURN(vid_conf && slot<vid_conf->opt.max_slot_cnt, PJ_EINVAL);
382
383 pj_mutex_lock(vid_conf->mutex);
384
385 /* Port must be valid. */
386 cport = vid_conf->ports[slot];
387 if (cport == NULL) {
388 pj_mutex_unlock(vid_conf->mutex);
389 return PJ_EINVAL;
390 }
391
392 /* Disconnect slot -> listeners */
393 while (cport->listener_cnt) {
394 pjmedia_vid_conf_disconnect_port(vid_conf, slot,
395 cport->listener_slots[0]);
396 }
397
398 /* Disconnect transmitters -> slot */
399 while (cport->transmitter_cnt) {
400 pjmedia_vid_conf_disconnect_port(vid_conf,
401 cport->transmitter_slots[0], slot);
402 }
403
404 /* Remove the port. */
405 vid_conf->ports[slot] = NULL;
406 --vid_conf->port_cnt;
407
408 PJ_LOG(4,(THIS_FILE,"Removed port %d (%.*s)",
409 slot, (int)cport->name.slen, cport->name.ptr));
410
411 /* Release pool */
412 pj_pool_safe_release(&cport->pool);
413
414 if (AUTO_STOP_CLOCK && vid_conf->connect_cnt == 0) {
415 pj_status_t status;
416
417 /* Warning: will stuck if this is called from the clock thread */
418 status = pjmedia_clock_stop(vid_conf->clock);
419 if (status != PJ_SUCCESS) {
420 PJ_PERROR(4, (THIS_FILE, status, "Failed to stop clock"));
421 return status;
422 }
423 }
424
425 pj_mutex_unlock(vid_conf->mutex);
426
427 return PJ_SUCCESS;
428 }
429
430
431 /*
432 * Get number of ports currently registered in the video conference bridge.
433 */
pjmedia_vid_conf_get_port_count(pjmedia_vid_conf * vid_conf)434 PJ_DEF(unsigned) pjmedia_vid_conf_get_port_count(pjmedia_vid_conf *vid_conf)
435 {
436 return vid_conf->port_cnt;
437 }
438
439
440 /*
441 * Enumerate occupied slots in the video conference bridge.
442 */
pjmedia_vid_conf_enum_ports(pjmedia_vid_conf * vid_conf,unsigned slots[],unsigned * count)443 PJ_DEF(pj_status_t) pjmedia_vid_conf_enum_ports( pjmedia_vid_conf *vid_conf,
444 unsigned slots[],
445 unsigned *count)
446 {
447 unsigned i, tmp_count=0;
448
449 PJ_ASSERT_RETURN(vid_conf && slots && count, PJ_EINVAL);
450
451 /* Lock mutex */
452 pj_mutex_lock(vid_conf->mutex);
453
454 for (i=0; i<vid_conf->opt.max_slot_cnt && tmp_count<*count; ++i) {
455 if (!vid_conf->ports[i])
456 continue;
457
458 slots[tmp_count++] = i;
459 }
460
461 /* Unlock mutex */
462 pj_mutex_unlock(vid_conf->mutex);
463
464 *count = tmp_count;
465
466 return PJ_SUCCESS;
467 }
468
469
470 /*
471 * Get port info.
472 */
pjmedia_vid_conf_get_port_info(pjmedia_vid_conf * vid_conf,unsigned slot,pjmedia_vid_conf_port_info * info)473 PJ_DEF(pj_status_t) pjmedia_vid_conf_get_port_info(
474 pjmedia_vid_conf *vid_conf,
475 unsigned slot,
476 pjmedia_vid_conf_port_info *info)
477 {
478 vconf_port *cp;
479
480 /* Check arguments */
481 PJ_ASSERT_RETURN(vid_conf && slot<vid_conf->opt.max_slot_cnt, PJ_EINVAL);
482
483 /* Lock mutex */
484 pj_mutex_lock(vid_conf->mutex);
485
486 /* Port must be valid. */
487 cp = vid_conf->ports[slot];
488 if (cp == NULL) {
489 pj_mutex_unlock(vid_conf->mutex);
490 return PJ_EINVAL;
491 }
492
493 info->slot = slot;
494 info->name = cp->name;
495 pjmedia_format_copy(&info->format, &cp->port->info.fmt);
496 info->listener_cnt = cp->listener_cnt;
497 info->listener_slots = cp->listener_slots;
498 info->transmitter_cnt = cp->transmitter_cnt;
499 info->transmitter_slots = cp->transmitter_slots;
500
501 /* Unlock mutex */
502 pj_mutex_unlock(vid_conf->mutex);
503
504 return PJ_SUCCESS;
505 }
506
507
508 /*
509 * Enable unidirectional video flow from the specified source slot to
510 * the specified sink slot.
511 */
pjmedia_vid_conf_connect_port(pjmedia_vid_conf * vid_conf,unsigned src_slot,unsigned sink_slot,void * opt)512 PJ_DEF(pj_status_t) pjmedia_vid_conf_connect_port(
513 pjmedia_vid_conf *vid_conf,
514 unsigned src_slot,
515 unsigned sink_slot,
516 void *opt)
517 {
518 vconf_port *src_port, *dst_port;
519 unsigned i;
520
521 /* Check arguments */
522 PJ_ASSERT_RETURN(vid_conf &&
523 src_slot<vid_conf->opt.max_slot_cnt &&
524 sink_slot<vid_conf->opt.max_slot_cnt, PJ_EINVAL);
525 PJ_UNUSED_ARG(opt);
526
527 pj_mutex_lock(vid_conf->mutex);
528
529 /* Ports must be valid. */
530 src_port = vid_conf->ports[src_slot];
531 dst_port = vid_conf->ports[sink_slot];
532 if (!src_port || !src_port->port->get_frame ||
533 !dst_port || !dst_port->port->put_frame)
534 {
535 PJ_LOG(4,(THIS_FILE,"Failed connecting video ports, make sure that "
536 "source has get_frame() & sink has put_frame()"));
537 pj_mutex_unlock(vid_conf->mutex);
538 return PJ_EINVAL;
539 }
540
541 /* Check if connection has been made */
542 for (i=0; i<src_port->listener_cnt; ++i) {
543 if (src_port->listener_slots[i] == sink_slot)
544 break;
545 }
546
547 if (i == src_port->listener_cnt) {
548 src_port->listener_slots[src_port->listener_cnt] = sink_slot;
549 dst_port->transmitter_slots[dst_port->transmitter_cnt] = src_slot;
550 ++src_port->listener_cnt;
551 ++dst_port->transmitter_cnt;
552
553 if (src_port->listener_cnt == 1) {
554 /* If this is the first listener, initialize source's buffer
555 * with black color.
556 */
557 const pjmedia_video_format_info *vfi;
558 pjmedia_video_apply_fmt_param vafp;
559
560 vfi = pjmedia_get_video_format_info(NULL,
561 src_port->port->info.fmt.id);
562 pj_bzero(&vafp, sizeof(vafp));
563 vafp.size = src_port->port->info.fmt.det.vid.size;
564 (*vfi->apply_fmt)(vfi, &vafp);
565
566 if (vfi->color_model == PJMEDIA_COLOR_MODEL_RGB) {
567 pj_memset(src_port->get_buf, 0, vafp.framebytes);
568 } else if (src_port->port->info.fmt.id == PJMEDIA_FORMAT_I420 ||
569 src_port->port->info.fmt.id == PJMEDIA_FORMAT_YV12)
570 {
571 pj_memset(src_port->get_buf, 16, vafp.plane_bytes[0]);
572 pj_memset((pj_uint8_t*)src_port->get_buf + vafp.plane_bytes[0],
573 0x80, vafp.plane_bytes[1] * 2);
574 }
575 }
576
577 update_render_state(vid_conf, dst_port);
578
579 ++vid_conf->connect_cnt;
580 if (vid_conf->connect_cnt == 1) {
581 pj_status_t status;
582 status = pjmedia_clock_start(vid_conf->clock);
583 if (status != PJ_SUCCESS) {
584 PJ_PERROR(4, (THIS_FILE, status, "Failed to start clock"));
585 return status;
586 }
587 }
588
589 PJ_LOG(4,(THIS_FILE,"Port %d (%.*s) transmitting to port %d (%.*s)",
590 src_slot,
591 (int)src_port->name.slen,
592 src_port->name.ptr,
593 sink_slot,
594 (int)dst_port->name.slen,
595 dst_port->name.ptr));
596 }
597
598 pj_mutex_unlock(vid_conf->mutex);
599
600 return PJ_SUCCESS;
601 }
602
603
604 /*
605 * Disconnect unidirectional video flow from the specified source to
606 * the specified sink slot.
607 */
pjmedia_vid_conf_disconnect_port(pjmedia_vid_conf * vid_conf,unsigned src_slot,unsigned sink_slot)608 PJ_DEF(pj_status_t) pjmedia_vid_conf_disconnect_port(
609 pjmedia_vid_conf *vid_conf,
610 unsigned src_slot,
611 unsigned sink_slot)
612 {
613 vconf_port *src_port, *dst_port;
614 unsigned i, j;
615
616 /* Check arguments */
617 PJ_ASSERT_RETURN(vid_conf &&
618 src_slot<vid_conf->opt.max_slot_cnt &&
619 sink_slot<vid_conf->opt.max_slot_cnt, PJ_EINVAL);
620
621 pj_mutex_lock(vid_conf->mutex);
622
623 /* Ports must be valid. */
624 src_port = vid_conf->ports[src_slot];
625 dst_port = vid_conf->ports[sink_slot];
626 if (!src_port || !dst_port) {
627 pj_mutex_unlock(vid_conf->mutex);
628 return PJ_EINVAL;
629 }
630
631 /* Check if connection has been made */
632 for (i=0; i<src_port->listener_cnt; ++i) {
633 if (src_port->listener_slots[i] == sink_slot)
634 break;
635 }
636 for (j=0; j<dst_port->transmitter_cnt; ++j) {
637 if (dst_port->transmitter_slots[j] == src_slot)
638 break;
639 }
640
641 if (i != src_port->listener_cnt && j != dst_port->transmitter_cnt) {
642 unsigned k;
643
644 pj_assert(src_port->listener_cnt > 0 &&
645 src_port->listener_cnt < vid_conf->opt.max_slot_cnt);
646 pj_assert(dst_port->transmitter_cnt > 0 &&
647 dst_port->transmitter_cnt < vid_conf->opt.max_slot_cnt);
648
649 /* Cleanup all render states of the sink */
650 for (k=0; k<dst_port->transmitter_cnt; ++k)
651 cleanup_render_state(dst_port, k);
652
653 /* Update listeners array of the source and transmitters array of
654 * the sink.
655 */
656 pj_array_erase(src_port->listener_slots, sizeof(unsigned),
657 src_port->listener_cnt, i);
658 pj_array_erase(dst_port->transmitter_slots, sizeof(unsigned),
659 dst_port->transmitter_cnt, j);
660 --src_port->listener_cnt;
661 --dst_port->transmitter_cnt;
662
663 /* Update render states of the sink */
664 update_render_state(vid_conf, dst_port);
665
666 --vid_conf->connect_cnt;
667
668 if (AUTO_STOP_CLOCK && vid_conf->connect_cnt == 0) {
669 pj_status_t status;
670 /* Warning: will stuck if this is called from the clock thread */
671 status = pjmedia_clock_stop(vid_conf->clock);
672 if (status != PJ_SUCCESS) {
673 PJ_PERROR(4, (THIS_FILE, status, "Failed to stop clock"));
674 return status;
675 }
676 }
677
678 PJ_LOG(4,(THIS_FILE,
679 "Port %d (%.*s) stop transmitting to port %d (%.*s)",
680 src_slot,
681 (int)src_port->name.slen,
682 src_port->name.ptr,
683 sink_slot,
684 (int)dst_port->name.slen,
685 dst_port->name.ptr));
686 }
687
688 pj_mutex_unlock(vid_conf->mutex);
689
690 return PJ_SUCCESS;
691 }
692
693
694 /*
695 * Internal functions.
696 */
697
on_clock_tick(const pj_timestamp * now,void * user_data)698 static void on_clock_tick(const pj_timestamp *now, void *user_data)
699 {
700 pjmedia_vid_conf *vid_conf = (pjmedia_vid_conf*)user_data;
701 unsigned ci, i;
702 pj_int32_t ts_diff;
703 pjmedia_frame frame;
704 pj_status_t status;
705
706 pj_mutex_lock(vid_conf->mutex);
707
708 /* Iterate all (sink) ports */
709 for (i=0, ci=0; i<vid_conf->opt.max_slot_cnt &&
710 ci<vid_conf->port_cnt; ++i)
711 {
712 unsigned j;
713 pj_bool_t got_frame = PJ_FALSE;
714 pj_bool_t ts_incremented = PJ_FALSE;
715 vconf_port *sink = vid_conf->ports[i];
716
717 /* Skip empty port */
718 if (!sink)
719 continue;
720
721 /* Increment occupied port counter */
722 ++ci;
723
724 /* Skip non-sink port */
725 if (!sink->port->put_frame)
726 continue;
727
728 if (sink->ts_next.u64 == 0)
729 sink->ts_next = *now;
730
731 /* Skip if too early for put_frame(), note:
732 * early = (now < ts_next)
733 * But careful for timestamp wrapped around.
734 */
735 ts_diff = pj_timestamp_diff32(&sink->ts_next, now);
736 if (ts_diff < 0 || ts_diff > TS_CLOCK_RATE)
737 continue;
738
739 /* Clean up sink put buffer (should we draw black instead?) */
740 //pj_bzero(sink->put_buf, sink->put_buf_size);
741
742 /* Iterate transmitters of this sink port */
743 for (j=0; j < sink->transmitter_cnt; ++j) {
744 vconf_port *src = vid_conf->ports[sink->transmitter_slots[j]];
745 pj_int32_t src_ts_diff;
746
747 if (src->ts_next.u64 == 0)
748 src->ts_next = *now;
749
750 /* Is it time for src->get_frame()? yes, if (now >= ts_next) */
751 src_ts_diff = pj_timestamp_diff32(&src->ts_next, now);
752 if (src_ts_diff >= 0) {
753
754 /* Call src->get_frame().
755 * Possible optimization: if this src only has one listener,
756 * perhaps we can skip this src buffer and directly render it
757 * to sink buffer (but still need buffer if conversion any).
758 */
759 pj_bzero(&frame, sizeof(frame));
760 frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
761 frame.timestamp = *now;
762 frame.buf = src->get_buf;
763 frame.size = src->get_buf_size;
764 status = pjmedia_port_get_frame(src->port, &frame);
765 if (status != PJ_SUCCESS) {
766 PJ_PERROR(5, (THIS_FILE, status,
767 "Failed to get frame from port %d [%s]!",
768 src->idx, src->port->info.name.ptr));
769 }
770
771 /* Update next src put/get */
772 pj_add_timestamp32(&src->ts_next, src->ts_interval);
773 ts_incremented = src==sink;
774 }
775
776 /* Render src get buffer to sink put buffer (based on sink layout
777 * settings, if any)
778 */
779 status = render_src_frame(src, sink, j);
780 if (status != PJ_SUCCESS) {
781 PJ_PERROR(5, (THIS_FILE, status,
782 "Failed to render frame from port %d [%s] to "
783 "%d [%s]",
784 src->idx, src->port->info.name.ptr,
785 sink->idx, sink->port->info.name.ptr));
786 }
787
788 got_frame = PJ_TRUE;
789 }
790
791 /* Call sink->put_frame()
792 * Note that if transmitter_cnt==0, we should still call put_frame()
793 * with zero frame size, as sink may need to send keep-alive packets
794 * and get timestamp update.
795 */
796 pj_bzero(&frame, sizeof(frame));
797 frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
798 frame.timestamp = *now;
799 if (got_frame) {
800 frame.buf = sink->put_buf;
801 frame.size = sink->put_buf_size;
802 }
803 status = pjmedia_port_put_frame(sink->port, &frame);
804 if (got_frame && status != PJ_SUCCESS) {
805 sink->last_err_cnt++;
806 if (sink->last_err != status ||
807 sink->last_err_cnt % MAX_ERR_COUNT == 0)
808 {
809 if (sink->last_err != status)
810 sink->last_err_cnt = 1;
811 sink->last_err = status;
812 PJ_PERROR(5, (THIS_FILE, status,
813 "Failed (%d time(s)) to put frame to port %d"
814 " [%s]!", sink->last_err_cnt,
815 sink->idx, sink->port->info.name.ptr));
816 }
817 } else {
818 sink->last_err = status;
819 sink->last_err_cnt = 0;
820 }
821
822 /* Update next put/get, careful that it may have been updated
823 * if this port transmits to itself!
824 */
825 if (!ts_incremented) {
826 pj_add_timestamp32(&sink->ts_next, sink->ts_interval);
827 }
828 }
829
830 pj_mutex_unlock(vid_conf->mutex);
831 }
832
is_landscape(const pjmedia_rect_size * size)833 static pj_bool_t is_landscape(const pjmedia_rect_size *size) {
834 return (size->w >= size->h);
835 }
836
837 /* Adjust a frame size to match ratio specified in the ref_size.
838 * Either dimension of the frame may be cropped (but will never be
839 * expanded).
840 */
match_ratio_crop(pjmedia_rect_size * size,const pjmedia_rect_size * ref_size)841 static void match_ratio_crop(pjmedia_rect_size *size,
842 const pjmedia_rect_size *ref_size)
843 {
844 pjmedia_rect_size res;
845
846 /* Try match width first */
847 res.w = size->w;
848 res.h = ref_size->h * size->w / ref_size->w;
849
850 /* If original height turns out to be shorther, match height */
851 if (size->h < res.h) {
852 res.w = ref_size->w * size->h / ref_size->h;
853 res.h = size->h;
854 }
855
856 *size = res;
857 return;
858 }
859
860 /* Cleanup rendering states, called when a transmitter is disconnected
861 * from a listener, or before reinit-ing rendering state of a listener
862 * when new connection has just been made.
863 */
cleanup_render_state(vconf_port * cp,unsigned transmitter_idx)864 static void cleanup_render_state(vconf_port *cp,
865 unsigned transmitter_idx)
866 {
867 render_state *rs = cp->render_states[transmitter_idx];
868 if (rs && rs->converter)
869 {
870 pjmedia_converter_destroy(rs->converter);
871 rs->converter = NULL;
872 }
873 cp->render_states[transmitter_idx] = NULL;
874
875 if (cp->render_pool[transmitter_idx]) {
876 pj_pool_safe_release(&cp->render_pool[transmitter_idx]);
877
878 TRACE_((THIS_FILE, "Cleaned up render state for connection %d->%d",
879 cp->transmitter_slots[transmitter_idx], cp->idx));
880 }
881 }
882
883 /* This function will do:
884 * - Recalculate layout setting, i.e: get video pos and size
885 * for each transmitter
886 * - Check if converter is needed and setup it.
887 * - Those above things will be stored in render_state and
888 * will be used by render_src_frame()
889 */
update_render_state(pjmedia_vid_conf * vid_conf,vconf_port * cp)890 static void update_render_state(pjmedia_vid_conf *vid_conf, vconf_port *cp)
891 {
892 pjmedia_format_id fmt_id, tr_fmt_id[4];
893 pjmedia_rect_size size, tr_size[4];
894 unsigned i;
895 pj_status_t status;
896
897 /* Nothing to render, just return */
898 if (cp->transmitter_cnt == 0)
899 return;
900
901 TRACE_((THIS_FILE, "Updating render state for port id %d (%d sources)..",
902 cp->idx, cp->transmitter_cnt));
903
904 fmt_id = cp->port->info.fmt.id;
905 size = cp->port->info.fmt.det.vid.size;
906 for (i = 0; i < cp->transmitter_cnt; ++i) {
907 vconf_port *tr = vid_conf->ports[cp->transmitter_slots[i]];
908
909 /* Cleanup render states & pool */
910 cleanup_render_state(cp, i);
911
912 /* Gather format ID, size of each transmitter */
913 tr_fmt_id[i] = tr->port->info.fmt.id;
914 tr_size[i] = tr->port->info.fmt.det.vid.size;
915 }
916
917 /* If only one transmitter and it has matched format & size, just use
918 * plain memcpy(). Usually preview window or call stream window will
919 * have matched format & size with its source.
920 */
921 if (cp->transmitter_cnt == 1 && fmt_id == tr_fmt_id[0] &&
922 pj_memcmp(&size, &tr_size[0], sizeof(size))==0)
923 {
924 TRACE_((THIS_FILE, "This port only has single source with "
925 "matched format & size, no conversion needed"));
926 return;
927 }
928
929 for (i = 0; i < cp->transmitter_cnt && i < 4; ++i) {
930 pj_pool_t *pool;
931 render_state *rs;
932 pjmedia_conversion_param cparam;
933 char tmp_buf[32];
934
935 /* Create pool & render state */
936 pj_ansi_snprintf(tmp_buf, sizeof(tmp_buf), "vcport_rs_%d->%d",
937 cp->transmitter_slots[i], cp->idx);
938 pool = pj_pool_create(cp->pool->factory, tmp_buf, 128, 128, NULL);
939 cp->render_pool[i] = pool;
940 rs = cp->render_states[i] = PJ_POOL_ZALLOC_T(pool, render_state);
941
942 TRACE_((THIS_FILE, "Created render state for connection %d->%d",
943 cp->transmitter_slots[i], cp->idx));
944
945 /* Setup format & frame */
946 rs->src_fmt_id = tr_fmt_id[i];
947 rs->dst_fmt_id = fmt_id;
948 rs->src_frame_size = tr_size[i];
949 rs->dst_frame_size = size;
950
951 /* For now, draw the whole source frame, will adjust ratio later */
952 rs->src_rect.coord.x = rs->src_rect.coord.y = 0;
953 rs->src_rect.size = tr_size[i];
954
955 /* Setup layout */
956 if (cp->transmitter_cnt == 1) {
957 rs->dst_rect.coord.x = rs->dst_rect.coord.y = 0;
958 rs->dst_rect.size = size;
959 } else if (cp->transmitter_cnt == 2) {
960 if (is_landscape(&size)) {
961 /*
962 * |
963 * Source 0 | Source 1
964 * |
965 */
966 rs->dst_rect.coord.x = i * (size.w/2);
967 rs->dst_rect.coord.y = 0;
968 rs->dst_rect.size.w = size.w / 2;
969 rs->dst_rect.size.h = size.h;
970 } else {
971 /*
972 * Source 0
973 * --------
974 * Source 1
975 */
976 rs->dst_rect.coord.x = 0;
977 rs->dst_rect.coord.y = i * (size.h/2);
978 rs->dst_rect.size.w = size.w;
979 rs->dst_rect.size.h = size.h / 2;
980 }
981 } else if (cp->transmitter_cnt == 3) {
982 if (is_landscape(&size)) {
983 /*
984 * | Source 1
985 * Source 0 |---------
986 * | Source 2
987 */
988 rs->dst_rect.coord.x = (i==0)? 0 : size.w/2;
989 rs->dst_rect.coord.y = (i!=2)? 0 : size.h/2;
990 rs->dst_rect.size.w = size.w / 2;
991 rs->dst_rect.size.h = (i==0)? size.h : size.h/2;
992 } else {
993 /*
994 * Source 0
995 * --------
996 * Source 1
997 * --------
998 * Source 2
999 */
1000 rs->dst_rect.coord.x = 0;
1001 rs->dst_rect.coord.y = i * size.h/3;
1002 rs->dst_rect.size.w = size.w;
1003 rs->dst_rect.size.h = size.h/3;
1004 }
1005 } else if (cp->transmitter_cnt == 4) {
1006 if (is_landscape(&size)) {
1007 /*
1008 * Source 0 | Source 1
1009 * ---------|---------
1010 * Source 2 | Source 3
1011 */
1012 rs->dst_rect.coord.x = (i%2==0)? 0 : size.w/2;
1013 rs->dst_rect.coord.y = (i/2==0)? 0 : size.h/2;
1014 rs->dst_rect.size.w = size.w/2;
1015 rs->dst_rect.size.h = size.h/2;
1016 } else {
1017 /*
1018 * Source 0
1019 * --------
1020 * Source 1
1021 * --------
1022 * Source 2
1023 * --------
1024 * Source 3
1025 */
1026 rs->dst_rect.coord.x = 0;
1027 rs->dst_rect.coord.y = i * size.h/4;
1028 rs->dst_rect.size.w = size.w;
1029 rs->dst_rect.size.h = size.h/4;
1030 }
1031 }
1032
1033 /* Adjust source size to match aspect ratio of rendering space. */
1034 match_ratio_crop(&rs->src_rect.size, &rs->dst_rect.size);
1035
1036 /* Now adjust source position after source size adjustment. */
1037 if (rs->src_rect.size.w < tr_size[i].w)
1038 rs->src_rect.coord.x = (tr_size[i].w - rs->src_rect.size.w)/2;
1039 if (rs->src_rect.size.h < tr_size[i].h)
1040 rs->src_rect.coord.y = (tr_size[i].h - rs->src_rect.size.h)/2;
1041
1042 TRACE_((THIS_FILE, "src#%d=%s/%dx%d->%dx%d@%d,%d dst=%dx%d@%d,%d",
1043 i, pjmedia_fourcc_name(tr_fmt_id[i], tmp_buf),
1044 tr_size[i].w, tr_size[i].h,
1045 rs->src_rect.size.w, rs->src_rect.size.h,
1046 rs->src_rect.coord.x, rs->src_rect.coord.y,
1047 rs->dst_rect.size.w, rs->dst_rect.size.h,
1048 rs->dst_rect.coord.x, rs->dst_rect.coord.y));
1049
1050 /* Create converter */
1051 pjmedia_format_init_video(&cparam.src, rs->src_fmt_id,
1052 rs->src_rect.size.w,
1053 rs->src_rect.size.h,
1054 0, 1);
1055 pjmedia_format_init_video(&cparam.dst, rs->dst_fmt_id,
1056 rs->dst_rect.size.w,
1057 rs->dst_rect.size.h,
1058 0, 1);
1059 status = pjmedia_converter_create(NULL, pool, &cparam,
1060 &rs->converter);
1061 if (status != PJ_SUCCESS) {
1062 PJ_PERROR(4,(THIS_FILE, status,
1063 "Port %d failed creating converter "
1064 "for source %d", cp->idx, i));
1065 }
1066 }
1067 }
1068
1069 /* Render frame from source to sink buffer based on rendering settings. */
render_src_frame(vconf_port * src,vconf_port * sink,unsigned transmitter_idx)1070 static pj_status_t render_src_frame(vconf_port *src, vconf_port *sink,
1071 unsigned transmitter_idx)
1072 {
1073 pj_status_t status;
1074 render_state *rs = sink->render_states[transmitter_idx];
1075
1076 if (sink->transmitter_cnt == 1 && (!rs || !rs->converter)) {
1077 /* The only transmitter and no conversion needed */
1078 pj_assert(src->get_buf_size <= sink->put_buf_size);
1079 pj_memcpy(sink->put_buf, src->get_buf, src->get_buf_size);
1080 } else if (rs && rs->converter) {
1081 pjmedia_frame src_frame, dst_frame;
1082
1083 pj_bzero(&src_frame, sizeof(src_frame));
1084 src_frame.buf = src->get_buf;
1085 src_frame.size = src->get_buf_size;
1086
1087 pj_bzero(&dst_frame, sizeof(dst_frame));
1088 dst_frame.buf = sink->put_buf;
1089 dst_frame.size = sink->put_buf_size;
1090
1091 status = pjmedia_converter_convert2(rs->converter,
1092 &src_frame,
1093 &rs->src_frame_size,
1094 &rs->src_rect.coord,
1095 &dst_frame,
1096 &rs->dst_frame_size,
1097 &rs->dst_rect.coord,
1098 NULL);
1099 if (status != PJ_SUCCESS) {
1100 PJ_PERROR(4,(THIS_FILE, status,
1101 "Port id %d: converter failed in "
1102 "rendering frame from port id %d",
1103 sink->idx, transmitter_idx));
1104 return status;
1105 }
1106 }
1107
1108 return PJ_SUCCESS;
1109 }
1110
1111
1112 #endif /* PJMEDIA_HAS_VIDEO */
1113