1 /* $Id$ */
2 /*
3 * Copyright (C) 2008-2011 Teluu Inc. (http://www.teluu.com)
4 * Copyright (C) 2003-2008 Benny Prijono <benny@prijono.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21 #include <pjmedia/echo.h>
22 #include <pjmedia/delaybuf.h>
23 #include <pjmedia/frame.h>
24 #include <pjmedia/errno.h>
25 #include <pj/assert.h>
26 #include <pj/list.h>
27 #include <pj/log.h>
28 #include <pj/math.h>
29 #include <pj/pool.h>
30 #include "echo_internal.h"
31
32 #define THIS_FILE "echo_common.c"
33
34 typedef struct ec_operations ec_operations;
35
36 struct frame
37 {
38 PJ_DECL_LIST_MEMBER(struct frame);
39 short buf[1];
40 };
41
42 struct pjmedia_echo_state
43 {
44 pj_pool_t *pool;
45 char *obj_name;
46 unsigned samples_per_frame;
47 void *state;
48 ec_operations *op;
49
50 pj_bool_t lat_ready; /* lat_buf has been filled in. */
51 struct frame lat_buf; /* Frame queue for delayed playback */
52 struct frame lat_free; /* Free frame list. */
53
54 pjmedia_delay_buf *delay_buf;
55 pj_int16_t *frm_buf;
56 };
57
58
59 struct ec_operations
60 {
61 const char *name;
62
63 pj_status_t (*ec_create)(pj_pool_t *pool,
64 unsigned clock_rate,
65 unsigned channel_count,
66 unsigned samples_per_frame,
67 unsigned tail_ms,
68 unsigned options,
69 void **p_state );
70 pj_status_t (*ec_destroy)(void *state );
71 void (*ec_reset)(void *state );
72 pj_status_t (*ec_cancel)(void *state,
73 pj_int16_t *rec_frm,
74 const pj_int16_t *play_frm,
75 unsigned options,
76 void *reserved );
77 pj_status_t (*ec_playback)(void *state,
78 pj_int16_t *play_frm );
79 pj_status_t (*ec_capture)(void *state,
80 pj_int16_t *rec_frm,
81 unsigned options );
82 pj_status_t (*ec_get_stat)(void *state,
83 pjmedia_echo_stat *stat);
84 };
85
86
87 static struct ec_operations echo_supp_op =
88 {
89 "Echo suppressor",
90 &echo_supp_create,
91 &echo_supp_destroy,
92 &echo_supp_reset,
93 &echo_supp_cancel_echo,
94 NULL,
95 NULL,
96 &echo_supp_get_stat
97 };
98
99
100
101 /*
102 * Speex AEC prototypes
103 */
104 #if defined(PJMEDIA_HAS_SPEEX_AEC) && PJMEDIA_HAS_SPEEX_AEC!=0
105 static struct ec_operations speex_aec_op =
106 {
107 "Speex AEC",
108 &speex_aec_create,
109 &speex_aec_destroy,
110 &speex_aec_reset,
111 &speex_aec_cancel_echo,
112 &speex_aec_playback,
113 &speex_aec_capture
114 };
115 #endif
116
117
118 /*
119 * IPP AEC prototypes
120 */
121 #if defined(PJMEDIA_HAS_INTEL_IPP_AEC) && PJMEDIA_HAS_INTEL_IPP_AEC!=0
122 static struct ec_operations ipp_aec_op =
123 {
124 "IPP AEC",
125 &ipp_aec_create,
126 &ipp_aec_destroy,
127 &ipp_aec_reset,
128 &ipp_aec_cancel_echo
129 };
130 #endif
131
132 /*
133 * WebRTC AEC prototypes
134 */
135 #if defined(PJMEDIA_HAS_WEBRTC_AEC) && PJMEDIA_HAS_WEBRTC_AEC!=0
136 static struct ec_operations webrtc_aec_op =
137 {
138 "WebRTC AEC",
139 &webrtc_aec_create,
140 &webrtc_aec_destroy,
141 &webrtc_aec_reset,
142 &webrtc_aec_cancel_echo,
143 NULL,
144 NULL,
145 &webrtc_aec_get_stat
146 };
147 #endif
148
pjmedia_echo_stat_default(pjmedia_echo_stat * stat)149 PJ_DEF(void) pjmedia_echo_stat_default(pjmedia_echo_stat *stat)
150 {
151 pj_bzero(stat, sizeof(pjmedia_echo_stat));
152 stat->median = PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
153 stat->std = PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
154 stat->frac_delay = (float)PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
155 stat->duration = PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
156 stat->tail = PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
157 stat->min_factor = PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
158 stat->avg_factor = PJMEDIA_ECHO_STAT_NOT_SPECIFIED;
159 }
160
161 /*
162 * Create the echo canceller.
163 */
pjmedia_echo_create(pj_pool_t * pool,unsigned clock_rate,unsigned samples_per_frame,unsigned tail_ms,unsigned latency_ms,unsigned options,pjmedia_echo_state ** p_echo)164 PJ_DEF(pj_status_t) pjmedia_echo_create( pj_pool_t *pool,
165 unsigned clock_rate,
166 unsigned samples_per_frame,
167 unsigned tail_ms,
168 unsigned latency_ms,
169 unsigned options,
170 pjmedia_echo_state **p_echo )
171 {
172 return pjmedia_echo_create2(pool, clock_rate, 1, samples_per_frame,
173 tail_ms, latency_ms, options, p_echo);
174 }
175
176 /*
177 * Create the echo canceller.
178 */
pjmedia_echo_create2(pj_pool_t * pool,unsigned clock_rate,unsigned channel_count,unsigned samples_per_frame,unsigned tail_ms,unsigned latency_ms,unsigned options,pjmedia_echo_state ** p_echo)179 PJ_DEF(pj_status_t) pjmedia_echo_create2(pj_pool_t *pool,
180 unsigned clock_rate,
181 unsigned channel_count,
182 unsigned samples_per_frame,
183 unsigned tail_ms,
184 unsigned latency_ms,
185 unsigned options,
186 pjmedia_echo_state **p_echo )
187 {
188 unsigned ptime, lat_cnt;
189 unsigned delay_buf_opt = 0;
190 pjmedia_echo_state *ec;
191 pj_status_t status;
192
193 /* Create new pool and instantiate and init the EC */
194 pool = pj_pool_create(pool->factory, "ec%p", 256, 256, NULL);
195 ec = PJ_POOL_ZALLOC_T(pool, struct pjmedia_echo_state);
196 ec->pool = pool;
197 ec->obj_name = pool->obj_name;
198 ec->samples_per_frame = samples_per_frame;
199 ec->frm_buf = (pj_int16_t*)pj_pool_alloc(pool, samples_per_frame<<1);
200 pj_list_init(&ec->lat_buf);
201 pj_list_init(&ec->lat_free);
202
203 /* Select the backend algorithm */
204 if (0) {
205 /* Dummy */
206 ;
207 #if defined(PJMEDIA_HAS_SPEEX_AEC) && PJMEDIA_HAS_SPEEX_AEC!=0
208 } else if ((options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_SPEEX ||
209 (options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_DEFAULT)
210 {
211 ec->op = &speex_aec_op;
212 #endif
213
214 #if defined(PJMEDIA_HAS_INTEL_IPP_AEC) && PJMEDIA_HAS_INTEL_IPP_AEC!=0
215 } else if ((options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_IPP ||
216 (options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_DEFAULT)
217 {
218 ec->op = &ipp_aec_op;
219
220 #endif
221
222 #if defined(PJMEDIA_HAS_WEBRTC_AEC) && PJMEDIA_HAS_WEBRTC_AEC!=0
223 } else if ((options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_WEBRTC ||
224 (options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_DEFAULT)
225 {
226 ec->op = &webrtc_aec_op;
227 #endif
228
229 } else {
230 ec->op = &echo_supp_op;
231 }
232
233 /* Completeness check for EC operation playback and capture, they must
234 * be implemented both or none.
235 */
236 pj_assert(!ec->op->ec_capture == !ec->op->ec_playback);
237
238 PJ_LOG(5,(ec->obj_name, "Creating %s", ec->op->name));
239
240 /* Instantiate EC object */
241 status = (*ec->op->ec_create)(pool, clock_rate, channel_count,
242 samples_per_frame, tail_ms,
243 options, &ec->state);
244 if (status != PJ_SUCCESS) {
245 pj_pool_release(pool);
246 return status;
247 }
248
249 /* If EC algo does not have playback and capture callbakcs,
250 * create latency buffer and delay buffer to handle drift.
251 */
252 if (ec->op->ec_playback && ec->op->ec_capture) {
253 latency_ms = 0;
254 } else {
255 /* Create latency buffers */
256 ptime = samples_per_frame * 1000 / clock_rate;
257 if (latency_ms > ptime) {
258 /* Normalize latency with delaybuf/WSOLA latency */
259 latency_ms -= PJ_MIN(ptime, PJMEDIA_WSOLA_DELAY_MSEC);
260 }
261 if (latency_ms < ptime) {
262 /* Give at least one frame delay to simplify programming */
263 latency_ms = ptime;
264 }
265 lat_cnt = latency_ms / ptime;
266 while (lat_cnt--) {
267 struct frame *frm;
268
269 frm = (struct frame*) pj_pool_alloc(pool, (samples_per_frame<<1) +
270 sizeof(struct frame));
271 pj_list_push_back(&ec->lat_free, frm);
272 }
273
274 /* Create delay buffer to compensate drifts */
275 if (options & PJMEDIA_ECHO_USE_SIMPLE_FIFO)
276 delay_buf_opt |= PJMEDIA_DELAY_BUF_SIMPLE_FIFO;
277 status = pjmedia_delay_buf_create(ec->pool, ec->obj_name, clock_rate,
278 samples_per_frame, channel_count,
279 (PJMEDIA_SOUND_BUFFER_COUNT+1) * ptime,
280 delay_buf_opt, &ec->delay_buf);
281 if (status != PJ_SUCCESS) {
282 pj_pool_release(pool);
283 return status;
284 }
285 }
286
287 PJ_LOG(4,(ec->obj_name,
288 "%s created, clock_rate=%d, channel=%d, "
289 "samples per frame=%d, tail length=%d ms, "
290 "latency=%d ms",
291 ec->op->name, clock_rate, channel_count, samples_per_frame,
292 tail_ms, latency_ms));
293
294 /* Done */
295 *p_echo = ec;
296
297 return PJ_SUCCESS;
298 }
299
300
301 /*
302 * Destroy the Echo Canceller.
303 */
pjmedia_echo_destroy(pjmedia_echo_state * echo)304 PJ_DEF(pj_status_t) pjmedia_echo_destroy(pjmedia_echo_state *echo )
305 {
306 (*echo->op->ec_destroy)(echo->state);
307
308 if (echo->delay_buf) {
309 pjmedia_delay_buf_destroy(echo->delay_buf);
310 echo->delay_buf = NULL;
311 }
312
313 pj_pool_release(echo->pool);
314 return PJ_SUCCESS;
315 }
316
317
318 /*
319 * Reset the echo canceller.
320 */
pjmedia_echo_reset(pjmedia_echo_state * echo)321 PJ_DEF(pj_status_t) pjmedia_echo_reset(pjmedia_echo_state *echo )
322 {
323 while (!pj_list_empty(&echo->lat_buf)) {
324 struct frame *frm;
325 frm = echo->lat_buf.next;
326 pj_list_erase(frm);
327 pj_list_push_back(&echo->lat_free, frm);
328 }
329 echo->lat_ready = PJ_FALSE;
330 if (echo->delay_buf)
331 pjmedia_delay_buf_reset(echo->delay_buf);
332 echo->op->ec_reset(echo->state);
333 return PJ_SUCCESS;
334 }
335
336
337 /*
338 * Let the Echo Canceller know that a frame has been played to the speaker.
339 */
pjmedia_echo_playback(pjmedia_echo_state * echo,pj_int16_t * play_frm)340 PJ_DEF(pj_status_t) pjmedia_echo_playback( pjmedia_echo_state *echo,
341 pj_int16_t *play_frm )
342 {
343 /* If EC algo has playback handler, just pass the frame. */
344 if (echo->op->ec_playback) {
345 return (*echo->op->ec_playback)(echo->state, play_frm);
346 }
347
348 /* Playing frame should be stored, as it will be used by echo_capture()
349 * as reference frame, delay buffer is used for storing the playing frames
350 * as in case there was clock drift between mic & speaker.
351 *
352 * Ticket #830:
353 * Note that pjmedia_delay_buf_put() may modify the input frame and those
354 * modified frames may not be smooth, i.e: if there were two or more
355 * consecutive pjmedia_delay_buf_get() before next pjmedia_delay_buf_put(),
356 * so we'll just feed the delay buffer with the copy of playing frame,
357 * instead of the original playing frame. However this will cause the EC
358 * uses slight 'different' frames (for reference) than actually played
359 * by the speaker.
360 */
361 pjmedia_copy_samples(echo->frm_buf, play_frm,
362 echo->samples_per_frame);
363 pjmedia_delay_buf_put(echo->delay_buf, echo->frm_buf);
364
365 if (!echo->lat_ready) {
366 /* We've not built enough latency in the buffer, so put this frame
367 * in the latency buffer list.
368 */
369 struct frame *frm;
370
371 if (pj_list_empty(&echo->lat_free)) {
372 echo->lat_ready = PJ_TRUE;
373 PJ_LOG(5,(echo->obj_name, "Latency bufferring complete"));
374 return PJ_SUCCESS;
375 }
376
377 frm = echo->lat_free.prev;
378 pj_list_erase(frm);
379
380 /* Move one frame from delay buffer to the latency buffer. */
381 pjmedia_delay_buf_get(echo->delay_buf, echo->frm_buf);
382 pjmedia_copy_samples(frm->buf, echo->frm_buf, echo->samples_per_frame);
383 pj_list_push_back(&echo->lat_buf, frm);
384 }
385
386 return PJ_SUCCESS;
387 }
388
389
390 /*
391 * Let the Echo Canceller knows that a frame has been captured from
392 * the microphone.
393 */
pjmedia_echo_capture(pjmedia_echo_state * echo,pj_int16_t * rec_frm,unsigned options)394 PJ_DEF(pj_status_t) pjmedia_echo_capture( pjmedia_echo_state *echo,
395 pj_int16_t *rec_frm,
396 unsigned options )
397 {
398 struct frame *oldest_frm;
399 pj_status_t status, rc;
400
401 /* If EC algo has capture handler, just pass the frame. */
402 if (echo->op->ec_capture) {
403 return (*echo->op->ec_capture)(echo->state, rec_frm, options);
404 }
405
406 if (!echo->lat_ready) {
407 /* Prefetching to fill in the desired latency */
408 PJ_LOG(5,(echo->obj_name, "Prefetching.."));
409 return PJ_SUCCESS;
410 }
411
412 /* Retrieve oldest frame from the latency buffer */
413 oldest_frm = echo->lat_buf.next;
414 pj_list_erase(oldest_frm);
415
416 /* Cancel echo using this reference frame */
417 status = pjmedia_echo_cancel(echo, rec_frm, oldest_frm->buf,
418 options, NULL);
419
420 /* Move one frame from delay buffer to the latency buffer. */
421 rc = pjmedia_delay_buf_get(echo->delay_buf, oldest_frm->buf);
422 if (rc != PJ_SUCCESS) {
423 /* Ooops.. no frame! */
424 PJ_PERROR(5,(echo->obj_name, rc,
425 "No frame from delay buffer (this will upset EC later)"));
426 pjmedia_zero_samples(oldest_frm->buf, echo->samples_per_frame);
427 }
428 pj_list_push_back(&echo->lat_buf, oldest_frm);
429
430 return status;
431 }
432
433
434 /*
435 * Perform echo cancellation.
436 */
pjmedia_echo_cancel(pjmedia_echo_state * echo,pj_int16_t * rec_frm,const pj_int16_t * play_frm,unsigned options,void * reserved)437 PJ_DEF(pj_status_t) pjmedia_echo_cancel( pjmedia_echo_state *echo,
438 pj_int16_t *rec_frm,
439 const pj_int16_t *play_frm,
440 unsigned options,
441 void *reserved )
442 {
443 return (*echo->op->ec_cancel)( echo->state, rec_frm, play_frm, options,
444 reserved);
445 }
446
447
448 /*
449 * Get the Echo Canceller stats.
450 */
pjmedia_echo_get_stat(pjmedia_echo_state * echo,pjmedia_echo_stat * p_stat)451 PJ_DEF(pj_status_t) pjmedia_echo_get_stat(pjmedia_echo_state *echo,
452 pjmedia_echo_stat *p_stat)
453 {
454 PJ_ASSERT_RETURN(p_stat, PJ_EINVAL);
455
456 if (echo->op->ec_get_stat)
457 return (*echo->op->ec_get_stat)(echo->state, p_stat);
458
459 return PJ_ENOTSUP;
460 }
461
462