1 /*****************************************************************************
2 * opensles_android.c : audio output for android native code
3 *****************************************************************************
4 * Copyright © 2011-2012 VideoLAN
5 *
6 * Authors: Dominique Martinet <asmadeus@codewreck.org>
7 * Hugo Beauzée-Luyssen <hugo@beauzee.fr>
8 * Rafaël Carré <funman@videolanorg>
9 *
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU Lesser General Public License as published by
12 * the Free Software Foundation; either version 2.1 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public License
21 * along with this program; if not, write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23 *****************************************************************************/
24
25 /*****************************************************************************
26 * Preamble
27 *****************************************************************************/
28
29 #ifdef HAVE_CONFIG_H
30 # include "config.h"
31 #endif
32
33 #include <vlc_common.h>
34 #include <vlc_plugin.h>
35 #include <vlc_aout.h>
36 #include <assert.h>
37 #include <dlfcn.h>
38 #include <math.h>
39
40 // For native audio
41 #include <SLES/OpenSLES.h>
42 #include <SLES/OpenSLES_Android.h>
43
44 #include <jni.h>
45 JNIEnv *android_getEnv(vlc_object_t *p_obj, const char *psz_thread_name);
46
47 #define OPENSLES_BUFFERS 255 /* maximum number of buffers */
48 #define OPENSLES_BUFLEN 10 /* ms */
49 /*
50 * 10ms of precision when mesasuring latency should be enough,
51 * with 255 buffers we can buffer 2.55s of audio.
52 */
53
54 #define CHECK_OPENSL_ERROR(msg) \
55 if (unlikely(result != SL_RESULT_SUCCESS)) \
56 { \
57 msg_Err(aout, msg" (%lu)", result); \
58 goto error; \
59 }
60
61 typedef SLresult (*slCreateEngine_t)(
62 SLObjectItf*, SLuint32, const SLEngineOption*, SLuint32,
63 const SLInterfaceID*, const SLboolean*);
64
65 #define Destroy(a) (*a)->Destroy(a);
66 #define SetPlayState(a, b) (*a)->SetPlayState(a, b)
67 #define RegisterCallback(a, b, c) (*a)->RegisterCallback(a, b, c)
68 #define GetInterface(a, b, c) (*a)->GetInterface(a, b, c)
69 #define Realize(a, b) (*a)->Realize(a, b)
70 #define CreateOutputMix(a, b, c, d, e) (*a)->CreateOutputMix(a, b, c, d, e)
71 #define CreateAudioPlayer(a, b, c, d, e, f, g) \
72 (*a)->CreateAudioPlayer(a, b, c, d, e, f, g)
73 #define Enqueue(a, b, c) (*a)->Enqueue(a, b, c)
74 #define Clear(a) (*a)->Clear(a)
75 #define GetState(a, b) (*a)->GetState(a, b)
76 #define SetPositionUpdatePeriod(a, b) (*a)->SetPositionUpdatePeriod(a, b)
77 #define SetVolumeLevel(a, b) (*a)->SetVolumeLevel(a, b)
78 #define SetMute(a, b) (*a)->SetMute(a, b)
79
80 /*****************************************************************************
81 *
82 *****************************************************************************/
83 struct aout_sys_t
84 {
85 /* OpenSL objects */
86 SLObjectItf engineObject;
87 SLObjectItf outputMixObject;
88 SLAndroidSimpleBufferQueueItf playerBufferQueue;
89 SLObjectItf playerObject;
90 SLVolumeItf volumeItf;
91 SLEngineItf engineEngine;
92 SLPlayItf playerPlay;
93
94 /* OpenSL symbols */
95 void *p_so_handle;
96
97 slCreateEngine_t slCreateEnginePtr;
98 SLInterfaceID SL_IID_ENGINE;
99 SLInterfaceID SL_IID_ANDROIDSIMPLEBUFFERQUEUE;
100 SLInterfaceID SL_IID_VOLUME;
101 SLInterfaceID SL_IID_PLAY;
102
103 /* */
104
105 vlc_mutex_t lock;
106
107 /* audio buffered through opensles */
108 uint8_t *buf;
109 size_t samples_per_buf;
110 int next_buf;
111
112 int rate;
113
114 /* if we can measure latency already */
115 bool started;
116
117 /* audio not yet buffered through opensles */
118 block_t *p_buffer_chain;
119 block_t **pp_buffer_last;
120 size_t samples;
121 };
122
123 /*****************************************************************************
124 * Local prototypes.
125 *****************************************************************************/
126 static int Open (vlc_object_t *);
127 static void Close (vlc_object_t *);
128
129 /*****************************************************************************
130 * Module descriptor
131 *****************************************************************************/
132
133 vlc_module_begin ()
134 set_description("OpenSLES audio output")
135 set_shortname("OpenSLES")
set_category(CAT_AUDIO)136 set_category(CAT_AUDIO)
137 set_subcategory(SUBCAT_AUDIO_AOUT)
138
139 set_capability("audio output", 170)
140 add_shortcut("opensles", "android")
141 set_callbacks(Open, Close)
142 vlc_module_end ()
143
144 /*****************************************************************************
145 *
146 *****************************************************************************/
147
148 static inline int bytesPerSample(void)
149 {
150 return 2 /* S16 */ * 2 /* stereo */;
151 }
152
TimeGet(audio_output_t * aout,mtime_t * restrict drift)153 static int TimeGet(audio_output_t* aout, mtime_t* restrict drift)
154 {
155 aout_sys_t *sys = aout->sys;
156
157 SLAndroidSimpleBufferQueueState st;
158 SLresult res = GetState(sys->playerBufferQueue, &st);
159 if (unlikely(res != SL_RESULT_SUCCESS)) {
160 msg_Err(aout, "Could not query buffer queue state in TimeGet (%lu)", res);
161 return -1;
162 }
163
164 vlc_mutex_lock(&sys->lock);
165 bool started = sys->started;
166 vlc_mutex_unlock(&sys->lock);
167
168 if (!started)
169 return -1;
170
171 *drift = (CLOCK_FREQ * OPENSLES_BUFLEN * st.count / 1000)
172 + sys->samples * CLOCK_FREQ / sys->rate;
173
174 /* msg_Dbg(aout, "latency %"PRId64" ms, %d/%d buffers", *drift / 1000,
175 (int)st.count, OPENSLES_BUFFERS); */
176
177 return 0;
178 }
179
Flush(audio_output_t * aout,bool drain)180 static void Flush(audio_output_t *aout, bool drain)
181 {
182 aout_sys_t *sys = aout->sys;
183
184 if (drain) {
185 mtime_t delay;
186 if (!TimeGet(aout, &delay))
187 msleep(delay);
188 } else {
189 vlc_mutex_lock(&sys->lock);
190 SetPlayState(sys->playerPlay, SL_PLAYSTATE_STOPPED);
191 Clear(sys->playerBufferQueue);
192 SetPlayState(sys->playerPlay, SL_PLAYSTATE_PLAYING);
193
194 /* release audio data not yet written to opensles */
195 block_ChainRelease(sys->p_buffer_chain);
196 sys->p_buffer_chain = NULL;
197 sys->pp_buffer_last = &sys->p_buffer_chain;
198
199 sys->samples = 0;
200 sys->started = false;
201
202 vlc_mutex_unlock(&sys->lock);
203 }
204 }
205
VolumeSet(audio_output_t * aout,float vol)206 static int VolumeSet(audio_output_t *aout, float vol)
207 {
208 if (!aout->sys->volumeItf)
209 return -1;
210
211 /* Convert UI volume to linear factor (cube) */
212 vol = vol * vol * vol;
213
214 /* millibels from linear amplification */
215 int mb = lroundf(2000.f * log10f(vol));
216 if (mb < SL_MILLIBEL_MIN)
217 mb = SL_MILLIBEL_MIN;
218 else if (mb > 0)
219 mb = 0; /* maximum supported level could be higher: GetMaxVolumeLevel */
220
221 SLresult r = SetVolumeLevel(aout->sys->volumeItf, mb);
222 return (r == SL_RESULT_SUCCESS) ? 0 : -1;
223 }
224
MuteSet(audio_output_t * aout,bool mute)225 static int MuteSet(audio_output_t *aout, bool mute)
226 {
227 if (!aout->sys->volumeItf)
228 return -1;
229
230 SLresult r = SetMute(aout->sys->volumeItf, mute);
231 return (r == SL_RESULT_SUCCESS) ? 0 : -1;
232 }
233
Pause(audio_output_t * aout,bool pause,mtime_t date)234 static void Pause(audio_output_t *aout, bool pause, mtime_t date)
235 {
236 (void)date;
237 aout_sys_t *sys = aout->sys;
238 SetPlayState(sys->playerPlay,
239 pause ? SL_PLAYSTATE_PAUSED : SL_PLAYSTATE_PLAYING);
240 }
241
WriteBuffer(audio_output_t * aout)242 static int WriteBuffer(audio_output_t *aout)
243 {
244 aout_sys_t *sys = aout->sys;
245 const size_t unit_size = sys->samples_per_buf * bytesPerSample();
246
247 block_t *b = sys->p_buffer_chain;
248 if (!b)
249 return false;
250
251 /* Check if we can fill at least one buffer unit by chaining blocks */
252 if (b->i_buffer < unit_size) {
253 if (!b->p_next)
254 return false;
255 ssize_t needed = unit_size - b->i_buffer;
256 for (block_t *next = b->p_next; next; next = next->p_next) {
257 needed -= next->i_buffer;
258 if (needed <= 0)
259 break;
260 }
261
262 if (needed > 0)
263 return false;
264 }
265
266 SLAndroidSimpleBufferQueueState st;
267 SLresult res = GetState(sys->playerBufferQueue, &st);
268 if (unlikely(res != SL_RESULT_SUCCESS)) {
269 msg_Err(aout, "Could not query buffer queue state in %s (%lu)", __func__, res);
270 return false;
271 }
272
273 if (st.count == OPENSLES_BUFFERS)
274 return false;
275
276 size_t done = 0;
277 while (done < unit_size) {
278 size_t cur = b->i_buffer;
279 if (cur > unit_size - done)
280 cur = unit_size - done;
281
282 memcpy(&sys->buf[unit_size * sys->next_buf + done], b->p_buffer, cur);
283 b->i_buffer -= cur;
284 b->p_buffer += cur;
285 done += cur;
286
287 block_t *next = b->p_next;
288 if (b->i_buffer == 0) {
289 block_Release(b);
290 b = NULL;
291 }
292
293 if (done == unit_size)
294 break;
295 else
296 b = next;
297 }
298
299 sys->p_buffer_chain = b;
300 if (!b)
301 sys->pp_buffer_last = &sys->p_buffer_chain;
302
303 SLresult r = Enqueue(sys->playerBufferQueue,
304 &sys->buf[unit_size * sys->next_buf], unit_size);
305
306 sys->samples -= sys->samples_per_buf;
307
308 if (r == SL_RESULT_SUCCESS) {
309 if (++sys->next_buf == OPENSLES_BUFFERS)
310 sys->next_buf = 0;
311 return true;
312 } else {
313 /* XXX : if writing fails, we don't retry */
314 msg_Err(aout, "error %lu when writing %d bytes %s",
315 r, b->i_buffer,
316 (r == SL_RESULT_BUFFER_INSUFFICIENT) ? " (buffer insufficient)" : "");
317 return false;
318 }
319 }
320
321 /*****************************************************************************
322 * Play: play a sound
323 *****************************************************************************/
Play(audio_output_t * aout,block_t * p_buffer)324 static void Play(audio_output_t *aout, block_t *p_buffer)
325 {
326 aout_sys_t *sys = aout->sys;
327
328 p_buffer->p_next = NULL; /* Make sur our linked list doesn't use old references */
329 vlc_mutex_lock(&sys->lock);
330
331 sys->samples += p_buffer->i_buffer / bytesPerSample();
332
333 /* Hold this block until we can write it into the OpenSL buffer */
334 block_ChainLastAppend(&sys->pp_buffer_last, p_buffer);
335
336 /* Fill OpenSL buffer */
337 while (WriteBuffer(aout))
338 ;
339
340 vlc_mutex_unlock(&sys->lock);
341 }
342
PlayedCallback(SLAndroidSimpleBufferQueueItf caller,void * pContext)343 static void PlayedCallback (SLAndroidSimpleBufferQueueItf caller, void *pContext)
344 {
345 (void)caller;
346 audio_output_t *aout = pContext;
347 aout_sys_t *sys = aout->sys;
348
349 assert (caller == sys->playerBufferQueue);
350
351 vlc_mutex_lock(&sys->lock);
352 sys->started = true;
353 vlc_mutex_unlock(&sys->lock);
354 }
355
aout_get_native_sample_rate(audio_output_t * aout)356 static int aout_get_native_sample_rate(audio_output_t *aout)
357 {
358 JNIEnv *p_env;
359 if (!(p_env = android_getEnv(VLC_OBJECT(aout), "opensles")))
360 return -1;
361 jclass cls = (*p_env)->FindClass (p_env, "android/media/AudioTrack");
362 if ((*p_env)->ExceptionCheck(p_env))
363 {
364 (*p_env)->ExceptionClear(p_env);
365 return -1;
366 }
367 jmethodID method = (*p_env)->GetStaticMethodID(p_env, cls,
368 "getNativeOutputSampleRate",
369 "(I)I");
370 /* 3 for AudioManager.STREAM_MUSIC */
371 int sample_rate = (*p_env)->CallStaticIntMethod(p_env, cls, method, 3);
372 (*p_env)->DeleteLocalRef(p_env, cls);
373 fprintf(stderr, "aout_get_native_sample_rate: %d\n", sample_rate);
374 return sample_rate;
375 }
376
377 /*****************************************************************************
378 *
379 *****************************************************************************/
Start(audio_output_t * aout,audio_sample_format_t * restrict fmt)380 static int Start(audio_output_t *aout, audio_sample_format_t *restrict fmt)
381 {
382 if (aout_FormatNbChannels(fmt) == 0 || !AOUT_FMT_LINEAR(fmt))
383 return VLC_EGENERIC;
384
385 SLresult result;
386
387 aout_sys_t *sys = aout->sys;
388
389 // configure audio source - this defines the number of samples you can enqueue.
390 SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {
391 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
392 OPENSLES_BUFFERS
393 };
394
395 SLDataFormat_PCM format_pcm;
396 format_pcm.formatType = SL_DATAFORMAT_PCM;
397 format_pcm.numChannels = 2;
398 format_pcm.samplesPerSec = ((SLuint32) fmt->i_rate * 1000) ;
399 format_pcm.bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_16;
400 format_pcm.containerSize = SL_PCMSAMPLEFORMAT_FIXED_16;
401 format_pcm.channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
402 format_pcm.endianness = SL_BYTEORDER_LITTLEENDIAN;
403
404 SLDataSource audioSrc = {&loc_bufq, &format_pcm};
405
406 // configure audio sink
407 SLDataLocator_OutputMix loc_outmix = {
408 SL_DATALOCATOR_OUTPUTMIX,
409 sys->outputMixObject
410 };
411 SLDataSink audioSnk = {&loc_outmix, NULL};
412
413 //create audio player
414 const SLInterfaceID ids2[] = { sys->SL_IID_ANDROIDSIMPLEBUFFERQUEUE, sys->SL_IID_VOLUME };
415 static const SLboolean req2[] = { SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
416
417 if (aout_get_native_sample_rate(aout) >= fmt->i_rate) {
418 result = CreateAudioPlayer(sys->engineEngine, &sys->playerObject, &audioSrc,
419 &audioSnk, sizeof(ids2) / sizeof(*ids2),
420 ids2, req2);
421 } else {
422 // Don't try to play back a sample rate higher than the native one,
423 // since OpenSL ES will try to use the fast path, which AudioFlinger
424 // will reject (fast path can't do resampling), and will end up with
425 // too small buffers for the resampling. See http://b.android.com/59453
426 // for details. This bug is still present in 4.4. If it is fixed later
427 // this workaround could be made conditional.
428 result = SL_RESULT_UNKNOWN_ERROR;
429 }
430 if (unlikely(result != SL_RESULT_SUCCESS)) {
431 /* Try again with a more sensible samplerate */
432 fmt->i_rate = 44100;
433 format_pcm.samplesPerSec = ((SLuint32) 44100 * 1000) ;
434 result = CreateAudioPlayer(sys->engineEngine, &sys->playerObject, &audioSrc,
435 &audioSnk, sizeof(ids2) / sizeof(*ids2),
436 ids2, req2);
437 }
438 CHECK_OPENSL_ERROR("Failed to create audio player");
439
440 result = Realize(sys->playerObject, SL_BOOLEAN_FALSE);
441 CHECK_OPENSL_ERROR("Failed to realize player object.");
442
443 result = GetInterface(sys->playerObject, sys->SL_IID_PLAY, &sys->playerPlay);
444 CHECK_OPENSL_ERROR("Failed to get player interface.");
445
446 result = GetInterface(sys->playerObject, sys->SL_IID_VOLUME, &sys->volumeItf);
447 CHECK_OPENSL_ERROR("failed to get volume interface.");
448
449 result = GetInterface(sys->playerObject, sys->SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
450 &sys->playerBufferQueue);
451 CHECK_OPENSL_ERROR("Failed to get buff queue interface");
452
453 result = RegisterCallback(sys->playerBufferQueue, PlayedCallback,
454 (void*)aout);
455 CHECK_OPENSL_ERROR("Failed to register buff queue callback.");
456
457 // set the player's state to playing
458 result = SetPlayState(sys->playerPlay, SL_PLAYSTATE_PLAYING);
459 CHECK_OPENSL_ERROR("Failed to switch to playing state");
460
461 /* XXX: rounding shouldn't affect us at normal sampling rate */
462 sys->rate = fmt->i_rate;
463 sys->samples_per_buf = OPENSLES_BUFLEN * fmt->i_rate / 1000;
464 sys->buf = vlc_alloc(sys->samples_per_buf * bytesPerSample(), OPENSLES_BUFFERS);
465 if (!sys->buf)
466 goto error;
467
468 sys->started = false;
469 sys->next_buf = 0;
470
471 sys->p_buffer_chain = NULL;
472 sys->pp_buffer_last = &sys->p_buffer_chain;
473 sys->samples = 0;
474
475 // we want 16bit signed data native endian.
476 fmt->i_format = VLC_CODEC_S16N;
477 fmt->i_physical_channels = AOUT_CHAN_LEFT | AOUT_CHAN_RIGHT;
478 fmt->channel_type = AUDIO_CHANNEL_TYPE_BITMAP;
479
480 SetPositionUpdatePeriod(sys->playerPlay, AOUT_MIN_PREPARE_TIME * 1000 / CLOCK_FREQ);
481
482 aout_FormatPrepare(fmt);
483
484 return VLC_SUCCESS;
485
486 error:
487 if (sys->playerObject) {
488 Destroy(sys->playerObject);
489 sys->playerObject = NULL;
490 sys->playerBufferQueue = NULL;
491 sys->volumeItf = NULL;
492 sys->playerPlay = NULL;
493 }
494
495 return VLC_EGENERIC;
496 }
497
Stop(audio_output_t * aout)498 static void Stop(audio_output_t *aout)
499 {
500 aout_sys_t *sys = aout->sys;
501
502 SetPlayState(sys->playerPlay, SL_PLAYSTATE_STOPPED);
503 //Flush remaining buffers if any.
504 Clear(sys->playerBufferQueue);
505
506 free(sys->buf);
507 block_ChainRelease(sys->p_buffer_chain);
508
509 Destroy(sys->playerObject);
510 sys->playerObject = NULL;
511 sys->playerBufferQueue = NULL;
512 sys->volumeItf = NULL;
513 sys->playerPlay = NULL;
514 }
515
516 /*****************************************************************************
517 *
518 *****************************************************************************/
Close(vlc_object_t * obj)519 static void Close(vlc_object_t *obj)
520 {
521 audio_output_t *aout = (audio_output_t *)obj;
522 aout_sys_t *sys = aout->sys;
523
524 Destroy(sys->outputMixObject);
525 Destroy(sys->engineObject);
526 dlclose(sys->p_so_handle);
527 vlc_mutex_destroy(&sys->lock);
528 free(sys);
529 }
530
Open(vlc_object_t * obj)531 static int Open (vlc_object_t *obj)
532 {
533 audio_output_t *aout = (audio_output_t *)obj;
534 aout_sys_t *sys;
535 SLresult result;
536
537 aout->sys = sys = calloc(1, sizeof(*sys));
538 if (unlikely(sys == NULL))
539 return VLC_ENOMEM;
540
541 sys->p_so_handle = dlopen("libOpenSLES.so", RTLD_NOW);
542 if (sys->p_so_handle == NULL)
543 {
544 msg_Err(aout, "Failed to load libOpenSLES");
545 goto error;
546 }
547
548 sys->slCreateEnginePtr = dlsym(sys->p_so_handle, "slCreateEngine");
549 if (unlikely(sys->slCreateEnginePtr == NULL))
550 {
551 msg_Err(aout, "Failed to load symbol slCreateEngine");
552 goto error;
553 }
554
555 #define OPENSL_DLSYM(dest, name) \
556 do { \
557 const SLInterfaceID *sym = dlsym(sys->p_so_handle, "SL_IID_"name); \
558 if (unlikely(sym == NULL)) \
559 { \
560 msg_Err(aout, "Failed to load symbol SL_IID_"name); \
561 goto error; \
562 } \
563 sys->dest = *sym; \
564 } while(0)
565
566 OPENSL_DLSYM(SL_IID_ANDROIDSIMPLEBUFFERQUEUE, "ANDROIDSIMPLEBUFFERQUEUE");
567 OPENSL_DLSYM(SL_IID_ENGINE, "ENGINE");
568 OPENSL_DLSYM(SL_IID_PLAY, "PLAY");
569 OPENSL_DLSYM(SL_IID_VOLUME, "VOLUME");
570 #undef OPENSL_DLSYM
571
572 // create engine
573 result = sys->slCreateEnginePtr(&sys->engineObject, 0, NULL, 0, NULL, NULL);
574 CHECK_OPENSL_ERROR("Failed to create engine");
575
576 // realize the engine in synchronous mode
577 result = Realize(sys->engineObject, SL_BOOLEAN_FALSE);
578 CHECK_OPENSL_ERROR("Failed to realize engine");
579
580 // get the engine interface, needed to create other objects
581 result = GetInterface(sys->engineObject, sys->SL_IID_ENGINE, &sys->engineEngine);
582 CHECK_OPENSL_ERROR("Failed to get the engine interface");
583
584 // create output mix, with environmental reverb specified as a non-required interface
585 const SLInterfaceID ids1[] = { sys->SL_IID_VOLUME };
586 const SLboolean req1[] = { SL_BOOLEAN_FALSE };
587 result = CreateOutputMix(sys->engineEngine, &sys->outputMixObject, 1, ids1, req1);
588 CHECK_OPENSL_ERROR("Failed to create output mix");
589
590 // realize the output mix in synchronous mode
591 result = Realize(sys->outputMixObject, SL_BOOLEAN_FALSE);
592 CHECK_OPENSL_ERROR("Failed to realize output mix");
593
594 vlc_mutex_init(&sys->lock);
595
596 aout->start = Start;
597 aout->stop = Stop;
598 aout->time_get = TimeGet;
599 aout->play = Play;
600 aout->pause = Pause;
601 aout->flush = Flush;
602 aout->mute_set = MuteSet;
603 aout->volume_set = VolumeSet;
604
605 return VLC_SUCCESS;
606
607 error:
608 if (sys->outputMixObject)
609 Destroy(sys->outputMixObject);
610 if (sys->engineObject)
611 Destroy(sys->engineObject);
612 if (sys->p_so_handle)
613 dlclose(sys->p_so_handle);
614 free(sys);
615 return VLC_EGENERIC;
616 }
617