1 /*
2 * GPAC - Multimedia Framework C SDK
3 *
4 * Authors: Jean Le Feuvre
5 * Copyright (c) Telecom ParisTech 2000-2018
6 * All rights reserved
7 *
8 * This file is part of GPAC / Scene Compositor sub-project
9 *
10 * GPAC is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU Lesser General Public License as published by
12 * the Free Software Foundation; either version 2, or (at your option)
13 * any later version.
14 *
15 * GPAC is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with this library; see the file COPYING. If not, write to
22 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 *
24 */
25
26 #include <gpac/internal/compositor_dev.h>
27
gf_ar_rcfg_done(GF_Filter * filter,GF_FilterPid * pid,GF_FilterPacket * pck)28 void gf_ar_rcfg_done(GF_Filter *filter, GF_FilterPid *pid, GF_FilterPacket *pck)
29 {
30 u32 size;
31 GF_AudioRenderer *ar = (GF_AudioRenderer *) gf_filter_pck_get_data(pck, &size);
32 assert(!size);
33 assert(ar->wait_for_rcfg);
34 ar->wait_for_rcfg --;
35 if (ar->wait_for_rcfg) {
36 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfigure negociation %d still pending\n", ar->wait_for_rcfg));
37 } else {
38 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfigure negociation done\n"));
39 }
40 }
41
gf_ar_setup_output_format(GF_AudioRenderer * ar)42 static GF_Err gf_ar_setup_output_format(GF_AudioRenderer *ar)
43 {
44 u32 freq, a_fmt, nb_chan;
45 u64 ch_cfg;
46 u32 bsize;
47
48 freq = ar->compositor->asr;
49 a_fmt = ar->compositor->afmt;
50 nb_chan = ar->compositor->ach;
51 ch_cfg = ar->compositor->alayout;
52 bsize = ar->compositor->asize;
53 if (!bsize) bsize = 1024;
54
55 if (!freq || !a_fmt || !nb_chan || !ch_cfg) {
56 gf_mixer_get_config(ar->mixer, &freq, &nb_chan, &a_fmt, &ch_cfg);
57
58 /*user disabled multichannel audio*/
59 if (!ar->compositor->amc && (nb_chan>2) ) nb_chan = 2;
60 } else {
61 if (!ar->config_forced) ar->config_forced++;
62 }
63
64
65 gf_mixer_set_config(ar->mixer, freq, nb_chan, a_fmt, ch_cfg);
66
67 if (ar->samplerate) {
68 ar->time_at_last_config_sr = ar->current_time_sr * freq / ar->samplerate;
69 }
70 if (!ar->compositor->abuf) ar->compositor->abuf = 100;
71 ar->samplerate = freq;
72 ar->bytes_per_samp = nb_chan * gf_audio_fmt_bit_depth(a_fmt) / 8;
73 ar->bytes_per_second = freq * ar->bytes_per_samp;
74 ar->max_bytes_out = ar->bytes_per_second * ar->compositor->abuf / 1000;
75 while (ar->max_bytes_out % (2*ar->bytes_per_samp) ) ar->max_bytes_out++;
76 ar->buffer_size = ar->bytes_per_samp * bsize;
77
78 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfigure audio to %d Hz %d channels %s\n", freq, nb_chan, gf_audio_fmt_name(a_fmt) ));
79
80 if (ar->aout) {
81 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_SAMPLE_RATE, &PROP_UINT(freq) );
82 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_TIMESCALE, &PROP_UINT(freq) );
83 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_NUM_CHANNELS, &PROP_UINT(nb_chan) );
84 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_FORMAT, &PROP_UINT(a_fmt) );
85 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_CHANNEL_LAYOUT, &PROP_LONGUINT(ch_cfg) );
86 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_VOLUME, &PROP_UINT(ar->volume) );
87 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_PAN, &PROP_UINT(ar->pan) );
88
89 gf_filter_pid_set_max_buffer(ar->aout, 1000*ar->compositor->abuf);
90 }
91
92 ar->time_at_last_config = ar->current_time;
93 ar->bytes_requested = 0;
94
95 if (ar->aout) {
96 GF_FilterPacket *pck;
97 //issue a dummy packet to tag the point at which we reconfigured
98 pck = gf_filter_pck_new_shared(ar->aout, (u8 *) ar, 0, gf_ar_rcfg_done);
99 ar->wait_for_rcfg ++;
100 gf_filter_pck_set_readonly(pck);
101 gf_filter_pck_send(pck);
102 }
103 return GF_OK;
104 }
105
gf_ar_pause(GF_AudioRenderer * ar,Bool DoFreeze,Bool for_reconfig,Bool reset_hw_buffer)106 static void gf_ar_pause(GF_AudioRenderer *ar, Bool DoFreeze, Bool for_reconfig, Bool reset_hw_buffer)
107 {
108 GF_FilterEvent evt;
109 gf_mixer_lock(ar->mixer, GF_TRUE);
110 if (DoFreeze) {
111 if (!ar->Frozen) {
112 ar->freeze_time = gf_sys_clock_high_res();
113 if (!for_reconfig && ar->aout) {
114 GF_FEVT_INIT(evt, GF_FEVT_STOP, ar->aout);
115 gf_filter_pid_send_event(ar->aout, &evt);
116 }
117 GF_LOG(GF_LOG_DEBUG, GF_LOG_SYNC, ("[Audio] pausing master clock - time "LLD" (sys time "LLD")\n", ar->freeze_time, gf_sys_clock_high_res()));
118 ar->Frozen = GF_TRUE;
119 }
120 } else {
121 if (ar->Frozen) {
122 if (!for_reconfig && ar->aout) {
123 GF_FEVT_INIT(evt, GF_FEVT_PLAY, ar->aout);
124 evt.play.hw_buffer_reset = reset_hw_buffer ? 1 : 0;
125 gf_filter_pid_send_event(ar->aout, &evt);
126 }
127
128 ar->start_time += gf_sys_clock_high_res() - ar->freeze_time;
129 GF_LOG(GF_LOG_DEBUG, GF_LOG_SYNC, ("[Audio] resuming master clock - new time "LLD" (sys time "LLD") \n", ar->start_time, gf_sys_clock_high_res()));
130 ar->Frozen = GF_FALSE;
131 }
132 }
133 gf_mixer_lock(ar->mixer, GF_FALSE);
134 }
135
136
gf_sc_ar_load(GF_Compositor * compositor,u32 init_flags)137 GF_AudioRenderer *gf_sc_ar_load(GF_Compositor *compositor, u32 init_flags)
138 {
139 GF_AudioRenderer *ar;
140 ar = (GF_AudioRenderer *) gf_malloc(sizeof(GF_AudioRenderer));
141 memset(ar, 0, sizeof(GF_AudioRenderer));
142
143 ar->compositor = compositor;
144
145 ar->mixer = gf_mixer_new(ar);
146 ar->non_rt_output = GF_TRUE;
147 ar->volume = MIN(100, compositor->avol);
148 ar->pan = MIN(100, compositor->apan);
149 if (! (init_flags & GF_TERM_NO_AUDIO) ) {
150 gf_ar_setup_output_format(ar);
151 }
152 gf_mixer_set_max_speed(ar->mixer, compositor->max_aspeed);
153 ar->current_time = 0;
154 return ar;
155 }
156
gf_sc_ar_del(GF_AudioRenderer * ar)157 void gf_sc_ar_del(GF_AudioRenderer *ar)
158 {
159 if (!ar) return;
160
161 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Destroying compositor\n"));
162 /*resume if paused (might cause deadlock otherwise)*/
163 if (ar->Frozen) gf_sc_ar_control(ar, GF_SC_AR_RESUME);
164
165 gf_mixer_del(ar->mixer);
166 gf_free(ar);
167 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Renderer destroyed\n"));
168 }
169
170
gf_sc_ar_reset(GF_AudioRenderer * ar)171 void gf_sc_ar_reset(GF_AudioRenderer *ar)
172 {
173 if (!ar) return;
174
175 gf_mixer_remove_all(ar->mixer);
176 if (ar->scene_ready) {
177 ar->scene_ready = 0;
178 ar->current_time = 0;
179 ar->current_time_sr = 0;
180 ar->bytes_requested = 0;
181 ar->nb_audio_objects = 0;
182 }
183 }
184
gf_sc_ar_control(GF_AudioRenderer * ar,u32 PauseType)185 void gf_sc_ar_control(GF_AudioRenderer *ar, u32 PauseType)
186 {
187 gf_ar_pause(ar, (PauseType==GF_SC_AR_PAUSE) ? GF_TRUE : GF_FALSE, GF_FALSE, (PauseType==GF_SC_AR_RESET_HW_AND_PLAY) ? GF_TRUE : GF_FALSE);
188 }
189
gf_sc_ar_set_volume(GF_AudioRenderer * ar,u32 Volume)190 void gf_sc_ar_set_volume(GF_AudioRenderer *ar, u32 Volume)
191 {
192 if (Volume>100) Volume=100;
193 if (ar->volume==Volume) return;
194 if (ar->aout) gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_VOLUME, &PROP_UINT(ar->volume) );
195 }
196
gf_sc_ar_mute(GF_AudioRenderer * ar,Bool mute)197 void gf_sc_ar_mute(GF_AudioRenderer *ar, Bool mute)
198 {
199 ar->mute = mute;
200 if (ar->aout) gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_VOLUME, &PROP_UINT(mute ? 0 : ar->volume) );
201 }
202
gf_sc_ar_set_pan(GF_AudioRenderer * ar,u32 Balance)203 void gf_sc_ar_set_pan(GF_AudioRenderer *ar, u32 Balance)
204 {
205 if (Balance>100) Balance = 100;
206 if (ar->pan == Balance) return;
207 ar->pan = Balance;
208 if (ar->aout) gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_PAN, &PROP_UINT(ar->pan) );
209 }
210
211
212 void compositor_setup_aout(GF_Compositor *ctx);
gf_sc_ar_add_src(GF_AudioRenderer * ar,GF_AudioInterface * source)213 void gf_sc_ar_add_src(GF_AudioRenderer *ar, GF_AudioInterface *source)
214 {
215 Bool recfg;
216 if (!ar) return;
217
218 compositor_setup_aout(ar->compositor);
219
220 /*lock mixer*/
221 gf_mixer_lock(ar->mixer, GF_TRUE);
222 gf_mixer_add_input(ar->mixer, source);
223 /*if changed reconfig*/
224 recfg = gf_mixer_reconfig(ar->mixer);
225 if (!ar->need_reconfig) ar->need_reconfig = recfg;
226
227 if (!gf_mixer_empty(ar->mixer) && ar->aout) {
228 GF_FilterEvent evt;
229 GF_FEVT_INIT(evt, GF_FEVT_PLAY, ar->aout);
230 gf_filter_pid_send_event(ar->aout, &evt);
231 }
232
233 /*unlock mixer*/
234 gf_mixer_lock(ar->mixer, GF_FALSE);
235 }
236
gf_sc_ar_remove_src(GF_AudioRenderer * ar,GF_AudioInterface * source)237 void gf_sc_ar_remove_src(GF_AudioRenderer *ar, GF_AudioInterface *source)
238 {
239 if (ar) {
240 gf_mixer_remove_input(ar->mixer, source);
241 if (gf_mixer_empty(ar->mixer) && ar->aout) {
242 GF_FilterEvent evt;
243 GF_FEVT_INIT(evt, GF_FEVT_STOP, ar->aout);
244 gf_filter_pid_send_event(ar->aout, &evt);
245 }
246 }
247 }
248
249
250 #if 0 //unused
251 void gf_sc_ar_set_priority(GF_AudioRenderer *ar, u32 priority)
252 {
253 if (ar->aout)
254 gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_PRIORITY, &PROP_UINT(priority) );
255 }
256 #endif
257
gf_sc_ar_update_video_clock(GF_AudioRenderer * ar,u32 video_ts)258 void gf_sc_ar_update_video_clock(GF_AudioRenderer *ar, u32 video_ts)
259 {
260 ar->video_ts = video_ts;
261 }
262
gf_ar_pck_done(GF_Filter * filter,GF_FilterPid * pid,GF_FilterPacket * pck)263 static void gf_ar_pck_done(GF_Filter *filter, GF_FilterPid *pid, GF_FilterPacket *pck)
264 {
265 u32 size;
266 GF_Compositor *compositor = gf_filter_pid_get_udta(pid);
267 /*data = */gf_filter_pck_get_data(pck, &size);
268 if (!size) return;
269
270 assert(size <= compositor->audio_renderer->nb_bytes_out);
271 compositor->audio_renderer->nb_bytes_out -= size;
272 }
273
gf_ar_send_packets(GF_AudioRenderer * ar)274 void gf_ar_send_packets(GF_AudioRenderer *ar)
275 {
276 u32 written, max_send=100;
277 u64 now = gf_sys_clock_high_res();
278
279 if (!ar->aout) {
280 if (ar->compositor->player) {
281 ar->current_time = (u32) ( (now - ar->start_time)/1000);
282 }
283 return;
284 }
285 if (!ar->scene_ready) return;
286 if (ar->need_reconfig) return;
287 if (ar->Frozen) return;
288
289 //reconfiguration is pending, wait for the packet issued at reconfig to be consummed before issuing any new frame
290 if (ar->wait_for_rcfg) {
291 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Waiting for audio output reconfiguration\n"));
292 return;
293 }
294 if (ar->scene_ready) {
295 if (!ar->start_time) {
296 ar->start_time = now;
297 }
298 if (!ar->nb_audio_objects && !ar->non_rt_output) {
299 ar->current_time = (u32) ( (now - ar->start_time)/1000);
300 return;
301 }
302 }
303
304 while (max_send) {
305 u32 delay_ms = 0;
306 u8 *data;
307 u32 dur;
308 GF_FilterPacket *pck;
309
310 if (gf_filter_pid_would_block(ar->aout))
311 break;
312
313 pck = gf_filter_pck_new_alloc_destructor(ar->aout, ar->buffer_size, &data, gf_ar_pck_done);
314 if (!pck) break;
315
316 if (ar->compositor->async) {
317 GF_Fraction64 ref_ts;
318 gf_filter_get_clock_hint(ar->compositor->filter, NULL, &ref_ts);
319 //valid clock hint, compute delay between last known playback point and current time
320 if (ref_ts.den) {
321 if (ref_ts.den != ar->samplerate) {
322 delay_ms = (u32) (1000 * ar->current_time_sr / ar->samplerate);
323 delay_ms -= (u32) (1000 * ref_ts.num / ref_ts.den);
324 } else {
325 delay_ms = (u32) (1000 * (ar->current_time_sr - ref_ts.num) / ar->samplerate);
326 }
327 }
328 //unknown clock hint, use number of bytes out as delay
329 else {
330 delay_ms = (1000*ar->nb_bytes_out) / ar->bytes_per_second;
331 }
332 }
333
334 gf_mixer_lock(ar->mixer, GF_TRUE);
335 written = gf_mixer_get_output(ar->mixer, data, ar->buffer_size, delay_ms);
336 gf_mixer_lock(ar->mixer, GF_FALSE);
337
338 if (!written) {
339 if (!ar->non_rt_output) written = ar->buffer_size;
340 else if (ar->scene_ready && ar->nb_audio_objects && !gf_mixer_buffering(ar->mixer) ) written = ar->buffer_size;
341 else {
342 gf_filter_pck_truncate(pck, 0);
343 gf_filter_pck_discard(pck);
344 break;
345 }
346 }
347
348 if (written<ar->buffer_size) {
349 gf_filter_pck_truncate(pck, written);
350 }
351 gf_filter_pck_set_sap(pck, GF_FILTER_SAP_1);
352 gf_filter_pck_set_cts(pck, ar->current_time_sr);
353 dur = written / ar->bytes_per_samp;
354 gf_filter_pck_set_duration(pck, dur);
355 GF_LOG(GF_LOG_INFO, GF_LOG_AUDIO, ("[Compositor] Send audio frame TS "LLU" nb samples %d - AR clock %u - delay %d ms\n", ar->current_time_sr, dur, ar->current_time, delay_ms));
356
357 ar->nb_bytes_out += written;
358 gf_filter_pck_send(pck);
359
360 ar->bytes_requested += written;
361 ar->current_time_sr = ar->time_at_last_config_sr + (u32) (ar->bytes_requested / ar->bytes_per_samp);
362 ar->current_time = ar->time_at_last_config + (u32) (ar->bytes_requested * 1000 / ar->bytes_per_second);
363
364 max_send--;
365
366 //this is a safety for non blocking mode, otherwise the pid_would_block is enough
367 if (ar->nb_bytes_out > ar->max_bytes_out)
368 break;
369 }
370 }
371
372
gf_sc_ar_send_or_reconfig(GF_AudioRenderer * ar)373 void gf_sc_ar_send_or_reconfig(GF_AudioRenderer *ar)
374 {
375 Bool frozen;
376 if (ar->need_reconfig) {
377 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfiguring audio mixer\n"));
378 /*lock mixer*/
379 gf_mixer_lock(ar->mixer, GF_TRUE);
380
381 frozen = ar->Frozen;
382 if (!frozen )
383 gf_ar_pause(ar, GF_TRUE, GF_TRUE, GF_FALSE);
384
385 ar->need_reconfig = GF_FALSE;
386 gf_ar_setup_output_format(ar);
387
388 if (!frozen)
389 gf_ar_pause(ar, GF_FALSE, GF_TRUE, GF_FALSE);
390
391 /*unlock mixer*/
392 gf_mixer_lock(ar->mixer, GF_FALSE);
393 }
394 GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] sending audio packets\n"));
395 gf_ar_send_packets(ar);
396 }
397
398 #if 0 //unused
399 u32 gf_sc_ar_get_delay(GF_AudioRenderer *ar)
400 {
401 if (!ar->bytes_per_second) return 0;
402 //try to FIXME, this is not as precise as what we have before using ar->audio_out->GetAudioDelay(ar->audio_out)
403 // since we don't know how much of the first packet data out there has been consumed
404 return 1000 * ar->nb_bytes_out / ar->bytes_per_second;
405 }
406 #endif
407
gf_sc_ar_get_clock(GF_AudioRenderer * ar)408 u32 gf_sc_ar_get_clock(GF_AudioRenderer *ar)
409 {
410 return ar->current_time;
411 }
412
413
414
415