1 /*
2 * consumer_sdl_audio.c -- A Simple DirectMedia Layer audio-only consumer
3 * Copyright (C) 2009-2020 Meltytech, LLC
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20 #include <framework/mlt_consumer.h>
21 #include <framework/mlt_frame.h>
22 #include <framework/mlt_deque.h>
23 #include <framework/mlt_factory.h>
24 #include <framework/mlt_filter.h>
25 #include <framework/mlt_log.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <pthread.h>
30 #include <SDL.h>
31 #include <sys/time.h>
32 #include <stdatomic.h>
33
34 extern pthread_mutex_t mlt_sdl_mutex;
35
36 /** This classes definition.
37 */
38
39 typedef struct consumer_sdl_s *consumer_sdl;
40
41 struct consumer_sdl_s
42 {
43 struct mlt_consumer_s parent;
44 mlt_properties properties;
45 mlt_deque queue;
46 pthread_t thread;
47 int joined;
48 atomic_int running;
49 uint8_t audio_buffer[ 4096 * 10 ];
50 int audio_avail;
51 pthread_mutex_t audio_mutex;
52 pthread_cond_t audio_cond;
53 pthread_mutex_t video_mutex;
54 pthread_cond_t video_cond;
55 atomic_int playing;
56
57 pthread_cond_t refresh_cond;
58 pthread_mutex_t refresh_mutex;
59 int refresh_count;
60 int is_purge;
61 };
62
63 /** Forward references to static functions.
64 */
65
66 static int consumer_start( mlt_consumer parent );
67 static int consumer_stop( mlt_consumer parent );
68 static int consumer_is_stopped( mlt_consumer parent );
69 static void consumer_purge( mlt_consumer parent );
70 static void consumer_close( mlt_consumer parent );
71 static void *consumer_thread( void * );
72 static void consumer_refresh_cb( mlt_consumer sdl, mlt_consumer self, char *name );
73
74 /** This is what will be called by the factory - anything can be passed in
75 via the argument, but keep it simple.
76 */
77
consumer_sdl_audio_init(mlt_profile profile,mlt_service_type type,const char * id,char * arg)78 mlt_consumer consumer_sdl_audio_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
79 {
80 // Create the consumer object
81 consumer_sdl self = calloc( 1, sizeof( struct consumer_sdl_s ) );
82
83 // If no malloc'd and consumer init ok
84 if ( self != NULL && mlt_consumer_init( &self->parent, self, profile ) == 0 )
85 {
86 // Create the queue
87 self->queue = mlt_deque_init( );
88
89 // Get the parent consumer object
90 mlt_consumer parent = &self->parent;
91
92 // We have stuff to clean up, so override the close method
93 parent->close = consumer_close;
94
95 // get a handle on properties
96 mlt_service service = MLT_CONSUMER_SERVICE( parent );
97 self->properties = MLT_SERVICE_PROPERTIES( service );
98
99 // Set the default volume
100 mlt_properties_set_double( self->properties, "volume", 1.0 );
101
102 // This is the initialisation of the consumer
103 pthread_mutex_init( &self->audio_mutex, NULL );
104 pthread_cond_init( &self->audio_cond, NULL);
105 pthread_mutex_init( &self->video_mutex, NULL );
106 pthread_cond_init( &self->video_cond, NULL);
107
108 // Default scaler (for now we'll use nearest)
109 mlt_properties_set( self->properties, "rescale", "nearest" );
110 mlt_properties_set( self->properties, "deinterlace_method", "onefield" );
111 mlt_properties_set_int( self->properties, "top_field_first", -1 );
112
113 // Default buffer for low latency
114 mlt_properties_set_int( self->properties, "buffer", 1 );
115
116 // Default audio buffer
117 mlt_properties_set_int( self->properties, "audio_buffer", 2048 );
118 #if defined(_WIN32) && SDL_MAJOR_VERSION == 2
119 mlt_properties_set( self->properties, "audio_driver", "DirectSound" );
120 #endif
121
122 // Ensure we don't join on a non-running object
123 self->joined = 1;
124
125 // Allow thread to be started/stopped
126 parent->start = consumer_start;
127 parent->stop = consumer_stop;
128 parent->is_stopped = consumer_is_stopped;
129 parent->purge = consumer_purge;
130
131 // Initialize the refresh handler
132 pthread_cond_init( &self->refresh_cond, NULL );
133 pthread_mutex_init( &self->refresh_mutex, NULL );
134 mlt_events_listen( MLT_CONSUMER_PROPERTIES( parent ), self, "property-changed", ( mlt_listener )consumer_refresh_cb );
135
136 // Return the consumer produced
137 return parent;
138 }
139
140 // malloc or consumer init failed
141 free( self );
142
143 // Indicate failure
144 return NULL;
145 }
146
consumer_refresh_cb(mlt_consumer sdl,mlt_consumer parent,char * name)147 static void consumer_refresh_cb( mlt_consumer sdl, mlt_consumer parent, char *name )
148 {
149 if ( !strcmp( name, "refresh" ) )
150 {
151 consumer_sdl self = parent->child;
152 pthread_mutex_lock( &self->refresh_mutex );
153 if ( self->refresh_count < 2 )
154 self->refresh_count = self->refresh_count <= 0 ? 1 : self->refresh_count + 1;
155 pthread_cond_broadcast( &self->refresh_cond );
156 pthread_mutex_unlock( &self->refresh_mutex );
157 }
158 }
159
consumer_start(mlt_consumer parent)160 int consumer_start( mlt_consumer parent )
161 {
162 consumer_sdl self = parent->child;
163
164 if ( !self->running )
165 {
166 consumer_stop( parent );
167
168 mlt_properties properties = MLT_CONSUMER_PROPERTIES( parent );
169 char *audio_driver = mlt_properties_get( properties, "audio_driver" );
170 char *audio_device = mlt_properties_get( properties, "audio_device" );
171
172 if ( audio_driver && strcmp( audio_driver, "" ) )
173 setenv( "SDL_AUDIODRIVER", audio_driver, 1 );
174
175 if ( audio_device && strcmp( audio_device, "" ) )
176 setenv( "AUDIODEV", audio_device, 1 );
177
178 pthread_mutex_lock( &mlt_sdl_mutex );
179 int ret = SDL_Init( SDL_INIT_AUDIO | SDL_INIT_NOPARACHUTE );
180 pthread_mutex_unlock( &mlt_sdl_mutex );
181 if ( ret < 0 )
182 {
183 mlt_log_error( MLT_CONSUMER_SERVICE(parent), "Failed to initialize SDL: %s\n", SDL_GetError() );
184 return -1;
185 }
186
187 self->running = 1;
188 self->joined = 0;
189 pthread_create( &self->thread, NULL, consumer_thread, self );
190 }
191
192 return 0;
193 }
194
consumer_stop(mlt_consumer parent)195 int consumer_stop( mlt_consumer parent )
196 {
197 // Get the actual object
198 consumer_sdl self = parent->child;
199
200 if ( self->running && !self->joined )
201 {
202 // Kill the thread and clean up
203 self->joined = 1;
204 self->running = 0;
205
206 // Unlatch the consumer thread
207 pthread_mutex_lock( &self->refresh_mutex );
208 pthread_cond_broadcast( &self->refresh_cond );
209 pthread_mutex_unlock( &self->refresh_mutex );
210
211 // Cleanup the main thread
212 #ifndef _WIN32
213 if ( self->thread )
214 #endif
215 pthread_join( self->thread, NULL );
216
217 // Unlatch the video thread
218 pthread_mutex_lock( &self->video_mutex );
219 pthread_cond_broadcast( &self->video_cond );
220 pthread_mutex_unlock( &self->video_mutex );
221
222 // Unlatch the audio callback
223 pthread_mutex_lock( &self->audio_mutex );
224 pthread_cond_broadcast( &self->audio_cond );
225 pthread_mutex_unlock( &self->audio_mutex );
226
227 SDL_QuitSubSystem( SDL_INIT_AUDIO );
228 }
229
230 return 0;
231 }
232
consumer_is_stopped(mlt_consumer parent)233 int consumer_is_stopped( mlt_consumer parent )
234 {
235 consumer_sdl self = parent->child;
236 return !self->running;
237 }
238
consumer_purge(mlt_consumer parent)239 void consumer_purge( mlt_consumer parent )
240 {
241 consumer_sdl self = parent->child;
242 if ( self->running )
243 {
244 pthread_mutex_lock( &self->video_mutex );
245 mlt_frame frame = MLT_FRAME( mlt_deque_peek_back( self->queue ) );
246 // When playing rewind or fast forward then we need to keep one
247 // frame in the queue to prevent playback stalling.
248 double speed = frame? mlt_properties_get_double( MLT_FRAME_PROPERTIES(frame), "_speed" ) : 0;
249 int n = ( speed == 0.0 || speed == 1.0 ) ? 0 : 1;
250 while ( mlt_deque_count( self->queue ) > n )
251 mlt_frame_close( mlt_deque_pop_back( self->queue ) );
252 self->is_purge = 1;
253 pthread_cond_broadcast( &self->video_cond );
254 pthread_mutex_unlock( &self->video_mutex );
255 }
256 }
257
sdl_fill_audio(void * udata,uint8_t * stream,int len)258 static void sdl_fill_audio( void *udata, uint8_t *stream, int len )
259 {
260 consumer_sdl self = udata;
261
262 // Get the volume
263 double volume = mlt_properties_get_double( self->properties, "volume" );
264
265 // Wipe the stream first
266 memset( stream, 0, len );
267
268 pthread_mutex_lock( &self->audio_mutex );
269
270 if ( self->audio_avail >= len )
271 {
272 // Place in the audio buffer
273 if ( volume != 1.0 )
274 SDL_MixAudio( stream, self->audio_buffer, len, ( int )( ( float )SDL_MIX_MAXVOLUME * volume ) );
275 else
276 memcpy( stream, self->audio_buffer, len );
277
278 // Remove len from the audio available
279 self->audio_avail -= len;
280
281 // Remove the samples
282 memmove( self->audio_buffer, self->audio_buffer + len, self->audio_avail );
283 }
284 else
285 {
286 // Mix the audio
287 SDL_MixAudio( stream, self->audio_buffer, self->audio_avail,
288 ( int )( ( float )SDL_MIX_MAXVOLUME * volume ) );
289
290 // No audio left
291 self->audio_avail = 0;
292 }
293
294 // We're definitely playing now
295 self->playing = 1;
296
297 pthread_cond_broadcast( &self->audio_cond );
298 pthread_mutex_unlock( &self->audio_mutex );
299 }
300
consumer_play_audio(consumer_sdl self,mlt_frame frame,int init_audio,int * duration)301 static int consumer_play_audio( consumer_sdl self, mlt_frame frame, int init_audio, int *duration )
302 {
303 // Get the properties of self consumer
304 mlt_properties properties = self->properties;
305 mlt_audio_format afmt = mlt_audio_s16;
306
307 // Set the preferred params of the test card signal
308 int channels = mlt_properties_get_int( properties, "channels" );
309 int dest_channels = channels;
310 int frequency = mlt_properties_get_int( properties, "frequency" );
311 int scrub = mlt_properties_get_int( properties, "scrub_audio" );
312 static int counter = 0;
313
314 int samples = mlt_audio_calculate_frame_samples( mlt_properties_get_double( self->properties, "fps" ), frequency, counter++ );
315 int16_t *pcm;
316 mlt_frame_get_audio( frame, (void**) &pcm, &afmt, &frequency, &channels, &samples );
317 *duration = ( ( samples * 1000 ) / frequency );
318 pcm += mlt_properties_get_int( properties, "audio_offset" );
319
320 if ( mlt_properties_get_int( properties, "audio_off" ) )
321 {
322 self->playing = 1;
323 init_audio = 1;
324 return init_audio;
325 }
326
327 if ( init_audio == 1 )
328 {
329 SDL_AudioSpec request;
330 SDL_AudioSpec got;
331
332 int audio_buffer = mlt_properties_get_int( properties, "audio_buffer" );
333
334 // specify audio format
335 memset( &request, 0, sizeof( SDL_AudioSpec ) );
336 self->playing = 0;
337 request.freq = frequency;
338 request.format = AUDIO_S16SYS;
339 request.channels = dest_channels;
340 request.samples = audio_buffer;
341 request.callback = sdl_fill_audio;
342 request.userdata = (void *)self;
343 if ( SDL_OpenAudio( &request, &got ) != 0 )
344 {
345 mlt_log_error( MLT_CONSUMER_SERVICE( self ), "SDL failed to open audio: %s\n", SDL_GetError() );
346 init_audio = 2;
347 }
348 else if ( got.size != 0 )
349 {
350 SDL_PauseAudio( 0 );
351 init_audio = 0;
352 }
353 }
354
355 if ( init_audio == 0 )
356 {
357 mlt_properties properties = MLT_FRAME_PROPERTIES( frame );
358 int samples_copied = 0;
359 int dst_stride = dest_channels * sizeof( *pcm );
360
361 pthread_mutex_lock( &self->audio_mutex );
362
363 while ( self->running && samples_copied < samples )
364 {
365 int sample_space = ( sizeof( self->audio_buffer ) - self->audio_avail ) / dst_stride;
366 while ( self->running && sample_space == 0 )
367 {
368 pthread_cond_wait( &self->audio_cond, &self->audio_mutex );
369 sample_space = ( sizeof( self->audio_buffer ) - self->audio_avail ) / dst_stride;
370 }
371 if ( self->running )
372 {
373 int samples_to_copy = samples - samples_copied;
374 if ( samples_to_copy > sample_space )
375 {
376 samples_to_copy = sample_space;
377 }
378 int dst_bytes = samples_to_copy * dst_stride;
379
380 if ( scrub || mlt_properties_get_double( properties, "_speed" ) == 1 )
381 {
382 if ( channels == dest_channels )
383 {
384 memcpy( &self->audio_buffer[ self->audio_avail ], pcm, dst_bytes );
385 pcm += samples_to_copy * channels;
386 }
387 else
388 {
389 int16_t *dest = (int16_t*) &self->audio_buffer[ self->audio_avail ];
390 int i = samples_to_copy + 1;
391 while ( --i )
392 {
393 memcpy( dest, pcm, dst_stride );
394 pcm += channels;
395 dest += dest_channels;
396 }
397 }
398 }
399 else
400 {
401 memset( &self->audio_buffer[ self->audio_avail ], 0, dst_bytes );
402 pcm += samples_to_copy * channels;
403 }
404 self->audio_avail += dst_bytes;
405 samples_copied += samples_to_copy;
406 }
407 pthread_cond_broadcast( &self->audio_cond );
408 }
409 pthread_mutex_unlock( &self->audio_mutex );
410 }
411 else
412 {
413 self->playing = 1;
414 }
415
416 return init_audio;
417 }
418
consumer_play_video(consumer_sdl self,mlt_frame frame)419 static int consumer_play_video( consumer_sdl self, mlt_frame frame )
420 {
421 // Get the properties of this consumer
422 mlt_properties properties = self->properties;
423 mlt_events_fire( properties, "consumer-frame-show", frame, NULL );
424 return 0;
425 }
426
video_thread(void * arg)427 static void *video_thread( void *arg )
428 {
429 // Identify the arg
430 consumer_sdl self = arg;
431
432 // Obtain time of thread start
433 struct timeval now;
434 int64_t start = 0;
435 int64_t elapsed = 0;
436 struct timespec tm;
437 mlt_frame next = NULL;
438 mlt_properties properties = NULL;
439 double speed = 0;
440
441 // Get real time flag
442 int real_time = mlt_properties_get_int( self->properties, "real_time" );
443
444 // Get the current time
445 gettimeofday( &now, NULL );
446
447 // Determine start time
448 start = ( int64_t )now.tv_sec * 1000000 + now.tv_usec;
449
450 while ( self->running )
451 {
452 // Pop the next frame
453 pthread_mutex_lock( &self->video_mutex );
454 next = mlt_deque_pop_front( self->queue );
455 while ( next == NULL && self->running )
456 {
457 pthread_cond_wait( &self->video_cond, &self->video_mutex );
458 next = mlt_deque_pop_front( self->queue );
459 }
460 pthread_mutex_unlock( &self->video_mutex );
461
462 if ( !self->running || next == NULL ) break;
463
464 // Get the properties
465 properties = MLT_FRAME_PROPERTIES( next );
466
467 // Get the speed of the frame
468 speed = mlt_properties_get_double( properties, "_speed" );
469
470 // Get the current time
471 gettimeofday( &now, NULL );
472
473 // Get the elapsed time
474 elapsed = ( ( int64_t )now.tv_sec * 1000000 + now.tv_usec ) - start;
475
476 // See if we have to delay the display of the current frame
477 if ( mlt_properties_get_int( properties, "rendered" ) == 1 )
478 {
479 // Obtain the scheduled playout time
480 int64_t scheduled = mlt_properties_get_int( properties, "playtime" );
481
482 // Determine the difference between the elapsed time and the scheduled playout time
483 int64_t difference = scheduled - elapsed;
484
485 // Smooth playback a bit
486 if ( real_time && ( difference > 20000 && speed == 1.0 ) )
487 {
488 tm.tv_sec = difference / 1000000;
489 tm.tv_nsec = ( difference % 1000000 ) * 500;
490 nanosleep( &tm, NULL );
491 }
492
493 // Show current frame if not too old
494 if ( !real_time || ( difference > -10000 || speed != 1.0 || mlt_deque_count( self->queue ) < 2 ) )
495 consumer_play_video( self, next );
496
497 // If the queue is empty, recalculate start to allow build up again
498 if ( real_time && ( mlt_deque_count( self->queue ) == 0 && speed == 1.0 ) )
499 {
500 gettimeofday( &now, NULL );
501 start = ( ( int64_t )now.tv_sec * 1000000 + now.tv_usec ) - scheduled + 20000;
502 }
503 }
504
505 // This frame can now be closed
506 mlt_frame_close( next );
507 next = NULL;
508 }
509
510 // This consumer is stopping. But audio has already been played for all
511 // the frames in the queue. Spit out all the frames so that the display has
512 // the option to catch up with the audio.
513 if ( next != NULL ) {
514 consumer_play_video( self, next );
515 mlt_frame_close( next );
516 next = NULL;
517 }
518 while ( mlt_deque_count( self->queue ) > 0 ) {
519 next = mlt_deque_pop_front( self->queue );
520 consumer_play_video( self, next );
521 mlt_frame_close( next );
522 next = NULL;
523 }
524
525 mlt_consumer_stopped( &self->parent );
526
527 return NULL;
528 }
529
530 /** Threaded wrapper for pipe.
531 */
532
consumer_thread(void * arg)533 static void *consumer_thread( void *arg )
534 {
535 // Identify the arg
536 consumer_sdl self = arg;
537
538 // Get the consumer
539 mlt_consumer consumer = &self->parent;
540
541 // Get the properties
542 mlt_properties consumer_props = MLT_CONSUMER_PROPERTIES( consumer );
543
544 // Video thread
545 pthread_t thread;
546
547 // internal initialization
548 int init_audio = 1;
549 int init_video = 1;
550 mlt_frame frame = NULL;
551 mlt_properties properties = NULL;
552 int duration = 0;
553 int64_t playtime = 0;
554 struct timespec tm = { 0, 100000 };
555 // int last_position = -1;
556
557 pthread_mutex_lock( &self->refresh_mutex );
558 self->refresh_count = 0;
559 pthread_mutex_unlock( &self->refresh_mutex );
560
561 // Loop until told not to
562 while( self->running )
563 {
564 // Get a frame from the attached producer
565 frame = mlt_consumer_rt_frame( consumer );
566
567 // Ensure that we have a frame
568 if ( frame )
569 {
570 // Get the frame properties
571 properties = MLT_FRAME_PROPERTIES( frame );
572
573 // Get the speed of the frame
574 double speed = mlt_properties_get_double( properties, "_speed" );
575
576 // Clear refresh
577 mlt_events_block( consumer_props, consumer_props );
578 mlt_properties_set_int( consumer_props, "refresh", 0 );
579 mlt_events_unblock( consumer_props, consumer_props );
580
581 // Play audio
582 init_audio = consumer_play_audio( self, frame, init_audio, &duration );
583
584 // Determine the start time now
585 if ( self->playing && init_video )
586 {
587 // Create the video thread
588 pthread_create( &thread, NULL, video_thread, self );
589
590 // Video doesn't need to be initialised any more
591 init_video = 0;
592 }
593
594 // Set playtime for this frame
595 mlt_properties_set_int( properties, "playtime", playtime );
596
597 while ( self->running && speed != 0 && mlt_deque_count( self->queue ) > 15 )
598 nanosleep( &tm, NULL );
599
600 // Push this frame to the back of the video queue
601 if ( self->running && speed )
602 {
603 pthread_mutex_lock( &self->video_mutex );
604 if ( self->is_purge && speed == 1.0 )
605 {
606 mlt_frame_close( frame );
607 frame = NULL;
608 self->is_purge = 0;
609 }
610 else
611 {
612 mlt_deque_push_back( self->queue, frame );
613 pthread_cond_broadcast( &self->video_cond );
614 }
615 pthread_mutex_unlock( &self->video_mutex );
616
617 // Calculate the next playtime
618 playtime += ( duration * 1000 );
619 }
620 else if ( self->running )
621 {
622 pthread_mutex_lock( &self->refresh_mutex );
623 consumer_play_video( self, frame );
624 mlt_frame_close( frame );
625 frame = NULL;
626 self->refresh_count --;
627 if ( self->refresh_count <= 0 )
628 {
629 pthread_cond_wait( &self->refresh_cond, &self->refresh_mutex );
630 }
631 pthread_mutex_unlock( &self->refresh_mutex );
632 }
633
634 // Optimisation to reduce latency
635 if ( speed == 1.0 )
636 {
637 // TODO: disabled due to misbehavior on parallel-consumer
638 // if ( last_position != -1 && last_position + 1 != mlt_frame_get_position( frame ) )
639 // mlt_consumer_purge( consumer );
640 // last_position = mlt_frame_get_position( frame );
641 }
642 else if (speed == 0.0)
643 {
644 mlt_consumer_purge( consumer );
645 // last_position = -1;
646 }
647 }
648 }
649
650 // Kill the video thread
651 if ( init_video == 0 )
652 {
653 pthread_mutex_lock( &self->video_mutex );
654 pthread_cond_broadcast( &self->video_cond );
655 pthread_mutex_unlock( &self->video_mutex );
656 pthread_join( thread, NULL );
657 }
658
659 if ( frame )
660 {
661 // The video thread has cleared out the queue. But the audio was played
662 // for this frame. So play the video before stopping so the display has
663 // the option to catch up with the audio.
664 consumer_play_video( self, frame );
665 mlt_frame_close( frame );
666 frame = NULL;
667 }
668
669 pthread_mutex_lock( &self->audio_mutex );
670 self->audio_avail = 0;
671 pthread_mutex_unlock( &self->audio_mutex );
672
673 return NULL;
674 }
675
676 /** Callback to allow override of the close method.
677 */
678
consumer_close(mlt_consumer parent)679 static void consumer_close( mlt_consumer parent )
680 {
681 // Get the actual object
682 consumer_sdl self = parent->child;
683
684 // Stop the consumer
685 mlt_consumer_stop( parent );
686
687 // Now clean up the rest
688 mlt_consumer_close( parent );
689
690 // Close the queue
691 mlt_deque_close( self->queue );
692
693 // Destroy mutexes
694 pthread_mutex_destroy( &self->audio_mutex );
695 pthread_cond_destroy( &self->audio_cond );
696 pthread_mutex_destroy( &self->video_mutex );
697 pthread_cond_destroy( &self->video_cond );
698 pthread_mutex_destroy( &self->refresh_mutex );
699 pthread_cond_destroy( &self->refresh_cond );
700
701 // Finally clean up this
702 free( self );
703 }
704