1 /*
2     SPDX-FileCopyrightText: 2003 Fabrice Bellard
3     SPDX-FileCopyrightText: 2020 Mladen Milinkovic <max@smoothware.net>
4 
5     SPDX-License-Identifier: GPL-2.0-or-later
6 */
7 
8 #include "audiodecoder.h"
9 
10 #include <QMap>
11 
12 #include "videoplayer/backend/ffplayer.h"
13 #include "videoplayer/backend/framequeue.h"
14 #include "videoplayer/backend/videostate.h"
15 #include "videoplayer/backend/renderthread.h"
16 
17 extern "C" {
18 #include "libavutil/time.h"
19 #include "libswresample/swresample.h"
20 }
21 
22 #include <AL/al.h>
23 #include <AL/alc.h>
24 
25 // maximum audio speed change to get correct sync
26 #define SAMPLE_CORRECTION_PERCENT_MAX 10
27 
28 // we use about AUDIO_DIFF_AVG_NB A-V differences to make the average
29 #define AUDIO_DIFF_AVG_NB   20
30 
31 // Minimum audio buffer size, in samples.
32 #define AUDIO_MIN_BUFFER_SIZE 512
33 
34 
35 using namespace SubtitleComposer;
36 
AudioDecoder(VideoState * state,QObject * parent)37 AudioDecoder::AudioDecoder(VideoState *state, QObject *parent)
38 	: Decoder(parent),
39 	  m_vs(state),
40 	  m_swrCtx(nullptr),
41 	  m_audioBuf(nullptr),
42 	  m_bufSize(0),
43 	  m_audioBuf1(nullptr),
44 	  m_buf1Size(0),
45 	  m_alDev(nullptr),
46 	  m_alCtx(nullptr),
47 	  m_alSrc(0),
48 	  m_bufCnt(0),
49 	  m_bufFmt(0)
50 {
51 }
52 
53 void
destroy()54 AudioDecoder::destroy()
55 {
56 	Decoder::destroy();
57 	swr_free(&m_swrCtx);
58 
59 	av_freep(&m_audioBuf1);
60 	m_buf1Size = 0;
61 	m_audioBuf = nullptr;
62 }
63 
64 void
abort()65 AudioDecoder::abort()
66 {
67 	Decoder::abort();
68 	close();
69 }
70 
71 void
play()72 AudioDecoder::play()
73 {
74 	alSourcePlay(m_alSrc);
75 }
76 
77 void
pause()78 AudioDecoder::pause()
79 {
80 	alSourcePause(m_alSrc);
81 }
82 
83 void
setListenerGain(double gain)84 AudioDecoder::setListenerGain(double gain)
85 {
86 	alListenerf(AL_GAIN, gain);
87 }
88 
89 double
pitch() const90 AudioDecoder::pitch() const
91 {
92 	ALfloat pitch = 1.0;
93 	alGetSourcef(m_alSrc, AL_PITCH, &pitch);
94 	return pitch;
95 }
96 
97 void
setPitch(double pitch)98 AudioDecoder::setPitch(double pitch)
99 {
100 	alSourcef(m_alSrc, AL_PITCH, pitch);
101 	m_vs->notifySpeed();
102 }
103 
104 void
flush()105 AudioDecoder::flush()
106 {
107 	for(;;) {
108 		alSourceStop(m_alSrc);
109 
110 		ALint bufferCnt = 0;
111 		alGetSourcei(m_alSrc, AL_BUFFERS_QUEUED, &bufferCnt);
112 		if(bufferCnt == 0)
113 			break;
114 
115 		bufferCnt = 0;
116 		alGetSourcei(m_alSrc, AL_BUFFERS_PROCESSED, &bufferCnt);
117 		if(bufferCnt == 0) {
118 			av_log(nullptr, AV_LOG_WARNING, "openal: source didn't stop... retrying flush\n");
119 			av_usleep(10);
120 			continue;
121 		}
122 
123 		ALuint *bufs = new ALuint[bufferCnt];
124 		alSourceUnqueueBuffers(m_alSrc, bufferCnt, bufs);
125 		alDeleteBuffers(bufferCnt, bufs);
126 		delete[] bufs;
127 		m_hwBufQueueSize = 0;
128 	}
129 }
130 
131 void
close()132 AudioDecoder::close()
133 {
134 	flush();
135 	alcMakeContextCurrent(nullptr);
136 	if(m_alCtx) {
137 		alcDestroyContext(m_alCtx);
138 		m_alCtx = nullptr;
139 	}
140 	if(m_alDev) {
141 		alcCloseDevice(m_alDev);
142 		m_alDev = nullptr;
143 	}
144 	if(m_alSrc)
145 		alDeleteSources(1, &m_alSrc);
146 	m_alSrc = 0;
147 }
148 
149 bool
open(int64_t wantChLayout,int wantNbChan,int wantSampleRate)150 AudioDecoder::open(int64_t wantChLayout, int wantNbChan, int wantSampleRate)
151 {
152 	const static QMap<int, const char *> bufFmtMap = {
153 		{ 4, "AL_FORMAT_QUAD16" },
154 		{ 6, "AL_FORMAT_51CHN16" },
155 		{ 7, "AL_FORMAT_61CHN16" },
156 		{ 8, "AL_FORMAT_71CHN16" },
157 	};
158 
159 	int err;
160 
161 	if(wantSampleRate <= 0 || wantNbChan <= 0) {
162 		av_log(nullptr, AV_LOG_ERROR, "openal: invalid sample rate or channel count!\n");
163 		return -1;
164 	}
165 
166 	int availNbChan = wantNbChan;
167 	for(;;) {
168 		while(availNbChan > 2 && !bufFmtMap.contains(availNbChan))
169 			availNbChan--;
170 		if(availNbChan <= 2) {
171 			m_bufFmt = availNbChan == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16;
172 			break;
173 		}
174 		m_bufFmt = alGetEnumValue(bufFmtMap[wantNbChan]);
175 		if(m_bufFmt)
176 			break;
177 		availNbChan--;
178 	}
179 
180 	if(!wantChLayout || wantNbChan != availNbChan || wantNbChan != av_get_channel_layout_nb_channels(wantChLayout)) {
181 		wantChLayout = av_get_default_channel_layout(availNbChan);
182 		wantChLayout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;
183 	}
184 
185 	m_alDev = alcOpenDevice(nullptr);
186 	if(!m_alDev) {
187 		av_log(nullptr, AV_LOG_ERROR, "openal: error opening default audio device!\n");
188 		close();
189 		return false;
190 	}
191 
192 	m_alCtx = alcCreateContext(m_alDev, nullptr);
193 	if(!m_alCtx) {
194 		av_log(nullptr, AV_LOG_ERROR, "openal: error creating audio context!\n");
195 		close();
196 		return false;
197 	}
198 	if(!alcMakeContextCurrent(m_alCtx)) {
199 		av_log(nullptr, AV_LOG_ERROR, "openal: error setting current audio context!\n");
200 		close();
201 		return false;
202 	}
203 
204 	alGetError(); // clear error
205 
206 	alGenSources(1, &m_alSrc);
207 	if((err = alGetError()) != AL_NO_ERROR) {
208 		av_log(nullptr, AV_LOG_ERROR, "openal: error generating audio source: %d\n", err);
209 		close();
210 		return false;
211 	}
212 
213 	m_fmtTgt.fmt = AV_SAMPLE_FMT_S16;
214 	m_fmtTgt.freq = wantSampleRate;
215 	m_fmtTgt.channelLayout = wantChLayout;
216 	m_fmtTgt.channels = availNbChan;
217 	m_fmtTgt.frameSize = av_samples_get_buffer_size(nullptr, m_fmtTgt.channels, 1, m_fmtTgt.fmt, 1);
218 	m_fmtTgt.bytesPerSec = av_samples_get_buffer_size(nullptr, m_fmtTgt.channels, m_fmtTgt.freq, m_fmtTgt.fmt, 1);
219 	if(m_fmtTgt.bytesPerSec <= 0 || m_fmtTgt.frameSize <= 0) {
220 		av_log(nullptr, AV_LOG_ERROR, "av_samples_get_buffer_size failed\n");
221 		close();
222 		return false;
223 	}
224 
225 	alListenerf(AL_GAIN, m_vs->player->muted() ? 0. : m_vs->player->volume());
226 
227 	m_fmtSrc = m_fmtTgt;
228 	m_hwBufQueueSize = 0;
229 	m_bufSize = 0;
230 
231 	// init averaging filter
232 	m_diffAvgCoef = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
233 	m_diffAvgCount = 0;
234 
235 	return true;
236 }
237 
238 void
queueBuffer(uint8_t * data,int len)239 AudioDecoder::queueBuffer(uint8_t *data, int len)
240 {
241 	int err = alGetError(); // reset error
242 	ALuint buf;
243 
244 	// get next buffer to fill - maybe use alGetSourcei(AL_BUFFERS_QUEUED / AL_BUFFERS_PROCESSED)
245 	alSourceUnqueueBuffers(m_alSrc, 1, &buf);
246 	if((err = alGetError()) != AL_NO_ERROR) {
247 		if(err == AL_INVALID_VALUE) {
248 			// buffer can not be unqueued because it has not been processed yet.
249 			alGenBuffers(1, &buf);
250 			if((err = alGetError()) != AL_NO_ERROR) {
251 				av_log(nullptr, AV_LOG_ERROR, "openal: alGenBuffers() failed: %d\n", err);
252 				return;
253 			} else {
254 				m_bufCnt++;
255 			}
256 		} else {
257 			av_log(nullptr, AV_LOG_ERROR, "openal: alSourceUnqueueBuffers() failed: %d\n", err);
258 			return;
259 		}
260 	} else {
261 		ALint hwBufSize;
262 		alGetBufferi(buf, AL_SIZE, &hwBufSize);
263 		if((err = alGetError()) != AL_NO_ERROR) {
264 			av_log(nullptr, AV_LOG_ERROR, "openal: alGetBufferi(AL_SIZE) failed: %d\n", err);
265 			return;
266 		}
267 		m_hwBufQueueSize -= hwBufSize;
268 	}
269 
270 	// copy data to buffer
271 	alBufferData(buf, m_bufFmt, data, len, m_fmtTgt.freq);
272 	if((err = alGetError()) != AL_NO_ERROR) {
273 		av_log(nullptr, AV_LOG_ERROR, "openal: alBufferData() failed: %d\n", err);
274 		return;
275 	}
276 
277 	// queue buffer
278 	alSourceQueueBuffers(m_alSrc, 1, &buf); // stream
279 	if((err = alGetError()) != AL_NO_ERROR) {
280 		av_log(nullptr, AV_LOG_ERROR, "openal: alSourceQueueBuffers() failed: %d\n", err);
281 		return;
282 	}
283 
284 	m_hwBufQueueSize += len;
285 
286 	// get current state
287 	ALint state = AL_INITIAL;
288 	alGetSourcei(m_alSrc, AL_SOURCE_STATE, &state);
289 	if((err = alGetError()) != AL_NO_ERROR) {
290 		av_log(nullptr, AV_LOG_ERROR, "openal: alGetSourcei(AL_SOURCE_STATE) failed: %d\n", err);
291 		return;
292 	}
293 
294 	// start playing
295 	if(state != AL_PLAYING) {
296 		play();
297 		if(m_vs->paused || m_vs->step)
298 			pause();
299 	}
300 }
301 
302 /* return the wanted number of samples to get better sync if sync_type is video
303  * or external master clock */
304 int
syncAudio(int nbSamples)305 AudioDecoder::syncAudio(int nbSamples)
306 {
307 	int wantedNbSamples = nbSamples;
308 
309 	// if not master, then we try to remove or add samples to correct the clock
310 	if(m_vs->masterSyncType() != AV_SYNC_AUDIO_MASTER) {
311 		int minNbSamples, maxNbSamples;
312 
313 		double diff = m_vs->audClk.get() - m_vs->masterTime();
314 
315 		if(!std::isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD) {
316 			m_diffCum = diff + m_diffAvgCoef * m_diffCum;
317 			if(m_diffAvgCount < AUDIO_DIFF_AVG_NB) {
318 				// not enough measures to have a correct estimate
319 				m_diffAvgCount++;
320 			} else {
321 				// estimate the A-V difference
322 				const double avgDiff = m_diffCum * (1.0 - m_diffAvgCoef);
323 				if(avgDiff != 0.) {
324 					wantedNbSamples = nbSamples + (int)(diff * m_fmtSrc.freq);
325 					minNbSamples = ((nbSamples * (100 - SAMPLE_CORRECTION_PERCENT_MAX) / 100));
326 					maxNbSamples = ((nbSamples * (100 + SAMPLE_CORRECTION_PERCENT_MAX) / 100));
327 					wantedNbSamples = av_clip(wantedNbSamples, minNbSamples, maxNbSamples);
328 				}
329 				av_log(nullptr, AV_LOG_TRACE, "diff=%f adiff=%f sample_diff=%d\n",
330 					   diff, avgDiff, wantedNbSamples - nbSamples);
331 			}
332 		} else {
333 			// too big difference : may be initial PTS errors, so reset A-V filter
334 			m_diffAvgCount = 0;
335 			m_diffCum = 0;
336 		}
337 	}
338 
339 	return wantedNbSamples;
340 }
341 
342 /**
343  * Decode one audio frame and return its uncompressed size.
344  *
345  * The processed audio frame is decoded, converted if required, and
346  * stored in m_audioBuf, with size in bytes given by the return
347  * value.
348  */
349 int
decodeFrame(Frame * af)350 AudioDecoder::decodeFrame(Frame *af)
351 {
352 	if(af->serial != m_queue->serial())
353 		return -1;
354 
355 	int dataSize = av_samples_get_buffer_size(nullptr, af->frame->channels,
356 										   af->frame->nb_samples,
357 										   (AVSampleFormat)af->frame->format, 1);
358 	int resampledDataSize;
359 
360 	uint64_t decChannelLayout =
361 		(af->frame->channel_layout &&
362 		 af->frame->channels == av_get_channel_layout_nb_channels(af->frame->channel_layout)) ?
363 		af->frame->channel_layout : av_get_default_channel_layout(af->frame->channels);
364 	int wantedNbSamples = syncAudio(af->frame->nb_samples);
365 
366 	if(af->frame->format != m_fmtSrc.fmt
367 	|| decChannelLayout != m_fmtSrc.channelLayout
368 	|| af->frame->sample_rate != m_fmtSrc.freq
369 	|| (wantedNbSamples != af->frame->nb_samples && !m_swrCtx)) {
370 		swr_free(&m_swrCtx);
371 		m_swrCtx = swr_alloc_set_opts(nullptr,
372 										 m_fmtTgt.channelLayout, m_fmtTgt.fmt, m_fmtTgt.freq,
373 										 decChannelLayout, (AVSampleFormat)af->frame->format, af->frame->sample_rate,
374 										 0, nullptr);
375 		if(!m_swrCtx || swr_init(m_swrCtx) < 0) {
376 			av_log(nullptr, AV_LOG_ERROR,
377 				   "Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
378 				   af->frame->sample_rate, av_get_sample_fmt_name((AVSampleFormat)af->frame->format),
379 				   af->frame->channels,
380 				   m_fmtTgt.freq, av_get_sample_fmt_name(m_fmtTgt.fmt), m_fmtTgt.channels);
381 			swr_free(&m_swrCtx);
382 			return -1;
383 		}
384 		m_fmtSrc.channelLayout = decChannelLayout;
385 		m_fmtSrc.channels = af->frame->channels;
386 		m_fmtSrc.freq = af->frame->sample_rate;
387 		m_fmtSrc.fmt = (AVSampleFormat)af->frame->format;
388 	}
389 
390 	if(m_swrCtx) {
391 		const int outCount = (int64_t)wantedNbSamples * m_fmtTgt.freq / af->frame->sample_rate + 256;
392 		const int outSize = av_samples_get_buffer_size(nullptr, m_fmtTgt.channels, outCount, m_fmtTgt.fmt, 0);
393 		if(outSize < 0) {
394 			av_log(nullptr, AV_LOG_ERROR, "av_samples_get_buffer_size() failed\n");
395 			return -1;
396 		}
397 		if(wantedNbSamples != af->frame->nb_samples) {
398 			if(swr_set_compensation(m_swrCtx,
399 					(wantedNbSamples - af->frame->nb_samples) * m_fmtTgt.freq / af->frame->sample_rate,
400 					wantedNbSamples * m_fmtTgt.freq / af->frame->sample_rate) < 0) {
401 				av_log(nullptr, AV_LOG_ERROR, "swr_set_compensation() failed\n");
402 				return -1;
403 			}
404 		}
405 		av_fast_malloc(&m_audioBuf1, &m_buf1Size, outSize);
406 		if(!m_audioBuf1)
407 			return AVERROR(ENOMEM);
408 		const int outSamplesPerChannel = swr_convert(m_swrCtx, &m_audioBuf1, outCount,
409 						   (const uint8_t **)af->frame->extended_data, af->frame->nb_samples);
410 		if(outSamplesPerChannel < 0) {
411 			av_log(nullptr, AV_LOG_ERROR, "swr_convert() failed\n");
412 			return -1;
413 		}
414 		if(outSamplesPerChannel == outCount) {
415 			av_log(nullptr, AV_LOG_WARNING, "audio buffer is probably too small\n");
416 			if(swr_init(m_swrCtx) < 0)
417 				swr_free(&m_swrCtx);
418 		}
419 		m_audioBuf = m_audioBuf1;
420 		resampledDataSize = outSamplesPerChannel * m_fmtTgt.channels * av_get_bytes_per_sample(m_fmtTgt.fmt);
421 	} else {
422 		m_audioBuf = af->frame->data[0];
423 		resampledDataSize = dataSize;
424 	}
425 
426 	return resampledDataSize;
427 }
428 
429 int
getFrame(AVFrame * frame)430 AudioDecoder::getFrame(AVFrame *frame)
431 {
432 	const int gotFrame = Decoder::decodeFrame(frame, nullptr);
433 
434 	if(gotFrame <= 0 || frame->pts == AV_NOPTS_VALUE)
435 		return gotFrame;
436 
437 	const double dPts = double(frame->pts) / frame->sample_rate;
438 
439 	if(!std::isnan(dPts) && m_vs->seekDecoder > 0. && m_vs->seekDecoder > dPts) {
440 		av_frame_unref(frame);
441 		return 0;
442 	}
443 
444 	return gotFrame;
445 }
446 
447 void
queueFrame(Frame * af)448 AudioDecoder::queueFrame(Frame *af)
449 {
450 	int audioSize = decodeFrame(af);
451 	if(audioSize < 0) {
452 		// if error, just output silence
453 		m_audioBuf = nullptr;
454 		m_bufSize = (AUDIO_MIN_BUFFER_SIZE / m_fmtTgt.frameSize) * m_fmtTgt.frameSize;
455 	} else {
456 		if(m_vs->showMode != SHOW_MODE_VIDEO)
457 			m_vs->renderThread->updateSampleDisplay((int16_t *)(void *)m_audioBuf, audioSize);
458 		m_bufSize = audioSize;
459 	}
460 
461 	if(!m_audioBuf) {
462 		uint8_t *silence = new uint8_t[m_bufSize]();
463 		queueBuffer(silence, m_bufSize);
464 		delete[] silence;
465 	} else {
466 		queueBuffer((uint8_t *)m_audioBuf, audioSize);
467 	}
468 }
469 
470 void
run()471 AudioDecoder::run()
472 {
473 	AVFrame *frame = av_frame_alloc();
474 	Frame *af = new Frame();
475 
476 	if(!frame)
477 		return;
478 
479 	for(;;) {
480 		const int got_frame = getFrame(frame);
481 		Q_ASSERT(got_frame != AVERROR(EAGAIN));
482 		Q_ASSERT(got_frame != AVERROR_EOF);
483 		if(got_frame < 0)
484 			break;
485 
486 		if(got_frame) {
487 			if(!(af->frame = av_frame_alloc()))
488 				break;
489 
490 			af->pts = frame->pts == AV_NOPTS_VALUE ? NAN : double(frame->pts) / frame->sample_rate;
491 			af->pos = frame->pkt_pos;
492 			af->serial = m_pktSerial;
493 			af->duration = double(frame->nb_samples) / frame->sample_rate;
494 
495 			av_frame_move_ref(af->frame, frame);
496 
497 			// time to unqueue one sample in microseconds (AV_TIME_BASE)
498 			const int64_t sleepTime = int64_t(double(AUDIO_MIN_BUFFER_SIZE / m_fmtTgt.frameSize) / (m_fmtTgt.freq * m_vs->audClk.speed()) * AV_TIME_BASE);
499 			// bytes needed for 100ms of audio
500 			const ALint hwMinBytes = m_vs->audClk.speed() * m_fmtTgt.bytesPerSec * .100;
501 
502 			while(!m_vs->abortRequested) {
503 				ALint hwBufOffset = 0;
504 				alGetSourcei(m_alSrc, AL_BYTE_OFFSET, &hwBufOffset);
505 				if(!std::isnan(af->pts)) {
506 					m_vs->audClk.setAt(
507 								 af->pts - double(m_hwBufQueueSize - hwBufOffset) / m_fmtTgt.bytesPerSec,
508 								 af->serial,
509 								 av_gettime_relative() / double(AV_TIME_BASE));
510 					m_vs->extClk.syncTo(&m_vs->audClk);
511 				}
512 
513 				if(!m_vs->paused) {
514 					if(m_hwBufQueueSize - hwBufOffset < hwMinBytes)
515 						break;
516 
517 					ALint bufReady = 0;
518 					alGetSourcei(m_alSrc, AL_BUFFERS_PROCESSED, &bufReady);
519 					if(bufReady > 0)
520 						break;
521 				}
522 
523 				av_usleep(sleepTime);
524 			}
525 
526 			queueFrame(af);
527 
528 			av_frame_unref(af->frame);
529 			av_frame_free(&af->frame);
530 		}
531 	}
532 
533 	av_frame_free(&frame);
534 	delete af;
535 	return;
536 }
537