1 /*
2 SPDX-FileCopyrightText: 2003 Fabrice Bellard
3 SPDX-FileCopyrightText: 2020 Mladen Milinkovic <max@smoothware.net>
4
5 SPDX-License-Identifier: GPL-2.0-or-later
6 */
7
8 #include "audiodecoder.h"
9
10 #include <QMap>
11
12 #include "videoplayer/backend/ffplayer.h"
13 #include "videoplayer/backend/framequeue.h"
14 #include "videoplayer/backend/videostate.h"
15 #include "videoplayer/backend/renderthread.h"
16
17 extern "C" {
18 #include "libavutil/time.h"
19 #include "libswresample/swresample.h"
20 }
21
22 #include <AL/al.h>
23 #include <AL/alc.h>
24
25 // maximum audio speed change to get correct sync
26 #define SAMPLE_CORRECTION_PERCENT_MAX 10
27
28 // we use about AUDIO_DIFF_AVG_NB A-V differences to make the average
29 #define AUDIO_DIFF_AVG_NB 20
30
31 // Minimum audio buffer size, in samples.
32 #define AUDIO_MIN_BUFFER_SIZE 512
33
34
35 using namespace SubtitleComposer;
36
AudioDecoder(VideoState * state,QObject * parent)37 AudioDecoder::AudioDecoder(VideoState *state, QObject *parent)
38 : Decoder(parent),
39 m_vs(state),
40 m_swrCtx(nullptr),
41 m_audioBuf(nullptr),
42 m_bufSize(0),
43 m_audioBuf1(nullptr),
44 m_buf1Size(0),
45 m_alDev(nullptr),
46 m_alCtx(nullptr),
47 m_alSrc(0),
48 m_bufCnt(0),
49 m_bufFmt(0)
50 {
51 }
52
53 void
destroy()54 AudioDecoder::destroy()
55 {
56 Decoder::destroy();
57 swr_free(&m_swrCtx);
58
59 av_freep(&m_audioBuf1);
60 m_buf1Size = 0;
61 m_audioBuf = nullptr;
62 }
63
64 void
abort()65 AudioDecoder::abort()
66 {
67 Decoder::abort();
68 close();
69 }
70
71 void
play()72 AudioDecoder::play()
73 {
74 alSourcePlay(m_alSrc);
75 }
76
77 void
pause()78 AudioDecoder::pause()
79 {
80 alSourcePause(m_alSrc);
81 }
82
83 void
setListenerGain(double gain)84 AudioDecoder::setListenerGain(double gain)
85 {
86 alListenerf(AL_GAIN, gain);
87 }
88
89 double
pitch() const90 AudioDecoder::pitch() const
91 {
92 ALfloat pitch = 1.0;
93 alGetSourcef(m_alSrc, AL_PITCH, &pitch);
94 return pitch;
95 }
96
97 void
setPitch(double pitch)98 AudioDecoder::setPitch(double pitch)
99 {
100 alSourcef(m_alSrc, AL_PITCH, pitch);
101 m_vs->notifySpeed();
102 }
103
104 void
flush()105 AudioDecoder::flush()
106 {
107 for(;;) {
108 alSourceStop(m_alSrc);
109
110 ALint bufferCnt = 0;
111 alGetSourcei(m_alSrc, AL_BUFFERS_QUEUED, &bufferCnt);
112 if(bufferCnt == 0)
113 break;
114
115 bufferCnt = 0;
116 alGetSourcei(m_alSrc, AL_BUFFERS_PROCESSED, &bufferCnt);
117 if(bufferCnt == 0) {
118 av_log(nullptr, AV_LOG_WARNING, "openal: source didn't stop... retrying flush\n");
119 av_usleep(10);
120 continue;
121 }
122
123 ALuint *bufs = new ALuint[bufferCnt];
124 alSourceUnqueueBuffers(m_alSrc, bufferCnt, bufs);
125 alDeleteBuffers(bufferCnt, bufs);
126 delete[] bufs;
127 m_hwBufQueueSize = 0;
128 }
129 }
130
131 void
close()132 AudioDecoder::close()
133 {
134 flush();
135 alcMakeContextCurrent(nullptr);
136 if(m_alCtx) {
137 alcDestroyContext(m_alCtx);
138 m_alCtx = nullptr;
139 }
140 if(m_alDev) {
141 alcCloseDevice(m_alDev);
142 m_alDev = nullptr;
143 }
144 if(m_alSrc)
145 alDeleteSources(1, &m_alSrc);
146 m_alSrc = 0;
147 }
148
149 bool
open(int64_t wantChLayout,int wantNbChan,int wantSampleRate)150 AudioDecoder::open(int64_t wantChLayout, int wantNbChan, int wantSampleRate)
151 {
152 const static QMap<int, const char *> bufFmtMap = {
153 { 4, "AL_FORMAT_QUAD16" },
154 { 6, "AL_FORMAT_51CHN16" },
155 { 7, "AL_FORMAT_61CHN16" },
156 { 8, "AL_FORMAT_71CHN16" },
157 };
158
159 int err;
160
161 if(wantSampleRate <= 0 || wantNbChan <= 0) {
162 av_log(nullptr, AV_LOG_ERROR, "openal: invalid sample rate or channel count!\n");
163 return -1;
164 }
165
166 int availNbChan = wantNbChan;
167 for(;;) {
168 while(availNbChan > 2 && !bufFmtMap.contains(availNbChan))
169 availNbChan--;
170 if(availNbChan <= 2) {
171 m_bufFmt = availNbChan == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16;
172 break;
173 }
174 m_bufFmt = alGetEnumValue(bufFmtMap[wantNbChan]);
175 if(m_bufFmt)
176 break;
177 availNbChan--;
178 }
179
180 if(!wantChLayout || wantNbChan != availNbChan || wantNbChan != av_get_channel_layout_nb_channels(wantChLayout)) {
181 wantChLayout = av_get_default_channel_layout(availNbChan);
182 wantChLayout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;
183 }
184
185 m_alDev = alcOpenDevice(nullptr);
186 if(!m_alDev) {
187 av_log(nullptr, AV_LOG_ERROR, "openal: error opening default audio device!\n");
188 close();
189 return false;
190 }
191
192 m_alCtx = alcCreateContext(m_alDev, nullptr);
193 if(!m_alCtx) {
194 av_log(nullptr, AV_LOG_ERROR, "openal: error creating audio context!\n");
195 close();
196 return false;
197 }
198 if(!alcMakeContextCurrent(m_alCtx)) {
199 av_log(nullptr, AV_LOG_ERROR, "openal: error setting current audio context!\n");
200 close();
201 return false;
202 }
203
204 alGetError(); // clear error
205
206 alGenSources(1, &m_alSrc);
207 if((err = alGetError()) != AL_NO_ERROR) {
208 av_log(nullptr, AV_LOG_ERROR, "openal: error generating audio source: %d\n", err);
209 close();
210 return false;
211 }
212
213 m_fmtTgt.fmt = AV_SAMPLE_FMT_S16;
214 m_fmtTgt.freq = wantSampleRate;
215 m_fmtTgt.channelLayout = wantChLayout;
216 m_fmtTgt.channels = availNbChan;
217 m_fmtTgt.frameSize = av_samples_get_buffer_size(nullptr, m_fmtTgt.channels, 1, m_fmtTgt.fmt, 1);
218 m_fmtTgt.bytesPerSec = av_samples_get_buffer_size(nullptr, m_fmtTgt.channels, m_fmtTgt.freq, m_fmtTgt.fmt, 1);
219 if(m_fmtTgt.bytesPerSec <= 0 || m_fmtTgt.frameSize <= 0) {
220 av_log(nullptr, AV_LOG_ERROR, "av_samples_get_buffer_size failed\n");
221 close();
222 return false;
223 }
224
225 alListenerf(AL_GAIN, m_vs->player->muted() ? 0. : m_vs->player->volume());
226
227 m_fmtSrc = m_fmtTgt;
228 m_hwBufQueueSize = 0;
229 m_bufSize = 0;
230
231 // init averaging filter
232 m_diffAvgCoef = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
233 m_diffAvgCount = 0;
234
235 return true;
236 }
237
238 void
queueBuffer(uint8_t * data,int len)239 AudioDecoder::queueBuffer(uint8_t *data, int len)
240 {
241 int err = alGetError(); // reset error
242 ALuint buf;
243
244 // get next buffer to fill - maybe use alGetSourcei(AL_BUFFERS_QUEUED / AL_BUFFERS_PROCESSED)
245 alSourceUnqueueBuffers(m_alSrc, 1, &buf);
246 if((err = alGetError()) != AL_NO_ERROR) {
247 if(err == AL_INVALID_VALUE) {
248 // buffer can not be unqueued because it has not been processed yet.
249 alGenBuffers(1, &buf);
250 if((err = alGetError()) != AL_NO_ERROR) {
251 av_log(nullptr, AV_LOG_ERROR, "openal: alGenBuffers() failed: %d\n", err);
252 return;
253 } else {
254 m_bufCnt++;
255 }
256 } else {
257 av_log(nullptr, AV_LOG_ERROR, "openal: alSourceUnqueueBuffers() failed: %d\n", err);
258 return;
259 }
260 } else {
261 ALint hwBufSize;
262 alGetBufferi(buf, AL_SIZE, &hwBufSize);
263 if((err = alGetError()) != AL_NO_ERROR) {
264 av_log(nullptr, AV_LOG_ERROR, "openal: alGetBufferi(AL_SIZE) failed: %d\n", err);
265 return;
266 }
267 m_hwBufQueueSize -= hwBufSize;
268 }
269
270 // copy data to buffer
271 alBufferData(buf, m_bufFmt, data, len, m_fmtTgt.freq);
272 if((err = alGetError()) != AL_NO_ERROR) {
273 av_log(nullptr, AV_LOG_ERROR, "openal: alBufferData() failed: %d\n", err);
274 return;
275 }
276
277 // queue buffer
278 alSourceQueueBuffers(m_alSrc, 1, &buf); // stream
279 if((err = alGetError()) != AL_NO_ERROR) {
280 av_log(nullptr, AV_LOG_ERROR, "openal: alSourceQueueBuffers() failed: %d\n", err);
281 return;
282 }
283
284 m_hwBufQueueSize += len;
285
286 // get current state
287 ALint state = AL_INITIAL;
288 alGetSourcei(m_alSrc, AL_SOURCE_STATE, &state);
289 if((err = alGetError()) != AL_NO_ERROR) {
290 av_log(nullptr, AV_LOG_ERROR, "openal: alGetSourcei(AL_SOURCE_STATE) failed: %d\n", err);
291 return;
292 }
293
294 // start playing
295 if(state != AL_PLAYING) {
296 play();
297 if(m_vs->paused || m_vs->step)
298 pause();
299 }
300 }
301
302 /* return the wanted number of samples to get better sync if sync_type is video
303 * or external master clock */
304 int
syncAudio(int nbSamples)305 AudioDecoder::syncAudio(int nbSamples)
306 {
307 int wantedNbSamples = nbSamples;
308
309 // if not master, then we try to remove or add samples to correct the clock
310 if(m_vs->masterSyncType() != AV_SYNC_AUDIO_MASTER) {
311 int minNbSamples, maxNbSamples;
312
313 double diff = m_vs->audClk.get() - m_vs->masterTime();
314
315 if(!std::isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD) {
316 m_diffCum = diff + m_diffAvgCoef * m_diffCum;
317 if(m_diffAvgCount < AUDIO_DIFF_AVG_NB) {
318 // not enough measures to have a correct estimate
319 m_diffAvgCount++;
320 } else {
321 // estimate the A-V difference
322 const double avgDiff = m_diffCum * (1.0 - m_diffAvgCoef);
323 if(avgDiff != 0.) {
324 wantedNbSamples = nbSamples + (int)(diff * m_fmtSrc.freq);
325 minNbSamples = ((nbSamples * (100 - SAMPLE_CORRECTION_PERCENT_MAX) / 100));
326 maxNbSamples = ((nbSamples * (100 + SAMPLE_CORRECTION_PERCENT_MAX) / 100));
327 wantedNbSamples = av_clip(wantedNbSamples, minNbSamples, maxNbSamples);
328 }
329 av_log(nullptr, AV_LOG_TRACE, "diff=%f adiff=%f sample_diff=%d\n",
330 diff, avgDiff, wantedNbSamples - nbSamples);
331 }
332 } else {
333 // too big difference : may be initial PTS errors, so reset A-V filter
334 m_diffAvgCount = 0;
335 m_diffCum = 0;
336 }
337 }
338
339 return wantedNbSamples;
340 }
341
342 /**
343 * Decode one audio frame and return its uncompressed size.
344 *
345 * The processed audio frame is decoded, converted if required, and
346 * stored in m_audioBuf, with size in bytes given by the return
347 * value.
348 */
349 int
decodeFrame(Frame * af)350 AudioDecoder::decodeFrame(Frame *af)
351 {
352 if(af->serial != m_queue->serial())
353 return -1;
354
355 int dataSize = av_samples_get_buffer_size(nullptr, af->frame->channels,
356 af->frame->nb_samples,
357 (AVSampleFormat)af->frame->format, 1);
358 int resampledDataSize;
359
360 uint64_t decChannelLayout =
361 (af->frame->channel_layout &&
362 af->frame->channels == av_get_channel_layout_nb_channels(af->frame->channel_layout)) ?
363 af->frame->channel_layout : av_get_default_channel_layout(af->frame->channels);
364 int wantedNbSamples = syncAudio(af->frame->nb_samples);
365
366 if(af->frame->format != m_fmtSrc.fmt
367 || decChannelLayout != m_fmtSrc.channelLayout
368 || af->frame->sample_rate != m_fmtSrc.freq
369 || (wantedNbSamples != af->frame->nb_samples && !m_swrCtx)) {
370 swr_free(&m_swrCtx);
371 m_swrCtx = swr_alloc_set_opts(nullptr,
372 m_fmtTgt.channelLayout, m_fmtTgt.fmt, m_fmtTgt.freq,
373 decChannelLayout, (AVSampleFormat)af->frame->format, af->frame->sample_rate,
374 0, nullptr);
375 if(!m_swrCtx || swr_init(m_swrCtx) < 0) {
376 av_log(nullptr, AV_LOG_ERROR,
377 "Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
378 af->frame->sample_rate, av_get_sample_fmt_name((AVSampleFormat)af->frame->format),
379 af->frame->channels,
380 m_fmtTgt.freq, av_get_sample_fmt_name(m_fmtTgt.fmt), m_fmtTgt.channels);
381 swr_free(&m_swrCtx);
382 return -1;
383 }
384 m_fmtSrc.channelLayout = decChannelLayout;
385 m_fmtSrc.channels = af->frame->channels;
386 m_fmtSrc.freq = af->frame->sample_rate;
387 m_fmtSrc.fmt = (AVSampleFormat)af->frame->format;
388 }
389
390 if(m_swrCtx) {
391 const int outCount = (int64_t)wantedNbSamples * m_fmtTgt.freq / af->frame->sample_rate + 256;
392 const int outSize = av_samples_get_buffer_size(nullptr, m_fmtTgt.channels, outCount, m_fmtTgt.fmt, 0);
393 if(outSize < 0) {
394 av_log(nullptr, AV_LOG_ERROR, "av_samples_get_buffer_size() failed\n");
395 return -1;
396 }
397 if(wantedNbSamples != af->frame->nb_samples) {
398 if(swr_set_compensation(m_swrCtx,
399 (wantedNbSamples - af->frame->nb_samples) * m_fmtTgt.freq / af->frame->sample_rate,
400 wantedNbSamples * m_fmtTgt.freq / af->frame->sample_rate) < 0) {
401 av_log(nullptr, AV_LOG_ERROR, "swr_set_compensation() failed\n");
402 return -1;
403 }
404 }
405 av_fast_malloc(&m_audioBuf1, &m_buf1Size, outSize);
406 if(!m_audioBuf1)
407 return AVERROR(ENOMEM);
408 const int outSamplesPerChannel = swr_convert(m_swrCtx, &m_audioBuf1, outCount,
409 (const uint8_t **)af->frame->extended_data, af->frame->nb_samples);
410 if(outSamplesPerChannel < 0) {
411 av_log(nullptr, AV_LOG_ERROR, "swr_convert() failed\n");
412 return -1;
413 }
414 if(outSamplesPerChannel == outCount) {
415 av_log(nullptr, AV_LOG_WARNING, "audio buffer is probably too small\n");
416 if(swr_init(m_swrCtx) < 0)
417 swr_free(&m_swrCtx);
418 }
419 m_audioBuf = m_audioBuf1;
420 resampledDataSize = outSamplesPerChannel * m_fmtTgt.channels * av_get_bytes_per_sample(m_fmtTgt.fmt);
421 } else {
422 m_audioBuf = af->frame->data[0];
423 resampledDataSize = dataSize;
424 }
425
426 return resampledDataSize;
427 }
428
429 int
getFrame(AVFrame * frame)430 AudioDecoder::getFrame(AVFrame *frame)
431 {
432 const int gotFrame = Decoder::decodeFrame(frame, nullptr);
433
434 if(gotFrame <= 0 || frame->pts == AV_NOPTS_VALUE)
435 return gotFrame;
436
437 const double dPts = double(frame->pts) / frame->sample_rate;
438
439 if(!std::isnan(dPts) && m_vs->seekDecoder > 0. && m_vs->seekDecoder > dPts) {
440 av_frame_unref(frame);
441 return 0;
442 }
443
444 return gotFrame;
445 }
446
447 void
queueFrame(Frame * af)448 AudioDecoder::queueFrame(Frame *af)
449 {
450 int audioSize = decodeFrame(af);
451 if(audioSize < 0) {
452 // if error, just output silence
453 m_audioBuf = nullptr;
454 m_bufSize = (AUDIO_MIN_BUFFER_SIZE / m_fmtTgt.frameSize) * m_fmtTgt.frameSize;
455 } else {
456 if(m_vs->showMode != SHOW_MODE_VIDEO)
457 m_vs->renderThread->updateSampleDisplay((int16_t *)(void *)m_audioBuf, audioSize);
458 m_bufSize = audioSize;
459 }
460
461 if(!m_audioBuf) {
462 uint8_t *silence = new uint8_t[m_bufSize]();
463 queueBuffer(silence, m_bufSize);
464 delete[] silence;
465 } else {
466 queueBuffer((uint8_t *)m_audioBuf, audioSize);
467 }
468 }
469
470 void
run()471 AudioDecoder::run()
472 {
473 AVFrame *frame = av_frame_alloc();
474 Frame *af = new Frame();
475
476 if(!frame)
477 return;
478
479 for(;;) {
480 const int got_frame = getFrame(frame);
481 Q_ASSERT(got_frame != AVERROR(EAGAIN));
482 Q_ASSERT(got_frame != AVERROR_EOF);
483 if(got_frame < 0)
484 break;
485
486 if(got_frame) {
487 if(!(af->frame = av_frame_alloc()))
488 break;
489
490 af->pts = frame->pts == AV_NOPTS_VALUE ? NAN : double(frame->pts) / frame->sample_rate;
491 af->pos = frame->pkt_pos;
492 af->serial = m_pktSerial;
493 af->duration = double(frame->nb_samples) / frame->sample_rate;
494
495 av_frame_move_ref(af->frame, frame);
496
497 // time to unqueue one sample in microseconds (AV_TIME_BASE)
498 const int64_t sleepTime = int64_t(double(AUDIO_MIN_BUFFER_SIZE / m_fmtTgt.frameSize) / (m_fmtTgt.freq * m_vs->audClk.speed()) * AV_TIME_BASE);
499 // bytes needed for 100ms of audio
500 const ALint hwMinBytes = m_vs->audClk.speed() * m_fmtTgt.bytesPerSec * .100;
501
502 while(!m_vs->abortRequested) {
503 ALint hwBufOffset = 0;
504 alGetSourcei(m_alSrc, AL_BYTE_OFFSET, &hwBufOffset);
505 if(!std::isnan(af->pts)) {
506 m_vs->audClk.setAt(
507 af->pts - double(m_hwBufQueueSize - hwBufOffset) / m_fmtTgt.bytesPerSec,
508 af->serial,
509 av_gettime_relative() / double(AV_TIME_BASE));
510 m_vs->extClk.syncTo(&m_vs->audClk);
511 }
512
513 if(!m_vs->paused) {
514 if(m_hwBufQueueSize - hwBufOffset < hwMinBytes)
515 break;
516
517 ALint bufReady = 0;
518 alGetSourcei(m_alSrc, AL_BUFFERS_PROCESSED, &bufReady);
519 if(bufReady > 0)
520 break;
521 }
522
523 av_usleep(sleepTime);
524 }
525
526 queueFrame(af);
527
528 av_frame_unref(af->frame);
529 av_frame_free(&af->frame);
530 }
531 }
532
533 av_frame_free(&frame);
534 delete af;
535 return;
536 }
537