1 //
2 // libtgvoip is free and unencumbered public domain software.
3 // For more information, see http://unlicense.org or the UNLICENSE file
4 // you should have received with this source code distribution.
5 //
6
7 #include "OpusDecoder.h"
8 #include "audio/Resampler.h"
9 #include "logging.h"
10 #include <assert.h>
11 #include <math.h>
12 #include <algorithm>
13 #if TGVOIP_INCLUDE_OPUS_PACKAGE
14 #include <opus/opus.h>
15 #else
16 #include <opus.h>
17 #endif
18
19 #include "VoIPController.h"
20
21 #define PACKET_SIZE (960*2)
22
23 using namespace tgvoip;
24
OpusDecoder(const std::shared_ptr<MediaStreamItf> & dst,bool isAsync,bool needEC)25 tgvoip::OpusDecoder::OpusDecoder(const std::shared_ptr<MediaStreamItf>& dst, bool isAsync, bool needEC){
26 dst->SetCallback(OpusDecoder::Callback, this);
27 Initialize(isAsync, needEC);
28 }
29
OpusDecoder(const std::unique_ptr<MediaStreamItf> & dst,bool isAsync,bool needEC)30 tgvoip::OpusDecoder::OpusDecoder(const std::unique_ptr<MediaStreamItf>& dst, bool isAsync, bool needEC){
31 dst->SetCallback(OpusDecoder::Callback, this);
32 Initialize(isAsync, needEC);
33 }
34
OpusDecoder(MediaStreamItf * dst,bool isAsync,bool needEC)35 tgvoip::OpusDecoder::OpusDecoder(MediaStreamItf* dst, bool isAsync, bool needEC){
36 dst->SetCallback(OpusDecoder::Callback, this);
37 Initialize(isAsync, needEC);
38 }
39
Initialize(bool isAsync,bool needEC)40 void tgvoip::OpusDecoder::Initialize(bool isAsync, bool needEC){
41 async=isAsync;
42 if(async){
43 decodedQueue=new BlockingQueue<unsigned char*>(33);
44 bufferPool=new BufferPool(PACKET_SIZE, 32);
45 semaphore=new Semaphore(32, 0);
46 }else{
47 decodedQueue=NULL;
48 bufferPool=NULL;
49 semaphore=NULL;
50 }
51 dec=opus_decoder_create(48000, 1, NULL);
52 if(needEC)
53 ecDec=opus_decoder_create(48000, 1, NULL);
54 else
55 ecDec=NULL;
56 buffer=(unsigned char *) malloc(8192);
57 lastDecoded=NULL;
58 outputBufferSize=0;
59 echoCanceller=NULL;
60 frameDuration=20;
61 consecutiveLostPackets=0;
62 enableDTX=false;
63 silentPacketCount=0;
64 levelMeter=NULL;
65 nextLen=0;
66 running=false;
67 remainingDataLen=0;
68 processedBuffer=NULL;
69 prevWasEC=false;
70 prevLastSample=0;
71 }
72
~OpusDecoder()73 tgvoip::OpusDecoder::~OpusDecoder(){
74 opus_decoder_destroy(dec);
75 if(ecDec)
76 opus_decoder_destroy(ecDec);
77 free(buffer);
78 if(bufferPool)
79 delete bufferPool;
80 if(decodedQueue)
81 delete decodedQueue;
82 if(semaphore)
83 delete semaphore;
84 }
85
86
SetEchoCanceller(EchoCanceller * canceller)87 void tgvoip::OpusDecoder::SetEchoCanceller(EchoCanceller* canceller){
88 echoCanceller=canceller;
89 }
90
Callback(unsigned char * data,size_t len,void * param)91 size_t tgvoip::OpusDecoder::Callback(unsigned char *data, size_t len, void *param){
92 return ((OpusDecoder*)param)->HandleCallback(data, len);
93 }
94
HandleCallback(unsigned char * data,size_t len)95 size_t tgvoip::OpusDecoder::HandleCallback(unsigned char *data, size_t len){
96 if(async){
97 if(!running){
98 memset(data, 0, len);
99 return 0;
100 }
101 if(outputBufferSize==0){
102 outputBufferSize=len;
103 int packetsNeeded;
104 if(len>PACKET_SIZE)
105 packetsNeeded=len/PACKET_SIZE;
106 else
107 packetsNeeded=1;
108 packetsNeeded*=2;
109 semaphore->Release(packetsNeeded);
110 }
111 assert(outputBufferSize==len && "output buffer size is supposed to be the same throughout callbacks");
112 if(len==PACKET_SIZE){
113 lastDecoded=(unsigned char *) decodedQueue->GetBlocking();
114 if(!lastDecoded)
115 return 0;
116 memcpy(data, lastDecoded, PACKET_SIZE);
117 bufferPool->Reuse(lastDecoded);
118 semaphore->Release();
119 if(silentPacketCount>0){
120 silentPacketCount--;
121 if(levelMeter)
122 levelMeter->Update(reinterpret_cast<int16_t *>(data), 0);
123 return 0;
124 }
125 if(echoCanceller){
126 echoCanceller->SpeakerOutCallback(data, PACKET_SIZE);
127 }
128 }else{
129 LOGE("Opus decoder buffer length != 960 samples");
130 abort();
131 }
132 }else{
133 if(remainingDataLen==0 && silentPacketCount==0){
134 int duration=DecodeNextFrame();
135 remainingDataLen=(size_t) (duration/20*960*2);
136 }
137 if(silentPacketCount>0 || remainingDataLen==0 || !processedBuffer){
138 if(silentPacketCount>0)
139 silentPacketCount--;
140 memset(data, 0, 960*2);
141 if(levelMeter)
142 levelMeter->Update(reinterpret_cast<int16_t *>(data), 0);
143 return 0;
144 }
145 memcpy(data, processedBuffer, 960*2);
146 remainingDataLen-=960*2;
147 if(remainingDataLen>0){
148 memmove(processedBuffer, processedBuffer+960*2, remainingDataLen);
149 }
150 }
151 if(levelMeter)
152 levelMeter->Update(reinterpret_cast<int16_t *>(data), len/2);
153 return len;
154 }
155
156
Start()157 void tgvoip::OpusDecoder::Start(){
158 if(!async)
159 return;
160 running=true;
161 thread=new Thread(std::bind(&tgvoip::OpusDecoder::RunThread, this));
162 thread->SetName("opus_decoder");
163 thread->SetMaxPriority();
164 thread->Start();
165 }
166
Stop()167 void tgvoip::OpusDecoder::Stop(){
168 if(!running || !async)
169 return;
170 running=false;
171 semaphore->Release();
172 thread->Join();
173 delete thread;
174 }
175
RunThread()176 void tgvoip::OpusDecoder::RunThread(){
177 int i;
178 LOGI("decoder: packets per frame %d", packetsPerFrame);
179 while(running){
180 int playbackDuration=DecodeNextFrame();
181 for(i=0;i<playbackDuration/20;i++){
182 semaphore->Acquire();
183 if(!running){
184 LOGI("==== decoder exiting ====");
185 return;
186 }
187 unsigned char *buf=bufferPool->Get();
188 if(buf){
189 if(remainingDataLen>0){
190 for(effects::AudioEffect*& effect:postProcEffects){
191 effect->Process(reinterpret_cast<int16_t*>(processedBuffer+(PACKET_SIZE*i)), 960);
192 }
193 memcpy(buf, processedBuffer+(PACKET_SIZE*i), PACKET_SIZE);
194 }else{
195 //LOGE("Error decoding, result=%d", size);
196 memset(buf, 0, PACKET_SIZE);
197 }
198 decodedQueue->Put(buf);
199 }else{
200 LOGW("decoder: no buffers left!");
201 }
202 }
203 }
204 }
205
DecodeNextFrame()206 int tgvoip::OpusDecoder::DecodeNextFrame(){
207 int playbackDuration=0;
208 bool isEC=false;
209 size_t len=jitterBuffer->HandleOutput(buffer, 8192, 0, true, playbackDuration, isEC);
210 bool fec=false;
211 if(!len){
212 fec=true;
213 len=jitterBuffer->HandleOutput(buffer, 8192, 0, false, playbackDuration, isEC);
214 //if(len)
215 // LOGV("Trying FEC...");
216 }
217 int size;
218 if(len){
219 size=opus_decode(isEC ? ecDec : dec, buffer, len, (opus_int16 *) decodeBuffer, packetsPerFrame*960, fec ? 1 : 0);
220 consecutiveLostPackets=0;
221 if(prevWasEC!=isEC && size){
222 // It turns out the waveforms generated by the PLC feature are also great to help smooth out the
223 // otherwise audible transition between the frames from different decoders. Those are basically an extrapolation
224 // of the previous successfully decoded data -- which is exactly what we need here.
225 size=opus_decode(prevWasEC ? ecDec : dec, NULL, 0, (opus_int16*)nextBuffer, packetsPerFrame*960, 0);
226 if(size){
227 int16_t* plcSamples=reinterpret_cast<int16_t*>(nextBuffer);
228 int16_t* samples=reinterpret_cast<int16_t*>(decodeBuffer);
229 constexpr float coeffs[]={0.999802, 0.995062, 0.984031, 0.966778, 0.943413, 0.914084, 0.878975, 0.838309, 0.792344,
230 0.741368, 0.685706, 0.625708, 0.561754, 0.494249, 0.423619, 0.350311, 0.274788, 0.197527, 0.119018, 0.039757};
231 for(int i=0;i<20;i++){
232 samples[i]=(int16_t)round((plcSamples[i]*coeffs[i]+(float)samples[i]*(1.0-coeffs[i])));
233 }
234 }
235 }
236 prevWasEC=isEC;
237 prevLastSample=decodeBuffer[size-1];
238 }else{ // do packet loss concealment
239 consecutiveLostPackets++;
240 if(consecutiveLostPackets>2 && enableDTX){
241 silentPacketCount+=packetsPerFrame;
242 size=packetsPerFrame*960;
243 }else{
244 size=opus_decode(prevWasEC ? ecDec : dec, NULL, 0, (opus_int16 *) decodeBuffer, packetsPerFrame*960, 0);
245 //LOGV("PLC");
246 }
247 }
248 if(size<0)
249 LOGW("decoder: opus_decode error %d", size);
250 remainingDataLen=size;
251 if(playbackDuration==80){
252 processedBuffer=buffer;
253 audio::Resampler::Rescale60To80((int16_t*) decodeBuffer, (int16_t*) processedBuffer);
254 }else if(playbackDuration==40){
255 processedBuffer=buffer;
256 audio::Resampler::Rescale60To40((int16_t*) decodeBuffer, (int16_t*) processedBuffer);
257 }else{
258 processedBuffer=decodeBuffer;
259 }
260 return playbackDuration;
261 }
262
263
SetFrameDuration(uint32_t duration)264 void tgvoip::OpusDecoder::SetFrameDuration(uint32_t duration){
265 frameDuration=duration;
266 packetsPerFrame=frameDuration/20;
267 }
268
269
SetJitterBuffer(std::shared_ptr<JitterBuffer> jitterBuffer)270 void tgvoip::OpusDecoder::SetJitterBuffer(std::shared_ptr<JitterBuffer> jitterBuffer){
271 this->jitterBuffer=jitterBuffer;
272 }
273
SetDTX(bool enable)274 void tgvoip::OpusDecoder::SetDTX(bool enable){
275 enableDTX=enable;
276 }
277
SetLevelMeter(AudioLevelMeter * levelMeter)278 void tgvoip::OpusDecoder::SetLevelMeter(AudioLevelMeter *levelMeter){
279 this->levelMeter=levelMeter;
280 }
281
AddAudioEffect(effects::AudioEffect * effect)282 void tgvoip::OpusDecoder::AddAudioEffect(effects::AudioEffect *effect){
283 postProcEffects.push_back(effect);
284 }
285
RemoveAudioEffect(effects::AudioEffect * effect)286 void tgvoip::OpusDecoder::RemoveAudioEffect(effects::AudioEffect *effect){
287 std::vector<effects::AudioEffect*>::iterator i=std::find(postProcEffects.begin(), postProcEffects.end(), effect);
288 if(i!=postProcEffects.end())
289 postProcEffects.erase(i);
290 }
291