1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #include "OpusDecoder.h"
8 #include "OpusParser.h"
9 #include "TimeUnits.h"
10 #include "VorbisUtils.h"
11 #include "VorbisDecoder.h"  // For VorbisLayout
12 #include "mozilla/EndianUtils.h"
13 #include "mozilla/PodOperations.h"
14 #include "mozilla/SyncRunnable.h"
15 #include "VideoUtils.h"
16 
17 #include <inttypes.h>  // For PRId64
18 
19 #include "opus/opus.h"
20 extern "C" {
21 #include "opus/opus_multistream.h"
22 }
23 
24 #define OPUS_DEBUG(arg, ...)                                           \
25   DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
26             ##__VA_ARGS__)
27 
28 namespace mozilla {
29 
OpusDataDecoder(const CreateDecoderParams & aParams)30 OpusDataDecoder::OpusDataDecoder(const CreateDecoderParams& aParams)
31     : mInfo(aParams.AudioConfig()),
32       mTaskQueue(aParams.mTaskQueue),
33       mOpusDecoder(nullptr),
34       mSkip(0),
35       mDecodedHeader(false),
36       mPaddingDiscarded(false),
37       mFrames(0) {}
38 
~OpusDataDecoder()39 OpusDataDecoder::~OpusDataDecoder() {
40   if (mOpusDecoder) {
41     opus_multistream_decoder_destroy(mOpusDecoder);
42     mOpusDecoder = nullptr;
43   }
44 }
45 
Shutdown()46 RefPtr<ShutdownPromise> OpusDataDecoder::Shutdown() {
47   RefPtr<OpusDataDecoder> self = this;
48   return InvokeAsync(mTaskQueue, __func__, [self]() {
49     return ShutdownPromise::CreateAndResolve(true, __func__);
50   });
51 }
52 
AppendCodecDelay(MediaByteBuffer * config,uint64_t codecDelayUS)53 void OpusDataDecoder::AppendCodecDelay(MediaByteBuffer* config,
54                                        uint64_t codecDelayUS) {
55   uint8_t buffer[sizeof(uint64_t)];
56   BigEndian::writeUint64(buffer, codecDelayUS);
57   config->AppendElements(buffer, sizeof(uint64_t));
58 }
59 
Init()60 RefPtr<MediaDataDecoder::InitPromise> OpusDataDecoder::Init() {
61   size_t length = mInfo.mCodecSpecificConfig->Length();
62   uint8_t* p = mInfo.mCodecSpecificConfig->Elements();
63   if (length < sizeof(uint64_t)) {
64     OPUS_DEBUG("CodecSpecificConfig too short to read codecDelay!");
65     return InitPromise::CreateAndReject(
66         MediaResult(
67             NS_ERROR_DOM_MEDIA_FATAL_ERR,
68             RESULT_DETAIL("CodecSpecificConfig too short to read codecDelay!")),
69         __func__);
70   }
71   int64_t codecDelay = BigEndian::readUint64(p);
72   length -= sizeof(uint64_t);
73   p += sizeof(uint64_t);
74   if (NS_FAILED(DecodeHeader(p, length))) {
75     OPUS_DEBUG("Error decoding header!");
76     return InitPromise::CreateAndReject(
77         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
78                     RESULT_DETAIL("Error decoding header!")),
79         __func__);
80   }
81 
82   int r;
83   mOpusDecoder = opus_multistream_decoder_create(
84       mOpusParser->mRate, mOpusParser->mChannels, mOpusParser->mStreams,
85       mOpusParser->mCoupledStreams, mMappingTable, &r);
86   mSkip = mOpusParser->mPreSkip;
87   mPaddingDiscarded = false;
88 
89   if (codecDelay !=
90       FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) {
91     NS_WARNING("Invalid Opus header: CodecDelay and pre-skip do not match!");
92     return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
93   }
94 
95   if (mInfo.mRate != (uint32_t)mOpusParser->mRate) {
96     NS_WARNING("Invalid Opus header: container and codec rate do not match!");
97   }
98   if (mInfo.mChannels != (uint32_t)mOpusParser->mChannels) {
99     NS_WARNING(
100         "Invalid Opus header: container and codec channels do not match!");
101   }
102 
103   return r == OPUS_OK
104              ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__)
105              : InitPromise::CreateAndReject(
106                    MediaResult(
107                        NS_ERROR_DOM_MEDIA_FATAL_ERR,
108                        RESULT_DETAIL(
109                            "could not create opus multistream decoder!")),
110                    __func__);
111 }
112 
DecodeHeader(const unsigned char * aData,size_t aLength)113 nsresult OpusDataDecoder::DecodeHeader(const unsigned char* aData,
114                                        size_t aLength) {
115   MOZ_ASSERT(!mOpusParser);
116   MOZ_ASSERT(!mOpusDecoder);
117   MOZ_ASSERT(!mDecodedHeader);
118   mDecodedHeader = true;
119 
120   mOpusParser = new OpusParser;
121   if (!mOpusParser->DecodeHeader(const_cast<unsigned char*>(aData), aLength)) {
122     return NS_ERROR_FAILURE;
123   }
124   int channels = mOpusParser->mChannels;
125 
126   AudioConfig::ChannelLayout layout(channels);
127   if (!layout.IsValid()) {
128     OPUS_DEBUG("Invalid channel mapping. Source is %d channels", channels);
129     return NS_ERROR_FAILURE;
130   }
131 
132   AudioConfig::ChannelLayout vorbisLayout(
133       channels, VorbisDataDecoder::VorbisLayout(channels));
134   AudioConfig::ChannelLayout smpteLayout(channels);
135   static_assert(sizeof(mOpusParser->mMappingTable) /
136                         sizeof(mOpusParser->mMappingTable[0]) >=
137                     MAX_AUDIO_CHANNELS,
138                 "Invalid size set");
139   uint8_t map[sizeof(mOpusParser->mMappingTable) /
140               sizeof(mOpusParser->mMappingTable[0])];
141   if (vorbisLayout.MappingTable(smpteLayout, map)) {
142     for (int i = 0; i < channels; i++) {
143       mMappingTable[i] = mOpusParser->mMappingTable[map[i]];
144     }
145   } else {
146     // Should never get here as vorbis layout is always convertible to SMPTE
147     // default layout.
148     PodCopy(mMappingTable, mOpusParser->mMappingTable, MAX_AUDIO_CHANNELS);
149   }
150 
151   return NS_OK;
152 }
153 
Decode(MediaRawData * aSample)154 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Decode(
155     MediaRawData* aSample) {
156   return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
157                                     &OpusDataDecoder::ProcessDecode, aSample);
158 }
159 
ProcessDecode(MediaRawData * aSample)160 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::ProcessDecode(
161     MediaRawData* aSample) {
162   uint32_t channels = mOpusParser->mChannels;
163 
164   if (mPaddingDiscarded) {
165     // Discard padding should be used only on the final packet, so
166     // decoding after a padding discard is invalid.
167     OPUS_DEBUG("Opus error, discard padding on interstitial packet");
168     return DecodePromise::CreateAndReject(
169         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
170                     RESULT_DETAIL("Discard padding on interstitial packet")),
171         __func__);
172   }
173 
174   if (!mLastFrameTime ||
175       mLastFrameTime.ref() != aSample->mTime.ToMicroseconds()) {
176     // We are starting a new block.
177     mFrames = 0;
178     mLastFrameTime = Some(aSample->mTime.ToMicroseconds());
179   }
180 
181   // Maximum value is 63*2880, so there's no chance of overflow.
182   int frames_number =
183       opus_packet_get_nb_frames(aSample->Data(), aSample->Size());
184   if (frames_number <= 0) {
185     OPUS_DEBUG("Invalid packet header: r=%d length=%zu", frames_number,
186                aSample->Size());
187     return DecodePromise::CreateAndReject(
188         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
189                     RESULT_DETAIL("Invalid packet header: r=%d length=%u",
190                                   frames_number, uint32_t(aSample->Size()))),
191         __func__);
192   }
193 
194   int samples = opus_packet_get_samples_per_frame(
195       aSample->Data(), opus_int32(mOpusParser->mRate));
196 
197   // A valid Opus packet must be between 2.5 and 120 ms long (48kHz).
198   CheckedInt32 totalFrames =
199       CheckedInt32(frames_number) * CheckedInt32(samples);
200   if (!totalFrames.isValid()) {
201     return DecodePromise::CreateAndReject(
202         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
203                     RESULT_DETAIL("Frames count overflow")),
204         __func__);
205   }
206 
207   int frames = totalFrames.value();
208   if (frames < 120 || frames > 5760) {
209     OPUS_DEBUG("Invalid packet frames: %d", frames);
210     return DecodePromise::CreateAndReject(
211         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
212                     RESULT_DETAIL("Invalid packet frames:%d", frames)),
213         __func__);
214   }
215 
216   AlignedAudioBuffer buffer(frames * channels);
217   if (!buffer) {
218     return DecodePromise::CreateAndReject(
219         MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
220   }
221 
222     // Decode to the appropriate sample type.
223 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
224   int ret = opus_multistream_decode_float(mOpusDecoder, aSample->Data(),
225                                           aSample->Size(), buffer.get(), frames,
226                                           false);
227 #else
228   int ret =
229       opus_multistream_decode(mOpusDecoder, aSample->Data(), aSample->Size(),
230                               buffer.get(), frames, false);
231 #endif
232   if (ret < 0) {
233     return DecodePromise::CreateAndReject(
234         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
235                     RESULT_DETAIL("Opus decoding error:%d", ret)),
236         __func__);
237   }
238   NS_ASSERTION(ret == frames, "Opus decoded too few audio samples");
239   auto startTime = aSample->mTime;
240 
241   // Trim the initial frames while the decoder is settling.
242   if (mSkip > 0) {
243     int32_t skipFrames = std::min<int32_t>(mSkip, frames);
244     int32_t keepFrames = frames - skipFrames;
245     OPUS_DEBUG("Opus decoder skipping %d of %d frames", skipFrames, frames);
246     PodMove(buffer.get(), buffer.get() + skipFrames * channels,
247             keepFrames * channels);
248     startTime = startTime + FramesToTimeUnit(skipFrames, mOpusParser->mRate);
249     frames = keepFrames;
250     mSkip -= skipFrames;
251   }
252 
253   if (aSample->mDiscardPadding > 0) {
254     OPUS_DEBUG("Opus decoder discarding %u of %d frames",
255                aSample->mDiscardPadding, frames);
256     // Padding discard is only supposed to happen on the final packet.
257     // Record the discard so we can return an error if another packet is
258     // decoded.
259     if (aSample->mDiscardPadding > uint32_t(frames)) {
260       // Discarding more than the entire packet is invalid.
261       OPUS_DEBUG("Opus error, discard padding larger than packet");
262       return DecodePromise::CreateAndReject(
263           MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
264                       RESULT_DETAIL("Discard padding larger than packet")),
265           __func__);
266     }
267 
268     mPaddingDiscarded = true;
269     frames = frames - aSample->mDiscardPadding;
270   }
271 
272     // Apply the header gain if one was specified.
273 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
274   if (mOpusParser->mGain != 1.0f) {
275     float gain = mOpusParser->mGain;
276     uint32_t samples = frames * channels;
277     for (uint32_t i = 0; i < samples; i++) {
278       buffer[i] *= gain;
279     }
280   }
281 #else
282   if (mOpusParser->mGain_Q16 != 65536) {
283     int64_t gain_Q16 = mOpusParser->mGain_Q16;
284     uint32_t samples = frames * channels;
285     for (uint32_t i = 0; i < samples; i++) {
286       int32_t val = static_cast<int32_t>((gain_Q16 * buffer[i] + 32768) >> 16);
287       buffer[i] = static_cast<AudioDataValue>(MOZ_CLIP_TO_15(val));
288     }
289   }
290 #endif
291 
292   auto duration = FramesToTimeUnit(frames, mOpusParser->mRate);
293   if (!duration.IsValid()) {
294     return DecodePromise::CreateAndReject(
295         MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
296                     RESULT_DETAIL("Overflow converting WebM audio duration")),
297         __func__);
298   }
299   auto time = startTime -
300               FramesToTimeUnit(mOpusParser->mPreSkip, mOpusParser->mRate) +
301               FramesToTimeUnit(mFrames, mOpusParser->mRate);
302   if (!time.IsValid()) {
303     return DecodePromise::CreateAndReject(
304         MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
305                     RESULT_DETAIL("Overflow shifting tstamp by codec delay")),
306         __func__);
307   };
308 
309   mFrames += frames;
310 
311   return DecodePromise::CreateAndResolve(
312       DecodedData{new AudioData(aSample->mOffset, time, duration, frames,
313                                 Move(buffer), mOpusParser->mChannels,
314                                 mOpusParser->mRate)},
315       __func__);
316 }
317 
Drain()318 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Drain() {
319   RefPtr<OpusDataDecoder> self = this;
320   // InvokeAsync dispatches a task that will be run after any pending decode
321   // completes. As such, once the drain task run, there's nothing more to do.
322   return InvokeAsync(mTaskQueue, __func__, [] {
323     return DecodePromise::CreateAndResolve(DecodedData(), __func__);
324   });
325 }
326 
Flush()327 RefPtr<MediaDataDecoder::FlushPromise> OpusDataDecoder::Flush() {
328   if (!mOpusDecoder) {
329     return FlushPromise::CreateAndResolve(true, __func__);
330   }
331 
332   RefPtr<OpusDataDecoder> self = this;
333   return InvokeAsync(mTaskQueue, __func__, [self, this]() {
334     MOZ_ASSERT(mOpusDecoder);
335     // Reset the decoder.
336     opus_multistream_decoder_ctl(mOpusDecoder, OPUS_RESET_STATE);
337     mSkip = mOpusParser->mPreSkip;
338     mPaddingDiscarded = false;
339     mLastFrameTime.reset();
340     return FlushPromise::CreateAndResolve(true, __func__);
341   });
342 }
343 
344 /* static */
IsOpus(const nsACString & aMimeType)345 bool OpusDataDecoder::IsOpus(const nsACString& aMimeType) {
346   return aMimeType.EqualsLiteral("audio/opus");
347 }
348 
349 }  // namespace mozilla
350 #undef OPUS_DEBUG
351