1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #include "OpusDecoder.h"
8 #include "OpusParser.h"
9 #include "TimeUnits.h"
10 #include "VorbisUtils.h"
11 #include "VorbisDecoder.h"  // For VorbisLayout
12 #include "mozilla/EndianUtils.h"
13 #include "mozilla/PodOperations.h"
14 #include "mozilla/SyncRunnable.h"
15 #include "VideoUtils.h"
16 
17 #include <inttypes.h>  // For PRId64
18 
19 #include "opus/opus.h"
20 extern "C" {
21 #include "opus/opus_multistream.h"
22 }
23 
24 #define OPUS_DEBUG(arg, ...)                                           \
25   DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
26             ##__VA_ARGS__)
27 
28 namespace mozilla {
29 
OpusDataDecoder(const CreateDecoderParams & aParams)30 OpusDataDecoder::OpusDataDecoder(const CreateDecoderParams& aParams)
31     : mInfo(aParams.AudioConfig()),
32       mTaskQueue(aParams.mTaskQueue),
33       mOpusDecoder(nullptr),
34       mSkip(0),
35       mDecodedHeader(false),
36       mPaddingDiscarded(false),
37       mFrames(0),
38       mChannelMap(AudioConfig::ChannelLayout::UNKNOWN_MAP),
39       mDefaultPlaybackDeviceMono(aParams.mOptions.contains(
40           CreateDecoderParams::Option::DefaultPlaybackDeviceMono)) {}
41 
~OpusDataDecoder()42 OpusDataDecoder::~OpusDataDecoder() {
43   if (mOpusDecoder) {
44     opus_multistream_decoder_destroy(mOpusDecoder);
45     mOpusDecoder = nullptr;
46   }
47 }
48 
Shutdown()49 RefPtr<ShutdownPromise> OpusDataDecoder::Shutdown() {
50   RefPtr<OpusDataDecoder> self = this;
51   return InvokeAsync(mTaskQueue, __func__, [self]() {
52     return ShutdownPromise::CreateAndResolve(true, __func__);
53   });
54 }
55 
AppendCodecDelay(MediaByteBuffer * config,uint64_t codecDelayUS)56 void OpusDataDecoder::AppendCodecDelay(MediaByteBuffer* config,
57                                        uint64_t codecDelayUS) {
58   uint8_t buffer[sizeof(uint64_t)];
59   BigEndian::writeUint64(buffer, codecDelayUS);
60   config->AppendElements(buffer, sizeof(uint64_t));
61 }
62 
Init()63 RefPtr<MediaDataDecoder::InitPromise> OpusDataDecoder::Init() {
64   size_t length = mInfo.mCodecSpecificConfig->Length();
65   uint8_t* p = mInfo.mCodecSpecificConfig->Elements();
66   if (length < sizeof(uint64_t)) {
67     OPUS_DEBUG("CodecSpecificConfig too short to read codecDelay!");
68     return InitPromise::CreateAndReject(
69         MediaResult(
70             NS_ERROR_DOM_MEDIA_FATAL_ERR,
71             RESULT_DETAIL("CodecSpecificConfig too short to read codecDelay!")),
72         __func__);
73   }
74   int64_t codecDelay = BigEndian::readUint64(p);
75   length -= sizeof(uint64_t);
76   p += sizeof(uint64_t);
77   if (NS_FAILED(DecodeHeader(p, length))) {
78     OPUS_DEBUG("Error decoding header!");
79     return InitPromise::CreateAndReject(
80         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
81                     RESULT_DETAIL("Error decoding header!")),
82         __func__);
83   }
84 
85   MOZ_ASSERT(mMappingTable.Length() >= uint32_t(mOpusParser->mChannels));
86   int r;
87   mOpusDecoder = opus_multistream_decoder_create(
88       mOpusParser->mRate, mOpusParser->mChannels, mOpusParser->mStreams,
89       mOpusParser->mCoupledStreams, mMappingTable.Elements(), &r);
90 
91   if (!mOpusDecoder) {
92     OPUS_DEBUG("Error creating decoder!");
93     return InitPromise::CreateAndReject(
94         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
95                     RESULT_DETAIL("Error creating decoder!")),
96         __func__);
97   }
98 
99   // Opus has a special feature for stereo coding where it represent wide
100   // stereo channels by 180-degree out of phase. This improves quality, but
101   // needs to be disabled when the output is downmixed to mono. Playback number
102   // of channels are set in AudioSink, using the same method
103   // `DecideAudioPlaybackChannels()`, and triggers downmix if needed.
104   if (mDefaultPlaybackDeviceMono || DecideAudioPlaybackChannels(mInfo) == 1) {
105     opus_multistream_decoder_ctl(mOpusDecoder,
106                                  OPUS_SET_PHASE_INVERSION_DISABLED(1));
107   }
108 
109   mSkip = mOpusParser->mPreSkip;
110   mPaddingDiscarded = false;
111 
112   if (codecDelay !=
113       FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) {
114     NS_WARNING("Invalid Opus header: CodecDelay and pre-skip do not match!");
115     return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
116   }
117 
118   if (mInfo.mRate != (uint32_t)mOpusParser->mRate) {
119     NS_WARNING("Invalid Opus header: container and codec rate do not match!");
120   }
121   if (mInfo.mChannels != (uint32_t)mOpusParser->mChannels) {
122     NS_WARNING(
123         "Invalid Opus header: container and codec channels do not match!");
124   }
125 
126   return r == OPUS_OK
127              ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__)
128              : InitPromise::CreateAndReject(
129                    MediaResult(
130                        NS_ERROR_DOM_MEDIA_FATAL_ERR,
131                        RESULT_DETAIL(
132                            "could not create opus multistream decoder!")),
133                    __func__);
134 }
135 
DecodeHeader(const unsigned char * aData,size_t aLength)136 nsresult OpusDataDecoder::DecodeHeader(const unsigned char* aData,
137                                        size_t aLength) {
138   MOZ_ASSERT(!mOpusParser);
139   MOZ_ASSERT(!mOpusDecoder);
140   MOZ_ASSERT(!mDecodedHeader);
141   mDecodedHeader = true;
142 
143   mOpusParser = MakeUnique<OpusParser>();
144   if (!mOpusParser->DecodeHeader(const_cast<unsigned char*>(aData), aLength)) {
145     return NS_ERROR_FAILURE;
146   }
147   int channels = mOpusParser->mChannels;
148 
149   mMappingTable.SetLength(channels);
150   AudioConfig::ChannelLayout vorbisLayout(
151       channels, VorbisDataDecoder::VorbisLayout(channels));
152   if (vorbisLayout.IsValid()) {
153     mChannelMap = vorbisLayout.Map();
154 
155     AudioConfig::ChannelLayout smpteLayout(
156         AudioConfig::ChannelLayout::SMPTEDefault(vorbisLayout));
157 
158     AutoTArray<uint8_t, 8> map;
159     map.SetLength(channels);
160     if (mOpusParser->mChannelMapping == 1 &&
161         vorbisLayout.MappingTable(smpteLayout, &map)) {
162       for (int i = 0; i < channels; i++) {
163         mMappingTable[i] = mOpusParser->mMappingTable[map[i]];
164       }
165     } else {
166       // Use Opus set channel mapping and return channels as-is.
167       PodCopy(mMappingTable.Elements(), mOpusParser->mMappingTable, channels);
168     }
169   } else {
170     // Create a dummy mapping table so that channel ordering stay the same
171     // during decoding.
172     for (int i = 0; i < channels; i++) {
173       mMappingTable[i] = i;
174     }
175   }
176 
177   return NS_OK;
178 }
179 
Decode(MediaRawData * aSample)180 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Decode(
181     MediaRawData* aSample) {
182   return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
183                                     &OpusDataDecoder::ProcessDecode, aSample);
184 }
185 
ProcessDecode(MediaRawData * aSample)186 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::ProcessDecode(
187     MediaRawData* aSample) {
188   uint32_t channels = mOpusParser->mChannels;
189 
190   if (mPaddingDiscarded) {
191     // Discard padding should be used only on the final packet, so
192     // decoding after a padding discard is invalid.
193     OPUS_DEBUG("Opus error, discard padding on interstitial packet");
194     return DecodePromise::CreateAndReject(
195         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
196                     RESULT_DETAIL("Discard padding on interstitial packet")),
197         __func__);
198   }
199 
200   if (!mLastFrameTime ||
201       mLastFrameTime.ref() != aSample->mTime.ToMicroseconds()) {
202     // We are starting a new block.
203     mFrames = 0;
204     mLastFrameTime = Some(aSample->mTime.ToMicroseconds());
205   }
206 
207   // Maximum value is 63*2880, so there's no chance of overflow.
208   int frames_number =
209       opus_packet_get_nb_frames(aSample->Data(), aSample->Size());
210   if (frames_number <= 0) {
211     OPUS_DEBUG("Invalid packet header: r=%d length=%zu", frames_number,
212                aSample->Size());
213     return DecodePromise::CreateAndReject(
214         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
215                     RESULT_DETAIL("Invalid packet header: r=%d length=%u",
216                                   frames_number, uint32_t(aSample->Size()))),
217         __func__);
218   }
219 
220   int samples = opus_packet_get_samples_per_frame(
221       aSample->Data(), opus_int32(mOpusParser->mRate));
222 
223   // A valid Opus packet must be between 2.5 and 120 ms long (48kHz).
224   CheckedInt32 totalFrames =
225       CheckedInt32(frames_number) * CheckedInt32(samples);
226   if (!totalFrames.isValid()) {
227     return DecodePromise::CreateAndReject(
228         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
229                     RESULT_DETAIL("Frames count overflow")),
230         __func__);
231   }
232 
233   int frames = totalFrames.value();
234   if (frames < 120 || frames > 5760) {
235     OPUS_DEBUG("Invalid packet frames: %d", frames);
236     return DecodePromise::CreateAndReject(
237         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
238                     RESULT_DETAIL("Invalid packet frames:%d", frames)),
239         __func__);
240   }
241 
242   AlignedAudioBuffer buffer(frames * channels);
243   if (!buffer) {
244     return DecodePromise::CreateAndReject(
245         MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
246   }
247 
248   // Decode to the appropriate sample type.
249 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
250   int ret = opus_multistream_decode_float(mOpusDecoder, aSample->Data(),
251                                           aSample->Size(), buffer.get(), frames,
252                                           false);
253 #else
254   int ret =
255       opus_multistream_decode(mOpusDecoder, aSample->Data(), aSample->Size(),
256                               buffer.get(), frames, false);
257 #endif
258   if (ret < 0) {
259     return DecodePromise::CreateAndReject(
260         MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
261                     RESULT_DETAIL("Opus decoding error:%d", ret)),
262         __func__);
263   }
264   NS_ASSERTION(ret == frames, "Opus decoded too few audio samples");
265   auto startTime = aSample->mTime;
266 
267   // Trim the initial frames while the decoder is settling.
268   if (mSkip > 0) {
269     int32_t skipFrames = std::min<int32_t>(mSkip, frames);
270     int32_t keepFrames = frames - skipFrames;
271     OPUS_DEBUG("Opus decoder skipping %d of %d frames", skipFrames, frames);
272     PodMove(buffer.get(), buffer.get() + skipFrames * channels,
273             keepFrames * channels);
274     startTime = startTime + FramesToTimeUnit(skipFrames, mOpusParser->mRate);
275     frames = keepFrames;
276     mSkip -= skipFrames;
277   }
278 
279   if (aSample->mDiscardPadding > 0) {
280     OPUS_DEBUG("Opus decoder discarding %u of %d frames",
281                aSample->mDiscardPadding, frames);
282     // Padding discard is only supposed to happen on the final packet.
283     // Record the discard so we can return an error if another packet is
284     // decoded.
285     if (aSample->mDiscardPadding > uint32_t(frames)) {
286       // Discarding more than the entire packet is invalid.
287       OPUS_DEBUG("Opus error, discard padding larger than packet");
288       return DecodePromise::CreateAndReject(
289           MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
290                       RESULT_DETAIL("Discard padding larger than packet")),
291           __func__);
292     }
293 
294     mPaddingDiscarded = true;
295     frames = frames - aSample->mDiscardPadding;
296   }
297 
298   // Apply the header gain if one was specified.
299 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
300   if (mOpusParser->mGain != 1.0f) {
301     float gain = mOpusParser->mGain;
302     uint32_t samples = frames * channels;
303     for (uint32_t i = 0; i < samples; i++) {
304       buffer[i] *= gain;
305     }
306   }
307 #else
308   if (mOpusParser->mGain_Q16 != 65536) {
309     int64_t gain_Q16 = mOpusParser->mGain_Q16;
310     uint32_t samples = frames * channels;
311     for (uint32_t i = 0; i < samples; i++) {
312       int32_t val = static_cast<int32_t>((gain_Q16 * buffer[i] + 32768) >> 16);
313       buffer[i] = static_cast<AudioDataValue>(MOZ_CLIP_TO_15(val));
314     }
315   }
316 #endif
317 
318   auto duration = FramesToTimeUnit(frames, mOpusParser->mRate);
319   if (!duration.IsValid()) {
320     return DecodePromise::CreateAndReject(
321         MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
322                     RESULT_DETAIL("Overflow converting WebM audio duration")),
323         __func__);
324   }
325   auto time = startTime -
326               FramesToTimeUnit(mOpusParser->mPreSkip, mOpusParser->mRate) +
327               FramesToTimeUnit(mFrames, mOpusParser->mRate);
328   if (!time.IsValid()) {
329     return DecodePromise::CreateAndReject(
330         MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
331                     RESULT_DETAIL("Overflow shifting tstamp by codec delay")),
332         __func__);
333   };
334 
335   mFrames += frames;
336 
337   if (!frames) {
338     return DecodePromise::CreateAndResolve(DecodedData(), __func__);
339   }
340 
341   // Trim extra allocated frames.
342   buffer.SetLength(frames * channels);
343 
344   return DecodePromise::CreateAndResolve(
345       DecodedData{new AudioData(aSample->mOffset, time, std::move(buffer),
346                                 mOpusParser->mChannels, mOpusParser->mRate,
347                                 mChannelMap)},
348       __func__);
349 }
350 
Drain()351 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Drain() {
352   RefPtr<OpusDataDecoder> self = this;
353   // InvokeAsync dispatches a task that will be run after any pending decode
354   // completes. As such, once the drain task run, there's nothing more to do.
355   return InvokeAsync(mTaskQueue, __func__, [] {
356     return DecodePromise::CreateAndResolve(DecodedData(), __func__);
357   });
358 }
359 
Flush()360 RefPtr<MediaDataDecoder::FlushPromise> OpusDataDecoder::Flush() {
361   if (!mOpusDecoder) {
362     return FlushPromise::CreateAndResolve(true, __func__);
363   }
364 
365   RefPtr<OpusDataDecoder> self = this;
366   return InvokeAsync(mTaskQueue, __func__, [self, this]() {
367     MOZ_ASSERT(mOpusDecoder);
368     // Reset the decoder.
369     opus_multistream_decoder_ctl(mOpusDecoder, OPUS_RESET_STATE);
370     mSkip = mOpusParser->mPreSkip;
371     mPaddingDiscarded = false;
372     mLastFrameTime.reset();
373     return FlushPromise::CreateAndResolve(true, __func__);
374   });
375 }
376 
377 /* static */
IsOpus(const nsACString & aMimeType)378 bool OpusDataDecoder::IsOpus(const nsACString& aMimeType) {
379   return aMimeType.EqualsLiteral("audio/opus");
380 }
381 
382 }  // namespace mozilla
383 #undef OPUS_DEBUG
384