1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "OpusDecoder.h"
8 #include "OpusParser.h"
9 #include "TimeUnits.h"
10 #include "VorbisUtils.h"
11 #include "VorbisDecoder.h" // For VorbisLayout
12 #include "mozilla/EndianUtils.h"
13 #include "mozilla/PodOperations.h"
14 #include "mozilla/SyncRunnable.h"
15 #include "VideoUtils.h"
16
17 #include <inttypes.h> // For PRId64
18
19 #include "opus/opus.h"
20 extern "C" {
21 #include "opus/opus_multistream.h"
22 }
23
24 #define OPUS_DEBUG(arg, ...) \
25 DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
26 ##__VA_ARGS__)
27
28 namespace mozilla {
29
OpusDataDecoder(const CreateDecoderParams & aParams)30 OpusDataDecoder::OpusDataDecoder(const CreateDecoderParams& aParams)
31 : mInfo(aParams.AudioConfig()),
32 mTaskQueue(aParams.mTaskQueue),
33 mOpusDecoder(nullptr),
34 mSkip(0),
35 mDecodedHeader(false),
36 mPaddingDiscarded(false),
37 mFrames(0),
38 mChannelMap(AudioConfig::ChannelLayout::UNKNOWN_MAP),
39 mDefaultPlaybackDeviceMono(aParams.mOptions.contains(
40 CreateDecoderParams::Option::DefaultPlaybackDeviceMono)) {}
41
~OpusDataDecoder()42 OpusDataDecoder::~OpusDataDecoder() {
43 if (mOpusDecoder) {
44 opus_multistream_decoder_destroy(mOpusDecoder);
45 mOpusDecoder = nullptr;
46 }
47 }
48
Shutdown()49 RefPtr<ShutdownPromise> OpusDataDecoder::Shutdown() {
50 RefPtr<OpusDataDecoder> self = this;
51 return InvokeAsync(mTaskQueue, __func__, [self]() {
52 return ShutdownPromise::CreateAndResolve(true, __func__);
53 });
54 }
55
AppendCodecDelay(MediaByteBuffer * config,uint64_t codecDelayUS)56 void OpusDataDecoder::AppendCodecDelay(MediaByteBuffer* config,
57 uint64_t codecDelayUS) {
58 uint8_t buffer[sizeof(uint64_t)];
59 BigEndian::writeUint64(buffer, codecDelayUS);
60 config->AppendElements(buffer, sizeof(uint64_t));
61 }
62
Init()63 RefPtr<MediaDataDecoder::InitPromise> OpusDataDecoder::Init() {
64 size_t length = mInfo.mCodecSpecificConfig->Length();
65 uint8_t* p = mInfo.mCodecSpecificConfig->Elements();
66 if (length < sizeof(uint64_t)) {
67 OPUS_DEBUG("CodecSpecificConfig too short to read codecDelay!");
68 return InitPromise::CreateAndReject(
69 MediaResult(
70 NS_ERROR_DOM_MEDIA_FATAL_ERR,
71 RESULT_DETAIL("CodecSpecificConfig too short to read codecDelay!")),
72 __func__);
73 }
74 int64_t codecDelay = BigEndian::readUint64(p);
75 length -= sizeof(uint64_t);
76 p += sizeof(uint64_t);
77 if (NS_FAILED(DecodeHeader(p, length))) {
78 OPUS_DEBUG("Error decoding header!");
79 return InitPromise::CreateAndReject(
80 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
81 RESULT_DETAIL("Error decoding header!")),
82 __func__);
83 }
84
85 MOZ_ASSERT(mMappingTable.Length() >= uint32_t(mOpusParser->mChannels));
86 int r;
87 mOpusDecoder = opus_multistream_decoder_create(
88 mOpusParser->mRate, mOpusParser->mChannels, mOpusParser->mStreams,
89 mOpusParser->mCoupledStreams, mMappingTable.Elements(), &r);
90
91 if (!mOpusDecoder) {
92 OPUS_DEBUG("Error creating decoder!");
93 return InitPromise::CreateAndReject(
94 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
95 RESULT_DETAIL("Error creating decoder!")),
96 __func__);
97 }
98
99 // Opus has a special feature for stereo coding where it represent wide
100 // stereo channels by 180-degree out of phase. This improves quality, but
101 // needs to be disabled when the output is downmixed to mono. Playback number
102 // of channels are set in AudioSink, using the same method
103 // `DecideAudioPlaybackChannels()`, and triggers downmix if needed.
104 if (mDefaultPlaybackDeviceMono || DecideAudioPlaybackChannels(mInfo) == 1) {
105 opus_multistream_decoder_ctl(mOpusDecoder,
106 OPUS_SET_PHASE_INVERSION_DISABLED(1));
107 }
108
109 mSkip = mOpusParser->mPreSkip;
110 mPaddingDiscarded = false;
111
112 if (codecDelay !=
113 FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) {
114 NS_WARNING("Invalid Opus header: CodecDelay and pre-skip do not match!");
115 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
116 }
117
118 if (mInfo.mRate != (uint32_t)mOpusParser->mRate) {
119 NS_WARNING("Invalid Opus header: container and codec rate do not match!");
120 }
121 if (mInfo.mChannels != (uint32_t)mOpusParser->mChannels) {
122 NS_WARNING(
123 "Invalid Opus header: container and codec channels do not match!");
124 }
125
126 return r == OPUS_OK
127 ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__)
128 : InitPromise::CreateAndReject(
129 MediaResult(
130 NS_ERROR_DOM_MEDIA_FATAL_ERR,
131 RESULT_DETAIL(
132 "could not create opus multistream decoder!")),
133 __func__);
134 }
135
DecodeHeader(const unsigned char * aData,size_t aLength)136 nsresult OpusDataDecoder::DecodeHeader(const unsigned char* aData,
137 size_t aLength) {
138 MOZ_ASSERT(!mOpusParser);
139 MOZ_ASSERT(!mOpusDecoder);
140 MOZ_ASSERT(!mDecodedHeader);
141 mDecodedHeader = true;
142
143 mOpusParser = MakeUnique<OpusParser>();
144 if (!mOpusParser->DecodeHeader(const_cast<unsigned char*>(aData), aLength)) {
145 return NS_ERROR_FAILURE;
146 }
147 int channels = mOpusParser->mChannels;
148
149 mMappingTable.SetLength(channels);
150 AudioConfig::ChannelLayout vorbisLayout(
151 channels, VorbisDataDecoder::VorbisLayout(channels));
152 if (vorbisLayout.IsValid()) {
153 mChannelMap = vorbisLayout.Map();
154
155 AudioConfig::ChannelLayout smpteLayout(
156 AudioConfig::ChannelLayout::SMPTEDefault(vorbisLayout));
157
158 AutoTArray<uint8_t, 8> map;
159 map.SetLength(channels);
160 if (mOpusParser->mChannelMapping == 1 &&
161 vorbisLayout.MappingTable(smpteLayout, &map)) {
162 for (int i = 0; i < channels; i++) {
163 mMappingTable[i] = mOpusParser->mMappingTable[map[i]];
164 }
165 } else {
166 // Use Opus set channel mapping and return channels as-is.
167 PodCopy(mMappingTable.Elements(), mOpusParser->mMappingTable, channels);
168 }
169 } else {
170 // Create a dummy mapping table so that channel ordering stay the same
171 // during decoding.
172 for (int i = 0; i < channels; i++) {
173 mMappingTable[i] = i;
174 }
175 }
176
177 return NS_OK;
178 }
179
Decode(MediaRawData * aSample)180 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Decode(
181 MediaRawData* aSample) {
182 return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
183 &OpusDataDecoder::ProcessDecode, aSample);
184 }
185
ProcessDecode(MediaRawData * aSample)186 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::ProcessDecode(
187 MediaRawData* aSample) {
188 uint32_t channels = mOpusParser->mChannels;
189
190 if (mPaddingDiscarded) {
191 // Discard padding should be used only on the final packet, so
192 // decoding after a padding discard is invalid.
193 OPUS_DEBUG("Opus error, discard padding on interstitial packet");
194 return DecodePromise::CreateAndReject(
195 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
196 RESULT_DETAIL("Discard padding on interstitial packet")),
197 __func__);
198 }
199
200 if (!mLastFrameTime ||
201 mLastFrameTime.ref() != aSample->mTime.ToMicroseconds()) {
202 // We are starting a new block.
203 mFrames = 0;
204 mLastFrameTime = Some(aSample->mTime.ToMicroseconds());
205 }
206
207 // Maximum value is 63*2880, so there's no chance of overflow.
208 int frames_number =
209 opus_packet_get_nb_frames(aSample->Data(), aSample->Size());
210 if (frames_number <= 0) {
211 OPUS_DEBUG("Invalid packet header: r=%d length=%zu", frames_number,
212 aSample->Size());
213 return DecodePromise::CreateAndReject(
214 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
215 RESULT_DETAIL("Invalid packet header: r=%d length=%u",
216 frames_number, uint32_t(aSample->Size()))),
217 __func__);
218 }
219
220 int samples = opus_packet_get_samples_per_frame(
221 aSample->Data(), opus_int32(mOpusParser->mRate));
222
223 // A valid Opus packet must be between 2.5 and 120 ms long (48kHz).
224 CheckedInt32 totalFrames =
225 CheckedInt32(frames_number) * CheckedInt32(samples);
226 if (!totalFrames.isValid()) {
227 return DecodePromise::CreateAndReject(
228 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
229 RESULT_DETAIL("Frames count overflow")),
230 __func__);
231 }
232
233 int frames = totalFrames.value();
234 if (frames < 120 || frames > 5760) {
235 OPUS_DEBUG("Invalid packet frames: %d", frames);
236 return DecodePromise::CreateAndReject(
237 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
238 RESULT_DETAIL("Invalid packet frames:%d", frames)),
239 __func__);
240 }
241
242 AlignedAudioBuffer buffer(frames * channels);
243 if (!buffer) {
244 return DecodePromise::CreateAndReject(
245 MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
246 }
247
248 // Decode to the appropriate sample type.
249 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
250 int ret = opus_multistream_decode_float(mOpusDecoder, aSample->Data(),
251 aSample->Size(), buffer.get(), frames,
252 false);
253 #else
254 int ret =
255 opus_multistream_decode(mOpusDecoder, aSample->Data(), aSample->Size(),
256 buffer.get(), frames, false);
257 #endif
258 if (ret < 0) {
259 return DecodePromise::CreateAndReject(
260 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
261 RESULT_DETAIL("Opus decoding error:%d", ret)),
262 __func__);
263 }
264 NS_ASSERTION(ret == frames, "Opus decoded too few audio samples");
265 auto startTime = aSample->mTime;
266
267 // Trim the initial frames while the decoder is settling.
268 if (mSkip > 0) {
269 int32_t skipFrames = std::min<int32_t>(mSkip, frames);
270 int32_t keepFrames = frames - skipFrames;
271 OPUS_DEBUG("Opus decoder skipping %d of %d frames", skipFrames, frames);
272 PodMove(buffer.get(), buffer.get() + skipFrames * channels,
273 keepFrames * channels);
274 startTime = startTime + FramesToTimeUnit(skipFrames, mOpusParser->mRate);
275 frames = keepFrames;
276 mSkip -= skipFrames;
277 }
278
279 if (aSample->mDiscardPadding > 0) {
280 OPUS_DEBUG("Opus decoder discarding %u of %d frames",
281 aSample->mDiscardPadding, frames);
282 // Padding discard is only supposed to happen on the final packet.
283 // Record the discard so we can return an error if another packet is
284 // decoded.
285 if (aSample->mDiscardPadding > uint32_t(frames)) {
286 // Discarding more than the entire packet is invalid.
287 OPUS_DEBUG("Opus error, discard padding larger than packet");
288 return DecodePromise::CreateAndReject(
289 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
290 RESULT_DETAIL("Discard padding larger than packet")),
291 __func__);
292 }
293
294 mPaddingDiscarded = true;
295 frames = frames - aSample->mDiscardPadding;
296 }
297
298 // Apply the header gain if one was specified.
299 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
300 if (mOpusParser->mGain != 1.0f) {
301 float gain = mOpusParser->mGain;
302 uint32_t samples = frames * channels;
303 for (uint32_t i = 0; i < samples; i++) {
304 buffer[i] *= gain;
305 }
306 }
307 #else
308 if (mOpusParser->mGain_Q16 != 65536) {
309 int64_t gain_Q16 = mOpusParser->mGain_Q16;
310 uint32_t samples = frames * channels;
311 for (uint32_t i = 0; i < samples; i++) {
312 int32_t val = static_cast<int32_t>((gain_Q16 * buffer[i] + 32768) >> 16);
313 buffer[i] = static_cast<AudioDataValue>(MOZ_CLIP_TO_15(val));
314 }
315 }
316 #endif
317
318 auto duration = FramesToTimeUnit(frames, mOpusParser->mRate);
319 if (!duration.IsValid()) {
320 return DecodePromise::CreateAndReject(
321 MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
322 RESULT_DETAIL("Overflow converting WebM audio duration")),
323 __func__);
324 }
325 auto time = startTime -
326 FramesToTimeUnit(mOpusParser->mPreSkip, mOpusParser->mRate) +
327 FramesToTimeUnit(mFrames, mOpusParser->mRate);
328 if (!time.IsValid()) {
329 return DecodePromise::CreateAndReject(
330 MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
331 RESULT_DETAIL("Overflow shifting tstamp by codec delay")),
332 __func__);
333 };
334
335 mFrames += frames;
336
337 if (!frames) {
338 return DecodePromise::CreateAndResolve(DecodedData(), __func__);
339 }
340
341 // Trim extra allocated frames.
342 buffer.SetLength(frames * channels);
343
344 return DecodePromise::CreateAndResolve(
345 DecodedData{new AudioData(aSample->mOffset, time, std::move(buffer),
346 mOpusParser->mChannels, mOpusParser->mRate,
347 mChannelMap)},
348 __func__);
349 }
350
Drain()351 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Drain() {
352 RefPtr<OpusDataDecoder> self = this;
353 // InvokeAsync dispatches a task that will be run after any pending decode
354 // completes. As such, once the drain task run, there's nothing more to do.
355 return InvokeAsync(mTaskQueue, __func__, [] {
356 return DecodePromise::CreateAndResolve(DecodedData(), __func__);
357 });
358 }
359
Flush()360 RefPtr<MediaDataDecoder::FlushPromise> OpusDataDecoder::Flush() {
361 if (!mOpusDecoder) {
362 return FlushPromise::CreateAndResolve(true, __func__);
363 }
364
365 RefPtr<OpusDataDecoder> self = this;
366 return InvokeAsync(mTaskQueue, __func__, [self, this]() {
367 MOZ_ASSERT(mOpusDecoder);
368 // Reset the decoder.
369 opus_multistream_decoder_ctl(mOpusDecoder, OPUS_RESET_STATE);
370 mSkip = mOpusParser->mPreSkip;
371 mPaddingDiscarded = false;
372 mLastFrameTime.reset();
373 return FlushPromise::CreateAndResolve(true, __func__);
374 });
375 }
376
377 /* static */
IsOpus(const nsACString & aMimeType)378 bool OpusDataDecoder::IsOpus(const nsACString& aMimeType) {
379 return aMimeType.EqualsLiteral("audio/opus");
380 }
381
382 } // namespace mozilla
383 #undef OPUS_DEBUG
384