1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "OpusDecoder.h"
8 #include "OpusParser.h"
9 #include "TimeUnits.h"
10 #include "VorbisUtils.h"
11 #include "VorbisDecoder.h" // For VorbisLayout
12 #include "mozilla/EndianUtils.h"
13 #include "mozilla/PodOperations.h"
14 #include "mozilla/SyncRunnable.h"
15 #include "VideoUtils.h"
16
17 #include <inttypes.h> // For PRId64
18
19 #include "opus/opus.h"
20 extern "C" {
21 #include "opus/opus_multistream.h"
22 }
23
24 #define OPUS_DEBUG(arg, ...) \
25 DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
26 ##__VA_ARGS__)
27
28 namespace mozilla {
29
OpusDataDecoder(const CreateDecoderParams & aParams)30 OpusDataDecoder::OpusDataDecoder(const CreateDecoderParams& aParams)
31 : mInfo(aParams.AudioConfig()),
32 mTaskQueue(aParams.mTaskQueue),
33 mOpusDecoder(nullptr),
34 mSkip(0),
35 mDecodedHeader(false),
36 mPaddingDiscarded(false),
37 mFrames(0) {}
38
~OpusDataDecoder()39 OpusDataDecoder::~OpusDataDecoder() {
40 if (mOpusDecoder) {
41 opus_multistream_decoder_destroy(mOpusDecoder);
42 mOpusDecoder = nullptr;
43 }
44 }
45
Shutdown()46 RefPtr<ShutdownPromise> OpusDataDecoder::Shutdown() {
47 RefPtr<OpusDataDecoder> self = this;
48 return InvokeAsync(mTaskQueue, __func__, [self]() {
49 return ShutdownPromise::CreateAndResolve(true, __func__);
50 });
51 }
52
AppendCodecDelay(MediaByteBuffer * config,uint64_t codecDelayUS)53 void OpusDataDecoder::AppendCodecDelay(MediaByteBuffer* config,
54 uint64_t codecDelayUS) {
55 uint8_t buffer[sizeof(uint64_t)];
56 BigEndian::writeUint64(buffer, codecDelayUS);
57 config->AppendElements(buffer, sizeof(uint64_t));
58 }
59
Init()60 RefPtr<MediaDataDecoder::InitPromise> OpusDataDecoder::Init() {
61 size_t length = mInfo.mCodecSpecificConfig->Length();
62 uint8_t* p = mInfo.mCodecSpecificConfig->Elements();
63 if (length < sizeof(uint64_t)) {
64 OPUS_DEBUG("CodecSpecificConfig too short to read codecDelay!");
65 return InitPromise::CreateAndReject(
66 MediaResult(
67 NS_ERROR_DOM_MEDIA_FATAL_ERR,
68 RESULT_DETAIL("CodecSpecificConfig too short to read codecDelay!")),
69 __func__);
70 }
71 int64_t codecDelay = BigEndian::readUint64(p);
72 length -= sizeof(uint64_t);
73 p += sizeof(uint64_t);
74 if (NS_FAILED(DecodeHeader(p, length))) {
75 OPUS_DEBUG("Error decoding header!");
76 return InitPromise::CreateAndReject(
77 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
78 RESULT_DETAIL("Error decoding header!")),
79 __func__);
80 }
81
82 int r;
83 mOpusDecoder = opus_multistream_decoder_create(
84 mOpusParser->mRate, mOpusParser->mChannels, mOpusParser->mStreams,
85 mOpusParser->mCoupledStreams, mMappingTable, &r);
86 mSkip = mOpusParser->mPreSkip;
87 mPaddingDiscarded = false;
88
89 if (codecDelay !=
90 FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) {
91 NS_WARNING("Invalid Opus header: CodecDelay and pre-skip do not match!");
92 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
93 }
94
95 if (mInfo.mRate != (uint32_t)mOpusParser->mRate) {
96 NS_WARNING("Invalid Opus header: container and codec rate do not match!");
97 }
98 if (mInfo.mChannels != (uint32_t)mOpusParser->mChannels) {
99 NS_WARNING(
100 "Invalid Opus header: container and codec channels do not match!");
101 }
102
103 return r == OPUS_OK
104 ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__)
105 : InitPromise::CreateAndReject(
106 MediaResult(
107 NS_ERROR_DOM_MEDIA_FATAL_ERR,
108 RESULT_DETAIL(
109 "could not create opus multistream decoder!")),
110 __func__);
111 }
112
DecodeHeader(const unsigned char * aData,size_t aLength)113 nsresult OpusDataDecoder::DecodeHeader(const unsigned char* aData,
114 size_t aLength) {
115 MOZ_ASSERT(!mOpusParser);
116 MOZ_ASSERT(!mOpusDecoder);
117 MOZ_ASSERT(!mDecodedHeader);
118 mDecodedHeader = true;
119
120 mOpusParser = new OpusParser;
121 if (!mOpusParser->DecodeHeader(const_cast<unsigned char*>(aData), aLength)) {
122 return NS_ERROR_FAILURE;
123 }
124 int channels = mOpusParser->mChannels;
125
126 AudioConfig::ChannelLayout layout(channels);
127 if (!layout.IsValid()) {
128 OPUS_DEBUG("Invalid channel mapping. Source is %d channels", channels);
129 return NS_ERROR_FAILURE;
130 }
131
132 AudioConfig::ChannelLayout vorbisLayout(
133 channels, VorbisDataDecoder::VorbisLayout(channels));
134 AudioConfig::ChannelLayout smpteLayout(channels);
135 static_assert(sizeof(mOpusParser->mMappingTable) /
136 sizeof(mOpusParser->mMappingTable[0]) >=
137 MAX_AUDIO_CHANNELS,
138 "Invalid size set");
139 uint8_t map[sizeof(mOpusParser->mMappingTable) /
140 sizeof(mOpusParser->mMappingTable[0])];
141 if (vorbisLayout.MappingTable(smpteLayout, map)) {
142 for (int i = 0; i < channels; i++) {
143 mMappingTable[i] = mOpusParser->mMappingTable[map[i]];
144 }
145 } else {
146 // Should never get here as vorbis layout is always convertible to SMPTE
147 // default layout.
148 PodCopy(mMappingTable, mOpusParser->mMappingTable, MAX_AUDIO_CHANNELS);
149 }
150
151 return NS_OK;
152 }
153
Decode(MediaRawData * aSample)154 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Decode(
155 MediaRawData* aSample) {
156 return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
157 &OpusDataDecoder::ProcessDecode, aSample);
158 }
159
ProcessDecode(MediaRawData * aSample)160 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::ProcessDecode(
161 MediaRawData* aSample) {
162 uint32_t channels = mOpusParser->mChannels;
163
164 if (mPaddingDiscarded) {
165 // Discard padding should be used only on the final packet, so
166 // decoding after a padding discard is invalid.
167 OPUS_DEBUG("Opus error, discard padding on interstitial packet");
168 return DecodePromise::CreateAndReject(
169 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
170 RESULT_DETAIL("Discard padding on interstitial packet")),
171 __func__);
172 }
173
174 if (!mLastFrameTime ||
175 mLastFrameTime.ref() != aSample->mTime.ToMicroseconds()) {
176 // We are starting a new block.
177 mFrames = 0;
178 mLastFrameTime = Some(aSample->mTime.ToMicroseconds());
179 }
180
181 // Maximum value is 63*2880, so there's no chance of overflow.
182 int frames_number =
183 opus_packet_get_nb_frames(aSample->Data(), aSample->Size());
184 if (frames_number <= 0) {
185 OPUS_DEBUG("Invalid packet header: r=%d length=%zu", frames_number,
186 aSample->Size());
187 return DecodePromise::CreateAndReject(
188 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
189 RESULT_DETAIL("Invalid packet header: r=%d length=%u",
190 frames_number, uint32_t(aSample->Size()))),
191 __func__);
192 }
193
194 int samples = opus_packet_get_samples_per_frame(
195 aSample->Data(), opus_int32(mOpusParser->mRate));
196
197 // A valid Opus packet must be between 2.5 and 120 ms long (48kHz).
198 CheckedInt32 totalFrames =
199 CheckedInt32(frames_number) * CheckedInt32(samples);
200 if (!totalFrames.isValid()) {
201 return DecodePromise::CreateAndReject(
202 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
203 RESULT_DETAIL("Frames count overflow")),
204 __func__);
205 }
206
207 int frames = totalFrames.value();
208 if (frames < 120 || frames > 5760) {
209 OPUS_DEBUG("Invalid packet frames: %d", frames);
210 return DecodePromise::CreateAndReject(
211 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
212 RESULT_DETAIL("Invalid packet frames:%d", frames)),
213 __func__);
214 }
215
216 AlignedAudioBuffer buffer(frames * channels);
217 if (!buffer) {
218 return DecodePromise::CreateAndReject(
219 MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
220 }
221
222 // Decode to the appropriate sample type.
223 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
224 int ret = opus_multistream_decode_float(mOpusDecoder, aSample->Data(),
225 aSample->Size(), buffer.get(), frames,
226 false);
227 #else
228 int ret =
229 opus_multistream_decode(mOpusDecoder, aSample->Data(), aSample->Size(),
230 buffer.get(), frames, false);
231 #endif
232 if (ret < 0) {
233 return DecodePromise::CreateAndReject(
234 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
235 RESULT_DETAIL("Opus decoding error:%d", ret)),
236 __func__);
237 }
238 NS_ASSERTION(ret == frames, "Opus decoded too few audio samples");
239 auto startTime = aSample->mTime;
240
241 // Trim the initial frames while the decoder is settling.
242 if (mSkip > 0) {
243 int32_t skipFrames = std::min<int32_t>(mSkip, frames);
244 int32_t keepFrames = frames - skipFrames;
245 OPUS_DEBUG("Opus decoder skipping %d of %d frames", skipFrames, frames);
246 PodMove(buffer.get(), buffer.get() + skipFrames * channels,
247 keepFrames * channels);
248 startTime = startTime + FramesToTimeUnit(skipFrames, mOpusParser->mRate);
249 frames = keepFrames;
250 mSkip -= skipFrames;
251 }
252
253 if (aSample->mDiscardPadding > 0) {
254 OPUS_DEBUG("Opus decoder discarding %u of %d frames",
255 aSample->mDiscardPadding, frames);
256 // Padding discard is only supposed to happen on the final packet.
257 // Record the discard so we can return an error if another packet is
258 // decoded.
259 if (aSample->mDiscardPadding > uint32_t(frames)) {
260 // Discarding more than the entire packet is invalid.
261 OPUS_DEBUG("Opus error, discard padding larger than packet");
262 return DecodePromise::CreateAndReject(
263 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
264 RESULT_DETAIL("Discard padding larger than packet")),
265 __func__);
266 }
267
268 mPaddingDiscarded = true;
269 frames = frames - aSample->mDiscardPadding;
270 }
271
272 // Apply the header gain if one was specified.
273 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
274 if (mOpusParser->mGain != 1.0f) {
275 float gain = mOpusParser->mGain;
276 uint32_t samples = frames * channels;
277 for (uint32_t i = 0; i < samples; i++) {
278 buffer[i] *= gain;
279 }
280 }
281 #else
282 if (mOpusParser->mGain_Q16 != 65536) {
283 int64_t gain_Q16 = mOpusParser->mGain_Q16;
284 uint32_t samples = frames * channels;
285 for (uint32_t i = 0; i < samples; i++) {
286 int32_t val = static_cast<int32_t>((gain_Q16 * buffer[i] + 32768) >> 16);
287 buffer[i] = static_cast<AudioDataValue>(MOZ_CLIP_TO_15(val));
288 }
289 }
290 #endif
291
292 auto duration = FramesToTimeUnit(frames, mOpusParser->mRate);
293 if (!duration.IsValid()) {
294 return DecodePromise::CreateAndReject(
295 MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
296 RESULT_DETAIL("Overflow converting WebM audio duration")),
297 __func__);
298 }
299 auto time = startTime -
300 FramesToTimeUnit(mOpusParser->mPreSkip, mOpusParser->mRate) +
301 FramesToTimeUnit(mFrames, mOpusParser->mRate);
302 if (!time.IsValid()) {
303 return DecodePromise::CreateAndReject(
304 MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
305 RESULT_DETAIL("Overflow shifting tstamp by codec delay")),
306 __func__);
307 };
308
309 mFrames += frames;
310
311 return DecodePromise::CreateAndResolve(
312 DecodedData{new AudioData(aSample->mOffset, time, duration, frames,
313 Move(buffer), mOpusParser->mChannels,
314 mOpusParser->mRate)},
315 __func__);
316 }
317
Drain()318 RefPtr<MediaDataDecoder::DecodePromise> OpusDataDecoder::Drain() {
319 RefPtr<OpusDataDecoder> self = this;
320 // InvokeAsync dispatches a task that will be run after any pending decode
321 // completes. As such, once the drain task run, there's nothing more to do.
322 return InvokeAsync(mTaskQueue, __func__, [] {
323 return DecodePromise::CreateAndResolve(DecodedData(), __func__);
324 });
325 }
326
Flush()327 RefPtr<MediaDataDecoder::FlushPromise> OpusDataDecoder::Flush() {
328 if (!mOpusDecoder) {
329 return FlushPromise::CreateAndResolve(true, __func__);
330 }
331
332 RefPtr<OpusDataDecoder> self = this;
333 return InvokeAsync(mTaskQueue, __func__, [self, this]() {
334 MOZ_ASSERT(mOpusDecoder);
335 // Reset the decoder.
336 opus_multistream_decoder_ctl(mOpusDecoder, OPUS_RESET_STATE);
337 mSkip = mOpusParser->mPreSkip;
338 mPaddingDiscarded = false;
339 mLastFrameTime.reset();
340 return FlushPromise::CreateAndResolve(true, __func__);
341 });
342 }
343
344 /* static */
IsOpus(const nsACString & aMimeType)345 bool OpusDataDecoder::IsOpus(const nsACString& aMimeType) {
346 return aMimeType.EqualsLiteral("audio/opus");
347 }
348
349 } // namespace mozilla
350 #undef OPUS_DEBUG
351