1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/cdm/library_cdm/clear_key_cdm/ffmpeg_cdm_audio_decoder.h"
6
7 #include <stddef.h>
8
9 #include <algorithm>
10
11 #include "base/bind.h"
12 #include "base/bind_helpers.h"
13 #include "base/callback.h"
14 #include "base/logging.h"
15 #include "media/base/audio_bus.h"
16 #include "media/base/audio_timestamp_helper.h"
17 #include "media/base/data_buffer.h"
18 #include "media/base/limits.h"
19 #include "media/cdm/library_cdm/cdm_host_proxy.h"
20 #include "media/ffmpeg/ffmpeg_common.h"
21 #include "media/ffmpeg/ffmpeg_decoding_loop.h"
22
23 namespace media {
24
25 namespace {
26
27 // Maximum number of channels with defined layout in src/media.
28 static const int kMaxChannels = 8;
29
IsValidConfig(const cdm::AudioDecoderConfig_2 & config)30 bool IsValidConfig(const cdm::AudioDecoderConfig_2& config) {
31 // No need to check |encryption_scheme| as the buffers will always be
32 // decrypted before calling DecodeBuffer().
33 return config.codec != cdm::kUnknownAudioCodec && config.channel_count > 0 &&
34 config.channel_count <= kMaxChannels && config.bits_per_channel > 0 &&
35 config.bits_per_channel <= limits::kMaxBitsPerSample &&
36 config.samples_per_second > 0 &&
37 config.samples_per_second <= limits::kMaxSampleRate;
38 }
39
CdmAudioCodecToCodecID(cdm::AudioCodec audio_codec)40 AVCodecID CdmAudioCodecToCodecID(cdm::AudioCodec audio_codec) {
41 switch (audio_codec) {
42 case cdm::kCodecVorbis:
43 return AV_CODEC_ID_VORBIS;
44 case cdm::kCodecAac:
45 return AV_CODEC_ID_AAC;
46 case cdm::kUnknownAudioCodec:
47 default:
48 NOTREACHED() << "Unsupported cdm::AudioCodec: " << audio_codec;
49 return AV_CODEC_ID_NONE;
50 }
51 }
52
CdmAudioDecoderConfigToAVCodecContext(const cdm::AudioDecoderConfig_2 & config,AVCodecContext * codec_context)53 void CdmAudioDecoderConfigToAVCodecContext(
54 const cdm::AudioDecoderConfig_2& config,
55 AVCodecContext* codec_context) {
56 codec_context->codec_type = AVMEDIA_TYPE_AUDIO;
57 codec_context->codec_id = CdmAudioCodecToCodecID(config.codec);
58
59 switch (config.bits_per_channel) {
60 case 8:
61 codec_context->sample_fmt = AV_SAMPLE_FMT_U8;
62 break;
63 case 16:
64 codec_context->sample_fmt = AV_SAMPLE_FMT_S16;
65 break;
66 case 32:
67 codec_context->sample_fmt = AV_SAMPLE_FMT_S32;
68 break;
69 default:
70 DVLOG(1) << "CdmAudioDecoderConfigToAVCodecContext() Unsupported bits "
71 "per channel: "
72 << config.bits_per_channel;
73 codec_context->sample_fmt = AV_SAMPLE_FMT_NONE;
74 }
75
76 codec_context->channels = config.channel_count;
77 codec_context->sample_rate = config.samples_per_second;
78
79 if (config.extra_data) {
80 codec_context->extradata_size = config.extra_data_size;
81 codec_context->extradata = reinterpret_cast<uint8_t*>(
82 av_malloc(config.extra_data_size + AV_INPUT_BUFFER_PADDING_SIZE));
83 memcpy(codec_context->extradata, config.extra_data, config.extra_data_size);
84 memset(codec_context->extradata + config.extra_data_size, '\0',
85 AV_INPUT_BUFFER_PADDING_SIZE);
86 } else {
87 codec_context->extradata = NULL;
88 codec_context->extradata_size = 0;
89 }
90 }
91
AVSampleFormatToCdmAudioFormat(AVSampleFormat sample_format)92 cdm::AudioFormat AVSampleFormatToCdmAudioFormat(AVSampleFormat sample_format) {
93 switch (sample_format) {
94 case AV_SAMPLE_FMT_U8:
95 return cdm::kAudioFormatU8;
96 case AV_SAMPLE_FMT_S16:
97 return cdm::kAudioFormatS16;
98 case AV_SAMPLE_FMT_S32:
99 return cdm::kAudioFormatS32;
100 case AV_SAMPLE_FMT_FLT:
101 return cdm::kAudioFormatF32;
102 case AV_SAMPLE_FMT_S16P:
103 return cdm::kAudioFormatPlanarS16;
104 case AV_SAMPLE_FMT_FLTP:
105 return cdm::kAudioFormatPlanarF32;
106 default:
107 DVLOG(1) << "Unknown AVSampleFormat: " << sample_format;
108 }
109 return cdm::kUnknownAudioFormat;
110 }
111
CopySamples(cdm::AudioFormat cdm_format,int decoded_audio_size,const AVFrame & av_frame,uint8_t * output_buffer)112 void CopySamples(cdm::AudioFormat cdm_format,
113 int decoded_audio_size,
114 const AVFrame& av_frame,
115 uint8_t* output_buffer) {
116 switch (cdm_format) {
117 case cdm::kAudioFormatU8:
118 case cdm::kAudioFormatS16:
119 case cdm::kAudioFormatS32:
120 case cdm::kAudioFormatF32:
121 memcpy(output_buffer, av_frame.data[0], decoded_audio_size);
122 break;
123 case cdm::kAudioFormatPlanarS16:
124 case cdm::kAudioFormatPlanarF32: {
125 const int decoded_size_per_channel =
126 decoded_audio_size / av_frame.channels;
127 for (int i = 0; i < av_frame.channels; ++i) {
128 memcpy(output_buffer, av_frame.extended_data[i],
129 decoded_size_per_channel);
130 output_buffer += decoded_size_per_channel;
131 }
132 break;
133 }
134 default:
135 NOTREACHED() << "Unsupported CDM Audio Format!";
136 memset(output_buffer, 0, decoded_audio_size);
137 }
138 }
139
140 } // namespace
141
FFmpegCdmAudioDecoder(CdmHostProxy * cdm_host_proxy)142 FFmpegCdmAudioDecoder::FFmpegCdmAudioDecoder(CdmHostProxy* cdm_host_proxy)
143 : cdm_host_proxy_(cdm_host_proxy) {}
144
~FFmpegCdmAudioDecoder()145 FFmpegCdmAudioDecoder::~FFmpegCdmAudioDecoder() {
146 ReleaseFFmpegResources();
147 }
148
Initialize(const cdm::AudioDecoderConfig_2 & config)149 bool FFmpegCdmAudioDecoder::Initialize(
150 const cdm::AudioDecoderConfig_2& config) {
151 DVLOG(1) << "Initialize()";
152 if (!IsValidConfig(config)) {
153 LOG(ERROR) << "Initialize(): invalid audio decoder configuration.";
154 return false;
155 }
156
157 if (is_initialized_) {
158 LOG(ERROR) << "Initialize(): Already initialized.";
159 return false;
160 }
161
162 // Initialize AVCodecContext structure.
163 codec_context_.reset(avcodec_alloc_context3(NULL));
164 CdmAudioDecoderConfigToAVCodecContext(config, codec_context_.get());
165
166 // MP3 decodes to S16P which we don't support, tell it to use S16 instead.
167 if (codec_context_->sample_fmt == AV_SAMPLE_FMT_S16P)
168 codec_context_->request_sample_fmt = AV_SAMPLE_FMT_S16;
169
170 AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
171 if (!codec || avcodec_open2(codec_context_.get(), codec, NULL) < 0) {
172 DLOG(ERROR) << "Could not initialize audio decoder: "
173 << codec_context_->codec_id;
174 return false;
175 }
176
177 // Ensure avcodec_open2() respected our format request.
178 if (codec_context_->sample_fmt == AV_SAMPLE_FMT_S16P) {
179 DLOG(ERROR) << "Unable to configure a supported sample format: "
180 << codec_context_->sample_fmt;
181 return false;
182 }
183
184 // Success!
185 decoding_loop_.reset(new FFmpegDecodingLoop(codec_context_.get()));
186 samples_per_second_ = config.samples_per_second;
187 bytes_per_frame_ = codec_context_->channels * config.bits_per_channel / 8;
188 output_timestamp_helper_.reset(
189 new AudioTimestampHelper(config.samples_per_second));
190 is_initialized_ = true;
191
192 // Store initial values to guard against midstream configuration changes.
193 channels_ = codec_context_->channels;
194 av_sample_format_ = codec_context_->sample_fmt;
195
196 return true;
197 }
198
Deinitialize()199 void FFmpegCdmAudioDecoder::Deinitialize() {
200 DVLOG(1) << "Deinitialize()";
201 ReleaseFFmpegResources();
202 is_initialized_ = false;
203 ResetTimestampState();
204 }
205
Reset()206 void FFmpegCdmAudioDecoder::Reset() {
207 DVLOG(1) << "Reset()";
208 avcodec_flush_buffers(codec_context_.get());
209 ResetTimestampState();
210 }
211
DecodeBuffer(const uint8_t * compressed_buffer,int32_t compressed_buffer_size,int64_t input_timestamp,cdm::AudioFrames * decoded_frames)212 cdm::Status FFmpegCdmAudioDecoder::DecodeBuffer(
213 const uint8_t* compressed_buffer,
214 int32_t compressed_buffer_size,
215 int64_t input_timestamp,
216 cdm::AudioFrames* decoded_frames) {
217 DVLOG(1) << "DecodeBuffer()";
218 const bool is_end_of_stream = !compressed_buffer;
219 base::TimeDelta timestamp =
220 base::TimeDelta::FromMicroseconds(input_timestamp);
221
222 if (!is_end_of_stream && timestamp != kNoTimestamp) {
223 if (last_input_timestamp_ != kNoTimestamp &&
224 timestamp < last_input_timestamp_) {
225 base::TimeDelta diff = timestamp - last_input_timestamp_;
226 DVLOG(1) << "Input timestamps are not monotonically increasing! "
227 << " ts " << timestamp.InMicroseconds() << " us"
228 << " diff " << diff.InMicroseconds() << " us";
229 return cdm::kDecodeError;
230 }
231 last_input_timestamp_ = timestamp;
232 }
233
234 size_t total_size = 0u;
235 std::vector<std::unique_ptr<AVFrame, ScopedPtrAVFreeFrame>> audio_frames;
236
237 AVPacket packet;
238 av_init_packet(&packet);
239 packet.data = const_cast<uint8_t*>(compressed_buffer);
240 packet.size = compressed_buffer_size;
241
242 switch (decoding_loop_->DecodePacket(
243 &packet, base::BindRepeating(&FFmpegCdmAudioDecoder::OnNewFrame,
244 base::Unretained(this), &total_size,
245 &audio_frames))) {
246 case FFmpegDecodingLoop::DecodeStatus::kSendPacketFailed:
247 return cdm::kDecodeError;
248 case FFmpegDecodingLoop::DecodeStatus::kFrameProcessingFailed:
249 NOTREACHED();
250 FALLTHROUGH;
251 case FFmpegDecodingLoop::DecodeStatus::kDecodeFrameFailed:
252 DLOG(WARNING) << " failed to decode an audio buffer: "
253 << timestamp.InMicroseconds();
254 break;
255 case FFmpegDecodingLoop::DecodeStatus::kOkay:
256 break;
257 }
258
259 if (output_timestamp_helper_->base_timestamp() == kNoTimestamp &&
260 !is_end_of_stream) {
261 DCHECK(timestamp != kNoTimestamp);
262 output_timestamp_helper_->SetBaseTimestamp(timestamp);
263 }
264
265 if (audio_frames.empty())
266 return cdm::kNeedMoreData;
267
268 const size_t allocation_size = total_size + 2 * sizeof(int64_t);
269 decoded_frames->SetFrameBuffer(cdm_host_proxy_->Allocate(allocation_size));
270 if (!decoded_frames->FrameBuffer()) {
271 LOG(ERROR) << "DecodeBuffer() ClearKeyCdmHost::Allocate failed.";
272 return cdm::kDecodeError;
273 }
274 decoded_frames->FrameBuffer()->SetSize(allocation_size);
275
276 // Tell the CDM what AudioFormat we're using.
277 const cdm::AudioFormat cdm_format = AVSampleFormatToCdmAudioFormat(
278 static_cast<AVSampleFormat>(av_sample_format_));
279 DCHECK_NE(cdm_format, cdm::kUnknownAudioFormat);
280 decoded_frames->SetFormat(cdm_format);
281
282 uint8_t* output_buffer = decoded_frames->FrameBuffer()->Data();
283 SerializeInt64(output_timestamp_helper_->GetTimestamp().InMicroseconds(),
284 output_buffer);
285 output_buffer += sizeof(int64_t);
286 SerializeInt64(total_size, output_buffer);
287 output_buffer += sizeof(int64_t);
288 output_timestamp_helper_->AddFrames(total_size / bytes_per_frame_);
289
290 for (auto& frame : audio_frames) {
291 int decoded_audio_size = 0;
292 if (frame->sample_rate != samples_per_second_ ||
293 frame->channels != channels_ || frame->format != av_sample_format_) {
294 DLOG(ERROR) << "Unsupported midstream configuration change!"
295 << " Sample Rate: " << frame->sample_rate << " vs "
296 << samples_per_second_ << ", Channels: " << frame->channels
297 << " vs " << channels_ << ", Sample Format: " << frame->format
298 << " vs " << av_sample_format_;
299 return cdm::kDecodeError;
300 }
301
302 decoded_audio_size = av_samples_get_buffer_size(
303 nullptr, codec_context_->channels, frame->nb_samples,
304 codec_context_->sample_fmt, 1);
305 if (!decoded_audio_size)
306 continue;
307
308 DCHECK_EQ(decoded_audio_size % bytes_per_frame_, 0)
309 << "Decoder didn't output full frames";
310
311 CopySamples(cdm_format, decoded_audio_size, *frame, output_buffer);
312 output_buffer += decoded_audio_size;
313 }
314
315 return cdm::kSuccess;
316 }
317
OnNewFrame(size_t * total_size,std::vector<std::unique_ptr<AVFrame,ScopedPtrAVFreeFrame>> * audio_frames,AVFrame * frame)318 bool FFmpegCdmAudioDecoder::OnNewFrame(
319 size_t* total_size,
320 std::vector<std::unique_ptr<AVFrame, ScopedPtrAVFreeFrame>>* audio_frames,
321 AVFrame* frame) {
322 *total_size += av_samples_get_buffer_size(nullptr, codec_context_->channels,
323 frame->nb_samples,
324 codec_context_->sample_fmt, 1);
325 audio_frames->emplace_back(av_frame_clone(frame));
326 return true;
327 }
328
ResetTimestampState()329 void FFmpegCdmAudioDecoder::ResetTimestampState() {
330 output_timestamp_helper_->SetBaseTimestamp(kNoTimestamp);
331 last_input_timestamp_ = kNoTimestamp;
332 }
333
ReleaseFFmpegResources()334 void FFmpegCdmAudioDecoder::ReleaseFFmpegResources() {
335 DVLOG(1) << "ReleaseFFmpegResources()";
336
337 decoding_loop_.reset();
338 codec_context_.reset();
339 }
340
SerializeInt64(int64_t value,uint8_t * dest)341 void FFmpegCdmAudioDecoder::SerializeInt64(int64_t value, uint8_t* dest) {
342 memcpy(dest, &value, sizeof(value));
343 }
344
345 } // namespace media
346