1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/renderer/pepper/pepper_audio_encoder_host.h"
6
7 #include <stddef.h>
8 #include <utility>
9
10 #include "base/bind.h"
11 #include "base/memory/unsafe_shared_memory_region.h"
12 #include "base/stl_util.h"
13 #include "content/public/renderer/renderer_ppapi_host.h"
14 #include "content/renderer/pepper/host_globals.h"
15 #include "content/renderer/render_thread_impl.h"
16 #include "media/base/bind_to_current_loop.h"
17 #include "ppapi/c/pp_codecs.h"
18 #include "ppapi/c/pp_errors.h"
19 #include "ppapi/host/dispatch_host_message.h"
20 #include "ppapi/host/ppapi_host.h"
21 #include "ppapi/proxy/ppapi_messages.h"
22 #include "ppapi/shared_impl/media_stream_buffer.h"
23 #include "third_party/opus/src/include/opus.h"
24
25 using ppapi::proxy::SerializedHandle;
26
27 namespace content {
28
29 namespace {
30
31 // Buffer up to 150ms (15 x 10ms per frame).
32 const uint32_t kDefaultNumberOfAudioBuffers = 15;
33
PP_HardwareAccelerationCompatibleAudio(bool accelerated,PP_HardwareAcceleration requested)34 bool PP_HardwareAccelerationCompatibleAudio(bool accelerated,
35 PP_HardwareAcceleration requested) {
36 switch (requested) {
37 case PP_HARDWAREACCELERATION_ONLY:
38 return accelerated;
39 case PP_HARDWAREACCELERATION_NONE:
40 return !accelerated;
41 case PP_HARDWAREACCELERATION_WITHFALLBACK:
42 return true;
43 // No default case, to catch unhandled PP_HardwareAcceleration values.
44 }
45 return false;
46 }
47
48 } // namespace
49
50 // This class should be constructed and initialized on the main renderer
51 // thread, used and destructed on the media thread.
52 class PepperAudioEncoderHost::AudioEncoderImpl {
53 public:
54 // Callback used to signal encoded data. If |size| is negative, an error
55 // occurred.
56 using BitstreamBufferReadyCB = base::OnceCallback<void(int32_t size)>;
57
58 AudioEncoderImpl();
59 ~AudioEncoderImpl();
60
61 // Used on the renderer thread.
62 static std::vector<PP_AudioProfileDescription> GetSupportedProfiles();
63 bool Initialize(const ppapi::proxy::PPB_AudioEncodeParameters& parameters);
64 int32_t GetNumberOfSamplesPerFrame();
65
66 // Used on the media thread.
67 void Encode(uint8_t* input_data,
68 size_t input_size,
69 uint8_t* output_data,
70 size_t output_size,
71 BitstreamBufferReadyCB callback);
72 void RequestBitrateChange(uint32_t bitrate);
73
74 private:
75 std::unique_ptr<uint8_t[]> encoder_memory_;
76 OpusEncoder* opus_encoder_;
77
78 // Initialization parameters, only valid if |encoder_memory_| is not
79 // nullptr.
80 ppapi::proxy::PPB_AudioEncodeParameters parameters_;
81
82 DISALLOW_COPY_AND_ASSIGN(AudioEncoderImpl);
83 };
84
AudioEncoderImpl()85 PepperAudioEncoderHost::AudioEncoderImpl::AudioEncoderImpl()
86 : opus_encoder_(nullptr) {}
87
~AudioEncoderImpl()88 PepperAudioEncoderHost::AudioEncoderImpl::~AudioEncoderImpl() {}
89
90 // static
91 std::vector<PP_AudioProfileDescription>
GetSupportedProfiles()92 PepperAudioEncoderHost::AudioEncoderImpl::GetSupportedProfiles() {
93 std::vector<PP_AudioProfileDescription> profiles;
94 static const uint32_t sampling_rates[] = {8000, 12000, 16000, 24000, 48000};
95
96 for (uint32_t i = 0; i < base::size(sampling_rates); ++i) {
97 PP_AudioProfileDescription profile;
98 profile.profile = PP_AUDIOPROFILE_OPUS;
99 profile.max_channels = 2;
100 profile.sample_size = PP_AUDIOBUFFER_SAMPLESIZE_16_BITS;
101 profile.sample_rate = sampling_rates[i];
102 profile.hardware_accelerated = PP_FALSE;
103 profiles.push_back(profile);
104 }
105 return profiles;
106 }
107
Initialize(const ppapi::proxy::PPB_AudioEncodeParameters & parameters)108 bool PepperAudioEncoderHost::AudioEncoderImpl::Initialize(
109 const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
110 if (parameters.output_profile != PP_AUDIOPROFILE_OPUS)
111 return false;
112
113 DCHECK(!encoder_memory_);
114
115 int32_t encoder_size = opus_encoder_get_size(parameters.channels);
116 if (encoder_size < 1)
117 return false;
118
119 std::unique_ptr<uint8_t[]> encoder_memory(new uint8_t[encoder_size]);
120 opus_encoder_ = reinterpret_cast<OpusEncoder*>(encoder_memory.get());
121
122 if (opus_encoder_init(opus_encoder_, parameters.input_sample_rate,
123 parameters.channels, OPUS_APPLICATION_AUDIO) != OPUS_OK)
124 return false;
125
126 if (opus_encoder_ctl(opus_encoder_,
127 OPUS_SET_BITRATE(parameters.initial_bitrate <= 0
128 ? OPUS_AUTO
129 : parameters.initial_bitrate)) !=
130 OPUS_OK)
131 return false;
132
133 encoder_memory_.swap(encoder_memory);
134 parameters_ = parameters;
135
136 return true;
137 }
138
GetNumberOfSamplesPerFrame()139 int32_t PepperAudioEncoderHost::AudioEncoderImpl::GetNumberOfSamplesPerFrame() {
140 DCHECK(encoder_memory_);
141 // Opus supports 2.5, 5, 10, 20, 40 or 60ms audio frames. We take
142 // 10ms by default.
143 return parameters_.input_sample_rate / 100;
144 }
145
Encode(uint8_t * input_data,size_t input_size,uint8_t * output_data,size_t output_size,BitstreamBufferReadyCB callback)146 void PepperAudioEncoderHost::AudioEncoderImpl::Encode(
147 uint8_t* input_data,
148 size_t input_size,
149 uint8_t* output_data,
150 size_t output_size,
151 BitstreamBufferReadyCB callback) {
152 DCHECK(encoder_memory_);
153 int32_t result = opus_encode(
154 opus_encoder_, reinterpret_cast<opus_int16*>(input_data),
155 (input_size / parameters_.channels) / parameters_.input_sample_size,
156 output_data, output_size);
157 std::move(callback).Run(result);
158 }
159
RequestBitrateChange(uint32_t bitrate)160 void PepperAudioEncoderHost::AudioEncoderImpl::RequestBitrateChange(
161 uint32_t bitrate) {
162 DCHECK(encoder_memory_);
163 opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate));
164 }
165
PepperAudioEncoderHost(RendererPpapiHost * host,PP_Instance instance,PP_Resource resource)166 PepperAudioEncoderHost::PepperAudioEncoderHost(RendererPpapiHost* host,
167 PP_Instance instance,
168 PP_Resource resource)
169 : ResourceHost(host->GetPpapiHost(), instance, resource),
170 renderer_ppapi_host_(host),
171 initialized_(false),
172 encoder_last_error_(PP_ERROR_FAILED),
173 media_task_runner_(
174 RenderThreadImpl::current()->GetMediaThreadTaskRunner()) {}
175
~PepperAudioEncoderHost()176 PepperAudioEncoderHost::~PepperAudioEncoderHost() {
177 Close();
178 }
179
OnResourceMessageReceived(const IPC::Message & msg,ppapi::host::HostMessageContext * context)180 int32_t PepperAudioEncoderHost::OnResourceMessageReceived(
181 const IPC::Message& msg,
182 ppapi::host::HostMessageContext* context) {
183 PPAPI_BEGIN_MESSAGE_MAP(PepperAudioEncoderHost, msg)
184 PPAPI_DISPATCH_HOST_RESOURCE_CALL_0(
185 PpapiHostMsg_AudioEncoder_GetSupportedProfiles,
186 OnHostMsgGetSupportedProfiles)
187 PPAPI_DISPATCH_HOST_RESOURCE_CALL(PpapiHostMsg_AudioEncoder_Initialize,
188 OnHostMsgInitialize)
189 PPAPI_DISPATCH_HOST_RESOURCE_CALL(PpapiHostMsg_AudioEncoder_Encode,
190 OnHostMsgEncode)
191 PPAPI_DISPATCH_HOST_RESOURCE_CALL(
192 PpapiHostMsg_AudioEncoder_RecycleBitstreamBuffer,
193 OnHostMsgRecycleBitstreamBuffer)
194 PPAPI_DISPATCH_HOST_RESOURCE_CALL(
195 PpapiHostMsg_AudioEncoder_RequestBitrateChange,
196 OnHostMsgRequestBitrateChange)
197 PPAPI_DISPATCH_HOST_RESOURCE_CALL_0(PpapiHostMsg_AudioEncoder_Close,
198 OnHostMsgClose)
199 PPAPI_END_MESSAGE_MAP()
200 return PP_ERROR_FAILED;
201 }
202
OnHostMsgGetSupportedProfiles(ppapi::host::HostMessageContext * context)203 int32_t PepperAudioEncoderHost::OnHostMsgGetSupportedProfiles(
204 ppapi::host::HostMessageContext* context) {
205 std::vector<PP_AudioProfileDescription> profiles;
206 GetSupportedProfiles(&profiles);
207
208 host()->SendReply(
209 context->MakeReplyMessageContext(),
210 PpapiPluginMsg_AudioEncoder_GetSupportedProfilesReply(profiles));
211
212 return PP_OK_COMPLETIONPENDING;
213 }
214
OnHostMsgInitialize(ppapi::host::HostMessageContext * context,const ppapi::proxy::PPB_AudioEncodeParameters & parameters)215 int32_t PepperAudioEncoderHost::OnHostMsgInitialize(
216 ppapi::host::HostMessageContext* context,
217 const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
218 if (initialized_)
219 return PP_ERROR_FAILED;
220
221 if (!IsInitializationValid(parameters))
222 return PP_ERROR_NOTSUPPORTED;
223
224 std::unique_ptr<AudioEncoderImpl> encoder(new AudioEncoderImpl);
225 if (!encoder->Initialize(parameters))
226 return PP_ERROR_FAILED;
227 if (!AllocateBuffers(parameters, encoder->GetNumberOfSamplesPerFrame()))
228 return PP_ERROR_NOMEMORY;
229
230 initialized_ = true;
231 encoder_last_error_ = PP_OK;
232 encoder_.swap(encoder);
233
234 ppapi::host::ReplyMessageContext reply_context =
235 context->MakeReplyMessageContext();
236 reply_context.params.AppendHandle(SerializedHandle(
237 renderer_ppapi_host_->ShareUnsafeSharedMemoryRegionWithRemote(
238 audio_buffer_manager_->region())));
239 reply_context.params.AppendHandle(SerializedHandle(
240 renderer_ppapi_host_->ShareUnsafeSharedMemoryRegionWithRemote(
241 bitstream_buffer_manager_->region())));
242 host()->SendReply(reply_context,
243 PpapiPluginMsg_AudioEncoder_InitializeReply(
244 encoder_->GetNumberOfSamplesPerFrame(),
245 audio_buffer_manager_->number_of_buffers(),
246 audio_buffer_manager_->buffer_size(),
247 bitstream_buffer_manager_->number_of_buffers(),
248 bitstream_buffer_manager_->buffer_size()));
249
250 return PP_OK_COMPLETIONPENDING;
251 }
252
OnHostMsgEncode(ppapi::host::HostMessageContext * context,int32_t buffer_id)253 int32_t PepperAudioEncoderHost::OnHostMsgEncode(
254 ppapi::host::HostMessageContext* context,
255 int32_t buffer_id) {
256 if (encoder_last_error_)
257 return encoder_last_error_;
258
259 if (buffer_id < 0 || buffer_id >= audio_buffer_manager_->number_of_buffers())
260 return PP_ERROR_FAILED;
261
262 audio_buffer_manager_->EnqueueBuffer(buffer_id);
263
264 DoEncode();
265
266 return PP_OK_COMPLETIONPENDING;
267 }
268
OnHostMsgRecycleBitstreamBuffer(ppapi::host::HostMessageContext * context,int32_t buffer_id)269 int32_t PepperAudioEncoderHost::OnHostMsgRecycleBitstreamBuffer(
270 ppapi::host::HostMessageContext* context,
271 int32_t buffer_id) {
272 if (encoder_last_error_)
273 return encoder_last_error_;
274
275 if (buffer_id < 0 ||
276 buffer_id >= bitstream_buffer_manager_->number_of_buffers())
277 return PP_ERROR_FAILED;
278
279 bitstream_buffer_manager_->EnqueueBuffer(buffer_id);
280
281 DoEncode();
282
283 return PP_OK;
284 }
285
OnHostMsgRequestBitrateChange(ppapi::host::HostMessageContext * context,uint32_t bitrate)286 int32_t PepperAudioEncoderHost::OnHostMsgRequestBitrateChange(
287 ppapi::host::HostMessageContext* context,
288 uint32_t bitrate) {
289 if (encoder_last_error_)
290 return encoder_last_error_;
291
292 media_task_runner_->PostTask(
293 FROM_HERE, base::BindOnce(&AudioEncoderImpl::RequestBitrateChange,
294 base::Unretained(encoder_.get()), bitrate));
295
296 return PP_OK;
297 }
298
OnHostMsgClose(ppapi::host::HostMessageContext * context)299 int32_t PepperAudioEncoderHost::OnHostMsgClose(
300 ppapi::host::HostMessageContext* context) {
301 encoder_last_error_ = PP_ERROR_FAILED;
302 Close();
303
304 return PP_OK;
305 }
306
GetSupportedProfiles(std::vector<PP_AudioProfileDescription> * profiles)307 void PepperAudioEncoderHost::GetSupportedProfiles(
308 std::vector<PP_AudioProfileDescription>* profiles) {
309 DCHECK(RenderThreadImpl::current());
310
311 *profiles = AudioEncoderImpl::GetSupportedProfiles();
312 }
313
IsInitializationValid(const ppapi::proxy::PPB_AudioEncodeParameters & parameters)314 bool PepperAudioEncoderHost::IsInitializationValid(
315 const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
316 DCHECK(RenderThreadImpl::current());
317
318 std::vector<PP_AudioProfileDescription> profiles;
319 GetSupportedProfiles(&profiles);
320
321 for (const PP_AudioProfileDescription& profile : profiles) {
322 if (parameters.output_profile == profile.profile &&
323 parameters.input_sample_size == profile.sample_size &&
324 parameters.input_sample_rate == profile.sample_rate &&
325 parameters.channels <= profile.max_channels &&
326 PP_HardwareAccelerationCompatibleAudio(
327 profile.hardware_accelerated == PP_TRUE, parameters.acceleration))
328 return true;
329 }
330
331 return false;
332 }
333
AllocateBuffers(const ppapi::proxy::PPB_AudioEncodeParameters & parameters,int32_t samples_per_frame)334 bool PepperAudioEncoderHost::AllocateBuffers(
335 const ppapi::proxy::PPB_AudioEncodeParameters& parameters,
336 int32_t samples_per_frame) {
337 DCHECK(RenderThreadImpl::current());
338
339 // Audio buffers size computation.
340 base::CheckedNumeric<uint32_t> audio_buffer_size = samples_per_frame;
341 audio_buffer_size *= parameters.channels;
342 audio_buffer_size *= parameters.input_sample_size;
343
344 base::CheckedNumeric<uint32_t> total_audio_buffer_size = audio_buffer_size;
345 total_audio_buffer_size += sizeof(ppapi::MediaStreamBuffer::Audio);
346 base::CheckedNumeric<size_t> total_audio_memory_size =
347 total_audio_buffer_size;
348 total_audio_memory_size *= kDefaultNumberOfAudioBuffers;
349
350 // Bitstream buffers size computation (individual bitstream buffers are
351 // twice the size of the raw data, to handle the worst case where
352 // compression doesn't work).
353 base::CheckedNumeric<uint32_t> bitstream_buffer_size = audio_buffer_size;
354 bitstream_buffer_size *= 2;
355 bitstream_buffer_size += sizeof(ppapi::MediaStreamBuffer::Bitstream);
356 base::CheckedNumeric<size_t> total_bitstream_memory_size =
357 bitstream_buffer_size;
358 total_bitstream_memory_size *= kDefaultNumberOfAudioBuffers;
359
360 if (!total_audio_memory_size.IsValid() ||
361 !total_bitstream_memory_size.IsValid())
362 return false;
363
364 base::UnsafeSharedMemoryRegion audio_region =
365 base::UnsafeSharedMemoryRegion::Create(
366 total_audio_memory_size.ValueOrDie());
367 if (!audio_region.IsValid())
368 return false;
369 std::unique_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager(
370 new ppapi::MediaStreamBufferManager(this));
371 if (!audio_buffer_manager->SetBuffers(
372 kDefaultNumberOfAudioBuffers,
373 base::ValueOrDieForType<int32_t>(total_audio_buffer_size),
374 std::move(audio_region), false))
375 return false;
376
377 for (int32_t i = 0; i < audio_buffer_manager->number_of_buffers(); ++i) {
378 ppapi::MediaStreamBuffer::Audio* buffer =
379 &(audio_buffer_manager->GetBufferPointer(i)->audio);
380 buffer->header.size = total_audio_buffer_size.ValueOrDie();
381 buffer->header.type = ppapi::MediaStreamBuffer::TYPE_AUDIO;
382 buffer->sample_rate =
383 static_cast<PP_AudioBuffer_SampleRate>(parameters.input_sample_rate);
384 buffer->number_of_channels = parameters.channels;
385 buffer->number_of_samples = samples_per_frame;
386 buffer->data_size = audio_buffer_size.ValueOrDie();
387 }
388
389 base::UnsafeSharedMemoryRegion bitstream_region =
390 base::UnsafeSharedMemoryRegion::Create(
391 total_bitstream_memory_size.ValueOrDie());
392 if (!bitstream_region.IsValid())
393 return false;
394 std::unique_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager(
395 new ppapi::MediaStreamBufferManager(this));
396 if (!bitstream_buffer_manager->SetBuffers(
397 kDefaultNumberOfAudioBuffers,
398 base::ValueOrDieForType<int32_t>(bitstream_buffer_size),
399 std::move(bitstream_region), true))
400 return false;
401
402 for (int32_t i = 0; i < bitstream_buffer_manager->number_of_buffers(); ++i) {
403 ppapi::MediaStreamBuffer::Bitstream* buffer =
404 &(bitstream_buffer_manager->GetBufferPointer(i)->bitstream);
405 buffer->header.size = bitstream_buffer_size.ValueOrDie();
406 buffer->header.type = ppapi::MediaStreamBuffer::TYPE_BITSTREAM;
407 }
408
409 audio_buffer_manager_.swap(audio_buffer_manager);
410 bitstream_buffer_manager_.swap(bitstream_buffer_manager);
411
412 return true;
413 }
414
DoEncode()415 void PepperAudioEncoderHost::DoEncode() {
416 DCHECK(RenderThreadImpl::current());
417 DCHECK(!encoder_last_error_);
418
419 if (!audio_buffer_manager_->HasAvailableBuffer() ||
420 !bitstream_buffer_manager_->HasAvailableBuffer())
421 return;
422
423 int32_t audio_buffer_id = audio_buffer_manager_->DequeueBuffer();
424 int32_t bitstream_buffer_id = bitstream_buffer_manager_->DequeueBuffer();
425
426 ppapi::MediaStreamBuffer* audio_buffer =
427 audio_buffer_manager_->GetBufferPointer(audio_buffer_id);
428 ppapi::MediaStreamBuffer* bitstream_buffer =
429 bitstream_buffer_manager_->GetBufferPointer(bitstream_buffer_id);
430
431 media_task_runner_->PostTask(
432 FROM_HERE,
433 base::BindOnce(&AudioEncoderImpl::Encode,
434 base::Unretained(encoder_.get()),
435 static_cast<uint8_t*>(audio_buffer->audio.data),
436 audio_buffer_manager_->buffer_size() -
437 sizeof(ppapi::MediaStreamBuffer::Audio),
438 static_cast<uint8_t*>(bitstream_buffer->bitstream.data),
439 bitstream_buffer_manager_->buffer_size() -
440 sizeof(ppapi::MediaStreamBuffer::Bitstream),
441 media::BindToCurrentLoop(base::BindOnce(
442 &PepperAudioEncoderHost::BitstreamBufferReady,
443 weak_ptr_factory_.GetWeakPtr(), audio_buffer_id,
444 bitstream_buffer_id))));
445 }
446
BitstreamBufferReady(int32_t audio_buffer_id,int32_t bitstream_buffer_id,int32_t result)447 void PepperAudioEncoderHost::BitstreamBufferReady(int32_t audio_buffer_id,
448 int32_t bitstream_buffer_id,
449 int32_t result) {
450 DCHECK(RenderThreadImpl::current());
451
452 if (encoder_last_error_)
453 return;
454
455 if (result < 0) {
456 NotifyPepperError(PP_ERROR_FAILED);
457 return;
458 }
459
460 host()->SendUnsolicitedReply(
461 pp_resource(), PpapiPluginMsg_AudioEncoder_EncodeReply(audio_buffer_id));
462
463 ppapi::MediaStreamBuffer::Bitstream* buffer =
464 &(bitstream_buffer_manager_->GetBufferPointer(bitstream_buffer_id)
465 ->bitstream);
466 buffer->data_size = static_cast<uint32_t>(result);
467
468 host()->SendUnsolicitedReply(
469 pp_resource(),
470 PpapiPluginMsg_AudioEncoder_BitstreamBufferReady(bitstream_buffer_id));
471 }
472
NotifyPepperError(int32_t error)473 void PepperAudioEncoderHost::NotifyPepperError(int32_t error) {
474 DCHECK(RenderThreadImpl::current());
475
476 encoder_last_error_ = error;
477 Close();
478 host()->SendUnsolicitedReply(
479 pp_resource(),
480 PpapiPluginMsg_AudioEncoder_NotifyError(encoder_last_error_));
481 }
482
Close()483 void PepperAudioEncoderHost::Close() {
484 DCHECK(RenderThreadImpl::current());
485
486 // Destroy the encoder and the audio/bitstream buffers on the media thread
487 // to avoid freeing memory the encoder might still be working on.
488 media_task_runner_->PostTask(
489 FROM_HERE, base::BindOnce(&StopAudioEncoder, std::move(encoder_),
490 std::move(audio_buffer_manager_),
491 std::move(bitstream_buffer_manager_)));
492 }
493
494 // static
StopAudioEncoder(std::unique_ptr<AudioEncoderImpl> encoder,std::unique_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager,std::unique_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager)495 void PepperAudioEncoderHost::StopAudioEncoder(
496 std::unique_ptr<AudioEncoderImpl> encoder,
497 std::unique_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager,
498 std::unique_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager) {
499 }
500
501 } // namespace content
502