1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "content/renderer/pepper/pepper_audio_encoder_host.h"
6 
7 #include <stddef.h>
8 #include <utility>
9 
10 #include "base/bind.h"
11 #include "base/memory/unsafe_shared_memory_region.h"
12 #include "base/stl_util.h"
13 #include "content/public/renderer/renderer_ppapi_host.h"
14 #include "content/renderer/pepper/host_globals.h"
15 #include "content/renderer/render_thread_impl.h"
16 #include "media/base/bind_to_current_loop.h"
17 #include "ppapi/c/pp_codecs.h"
18 #include "ppapi/c/pp_errors.h"
19 #include "ppapi/host/dispatch_host_message.h"
20 #include "ppapi/host/ppapi_host.h"
21 #include "ppapi/proxy/ppapi_messages.h"
22 #include "ppapi/shared_impl/media_stream_buffer.h"
23 #include "third_party/opus/src/include/opus.h"
24 
25 using ppapi::proxy::SerializedHandle;
26 
27 namespace content {
28 
29 namespace {
30 
31 // Buffer up to 150ms (15 x 10ms per frame).
32 const uint32_t kDefaultNumberOfAudioBuffers = 15;
33 
PP_HardwareAccelerationCompatibleAudio(bool accelerated,PP_HardwareAcceleration requested)34 bool PP_HardwareAccelerationCompatibleAudio(bool accelerated,
35                                             PP_HardwareAcceleration requested) {
36   switch (requested) {
37     case PP_HARDWAREACCELERATION_ONLY:
38       return accelerated;
39     case PP_HARDWAREACCELERATION_NONE:
40       return !accelerated;
41     case PP_HARDWAREACCELERATION_WITHFALLBACK:
42       return true;
43       // No default case, to catch unhandled PP_HardwareAcceleration values.
44   }
45   return false;
46 }
47 
48 }  // namespace
49 
50 // This class should be constructed and initialized on the main renderer
51 // thread, used and destructed on the media thread.
52 class PepperAudioEncoderHost::AudioEncoderImpl {
53  public:
54   // Callback used to signal encoded data. If |size| is negative, an error
55   // occurred.
56   using BitstreamBufferReadyCB = base::OnceCallback<void(int32_t size)>;
57 
58   AudioEncoderImpl();
59   ~AudioEncoderImpl();
60 
61   // Used on the renderer thread.
62   static std::vector<PP_AudioProfileDescription> GetSupportedProfiles();
63   bool Initialize(const ppapi::proxy::PPB_AudioEncodeParameters& parameters);
64   int32_t GetNumberOfSamplesPerFrame();
65 
66   // Used on the media thread.
67   void Encode(uint8_t* input_data,
68               size_t input_size,
69               uint8_t* output_data,
70               size_t output_size,
71               BitstreamBufferReadyCB callback);
72   void RequestBitrateChange(uint32_t bitrate);
73 
74  private:
75   std::unique_ptr<uint8_t[]> encoder_memory_;
76   OpusEncoder* opus_encoder_;
77 
78   // Initialization parameters, only valid if |encoder_memory_| is not
79   // nullptr.
80   ppapi::proxy::PPB_AudioEncodeParameters parameters_;
81 
82   DISALLOW_COPY_AND_ASSIGN(AudioEncoderImpl);
83 };
84 
AudioEncoderImpl()85 PepperAudioEncoderHost::AudioEncoderImpl::AudioEncoderImpl()
86     : opus_encoder_(nullptr) {}
87 
~AudioEncoderImpl()88 PepperAudioEncoderHost::AudioEncoderImpl::~AudioEncoderImpl() {}
89 
90 // static
91 std::vector<PP_AudioProfileDescription>
GetSupportedProfiles()92 PepperAudioEncoderHost::AudioEncoderImpl::GetSupportedProfiles() {
93   std::vector<PP_AudioProfileDescription> profiles;
94   static const uint32_t sampling_rates[] = {8000, 12000, 16000, 24000, 48000};
95 
96   for (uint32_t i = 0; i < base::size(sampling_rates); ++i) {
97     PP_AudioProfileDescription profile;
98     profile.profile = PP_AUDIOPROFILE_OPUS;
99     profile.max_channels = 2;
100     profile.sample_size = PP_AUDIOBUFFER_SAMPLESIZE_16_BITS;
101     profile.sample_rate = sampling_rates[i];
102     profile.hardware_accelerated = PP_FALSE;
103     profiles.push_back(profile);
104   }
105   return profiles;
106 }
107 
Initialize(const ppapi::proxy::PPB_AudioEncodeParameters & parameters)108 bool PepperAudioEncoderHost::AudioEncoderImpl::Initialize(
109     const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
110   if (parameters.output_profile != PP_AUDIOPROFILE_OPUS)
111     return false;
112 
113   DCHECK(!encoder_memory_);
114 
115   int32_t encoder_size = opus_encoder_get_size(parameters.channels);
116   if (encoder_size < 1)
117     return false;
118 
119   std::unique_ptr<uint8_t[]> encoder_memory(new uint8_t[encoder_size]);
120   opus_encoder_ = reinterpret_cast<OpusEncoder*>(encoder_memory.get());
121 
122   if (opus_encoder_init(opus_encoder_, parameters.input_sample_rate,
123                         parameters.channels, OPUS_APPLICATION_AUDIO) != OPUS_OK)
124     return false;
125 
126   if (opus_encoder_ctl(opus_encoder_,
127                        OPUS_SET_BITRATE(parameters.initial_bitrate <= 0
128                                             ? OPUS_AUTO
129                                             : parameters.initial_bitrate)) !=
130       OPUS_OK)
131     return false;
132 
133   encoder_memory_.swap(encoder_memory);
134   parameters_ = parameters;
135 
136   return true;
137 }
138 
GetNumberOfSamplesPerFrame()139 int32_t PepperAudioEncoderHost::AudioEncoderImpl::GetNumberOfSamplesPerFrame() {
140   DCHECK(encoder_memory_);
141   // Opus supports 2.5, 5, 10, 20, 40 or 60ms audio frames. We take
142   // 10ms by default.
143   return parameters_.input_sample_rate / 100;
144 }
145 
Encode(uint8_t * input_data,size_t input_size,uint8_t * output_data,size_t output_size,BitstreamBufferReadyCB callback)146 void PepperAudioEncoderHost::AudioEncoderImpl::Encode(
147     uint8_t* input_data,
148     size_t input_size,
149     uint8_t* output_data,
150     size_t output_size,
151     BitstreamBufferReadyCB callback) {
152   DCHECK(encoder_memory_);
153   int32_t result = opus_encode(
154       opus_encoder_, reinterpret_cast<opus_int16*>(input_data),
155       (input_size / parameters_.channels) / parameters_.input_sample_size,
156       output_data, output_size);
157   std::move(callback).Run(result);
158 }
159 
RequestBitrateChange(uint32_t bitrate)160 void PepperAudioEncoderHost::AudioEncoderImpl::RequestBitrateChange(
161     uint32_t bitrate) {
162   DCHECK(encoder_memory_);
163   opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate));
164 }
165 
PepperAudioEncoderHost(RendererPpapiHost * host,PP_Instance instance,PP_Resource resource)166 PepperAudioEncoderHost::PepperAudioEncoderHost(RendererPpapiHost* host,
167                                                PP_Instance instance,
168                                                PP_Resource resource)
169     : ResourceHost(host->GetPpapiHost(), instance, resource),
170       renderer_ppapi_host_(host),
171       initialized_(false),
172       encoder_last_error_(PP_ERROR_FAILED),
173       media_task_runner_(
174           RenderThreadImpl::current()->GetMediaThreadTaskRunner()) {}
175 
~PepperAudioEncoderHost()176 PepperAudioEncoderHost::~PepperAudioEncoderHost() {
177   Close();
178 }
179 
OnResourceMessageReceived(const IPC::Message & msg,ppapi::host::HostMessageContext * context)180 int32_t PepperAudioEncoderHost::OnResourceMessageReceived(
181     const IPC::Message& msg,
182     ppapi::host::HostMessageContext* context) {
183   PPAPI_BEGIN_MESSAGE_MAP(PepperAudioEncoderHost, msg)
184     PPAPI_DISPATCH_HOST_RESOURCE_CALL_0(
185         PpapiHostMsg_AudioEncoder_GetSupportedProfiles,
186         OnHostMsgGetSupportedProfiles)
187     PPAPI_DISPATCH_HOST_RESOURCE_CALL(PpapiHostMsg_AudioEncoder_Initialize,
188                                       OnHostMsgInitialize)
189     PPAPI_DISPATCH_HOST_RESOURCE_CALL(PpapiHostMsg_AudioEncoder_Encode,
190                                       OnHostMsgEncode)
191     PPAPI_DISPATCH_HOST_RESOURCE_CALL(
192         PpapiHostMsg_AudioEncoder_RecycleBitstreamBuffer,
193         OnHostMsgRecycleBitstreamBuffer)
194     PPAPI_DISPATCH_HOST_RESOURCE_CALL(
195         PpapiHostMsg_AudioEncoder_RequestBitrateChange,
196         OnHostMsgRequestBitrateChange)
197     PPAPI_DISPATCH_HOST_RESOURCE_CALL_0(PpapiHostMsg_AudioEncoder_Close,
198                                         OnHostMsgClose)
199   PPAPI_END_MESSAGE_MAP()
200   return PP_ERROR_FAILED;
201 }
202 
OnHostMsgGetSupportedProfiles(ppapi::host::HostMessageContext * context)203 int32_t PepperAudioEncoderHost::OnHostMsgGetSupportedProfiles(
204     ppapi::host::HostMessageContext* context) {
205   std::vector<PP_AudioProfileDescription> profiles;
206   GetSupportedProfiles(&profiles);
207 
208   host()->SendReply(
209       context->MakeReplyMessageContext(),
210       PpapiPluginMsg_AudioEncoder_GetSupportedProfilesReply(profiles));
211 
212   return PP_OK_COMPLETIONPENDING;
213 }
214 
OnHostMsgInitialize(ppapi::host::HostMessageContext * context,const ppapi::proxy::PPB_AudioEncodeParameters & parameters)215 int32_t PepperAudioEncoderHost::OnHostMsgInitialize(
216     ppapi::host::HostMessageContext* context,
217     const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
218   if (initialized_)
219     return PP_ERROR_FAILED;
220 
221   if (!IsInitializationValid(parameters))
222     return PP_ERROR_NOTSUPPORTED;
223 
224   std::unique_ptr<AudioEncoderImpl> encoder(new AudioEncoderImpl);
225   if (!encoder->Initialize(parameters))
226     return PP_ERROR_FAILED;
227   if (!AllocateBuffers(parameters, encoder->GetNumberOfSamplesPerFrame()))
228     return PP_ERROR_NOMEMORY;
229 
230   initialized_ = true;
231   encoder_last_error_ = PP_OK;
232   encoder_.swap(encoder);
233 
234   ppapi::host::ReplyMessageContext reply_context =
235       context->MakeReplyMessageContext();
236   reply_context.params.AppendHandle(SerializedHandle(
237       renderer_ppapi_host_->ShareUnsafeSharedMemoryRegionWithRemote(
238           audio_buffer_manager_->region())));
239   reply_context.params.AppendHandle(SerializedHandle(
240       renderer_ppapi_host_->ShareUnsafeSharedMemoryRegionWithRemote(
241           bitstream_buffer_manager_->region())));
242   host()->SendReply(reply_context,
243                     PpapiPluginMsg_AudioEncoder_InitializeReply(
244                         encoder_->GetNumberOfSamplesPerFrame(),
245                         audio_buffer_manager_->number_of_buffers(),
246                         audio_buffer_manager_->buffer_size(),
247                         bitstream_buffer_manager_->number_of_buffers(),
248                         bitstream_buffer_manager_->buffer_size()));
249 
250   return PP_OK_COMPLETIONPENDING;
251 }
252 
OnHostMsgEncode(ppapi::host::HostMessageContext * context,int32_t buffer_id)253 int32_t PepperAudioEncoderHost::OnHostMsgEncode(
254     ppapi::host::HostMessageContext* context,
255     int32_t buffer_id) {
256   if (encoder_last_error_)
257     return encoder_last_error_;
258 
259   if (buffer_id < 0 || buffer_id >= audio_buffer_manager_->number_of_buffers())
260     return PP_ERROR_FAILED;
261 
262   audio_buffer_manager_->EnqueueBuffer(buffer_id);
263 
264   DoEncode();
265 
266   return PP_OK_COMPLETIONPENDING;
267 }
268 
OnHostMsgRecycleBitstreamBuffer(ppapi::host::HostMessageContext * context,int32_t buffer_id)269 int32_t PepperAudioEncoderHost::OnHostMsgRecycleBitstreamBuffer(
270     ppapi::host::HostMessageContext* context,
271     int32_t buffer_id) {
272   if (encoder_last_error_)
273     return encoder_last_error_;
274 
275   if (buffer_id < 0 ||
276       buffer_id >= bitstream_buffer_manager_->number_of_buffers())
277     return PP_ERROR_FAILED;
278 
279   bitstream_buffer_manager_->EnqueueBuffer(buffer_id);
280 
281   DoEncode();
282 
283   return PP_OK;
284 }
285 
OnHostMsgRequestBitrateChange(ppapi::host::HostMessageContext * context,uint32_t bitrate)286 int32_t PepperAudioEncoderHost::OnHostMsgRequestBitrateChange(
287     ppapi::host::HostMessageContext* context,
288     uint32_t bitrate) {
289   if (encoder_last_error_)
290     return encoder_last_error_;
291 
292   media_task_runner_->PostTask(
293       FROM_HERE, base::BindOnce(&AudioEncoderImpl::RequestBitrateChange,
294                                 base::Unretained(encoder_.get()), bitrate));
295 
296   return PP_OK;
297 }
298 
OnHostMsgClose(ppapi::host::HostMessageContext * context)299 int32_t PepperAudioEncoderHost::OnHostMsgClose(
300     ppapi::host::HostMessageContext* context) {
301   encoder_last_error_ = PP_ERROR_FAILED;
302   Close();
303 
304   return PP_OK;
305 }
306 
GetSupportedProfiles(std::vector<PP_AudioProfileDescription> * profiles)307 void PepperAudioEncoderHost::GetSupportedProfiles(
308     std::vector<PP_AudioProfileDescription>* profiles) {
309   DCHECK(RenderThreadImpl::current());
310 
311   *profiles = AudioEncoderImpl::GetSupportedProfiles();
312 }
313 
IsInitializationValid(const ppapi::proxy::PPB_AudioEncodeParameters & parameters)314 bool PepperAudioEncoderHost::IsInitializationValid(
315     const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
316   DCHECK(RenderThreadImpl::current());
317 
318   std::vector<PP_AudioProfileDescription> profiles;
319   GetSupportedProfiles(&profiles);
320 
321   for (const PP_AudioProfileDescription& profile : profiles) {
322     if (parameters.output_profile == profile.profile &&
323         parameters.input_sample_size == profile.sample_size &&
324         parameters.input_sample_rate == profile.sample_rate &&
325         parameters.channels <= profile.max_channels &&
326         PP_HardwareAccelerationCompatibleAudio(
327             profile.hardware_accelerated == PP_TRUE, parameters.acceleration))
328       return true;
329   }
330 
331   return false;
332 }
333 
AllocateBuffers(const ppapi::proxy::PPB_AudioEncodeParameters & parameters,int32_t samples_per_frame)334 bool PepperAudioEncoderHost::AllocateBuffers(
335     const ppapi::proxy::PPB_AudioEncodeParameters& parameters,
336     int32_t samples_per_frame) {
337   DCHECK(RenderThreadImpl::current());
338 
339   // Audio buffers size computation.
340   base::CheckedNumeric<uint32_t> audio_buffer_size = samples_per_frame;
341   audio_buffer_size *= parameters.channels;
342   audio_buffer_size *= parameters.input_sample_size;
343 
344   base::CheckedNumeric<uint32_t> total_audio_buffer_size = audio_buffer_size;
345   total_audio_buffer_size += sizeof(ppapi::MediaStreamBuffer::Audio);
346   base::CheckedNumeric<size_t> total_audio_memory_size =
347       total_audio_buffer_size;
348   total_audio_memory_size *= kDefaultNumberOfAudioBuffers;
349 
350   // Bitstream buffers size computation (individual bitstream buffers are
351   // twice the size of the raw data, to handle the worst case where
352   // compression doesn't work).
353   base::CheckedNumeric<uint32_t> bitstream_buffer_size = audio_buffer_size;
354   bitstream_buffer_size *= 2;
355   bitstream_buffer_size += sizeof(ppapi::MediaStreamBuffer::Bitstream);
356   base::CheckedNumeric<size_t> total_bitstream_memory_size =
357       bitstream_buffer_size;
358   total_bitstream_memory_size *= kDefaultNumberOfAudioBuffers;
359 
360   if (!total_audio_memory_size.IsValid() ||
361       !total_bitstream_memory_size.IsValid())
362     return false;
363 
364   base::UnsafeSharedMemoryRegion audio_region =
365       base::UnsafeSharedMemoryRegion::Create(
366           total_audio_memory_size.ValueOrDie());
367   if (!audio_region.IsValid())
368     return false;
369   std::unique_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager(
370       new ppapi::MediaStreamBufferManager(this));
371   if (!audio_buffer_manager->SetBuffers(
372           kDefaultNumberOfAudioBuffers,
373           base::ValueOrDieForType<int32_t>(total_audio_buffer_size),
374           std::move(audio_region), false))
375     return false;
376 
377   for (int32_t i = 0; i < audio_buffer_manager->number_of_buffers(); ++i) {
378     ppapi::MediaStreamBuffer::Audio* buffer =
379         &(audio_buffer_manager->GetBufferPointer(i)->audio);
380     buffer->header.size = total_audio_buffer_size.ValueOrDie();
381     buffer->header.type = ppapi::MediaStreamBuffer::TYPE_AUDIO;
382     buffer->sample_rate =
383         static_cast<PP_AudioBuffer_SampleRate>(parameters.input_sample_rate);
384     buffer->number_of_channels = parameters.channels;
385     buffer->number_of_samples = samples_per_frame;
386     buffer->data_size = audio_buffer_size.ValueOrDie();
387   }
388 
389   base::UnsafeSharedMemoryRegion bitstream_region =
390       base::UnsafeSharedMemoryRegion::Create(
391           total_bitstream_memory_size.ValueOrDie());
392   if (!bitstream_region.IsValid())
393     return false;
394   std::unique_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager(
395       new ppapi::MediaStreamBufferManager(this));
396   if (!bitstream_buffer_manager->SetBuffers(
397           kDefaultNumberOfAudioBuffers,
398           base::ValueOrDieForType<int32_t>(bitstream_buffer_size),
399           std::move(bitstream_region), true))
400     return false;
401 
402   for (int32_t i = 0; i < bitstream_buffer_manager->number_of_buffers(); ++i) {
403     ppapi::MediaStreamBuffer::Bitstream* buffer =
404         &(bitstream_buffer_manager->GetBufferPointer(i)->bitstream);
405     buffer->header.size = bitstream_buffer_size.ValueOrDie();
406     buffer->header.type = ppapi::MediaStreamBuffer::TYPE_BITSTREAM;
407   }
408 
409   audio_buffer_manager_.swap(audio_buffer_manager);
410   bitstream_buffer_manager_.swap(bitstream_buffer_manager);
411 
412   return true;
413 }
414 
DoEncode()415 void PepperAudioEncoderHost::DoEncode() {
416   DCHECK(RenderThreadImpl::current());
417   DCHECK(!encoder_last_error_);
418 
419   if (!audio_buffer_manager_->HasAvailableBuffer() ||
420       !bitstream_buffer_manager_->HasAvailableBuffer())
421     return;
422 
423   int32_t audio_buffer_id = audio_buffer_manager_->DequeueBuffer();
424   int32_t bitstream_buffer_id = bitstream_buffer_manager_->DequeueBuffer();
425 
426   ppapi::MediaStreamBuffer* audio_buffer =
427       audio_buffer_manager_->GetBufferPointer(audio_buffer_id);
428   ppapi::MediaStreamBuffer* bitstream_buffer =
429       bitstream_buffer_manager_->GetBufferPointer(bitstream_buffer_id);
430 
431   media_task_runner_->PostTask(
432       FROM_HERE,
433       base::BindOnce(&AudioEncoderImpl::Encode,
434                      base::Unretained(encoder_.get()),
435                      static_cast<uint8_t*>(audio_buffer->audio.data),
436                      audio_buffer_manager_->buffer_size() -
437                          sizeof(ppapi::MediaStreamBuffer::Audio),
438                      static_cast<uint8_t*>(bitstream_buffer->bitstream.data),
439                      bitstream_buffer_manager_->buffer_size() -
440                          sizeof(ppapi::MediaStreamBuffer::Bitstream),
441                      media::BindToCurrentLoop(base::BindOnce(
442                          &PepperAudioEncoderHost::BitstreamBufferReady,
443                          weak_ptr_factory_.GetWeakPtr(), audio_buffer_id,
444                          bitstream_buffer_id))));
445 }
446 
BitstreamBufferReady(int32_t audio_buffer_id,int32_t bitstream_buffer_id,int32_t result)447 void PepperAudioEncoderHost::BitstreamBufferReady(int32_t audio_buffer_id,
448                                                   int32_t bitstream_buffer_id,
449                                                   int32_t result) {
450   DCHECK(RenderThreadImpl::current());
451 
452   if (encoder_last_error_)
453     return;
454 
455   if (result < 0) {
456     NotifyPepperError(PP_ERROR_FAILED);
457     return;
458   }
459 
460   host()->SendUnsolicitedReply(
461       pp_resource(), PpapiPluginMsg_AudioEncoder_EncodeReply(audio_buffer_id));
462 
463   ppapi::MediaStreamBuffer::Bitstream* buffer =
464       &(bitstream_buffer_manager_->GetBufferPointer(bitstream_buffer_id)
465             ->bitstream);
466   buffer->data_size = static_cast<uint32_t>(result);
467 
468   host()->SendUnsolicitedReply(
469       pp_resource(),
470       PpapiPluginMsg_AudioEncoder_BitstreamBufferReady(bitstream_buffer_id));
471 }
472 
NotifyPepperError(int32_t error)473 void PepperAudioEncoderHost::NotifyPepperError(int32_t error) {
474   DCHECK(RenderThreadImpl::current());
475 
476   encoder_last_error_ = error;
477   Close();
478   host()->SendUnsolicitedReply(
479       pp_resource(),
480       PpapiPluginMsg_AudioEncoder_NotifyError(encoder_last_error_));
481 }
482 
Close()483 void PepperAudioEncoderHost::Close() {
484   DCHECK(RenderThreadImpl::current());
485 
486   // Destroy the encoder and the audio/bitstream buffers on the media thread
487   // to avoid freeing memory the encoder might still be working on.
488   media_task_runner_->PostTask(
489       FROM_HERE, base::BindOnce(&StopAudioEncoder, std::move(encoder_),
490                                 std::move(audio_buffer_manager_),
491                                 std::move(bitstream_buffer_manager_)));
492 }
493 
494 // static
StopAudioEncoder(std::unique_ptr<AudioEncoderImpl> encoder,std::unique_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager,std::unique_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager)495 void PepperAudioEncoderHost::StopAudioEncoder(
496     std::unique_ptr<AudioEncoderImpl> encoder,
497     std::unique_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager,
498     std::unique_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager) {
499 }
500 
501 }  // namespace content
502