1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "chromecast/media/cma/backend/mixer/audio_output_redirector.h"
6 
7 #include <algorithm>
8 #include <limits>
9 #include <utility>
10 
11 #include "base/bind.h"
12 #include "base/location.h"
13 #include "base/logging.h"
14 #include "base/numerics/ranges.h"
15 #include "base/sequenced_task_runner.h"
16 #include "base/strings/pattern.h"
17 #include "base/threading/sequenced_task_runner_handle.h"
18 #include "chromecast/media/audio/audio_fader.h"
19 #include "chromecast/media/audio/audio_log.h"
20 #include "chromecast/media/audio/mixer_service/conversions.h"
21 #include "chromecast/media/audio/mixer_service/mixer_service.pb.h"
22 #include "chromecast/media/audio/mixer_service/mixer_socket.h"
23 #include "chromecast/media/cma/backend/mixer/audio_output_redirector_input.h"
24 #include "chromecast/media/cma/backend/mixer/channel_layout.h"
25 #include "chromecast/media/cma/backend/mixer/mixer_input.h"
26 #include "chromecast/media/cma/backend/mixer/stream_mixer.h"
27 #include "chromecast/media/cma/base/decoder_config_adapter.h"
28 #include "media/base/audio_bus.h"
29 #include "media/base/channel_layout.h"
30 #include "media/base/channel_mixer.h"
31 
32 namespace chromecast {
33 namespace media {
34 
35 namespace {
36 
37 using Patterns = std::vector<std::pair<AudioContentType, std::string>>;
38 
39 constexpr int kDefaultBufferSize = 2048;
40 constexpr int kMaxChannels = 32;
41 
42 constexpr int kAudioMessageHeaderSize =
43     mixer_service::MixerSocket::kAudioMessageHeaderSize;
44 
GetMediaChannelLayout(chromecast::media::ChannelLayout layout,int num_channels)45 ::media::ChannelLayout GetMediaChannelLayout(
46     chromecast::media::ChannelLayout layout,
47     int num_channels) {
48   if (layout == media::ChannelLayout::UNSUPPORTED) {
49     return mixer::GuessChannelLayout(num_channels);
50   }
51   return DecoderConfigAdapter::ToMediaChannelLayout(layout);
52 }
53 
54 }  // namespace
55 
56 class AudioOutputRedirector::RedirectionConnection
57     : public mixer_service::MixerSocket::Delegate {
58  public:
RedirectionConnection(std::unique_ptr<mixer_service::MixerSocket> socket,scoped_refptr<base::SequencedTaskRunner> mixer_task_runner,base::WeakPtr<AudioOutputRedirector> redirector)59   explicit RedirectionConnection(
60       std::unique_ptr<mixer_service::MixerSocket> socket,
61       scoped_refptr<base::SequencedTaskRunner> mixer_task_runner,
62       base::WeakPtr<AudioOutputRedirector> redirector)
63       : socket_(std::move(socket)),
64         mixer_task_runner_(std::move(mixer_task_runner)),
65         redirector_(std::move(redirector)) {
66     DCHECK(socket_);
67     DCHECK(mixer_task_runner_);
68 
69     socket_->SetDelegate(this);
70   }
71 
72   ~RedirectionConnection() override = default;
73 
SetStreamConfig(SampleFormat sample_format,int sample_rate,int num_channels,int data_size)74   void SetStreamConfig(SampleFormat sample_format,
75                        int sample_rate,
76                        int num_channels,
77                        int data_size) {
78     mixer_service::Generic message;
79     mixer_service::StreamConfig* config = message.mutable_stream_config();
80     config->set_sample_format(
81         mixer_service::ConvertSampleFormat(sample_format));
82     config->set_sample_rate(sample_rate);
83     config->set_num_channels(num_channels);
84     config->set_data_size(data_size);
85     socket_->SendProto(message);
86 
87     sent_stream_config_ = true;
88   }
89 
SendAudio(scoped_refptr<net::IOBuffer> audio_buffer,int data_size_bytes,int64_t timestamp)90   void SendAudio(scoped_refptr<net::IOBuffer> audio_buffer,
91                  int data_size_bytes,
92                  int64_t timestamp) {
93     if (error_) {
94       return;
95     }
96     DCHECK(sent_stream_config_);
97     socket_->SendAudioBuffer(std::move(audio_buffer), data_size_bytes,
98                              timestamp);
99   }
100 
101   // mixer_service::MixerSocket::Delegate implementation:
HandleMetadata(const mixer_service::Generic & message)102   bool HandleMetadata(const mixer_service::Generic& message) override {
103     if (!message.has_redirected_stream_patterns()) {
104       return true;
105     }
106 
107     Patterns new_patterns;
108     for (const auto& p : message.redirected_stream_patterns().patterns()) {
109       new_patterns.emplace_back(ConvertContentType(p.content_type()),
110                                 p.device_id_pattern());
111     }
112     mixer_task_runner_->PostTask(
113         FROM_HERE, base::BindOnce(&AudioOutputRedirector::UpdatePatterns,
114                                   redirector_, std::move(new_patterns)));
115     return true;
116   }
117 
118  private:
HandleAudioData(char * data,size_t size,int64_t timestamp)119   bool HandleAudioData(char* data, size_t size, int64_t timestamp) override {
120     return true;
121   }
122 
OnConnectionError()123   void OnConnectionError() override {
124     if (error_) {
125       return;
126     }
127     error_ = true;
128     mixer_task_runner_->PostTask(
129         FROM_HERE,
130         base::BindOnce(&AudioOutputRedirector::OnConnectionError, redirector_));
131   }
132 
133   const std::unique_ptr<mixer_service::MixerSocket> socket_;
134   const scoped_refptr<base::SequencedTaskRunner> mixer_task_runner_;
135   const base::WeakPtr<AudioOutputRedirector> redirector_;
136 
137   bool error_ = false;
138   bool sent_stream_config_ = false;
139 
140   DISALLOW_COPY_AND_ASSIGN(RedirectionConnection);
141 };
142 
143 class AudioOutputRedirector::InputImpl : public AudioOutputRedirectorInput {
144  public:
145   using RenderingDelay = MediaPipelineBackend::AudioDecoder::RenderingDelay;
146 
147   InputImpl(AudioOutputRedirector* output_redirector, MixerInput* mixer_input);
148   ~InputImpl() override;
149 
150   // AudioOutputRedirectorInput implementation:
Order()151   int Order() override { return output_redirector_->order(); }
GetDelayMicroseconds()152   int64_t GetDelayMicroseconds() override {
153     return output_redirector_->extra_delay_microseconds();
154   }
155   void Redirect(::media::AudioBus* const buffer,
156                 int num_frames,
157                 RenderingDelay rendering_delay,
158                 bool redirected) override;
159 
160  private:
161   AudioOutputRedirector* const output_redirector_;
162   MixerInput* const mixer_input_;
163   const ::media::ChannelLayout output_channel_layout_;
164   const int num_output_channels_;
165 
166   bool previous_ended_in_silence_;
167 
168   std::unique_ptr<::media::ChannelMixer> channel_mixer_;
169   std::unique_ptr<::media::AudioBus> temp_buffer_;
170 
171   DISALLOW_COPY_AND_ASSIGN(InputImpl);
172 };
173 
InputImpl(AudioOutputRedirector * output_redirector,MixerInput * mixer_input)174 AudioOutputRedirector::InputImpl::InputImpl(
175     AudioOutputRedirector* output_redirector,
176     MixerInput* mixer_input)
177     : output_redirector_(output_redirector),
178       mixer_input_(mixer_input),
179       output_channel_layout_(output_redirector_->output_channel_layout()),
180       num_output_channels_(output_redirector_->num_output_channels()),
181       previous_ended_in_silence_(true) {
182   DCHECK_LE(num_output_channels_, kMaxChannels);
183   DCHECK(output_redirector_);
184   DCHECK(mixer_input_);
185 
186   if (mixer_input_->num_channels() != num_output_channels_) {
187     AUDIO_LOG(INFO) << "Remixing channels for " << mixer_input_->source()
188                     << " from " << mixer_input_->num_channels() << " to "
189                     << num_output_channels_;
190     channel_mixer_ = std::make_unique<::media::ChannelMixer>(
191         mixer::CreateAudioParametersForChannelMixer(
192             mixer_input_->channel_layout(), mixer_input_->num_channels()),
193         mixer::CreateAudioParametersForChannelMixer(output_channel_layout_,
194                                                     num_output_channels_));
195   }
196 
197   temp_buffer_ =
198       ::media::AudioBus::Create(num_output_channels_, kDefaultBufferSize);
199   temp_buffer_->Zero();
200 
201   mixer_input_->AddAudioOutputRedirector(this);
202 }
203 
~InputImpl()204 AudioOutputRedirector::InputImpl::~InputImpl() {
205   mixer_input_->RemoveAudioOutputRedirector(this);
206 }
207 
Redirect(::media::AudioBus * const buffer,int num_frames,RenderingDelay rendering_delay,bool redirected)208 void AudioOutputRedirector::InputImpl::Redirect(::media::AudioBus* const buffer,
209                                                 int num_frames,
210                                                 RenderingDelay rendering_delay,
211                                                 bool redirected) {
212   if (num_frames == 0) {
213     return;
214   }
215 
216   if (previous_ended_in_silence_ && redirected) {
217     // Previous buffer ended in silence, and the current buffer was redirected
218     // by a previous output splitter, so maintain silence.
219     return;
220   }
221   if (!previous_ended_in_silence_ && !redirected && !channel_mixer_) {
222     // No fading or channel mixing required, just mix directly.
223     output_redirector_->MixInput(mixer_input_, buffer, num_frames,
224                                  rendering_delay);
225     return;
226   }
227 
228   if (temp_buffer_->frames() < num_frames) {
229     temp_buffer_ = ::media::AudioBus::Create(
230         num_output_channels_, std::max(num_frames, kDefaultBufferSize));
231   }
232 
233   if (channel_mixer_) {
234     channel_mixer_->TransformPartial(buffer, num_frames, temp_buffer_.get());
235   } else {
236     buffer->CopyPartialFramesTo(0, num_frames, 0, temp_buffer_.get());
237   }
238 
239   float* channels[kMaxChannels];
240   for (int c = 0; c < num_output_channels_; ++c) {
241     channels[c] = temp_buffer_->channel(c);
242   }
243   if (previous_ended_in_silence_) {
244     if (!redirected) {
245       // Smoothly fade in from previous silence.
246       AudioFader::FadeInHelper(channels, num_output_channels_, num_frames,
247                                num_frames, num_frames);
248     }
249   } else if (redirected) {
250     // Smoothly fade out to silence, since output is now being redirected by a
251     // previous output splitter.
252     AudioFader::FadeOutHelper(channels, num_output_channels_, num_frames,
253                               num_frames, num_frames);
254   }
255   previous_ended_in_silence_ = redirected;
256 
257   output_redirector_->MixInput(mixer_input_, temp_buffer_.get(), num_frames,
258                                rendering_delay);
259 }
260 
261 // static
ParseConfig(const mixer_service::Generic & message)262 AudioOutputRedirector::Config AudioOutputRedirector::ParseConfig(
263     const mixer_service::Generic& message) {
264   Config config;
265   DCHECK(message.has_redirection_request());
266   const mixer_service::RedirectionRequest& request =
267       message.redirection_request();
268   if (request.has_num_channels()) {
269     config.num_output_channels = request.num_channels();
270   }
271   if (request.has_channel_layout()) {
272     config.output_channel_layout =
273         ConvertChannelLayout(request.channel_layout());
274   } else {
275     config.output_channel_layout = media::ChannelLayout::UNSUPPORTED;
276   }
277   if (request.has_order()) {
278     config.order = request.order();
279   }
280   if (request.has_apply_volume()) {
281     config.apply_volume = request.apply_volume();
282   }
283   if (request.has_extra_delay_microseconds()) {
284     config.extra_delay_microseconds = request.extra_delay_microseconds();
285   }
286   return config;
287 }
288 
AudioOutputRedirector(StreamMixer * mixer,std::unique_ptr<mixer_service::MixerSocket> socket,const mixer_service::Generic & message)289 AudioOutputRedirector::AudioOutputRedirector(
290     StreamMixer* mixer,
291     std::unique_ptr<mixer_service::MixerSocket> socket,
292     const mixer_service::Generic& message)
293     : mixer_(mixer),
294       config_(ParseConfig(message)),
295       output_channel_layout_(
296           GetMediaChannelLayout(config_.output_channel_layout,
297                                 config_.num_output_channels)),
298       io_task_runner_(base::SequencedTaskRunnerHandle::Get()),
299       buffer_pool_(
300           base::MakeRefCounted<IOBufferPool>(kDefaultBufferSize,
301                                              std::numeric_limits<size_t>::max(),
302                                              true /* threadsafe */)),
303       weak_factory_(this) {
304   DCHECK_GT(config_.num_output_channels, 0);
305 
306   output_ = std::make_unique<RedirectionConnection>(
307       std::move(socket), mixer_->task_runner(), weak_factory_.GetWeakPtr());
308   output_->HandleMetadata(message);
309 }
310 
~AudioOutputRedirector()311 AudioOutputRedirector::~AudioOutputRedirector() {
312   io_task_runner_->DeleteSoon(FROM_HERE, std::move(output_));
313 }
314 
AddInput(MixerInput * mixer_input)315 void AudioOutputRedirector::AddInput(MixerInput* mixer_input) {
316   if (ApplyToInput(mixer_input)) {
317     DCHECK_EQ(mixer_input->output_samples_per_second(), sample_rate_);
318     inputs_[mixer_input] = std::make_unique<InputImpl>(this, mixer_input);
319   } else {
320     non_redirected_inputs_.insert(mixer_input);
321   }
322 }
323 
RemoveInput(MixerInput * mixer_input)324 void AudioOutputRedirector::RemoveInput(MixerInput* mixer_input) {
325   inputs_.erase(mixer_input);
326   non_redirected_inputs_.erase(mixer_input);
327 }
328 
ApplyToInput(MixerInput * mixer_input)329 bool AudioOutputRedirector::ApplyToInput(MixerInput* mixer_input) {
330   if (!mixer_input->primary()) {
331     return false;
332   }
333 
334   for (const auto& pattern : patterns_) {
335     if (mixer_input->content_type() == pattern.first &&
336         base::MatchPattern(mixer_input->device_id(), pattern.second)) {
337       return true;
338     }
339   }
340 
341   return false;
342 }
343 
UpdatePatterns(std::vector<std::pair<AudioContentType,std::string>> patterns)344 void AudioOutputRedirector::UpdatePatterns(
345     std::vector<std::pair<AudioContentType, std::string>> patterns) {
346   patterns_ = std::move(patterns);
347   // Remove streams that no longer match.
348   for (auto it = inputs_.begin(); it != inputs_.end();) {
349     MixerInput* mixer_input = it->first;
350     if (!ApplyToInput(mixer_input)) {
351       non_redirected_inputs_.insert(mixer_input);
352       it = inputs_.erase(it);
353     } else {
354       ++it;
355     }
356   }
357 
358   // Add streams that previously didn't match.
359   for (auto it = non_redirected_inputs_.begin();
360        it != non_redirected_inputs_.end();) {
361     MixerInput* mixer_input = *it;
362     if (ApplyToInput(mixer_input)) {
363       inputs_[mixer_input] = std::make_unique<InputImpl>(this, mixer_input);
364       it = non_redirected_inputs_.erase(it);
365     } else {
366       ++it;
367     }
368   }
369 }
370 
OnConnectionError()371 void AudioOutputRedirector::OnConnectionError() {
372   mixer_->RemoveAudioOutputRedirector(this);
373 }
374 
SetSampleRate(int output_samples_per_second)375 void AudioOutputRedirector::SetSampleRate(int output_samples_per_second) {
376   sample_rate_ = output_samples_per_second;
377 
378   io_task_runner_->PostTask(
379       FROM_HERE,
380       base::BindOnce(&RedirectionConnection::SetStreamConfig,
381                      base::Unretained(output_.get()), kSampleFormatPlanarF32,
382                      sample_rate_, config_.num_output_channels,
383                      buffer_pool_->buffer_size() - kAudioMessageHeaderSize));
384 }
385 
PrepareNextBuffer(int num_frames)386 void AudioOutputRedirector::PrepareNextBuffer(int num_frames) {
387   size_t required_size_bytes =
388       kAudioMessageHeaderSize +
389       num_frames * config_.num_output_channels * sizeof(float);
390   if (buffer_pool_->buffer_size() < required_size_bytes) {
391     buffer_pool_ = base::MakeRefCounted<IOBufferPool>(
392         required_size_bytes, std::numeric_limits<size_t>::max(),
393         true /* threadsafe */);
394     io_task_runner_->PostTask(
395         FROM_HERE,
396         base::BindOnce(
397             &RedirectionConnection::SetStreamConfig,
398             base::Unretained(output_.get()), kSampleFormatPlanarF32,
399             sample_rate_, config_.num_output_channels,
400             num_frames * config_.num_output_channels * sizeof(float)));
401   }
402 
403   current_mix_buffer_ = buffer_pool_->GetBuffer();
404   current_mix_data_ = reinterpret_cast<float*>(current_mix_buffer_->data() +
405                                                kAudioMessageHeaderSize);
406   std::fill_n(current_mix_data_, num_frames * config_.num_output_channels,
407               0.0f);
408   next_output_timestamp_ = INT64_MIN;
409   next_num_frames_ = num_frames;
410   input_count_ = 0;
411 }
412 
MixInput(MixerInput * mixer_input,::media::AudioBus * data,int num_frames,RenderingDelay rendering_delay)413 void AudioOutputRedirector::MixInput(MixerInput* mixer_input,
414                                      ::media::AudioBus* data,
415                                      int num_frames,
416                                      RenderingDelay rendering_delay) {
417   DCHECK(current_mix_data_);
418   DCHECK_GE(next_num_frames_, num_frames);
419   DCHECK_EQ(config_.num_output_channels, data->channels());
420 
421   if (rendering_delay.timestamp_microseconds != INT64_MIN) {
422     int64_t output_timestamp = rendering_delay.timestamp_microseconds +
423                                rendering_delay.delay_microseconds +
424                                extra_delay_microseconds();
425     if (next_output_timestamp_ == INT64_MIN ||
426         output_timestamp < next_output_timestamp_) {
427       next_output_timestamp_ = output_timestamp;
428     }
429   }
430 
431   if (num_frames <= 0) {
432     return;
433   }
434 
435   ++input_count_;
436   for (int c = 0; c < config_.num_output_channels; ++c) {
437     float* dest_channel = current_mix_data_ + c * next_num_frames_;
438     if (config_.apply_volume) {
439       mixer_input->VolumeScaleAccumulate(data->channel(c), num_frames,
440                                          dest_channel);
441     } else {
442       const float* temp_channel = data->channel(c);
443       for (int i = 0; i < num_frames; ++i) {
444         dest_channel[i] += temp_channel[i];
445       }
446     }
447   }
448 }
449 
FinishBuffer()450 void AudioOutputRedirector::FinishBuffer() {
451   if (input_count_ == 0) {
452     return;
453   }
454 
455   // Hard limit to [1.0, -1.0].
456   for (int s = 0; s < config_.num_output_channels * next_num_frames_; ++s) {
457     current_mix_data_[s] =
458         base::ClampToRange(current_mix_data_[s], -1.0f, 1.0f);
459   }
460 
461   io_task_runner_->PostTask(
462       FROM_HERE,
463       base::BindOnce(
464           &RedirectionConnection::SendAudio, base::Unretained(output_.get()),
465           std::move(current_mix_buffer_),
466           config_.num_output_channels * next_num_frames_ * sizeof(float),
467           next_output_timestamp_));
468 }
469 
470 }  // namespace media
471 }  // namespace chromecast
472