1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "chromecast/media/cma/backend/mixer/mixer_input.h"
6 
7 #include <stdint.h>
8 
9 #include <algorithm>
10 #include <cmath>
11 #include <utility>
12 
13 #include "base/bind.h"
14 #include "base/callback_helpers.h"
15 #include "base/logging.h"
16 #include "base/numerics/ranges.h"
17 #include "chromecast/media/audio/audio_fader.h"
18 #include "chromecast/media/audio/audio_log.h"
19 #include "chromecast/media/cma/backend/mixer/audio_output_redirector_input.h"
20 #include "chromecast/media/cma/backend/mixer/channel_layout.h"
21 #include "chromecast/media/cma/backend/mixer/filter_group.h"
22 #include "media/base/audio_bus.h"
23 #include "media/base/audio_timestamp_helper.h"
24 #include "media/base/channel_mixer.h"
25 #include "media/base/multi_channel_resampler.h"
26 
27 namespace chromecast {
28 namespace media {
29 
30 namespace {
31 
32 const int64_t kMicrosecondsPerSecond = 1000 * 1000;
33 const int kDefaultSlewTimeMs = 50;
34 const int kDefaultFillBufferFrames = 2048;
35 
RoundUpMultiple(int value,int multiple)36 int RoundUpMultiple(int value, int multiple) {
37   return multiple * ((value + (multiple - 1)) / multiple);
38 }
39 
40 }  // namespace
41 
MixerInput(Source * source,FilterGroup * filter_group)42 MixerInput::MixerInput(Source* source, FilterGroup* filter_group)
43     : source_(source),
44       num_channels_(source->num_channels()),
45       channel_layout_(source->channel_layout()),
46       input_samples_per_second_(source->sample_rate()),
47       output_samples_per_second_(filter_group->input_samples_per_second()),
48       primary_(source->primary()),
49       device_id_(source->device_id()),
50       content_type_(source->content_type()),
51       slew_volume_(kDefaultSlewTimeMs, true),
52       volume_applied_(false),
53       previous_ended_in_silence_(false),
54       first_buffer_(true),
55       resampler_buffered_frames_(0.0) {
56   DCHECK(source_);
57   DCHECK_GT(num_channels_, 0);
58   DCHECK_GT(input_samples_per_second_, 0);
59 
60   fill_buffer_ =
61       ::media::AudioBus::Create(num_channels_, kDefaultFillBufferFrames);
62   fill_buffer_->Zero();
63 
64   MediaPipelineBackend::AudioDecoder::RenderingDelay initial_rendering_delay =
65       filter_group->GetRenderingDelayToOutput();
66 
67   int source_read_size = filter_group->input_frames_per_write();
68   if (output_samples_per_second_ > 0 &&
69       output_samples_per_second_ != input_samples_per_second_) {
70     // Round up to nearest multiple of SincResampler::kKernelSize. The read size
71     // must be > kKernelSize, so we round up to at least 2 * kKernelSize.
72     source_read_size = std::max(source_->desired_read_size(),
73                                 ::media::SincResampler::kKernelSize + 1);
74     source_read_size =
75         RoundUpMultiple(source_read_size, ::media::SincResampler::kKernelSize);
76     double resample_ratio = static_cast<double>(input_samples_per_second_) /
77                             output_samples_per_second_;
78     resampler_ = std::make_unique<::media::MultiChannelResampler>(
79         num_channels_, resample_ratio, source_read_size,
80         base::BindRepeating(&MixerInput::ResamplerReadCallback,
81                             base::Unretained(this)));
82     resampler_->PrimeWithSilence();
83 
84     double resampler_queued_frames = resampler_->BufferedFrames();
85     initial_rendering_delay.delay_microseconds +=
86         static_cast<int64_t>(resampler_queued_frames * kMicrosecondsPerSecond /
87                              input_samples_per_second_);
88   }
89 
90   if (output_samples_per_second_ != 0) {
91     // If output_samples_per_second_ is 0, this stream will be unusable.
92     // OnError() will be called shortly.
93     slew_volume_.SetSampleRate(output_samples_per_second_);
94   }
95   source_->InitializeAudioPlayback(source_read_size, initial_rendering_delay);
96 
97   SetFilterGroup(filter_group);
98 }
99 
~MixerInput()100 MixerInput::~MixerInput() {
101   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
102   SetFilterGroup(nullptr);
103   source_->FinalizeAudioPlayback();
104 }
105 
SetFilterGroup(FilterGroup * filter_group)106 void MixerInput::SetFilterGroup(FilterGroup* filter_group) {
107   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
108   DCHECK(!filter_group || !filter_group_);
109 
110   if (filter_group == filter_group_) {
111     return;
112   }
113   if (filter_group_) {
114     filter_group_->RemoveInput(this);
115   }
116   if (filter_group) {
117     filter_group->AddInput(this);
118     if (filter_group->num_channels() == num_channels_) {
119       channel_mixer_.reset();
120     } else {
121       AUDIO_LOG(INFO) << "Remixing channels for " << source_ << " from "
122                       << num_channels_ << " to "
123                       << filter_group->num_channels();
124       channel_mixer_ = std::make_unique<::media::ChannelMixer>(
125           mixer::CreateAudioParametersForChannelMixer(channel_layout_,
126                                                       num_channels_),
127           mixer::CreateAudioParametersForChannelMixer(
128               ::media::CHANNEL_LAYOUT_NONE, filter_group->num_channels()));
129     }
130   }
131   filter_group_ = filter_group;
132 }
133 
AddAudioOutputRedirector(AudioOutputRedirectorInput * redirector)134 void MixerInput::AddAudioOutputRedirector(
135     AudioOutputRedirectorInput* redirector) {
136   AUDIO_LOG(INFO) << "Add redirector to " << device_id_ << "(" << source_
137                   << ")";
138   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
139   DCHECK(redirector);
140   audio_output_redirectors_.insert(
141       std::upper_bound(
142           audio_output_redirectors_.begin(), audio_output_redirectors_.end(),
143           redirector,
144           [](AudioOutputRedirectorInput* a, AudioOutputRedirectorInput* b) {
145             return (a->Order() < b->Order());
146           }),
147       redirector);
148 }
149 
RemoveAudioOutputRedirector(AudioOutputRedirectorInput * redirector)150 void MixerInput::RemoveAudioOutputRedirector(
151     AudioOutputRedirectorInput* redirector) {
152   AUDIO_LOG(INFO) << "Remove redirector from " << device_id_ << "(" << source_
153                   << ")";
154   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
155   DCHECK(redirector);
156   audio_output_redirectors_.erase(
157       std::remove(audio_output_redirectors_.begin(),
158                   audio_output_redirectors_.end(), redirector),
159       audio_output_redirectors_.end());
160 }
161 
FillAudioData(int num_frames,RenderingDelay rendering_delay,::media::AudioBus * dest)162 int MixerInput::FillAudioData(int num_frames,
163                               RenderingDelay rendering_delay,
164                               ::media::AudioBus* dest) {
165   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
166   DCHECK(dest);
167   DCHECK_GE(dest->frames(), num_frames);
168 
169   ::media::AudioBus* fill_dest;
170   if (channel_mixer_) {
171     if (num_frames > fill_buffer_->frames()) {
172       fill_buffer_ = ::media::AudioBus::Create(num_channels_, dest->frames());
173     }
174     fill_dest = fill_buffer_.get();
175   } else {
176     fill_dest = dest;
177   }
178 
179   volume_applied_ = false;
180 
181   RenderingDelay redirected_delay = rendering_delay;
182   if (!audio_output_redirectors_.empty()) {
183     redirected_delay.delay_microseconds +=
184         audio_output_redirectors_[0]->GetDelayMicroseconds();
185   }
186   int filled = FillBuffer(num_frames, redirected_delay, fill_dest);
187 
188   bool redirected = false;
189   for (auto* redirector : audio_output_redirectors_) {
190     redirector->Redirect(fill_dest, filled, rendering_delay, redirected);
191     redirected = true;
192   }
193 
194   float* channels[num_channels_];
195   for (int c = 0; c < num_channels_; ++c) {
196     channels[c] = fill_dest->channel(c);
197   }
198   if (first_buffer_ && redirected) {
199     // If the first buffer is redirected, don't provide any data to the mixer
200     // (we want to avoid a 'blip' of sound from the first buffer if it is being
201     // redirected).
202     filled = 0;
203   } else if (previous_ended_in_silence_) {
204     if (redirected) {
205       // Previous buffer ended in silence, and the current buffer was redirected
206       // by the output chain, so maintain silence.
207       filled = 0;
208     } else {
209       // Smoothly fade in from previous silence.
210       AudioFader::FadeInHelper(channels, num_channels_, filled, filled, filled);
211     }
212   } else if (redirected) {
213     // Smoothly fade out to silence, since output is now being redirected.
214     AudioFader::FadeOutHelper(channels, num_channels_, filled, filled, filled);
215   }
216   previous_ended_in_silence_ = redirected;
217   first_buffer_ = false;
218 
219   // TODO(kmackay): If we ever support channel selections other than L and R,
220   // we should remix channels to a format that includes the selected channel
221   // and then do channel selection. Currently if the input is mono we don't
222   // bother doing channel selection since the result would be the same as
223   // doing nothing anyway.
224   int playout_channel = source_->playout_channel();
225   if (playout_channel != kChannelAll && playout_channel < num_channels_) {
226     // Duplicate selected channel to all channels.
227     for (int c = 0; c < num_channels_; ++c) {
228       if (c != source_->playout_channel()) {
229         std::copy_n(fill_dest->channel(playout_channel), filled,
230                     fill_dest->channel(c));
231       }
232     }
233   }
234 
235   // Mix channels if necessary.
236   if (channel_mixer_) {
237     channel_mixer_->TransformPartial(fill_dest, filled, dest);
238   }
239 
240   if (filled != num_frames) {
241     slew_volume_.Interrupted();
242   }
243 
244   return filled;
245 }
246 
FillBuffer(int num_frames,RenderingDelay rendering_delay,::media::AudioBus * dest)247 int MixerInput::FillBuffer(int num_frames,
248                            RenderingDelay rendering_delay,
249                            ::media::AudioBus* dest) {
250   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
251   DCHECK(dest);
252   DCHECK_EQ(num_channels_, dest->channels());
253   DCHECK_GE(dest->frames(), num_frames);
254 
255   if (resampler_) {
256     mixer_rendering_delay_ = rendering_delay;
257     // resampler_->BufferedFrames() gives incorrect values in the read callback,
258     // so track the number of buffered frames ourselves.
259     resampler_buffered_frames_ = resampler_->BufferedFrames();
260     filled_for_resampler_ = 0;
261     tried_to_fill_resampler_ = false;
262     resampler_->Resample(num_frames, dest);
263     // If the source is not providing any audio anymore, we want to stop filling
264     // frames so we can reduce processing overhead. However, since the resampler
265     // fill size doesn't necessarily match the mixer's request size at all, we
266     // need to be careful. The resampler could have a lot of data buffered
267     // internally, so we only count cases where the resampler needed more data
268     // from the source but none was available. Then, to make sure all data is
269     // flushed out of the resampler, we require that to happen twice before we
270     // stop filling audio.
271     if (tried_to_fill_resampler_) {
272       if (filled_for_resampler_ == 0) {
273         resampled_silence_count_ = std::min(resampled_silence_count_ + 1, 2);
274       } else {
275         resampled_silence_count_ = 0;
276       }
277     }
278     if (resampled_silence_count_ > 1) {
279       return 0;
280     }
281     return num_frames;
282   } else {
283     return source_->FillAudioPlaybackFrames(num_frames, rendering_delay, dest);
284   }
285 }
286 
ResamplerReadCallback(int frame_delay,::media::AudioBus * output)287 void MixerInput::ResamplerReadCallback(int frame_delay,
288                                        ::media::AudioBus* output) {
289   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
290 
291   RenderingDelay delay = mixer_rendering_delay_;
292   int64_t resampler_delay =
293       std::round(resampler_buffered_frames_ * kMicrosecondsPerSecond /
294                  input_samples_per_second_);
295   delay.delay_microseconds += resampler_delay;
296 
297   const int needed_frames = output->frames();
298   tried_to_fill_resampler_ = true;
299   int filled = source_->FillAudioPlaybackFrames(needed_frames, delay, output);
300   filled_for_resampler_ += filled;
301   if (filled < needed_frames) {
302     output->ZeroFramesPartial(filled, needed_frames - filled);
303   }
304   resampler_buffered_frames_ += output->frames();
305 }
306 
SignalError(Source::MixerError error)307 void MixerInput::SignalError(Source::MixerError error) {
308   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
309   if (filter_group_) {
310     filter_group_->RemoveInput(this);
311     filter_group_ = nullptr;
312   }
313   source_->OnAudioPlaybackError(error);
314 }
315 
VolumeScaleAccumulate(const float * src,int frames,float * dest)316 void MixerInput::VolumeScaleAccumulate(const float* src,
317                                        int frames,
318                                        float* dest) {
319   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
320   slew_volume_.ProcessFMAC(volume_applied_ /* repeat_transition */, src, frames,
321                            1, dest);
322   volume_applied_ = true;
323 }
324 
SetVolumeMultiplier(float multiplier)325 void MixerInput::SetVolumeMultiplier(float multiplier) {
326   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
327   float old_target_volume = TargetVolume();
328   stream_volume_multiplier_ = std::max(0.0f, multiplier);
329   float target_volume = TargetVolume();
330   AUDIO_LOG(INFO) << device_id_ << "(" << source_
331                   << "): stream volume = " << stream_volume_multiplier_
332                   << ", effective multiplier = " << target_volume;
333   if (target_volume != old_target_volume) {
334     slew_volume_.SetMaxSlewTimeMs(kDefaultSlewTimeMs);
335     slew_volume_.SetVolume(target_volume);
336   }
337 }
338 
SetContentTypeVolume(float volume)339 void MixerInput::SetContentTypeVolume(float volume) {
340   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
341   DCHECK(content_type_ != AudioContentType::kOther);
342 
343   float old_target_volume = TargetVolume();
344   type_volume_multiplier_ = volume;
345   float target_volume = TargetVolume();
346   AUDIO_LOG(INFO) << device_id_ << "(" << source_
347                   << "): type volume = " << type_volume_multiplier_
348                   << ", effective multiplier = " << target_volume;
349   if (target_volume != old_target_volume) {
350     slew_volume_.SetMaxSlewTimeMs(kDefaultSlewTimeMs);
351     slew_volume_.SetVolume(target_volume);
352   }
353 }
354 
SetVolumeLimits(float volume_min,float volume_max)355 void MixerInput::SetVolumeLimits(float volume_min, float volume_max) {
356   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
357   float old_target_volume = TargetVolume();
358   volume_min_ = volume_min;
359   volume_max_ = volume_max;
360   float target_volume = TargetVolume();
361   AUDIO_LOG(INFO) << device_id_ << "(" << source_ << "): set volume limits to ["
362                   << volume_min_ << ", " << volume_max_ << "]";
363   if (target_volume != old_target_volume) {
364     slew_volume_.SetMaxSlewTimeMs(kDefaultSlewTimeMs);
365     slew_volume_.SetVolume(target_volume);
366   }
367 }
368 
SetOutputLimit(float limit,int fade_ms)369 void MixerInput::SetOutputLimit(float limit, int fade_ms) {
370   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
371   float old_target_volume = TargetVolume();
372   output_volume_limit_ = limit;
373   float target_volume = TargetVolume();
374   AUDIO_LOG(INFO) << device_id_ << "(" << source_
375                   << "): output limit = " << output_volume_limit_
376                   << ", effective multiplier = " << target_volume;
377   if (fade_ms < 0) {
378     fade_ms = kDefaultSlewTimeMs;
379   } else {
380     AUDIO_LOG(INFO) << "Fade over " << fade_ms << " ms";
381   }
382   if (target_volume != old_target_volume) {
383     slew_volume_.SetMaxSlewTimeMs(fade_ms);
384     slew_volume_.SetVolume(target_volume);
385   }
386 }
387 
SetMuted(bool muted)388 void MixerInput::SetMuted(bool muted) {
389   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
390   DCHECK(content_type_ != AudioContentType::kOther);
391 
392   float old_target_volume = TargetVolume();
393   mute_volume_multiplier_ = muted ? 0.0f : 1.0f;
394   float target_volume = TargetVolume();
395   AUDIO_LOG(INFO) << device_id_ << "(" << source_
396                   << "): mute volume = " << mute_volume_multiplier_
397                   << ", effective multiplier = " << target_volume;
398   if (target_volume != old_target_volume) {
399     slew_volume_.SetMaxSlewTimeMs(kDefaultSlewTimeMs);
400     slew_volume_.SetVolume(target_volume);
401   }
402 }
403 
TargetVolume()404 float MixerInput::TargetVolume() {
405   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
406   float output_volume = stream_volume_multiplier_ * type_volume_multiplier_;
407   float clamped_volume =
408       base::ClampToRange(output_volume, volume_min_, volume_max_);
409   float limited_volume = std::min(clamped_volume, output_volume_limit_);
410   float muted_volume = limited_volume * mute_volume_multiplier_;
411   // Volume is clamped after all gains have been multiplied, to avoid clipping.
412   // TODO(kmackay): Consider removing this clamp and use a postprocessor filter
413   // to avoid clipping instead.
414   return base::ClampToRange(muted_volume, 0.0f, 1.0f);
415 }
416 
InstantaneousVolume()417 float MixerInput::InstantaneousVolume() {
418   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
419   return slew_volume_.LastBufferMaxMultiplier();
420 }
421 
422 }  // namespace media
423 }  // namespace chromecast
424