1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/renderers/audio_renderer_impl.h"
6
7 #include <math.h>
8 #include <stddef.h>
9 #include <algorithm>
10 #include <memory>
11 #include <utility>
12
13 #include "base/bind.h"
14 #include "base/callback.h"
15 #include "base/callback_helpers.h"
16 #include "base/command_line.h"
17 #include "base/logging.h"
18 #include "base/metrics/histogram_macros.h"
19 #include "base/power_monitor/power_monitor.h"
20 #include "base/single_thread_task_runner.h"
21 #include "base/time/default_tick_clock.h"
22 #include "base/time/time.h"
23 #include "base/trace_event/trace_event.h"
24 #include "build/build_config.h"
25 #include "media/audio/null_audio_sink.h"
26 #include "media/base/audio_buffer.h"
27 #include "media/base/audio_buffer_converter.h"
28 #include "media/base/audio_latency.h"
29 #include "media/base/audio_parameters.h"
30 #include "media/base/bind_to_current_loop.h"
31 #include "media/base/channel_mixing_matrix.h"
32 #include "media/base/demuxer_stream.h"
33 #include "media/base/media_client.h"
34 #include "media/base/media_log.h"
35 #include "media/base/media_switches.h"
36 #include "media/base/renderer_client.h"
37 #include "media/base/timestamp_constants.h"
38 #include "media/filters/audio_clock.h"
39 #include "media/filters/decrypting_demuxer_stream.h"
40
41 namespace media {
42
AudioRendererImpl(const scoped_refptr<base::SingleThreadTaskRunner> & task_runner,AudioRendererSink * sink,const CreateAudioDecodersCB & create_audio_decoders_cb,MediaLog * media_log,SpeechRecognitionClient * speech_recognition_client)43 AudioRendererImpl::AudioRendererImpl(
44 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
45 AudioRendererSink* sink,
46 const CreateAudioDecodersCB& create_audio_decoders_cb,
47 MediaLog* media_log,
48 SpeechRecognitionClient* speech_recognition_client)
49 : task_runner_(task_runner),
50 expecting_config_changes_(false),
51 sink_(sink),
52 media_log_(media_log),
53 client_(nullptr),
54 tick_clock_(base::DefaultTickClock::GetInstance()),
55 last_audio_memory_usage_(0),
56 last_decoded_sample_rate_(0),
57 last_decoded_channel_layout_(CHANNEL_LAYOUT_NONE),
58 is_encrypted_(false),
59 last_decoded_channels_(0),
60 volume_(1.0f), // Default unmuted.
61 playback_rate_(0.0),
62 state_(kUninitialized),
63 create_audio_decoders_cb_(create_audio_decoders_cb),
64 buffering_state_(BUFFERING_HAVE_NOTHING),
65 rendering_(false),
66 sink_playing_(false),
67 pending_read_(false),
68 received_end_of_stream_(false),
69 rendered_end_of_stream_(false),
70 is_suspending_(false),
71 #if defined(OS_ANDROID)
72 is_passthrough_(false) {
73 #else
74 is_passthrough_(false),
75 speech_recognition_client_(speech_recognition_client) {
76 #endif
77 DCHECK(create_audio_decoders_cb_);
78
79 // PowerObserver's must be added and removed from the same thread, but we
80 // won't remove the observer until we're destructed on |task_runner_| so we
81 // must post it here if we're on the wrong thread.
82 if (task_runner_->BelongsToCurrentThread()) {
83 base::PowerMonitor::AddObserver(this);
84 } else {
85 // Safe to post this without a WeakPtr because this class must be destructed
86 // on the same thread and construction has not completed yet.
87 task_runner_->PostTask(
88 FROM_HERE,
89 base::BindOnce(IgnoreResult(&base::PowerMonitor::AddObserver), this));
90 }
91
92 // Do not add anything below this line since the above actions are only safe
93 // as the last lines of the constructor.
94 }
95
96 AudioRendererImpl::~AudioRendererImpl() {
97 DVLOG(1) << __func__;
98 DCHECK(task_runner_->BelongsToCurrentThread());
99 base::PowerMonitor::RemoveObserver(this);
100
101 // If Render() is in progress, this call will wait for Render() to finish.
102 // After this call, the |sink_| will not call back into |this| anymore.
103 sink_->Stop();
104 if (null_sink_)
105 null_sink_->Stop();
106
107 if (init_cb_)
108 FinishInitialization(PIPELINE_ERROR_ABORT);
109 }
110
111 void AudioRendererImpl::StartTicking() {
112 DVLOG(1) << __func__;
113 DCHECK(task_runner_->BelongsToCurrentThread());
114
115 base::AutoLock auto_lock(lock_);
116
117 DCHECK(!rendering_);
118 rendering_ = true;
119
120 // Wait for an eventual call to SetPlaybackRate() to start rendering.
121 if (playback_rate_ == 0) {
122 DCHECK(!sink_playing_);
123 return;
124 }
125
126 StartRendering_Locked();
127 }
128
129 void AudioRendererImpl::StartRendering_Locked() {
130 DVLOG(1) << __func__;
131 DCHECK(task_runner_->BelongsToCurrentThread());
132 DCHECK_EQ(state_, kPlaying);
133 DCHECK(!sink_playing_);
134 DCHECK_NE(playback_rate_, 0.0);
135 lock_.AssertAcquired();
136
137 sink_playing_ = true;
138
139 base::AutoUnlock auto_unlock(lock_);
140 if (volume_ || !null_sink_)
141 sink_->Play();
142 else
143 null_sink_->Play();
144 }
145
146 void AudioRendererImpl::StopTicking() {
147 DVLOG(1) << __func__;
148 DCHECK(task_runner_->BelongsToCurrentThread());
149
150 base::AutoLock auto_lock(lock_);
151
152 DCHECK(rendering_);
153 rendering_ = false;
154
155 // Rendering should have already been stopped with a zero playback rate.
156 if (playback_rate_ == 0) {
157 DCHECK(!sink_playing_);
158 return;
159 }
160
161 StopRendering_Locked();
162 }
163
164 void AudioRendererImpl::StopRendering_Locked() {
165 DCHECK(task_runner_->BelongsToCurrentThread());
166 DCHECK_EQ(state_, kPlaying);
167 DCHECK(sink_playing_);
168 lock_.AssertAcquired();
169
170 sink_playing_ = false;
171
172 base::AutoUnlock auto_unlock(lock_);
173 if (volume_ || !null_sink_)
174 sink_->Pause();
175 else
176 null_sink_->Pause();
177
178 stop_rendering_time_ = last_render_time_;
179 }
180
181 void AudioRendererImpl::SetMediaTime(base::TimeDelta time) {
182 DVLOG(1) << __func__ << "(" << time << ")";
183 DCHECK(task_runner_->BelongsToCurrentThread());
184
185 base::AutoLock auto_lock(lock_);
186 DCHECK(!rendering_);
187 DCHECK_EQ(state_, kFlushed);
188
189 start_timestamp_ = time;
190 ended_timestamp_ = kInfiniteDuration;
191 last_render_time_ = stop_rendering_time_ = base::TimeTicks();
192 first_packet_timestamp_ = kNoTimestamp;
193 audio_clock_.reset(new AudioClock(time, audio_parameters_.sample_rate()));
194 }
195
196 base::TimeDelta AudioRendererImpl::CurrentMediaTime() {
197 base::AutoLock auto_lock(lock_);
198
199 // Return the current time based on the known extents of the rendered audio
200 // data plus an estimate based on the last time those values were calculated.
201 base::TimeDelta current_media_time = audio_clock_->front_timestamp();
202 if (!last_render_time_.is_null()) {
203 current_media_time +=
204 (tick_clock_->NowTicks() - last_render_time_) * playback_rate_;
205 if (current_media_time > audio_clock_->back_timestamp())
206 current_media_time = audio_clock_->back_timestamp();
207 }
208
209 return current_media_time;
210 }
211
212 bool AudioRendererImpl::GetWallClockTimes(
213 const std::vector<base::TimeDelta>& media_timestamps,
214 std::vector<base::TimeTicks>* wall_clock_times) {
215 base::AutoLock auto_lock(lock_);
216 DCHECK(wall_clock_times->empty());
217
218 // When playback is paused (rate is zero), assume a rate of 1.0.
219 const double playback_rate = playback_rate_ ? playback_rate_ : 1.0;
220 const bool is_time_moving = sink_playing_ && playback_rate_ &&
221 !last_render_time_.is_null() &&
222 stop_rendering_time_.is_null() && !is_suspending_;
223
224 // Pre-compute the time until playback of the audio buffer extents, since
225 // these values are frequently used below.
226 const base::TimeDelta time_until_front =
227 audio_clock_->TimeUntilPlayback(audio_clock_->front_timestamp());
228 const base::TimeDelta time_until_back =
229 audio_clock_->TimeUntilPlayback(audio_clock_->back_timestamp());
230
231 if (media_timestamps.empty()) {
232 // Return the current media time as a wall clock time while accounting for
233 // frames which may be in the process of play out.
234 wall_clock_times->push_back(std::min(
235 std::max(tick_clock_->NowTicks(), last_render_time_ + time_until_front),
236 last_render_time_ + time_until_back));
237 return is_time_moving;
238 }
239
240 wall_clock_times->reserve(media_timestamps.size());
241 for (const auto& media_timestamp : media_timestamps) {
242 // When time was or is moving and the requested media timestamp is within
243 // range of played out audio, we can provide an exact conversion.
244 if (!last_render_time_.is_null() &&
245 media_timestamp >= audio_clock_->front_timestamp() &&
246 media_timestamp <= audio_clock_->back_timestamp()) {
247 wall_clock_times->push_back(
248 last_render_time_ + audio_clock_->TimeUntilPlayback(media_timestamp));
249 continue;
250 }
251
252 base::TimeDelta base_timestamp, time_until_playback;
253 if (media_timestamp < audio_clock_->front_timestamp()) {
254 base_timestamp = audio_clock_->front_timestamp();
255 time_until_playback = time_until_front;
256 } else {
257 base_timestamp = audio_clock_->back_timestamp();
258 time_until_playback = time_until_back;
259 }
260
261 // In practice, most calls will be estimates given the relatively small
262 // window in which clients can get the actual time.
263 wall_clock_times->push_back(last_render_time_ + time_until_playback +
264 (media_timestamp - base_timestamp) /
265 playback_rate);
266 }
267
268 return is_time_moving;
269 }
270
271 TimeSource* AudioRendererImpl::GetTimeSource() {
272 return this;
273 }
274
275 void AudioRendererImpl::Flush(base::OnceClosure callback) {
276 DVLOG(1) << __func__;
277 DCHECK(task_runner_->BelongsToCurrentThread());
278 TRACE_EVENT_ASYNC_BEGIN0("media", "AudioRendererImpl::Flush", this);
279
280 // Flush |sink_| now. |sink_| must only be accessed on |task_runner_| and not
281 // be called under |lock_|.
282 DCHECK(!sink_playing_);
283 if (volume_ || !null_sink_)
284 sink_->Flush();
285 else
286 null_sink_->Flush();
287
288 base::AutoLock auto_lock(lock_);
289 DCHECK_EQ(state_, kPlaying);
290 DCHECK(!flush_cb_);
291
292 flush_cb_ = std::move(callback);
293 ChangeState_Locked(kFlushing);
294
295 if (pending_read_)
296 return;
297
298 ChangeState_Locked(kFlushed);
299 DoFlush_Locked();
300 }
301
302 void AudioRendererImpl::DoFlush_Locked() {
303 DCHECK(task_runner_->BelongsToCurrentThread());
304 lock_.AssertAcquired();
305
306 DCHECK(!pending_read_);
307 DCHECK_EQ(state_, kFlushed);
308
309 ended_timestamp_ = kInfiniteDuration;
310 audio_decoder_stream_->Reset(base::BindOnce(
311 &AudioRendererImpl::ResetDecoderDone, weak_factory_.GetWeakPtr()));
312 }
313
314 void AudioRendererImpl::ResetDecoderDone() {
315 DCHECK(task_runner_->BelongsToCurrentThread());
316 {
317 base::AutoLock auto_lock(lock_);
318
319 DCHECK_EQ(state_, kFlushed);
320 DCHECK(flush_cb_);
321
322 received_end_of_stream_ = false;
323 rendered_end_of_stream_ = false;
324
325 // Flush() may have been called while underflowed/not fully buffered.
326 if (buffering_state_ != BUFFERING_HAVE_NOTHING)
327 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
328
329 if (buffer_converter_)
330 buffer_converter_->Reset();
331 algorithm_->FlushBuffers();
332 }
333
334 // Changes in buffering state are always posted. Flush callback must only be
335 // run after buffering state has been set back to nothing.
336 flush_cb_ = BindToCurrentLoop(std::move(flush_cb_));
337 FinishFlush();
338 }
339
340 void AudioRendererImpl::StartPlaying() {
341 DVLOG(1) << __func__;
342 DCHECK(task_runner_->BelongsToCurrentThread());
343
344 base::AutoLock auto_lock(lock_);
345 DCHECK(!sink_playing_);
346 DCHECK_EQ(state_, kFlushed);
347 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
348 DCHECK(!pending_read_) << "Pending read must complete before seeking";
349
350 ChangeState_Locked(kPlaying);
351 AttemptRead_Locked();
352 }
353
354 void AudioRendererImpl::Initialize(DemuxerStream* stream,
355 CdmContext* cdm_context,
356 RendererClient* client,
357 PipelineStatusCallback init_cb) {
358 DVLOG(1) << __func__;
359 DCHECK(task_runner_->BelongsToCurrentThread());
360 DCHECK(client);
361 DCHECK(stream);
362 DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
363 DCHECK(init_cb);
364 DCHECK(state_ == kUninitialized || state_ == kFlushed);
365 DCHECK(sink_);
366 TRACE_EVENT_ASYNC_BEGIN0("media", "AudioRendererImpl::Initialize", this);
367
368 // If we are re-initializing playback (e.g. switching media tracks), stop the
369 // sink first.
370 if (state_ == kFlushed) {
371 sink_->Stop();
372 if (null_sink_)
373 null_sink_->Stop();
374 }
375
376 state_ = kInitializing;
377 demuxer_stream_ = stream;
378 client_ = client;
379
380 // Always post |init_cb_| because |this| could be destroyed if initialization
381 // failed.
382 init_cb_ = BindToCurrentLoop(std::move(init_cb));
383
384 // Retrieve hardware device parameters asynchronously so we don't block the
385 // media thread on synchronous IPC.
386 sink_->GetOutputDeviceInfoAsync(
387 base::BindOnce(&AudioRendererImpl::OnDeviceInfoReceived,
388 weak_factory_.GetWeakPtr(), demuxer_stream_, cdm_context));
389
390 #if !defined(OS_ANDROID)
391 if (speech_recognition_client_) {
392 speech_recognition_client_->SetOnReadyCallback(BindToCurrentLoop(
393 base::BindOnce(&AudioRendererImpl::EnableSpeechRecognition,
394 weak_factory_.GetWeakPtr())));
395 }
396 #endif
397 }
398
399 void AudioRendererImpl::OnDeviceInfoReceived(
400 DemuxerStream* stream,
401 CdmContext* cdm_context,
402 OutputDeviceInfo output_device_info) {
403 DVLOG(1) << __func__;
404 DCHECK(task_runner_->BelongsToCurrentThread());
405 DCHECK(client_);
406 DCHECK(stream);
407 DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
408 DCHECK(init_cb_);
409 DCHECK_EQ(state_, kInitializing);
410
411 // Fall-back to a fake audio sink if the audio device can't be setup; this
412 // allows video playback in cases where there is no audio hardware.
413 //
414 // TODO(dalecurtis): We could disable the audio track here too.
415 UMA_HISTOGRAM_ENUMERATION("Media.AudioRendererImpl.SinkStatus",
416 output_device_info.device_status(),
417 OUTPUT_DEVICE_STATUS_MAX + 1);
418 if (output_device_info.device_status() != OUTPUT_DEVICE_STATUS_OK) {
419 MEDIA_LOG(ERROR, media_log_)
420 << "Output device error, falling back to null sink. device_status="
421 << output_device_info.device_status();
422 sink_ = new NullAudioSink(task_runner_);
423 output_device_info = sink_->GetOutputDeviceInfo();
424 } else if (base::FeatureList::IsEnabled(kSuspendMutedAudio)) {
425 // If playback is muted, we use a fake sink for output until it unmutes.
426 null_sink_ = new NullAudioSink(task_runner_);
427 }
428
429 current_decoder_config_ = stream->audio_decoder_config();
430 DCHECK(current_decoder_config_.IsValidConfig());
431
432 const AudioParameters& hw_params = output_device_info.output_params();
433 ChannelLayout hw_channel_layout =
434 hw_params.IsValid() ? hw_params.channel_layout() : CHANNEL_LAYOUT_NONE;
435
436 audio_decoder_stream_ = std::make_unique<AudioDecoderStream>(
437 std::make_unique<AudioDecoderStream::StreamTraits>(media_log_,
438 hw_channel_layout),
439 task_runner_, create_audio_decoders_cb_, media_log_);
440
441 audio_decoder_stream_->set_config_change_observer(base::BindRepeating(
442 &AudioRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
443
444 AudioCodec codec = stream->audio_decoder_config().codec();
445 if (auto* mc = GetMediaClient())
446 is_passthrough_ = mc->IsSupportedBitstreamAudioCodec(codec);
447 else
448 is_passthrough_ = false;
449 expecting_config_changes_ = stream->SupportsConfigChanges();
450
451 bool use_stream_params = !expecting_config_changes_ || !hw_params.IsValid() ||
452 hw_params.format() == AudioParameters::AUDIO_FAKE ||
453 !sink_->IsOptimizedForHardwareParameters();
454
455 if (stream->audio_decoder_config().channel_layout() ==
456 CHANNEL_LAYOUT_DISCRETE &&
457 sink_->IsOptimizedForHardwareParameters()) {
458 use_stream_params = false;
459 }
460
461 // Target ~20ms for our buffer size (which is optimal for power efficiency and
462 // responsiveness to play/pause events), but if the hardware needs something
463 // even larger (say for Bluetooth devices) prefer that.
464 //
465 // Even if |use_stream_params| is true we should choose a value here based on
466 // hardware parameters since it affects the initial buffer size used by
467 // AudioRendererAlgorithm. Too small and we will underflow if the hardware
468 // asks for a buffer larger than the initial algorithm capacity.
469 const int preferred_buffer_size =
470 std::max(2 * stream->audio_decoder_config().samples_per_second() / 100,
471 hw_params.IsValid() ? hw_params.frames_per_buffer() : 0);
472
473 if (is_passthrough_) {
474 AudioParameters::Format format = AudioParameters::AUDIO_FAKE;
475 if (codec == kCodecAC3) {
476 format = AudioParameters::AUDIO_BITSTREAM_AC3;
477 } else if (codec == kCodecEAC3) {
478 format = AudioParameters::AUDIO_BITSTREAM_EAC3;
479 } else {
480 NOTREACHED();
481 }
482
483 // If we want the precise PCM frame count here, we have to somehow peek the
484 // audio bitstream and parse the header ahead of time. Instead, we ensure
485 // audio bus being large enough to accommodate
486 // kMaxFramesPerCompressedAudioBuffer frames. The real data size and frame
487 // count for bitstream formats will be carried in additional fields of
488 // AudioBus.
489 const int buffer_size =
490 AudioParameters::kMaxFramesPerCompressedAudioBuffer *
491 stream->audio_decoder_config().bytes_per_frame();
492
493 audio_parameters_.Reset(
494 format, stream->audio_decoder_config().channel_layout(),
495 stream->audio_decoder_config().samples_per_second(), buffer_size);
496 buffer_converter_.reset();
497 } else if (use_stream_params) {
498 audio_parameters_.Reset(AudioParameters::AUDIO_PCM_LOW_LATENCY,
499 stream->audio_decoder_config().channel_layout(),
500 stream->audio_decoder_config().samples_per_second(),
501 preferred_buffer_size);
502 audio_parameters_.set_channels_for_discrete(
503 stream->audio_decoder_config().channels());
504 buffer_converter_.reset();
505 } else {
506 // To allow for seamless sample rate adaptations (i.e. changes from say
507 // 16kHz to 48kHz), always resample to the hardware rate.
508 int sample_rate = hw_params.sample_rate();
509
510 // If supported by the OS and the initial sample rate is not too low, let
511 // the OS level resampler handle resampling for power efficiency.
512 if (AudioLatency::IsResamplingPassthroughSupported(
513 AudioLatency::LATENCY_PLAYBACK) &&
514 stream->audio_decoder_config().samples_per_second() >= 44100) {
515 sample_rate = stream->audio_decoder_config().samples_per_second();
516 }
517
518 int stream_channel_count = stream->audio_decoder_config().channels();
519
520 bool try_supported_channel_layouts = false;
521 #if defined(OS_WIN)
522 try_supported_channel_layouts =
523 base::CommandLine::ForCurrentProcess()->HasSwitch(
524 switches::kTrySupportedChannelLayouts);
525 #endif
526
527 // We don't know how to up-mix for DISCRETE layouts (fancy multichannel
528 // hardware with non-standard speaker arrangement). Instead, pretend the
529 // hardware layout is stereo and let the OS take care of further up-mixing
530 // to the discrete layout (http://crbug.com/266674). Additionally, pretend
531 // hardware is stereo whenever kTrySupportedChannelLayouts is set. This flag
532 // is for savvy users who want stereo content to output in all surround
533 // speakers. Using the actual layout (likely 5.1 or higher) will mean our
534 // mixer will attempt to up-mix stereo source streams to just the left/right
535 // speaker of the 5.1 setup, nulling out the other channels
536 // (http://crbug.com/177872).
537 ChannelLayout hw_channel_layout =
538 hw_params.channel_layout() == CHANNEL_LAYOUT_DISCRETE ||
539 try_supported_channel_layouts
540 ? CHANNEL_LAYOUT_STEREO
541 : hw_params.channel_layout();
542 int hw_channel_count = ChannelLayoutToChannelCount(hw_channel_layout);
543
544 // The layout we pass to |audio_parameters_| will be used for the lifetime
545 // of this audio renderer, regardless of changes to hardware and/or stream
546 // properties. Below we choose the max of stream layout vs. hardware layout
547 // to leave room for changes to the hardware and/or stream (i.e. avoid
548 // premature down-mixing - http://crbug.com/379288).
549 // If stream_channels < hw_channels:
550 // Taking max means we up-mix to hardware layout. If stream later changes
551 // to have more channels, we aren't locked into down-mixing to the
552 // initial stream layout.
553 // If stream_channels > hw_channels:
554 // We choose to output stream's layout, meaning mixing is a no-op for the
555 // renderer. Browser-side will down-mix to the hardware config. If the
556 // hardware later changes to equal stream channels, browser-side will stop
557 // down-mixing and use the data from all stream channels.
558 ChannelLayout renderer_channel_layout =
559 hw_channel_count > stream_channel_count
560 ? hw_channel_layout
561 : stream->audio_decoder_config().channel_layout();
562
563 audio_parameters_.Reset(hw_params.format(), renderer_channel_layout,
564 sample_rate,
565 AudioLatency::GetHighLatencyBufferSize(
566 sample_rate, preferred_buffer_size));
567 }
568
569 audio_parameters_.set_effects(audio_parameters_.effects() |
570 AudioParameters::MULTIZONE);
571
572 audio_parameters_.set_latency_tag(AudioLatency::LATENCY_PLAYBACK);
573
574 if (!client_->IsVideoStreamAvailable()) {
575 // When video is not available, audio prefetch can be enabled. See
576 // crbug/988535.
577 audio_parameters_.set_effects(audio_parameters_.effects() |
578 AudioParameters::AUDIO_PREFETCH);
579 }
580
581 last_decoded_channel_layout_ =
582 stream->audio_decoder_config().channel_layout();
583
584 is_encrypted_ = stream->audio_decoder_config().is_encrypted();
585
586 last_decoded_channels_ = stream->audio_decoder_config().channels();
587
588 {
589 // Set the |audio_clock_| under lock in case this is a reinitialize and some
590 // external caller to GetWallClockTimes() exists.
591 base::AutoLock lock(lock_);
592 audio_clock_.reset(
593 new AudioClock(base::TimeDelta(), audio_parameters_.sample_rate()));
594 }
595
596 audio_decoder_stream_->Initialize(
597 stream,
598 base::BindOnce(&AudioRendererImpl::OnAudioDecoderStreamInitialized,
599 weak_factory_.GetWeakPtr()),
600 cdm_context,
601 base::BindRepeating(&AudioRendererImpl::OnStatisticsUpdate,
602 weak_factory_.GetWeakPtr()),
603 base::BindRepeating(&AudioRendererImpl::OnWaiting,
604 weak_factory_.GetWeakPtr()));
605 }
606
607 void AudioRendererImpl::OnAudioDecoderStreamInitialized(bool success) {
608 DVLOG(1) << __func__ << ": " << success;
609 DCHECK(task_runner_->BelongsToCurrentThread());
610 base::AutoLock auto_lock(lock_);
611
612 if (!success) {
613 state_ = kUninitialized;
614 FinishInitialization(DECODER_ERROR_NOT_SUPPORTED);
615 return;
616 }
617
618 if (!audio_parameters_.IsValid()) {
619 DVLOG(1) << __func__ << ": Invalid audio parameters: "
620 << audio_parameters_.AsHumanReadableString();
621 ChangeState_Locked(kUninitialized);
622
623 // TODO(flim): If the channel layout is discrete but channel count is 0, a
624 // possible cause is that the input stream has > 8 channels but there is no
625 // Web Audio renderer attached and no channel mixing matrices defined for
626 // hardware renderers. Adding one for previewing content could be useful.
627 FinishInitialization(PIPELINE_ERROR_INITIALIZATION_FAILED);
628 return;
629 }
630
631 if (expecting_config_changes_)
632 buffer_converter_.reset(new AudioBufferConverter(audio_parameters_));
633
634 // We're all good! Continue initializing the rest of the audio renderer
635 // based on the decoder format.
636 auto* media_client = GetMediaClient();
637 auto params =
638 (media_client ? media_client->GetAudioRendererAlgorithmParameters(
639 audio_parameters_)
640 : base::nullopt);
641 if (params && !client_->IsVideoStreamAvailable()) {
642 algorithm_ =
643 std::make_unique<AudioRendererAlgorithm>(media_log_, params.value());
644 } else {
645 algorithm_ = std::make_unique<AudioRendererAlgorithm>(media_log_);
646 }
647 algorithm_->Initialize(audio_parameters_, is_encrypted_);
648 if (latency_hint_)
649 algorithm_->SetLatencyHint(latency_hint_);
650
651 algorithm_->SetPreservesPitch(preserves_pitch_);
652 ConfigureChannelMask();
653
654 ChangeState_Locked(kFlushed);
655
656 {
657 base::AutoUnlock auto_unlock(lock_);
658 sink_->Initialize(audio_parameters_, this);
659 if (null_sink_) {
660 null_sink_->Initialize(audio_parameters_, this);
661 null_sink_->Start(); // Does nothing but reduce state bookkeeping.
662 real_sink_needs_start_ = true;
663 } else {
664 // Even when kSuspendMutedAudio is enabled, we can hit this path if we are
665 // exclusively using NullAudioSink due to OnDeviceInfoReceived() failure.
666 sink_->Start();
667 sink_->Pause(); // Sinks play on start.
668 }
669 SetVolume(volume_);
670 }
671
672 DCHECK(!sink_playing_);
673 FinishInitialization(PIPELINE_OK);
674 }
675
676 void AudioRendererImpl::FinishInitialization(PipelineStatus status) {
677 DCHECK(init_cb_);
678 TRACE_EVENT_ASYNC_END1("media", "AudioRendererImpl::Initialize", this,
679 "status", PipelineStatusToString(status));
680 std::move(init_cb_).Run(status);
681 }
682
683 void AudioRendererImpl::FinishFlush() {
684 DCHECK(flush_cb_);
685 TRACE_EVENT_ASYNC_END0("media", "AudioRendererImpl::Flush", this);
686 std::move(flush_cb_).Run();
687 }
688
689 void AudioRendererImpl::OnPlaybackError(PipelineStatus error) {
690 DCHECK(task_runner_->BelongsToCurrentThread());
691 client_->OnError(error);
692 }
693
694 void AudioRendererImpl::OnPlaybackEnded() {
695 DCHECK(task_runner_->BelongsToCurrentThread());
696 client_->OnEnded();
697 }
698
699 void AudioRendererImpl::OnStatisticsUpdate(const PipelineStatistics& stats) {
700 DCHECK(task_runner_->BelongsToCurrentThread());
701 client_->OnStatisticsUpdate(stats);
702 }
703
704 void AudioRendererImpl::OnBufferingStateChange(BufferingState buffering_state) {
705 DCHECK(task_runner_->BelongsToCurrentThread());
706
707 // "Underflow" is only possible when playing. This avoids noise like blaming
708 // the decoder for an "underflow" that is really just a seek.
709 BufferingStateChangeReason reason = BUFFERING_CHANGE_REASON_UNKNOWN;
710 if (state_ == kPlaying && buffering_state == BUFFERING_HAVE_NOTHING) {
711 reason = audio_decoder_stream_->is_demuxer_read_pending()
712 ? DEMUXER_UNDERFLOW
713 : DECODER_UNDERFLOW;
714 }
715
716 media_log_->AddEvent<MediaLogEvent::kBufferingStateChanged>(
717 SerializableBufferingState<SerializableBufferingStateType::kAudio>{
718 buffering_state, reason});
719
720 client_->OnBufferingStateChange(buffering_state, reason);
721 }
722
723 void AudioRendererImpl::OnWaiting(WaitingReason reason) {
724 DCHECK(task_runner_->BelongsToCurrentThread());
725 client_->OnWaiting(reason);
726 }
727
728 void AudioRendererImpl::SetVolume(float volume) {
729 DCHECK(task_runner_->BelongsToCurrentThread());
730 if (state_ == kUninitialized || state_ == kInitializing) {
731 volume_ = volume;
732 return;
733 }
734
735 sink_->SetVolume(volume);
736 if (!null_sink_) {
737 // Either null sink suspension is not enabled or we're already on the null
738 // sink due to failing to get device parameters.
739 return;
740 }
741
742 null_sink_->SetVolume(volume);
743
744 // Two cases to handle:
745 // 1. Changing from muted to unmuted state.
746 // 2. Unmuted startup case.
747 if ((!volume_ && volume) || (volume && real_sink_needs_start_)) {
748 // Suspend null audio sink (does nothing if unused).
749 null_sink_->Pause();
750
751 // Complete startup for the real sink if needed.
752 if (real_sink_needs_start_) {
753 sink_->Start();
754 if (!sink_playing_)
755 sink_->Pause(); // Sinks play on start.
756 real_sink_needs_start_ = false;
757 }
758
759 // Start sink playback if needed.
760 if (sink_playing_)
761 sink_->Play();
762 } else if (volume_ && !volume) {
763 // Suspend the real sink (does nothing if unused).
764 sink_->Pause();
765
766 // Start fake sink playback if needed.
767 if (sink_playing_)
768 null_sink_->Play();
769 }
770
771 volume_ = volume;
772 }
773
774 void AudioRendererImpl::SetLatencyHint(
775 base::Optional<base::TimeDelta> latency_hint) {
776 base::AutoLock auto_lock(lock_);
777
778 latency_hint_ = latency_hint;
779
780 if (algorithm_) {
781 algorithm_->SetLatencyHint(latency_hint);
782
783 // See if we need further reads to fill up to the new playback threshold.
784 // This may be needed if rendering isn't active to schedule regular reads.
785 AttemptRead_Locked();
786 }
787 }
788
789 void AudioRendererImpl::SetPreservesPitch(bool preserves_pitch) {
790 base::AutoLock auto_lock(lock_);
791
792 preserves_pitch_ = preserves_pitch;
793
794 if (algorithm_)
795 algorithm_->SetPreservesPitch(preserves_pitch);
796 }
797
798 void AudioRendererImpl::OnSuspend() {
799 base::AutoLock auto_lock(lock_);
800 is_suspending_ = true;
801 }
802
803 void AudioRendererImpl::OnResume() {
804 base::AutoLock auto_lock(lock_);
805 is_suspending_ = false;
806 }
807
808 void AudioRendererImpl::SetPlayDelayCBForTesting(PlayDelayCBForTesting cb) {
809 DCHECK_EQ(state_, kUninitialized);
810 play_delay_cb_for_testing_ = std::move(cb);
811 }
812
813 void AudioRendererImpl::DecodedAudioReady(AudioDecoderStream::ReadStatus status,
814 scoped_refptr<AudioBuffer> buffer) {
815 DVLOG(2) << __func__ << "(" << status << ")";
816 DCHECK(task_runner_->BelongsToCurrentThread());
817
818 base::AutoLock auto_lock(lock_);
819 DCHECK(state_ != kUninitialized);
820
821 CHECK(pending_read_);
822 pending_read_ = false;
823
824 if (status == AudioDecoderStream::ABORTED ||
825 status == AudioDecoderStream::DEMUXER_READ_ABORTED) {
826 HandleAbortedReadOrDecodeError(PIPELINE_OK);
827 return;
828 }
829
830 if (status == AudioDecoderStream::DECODE_ERROR) {
831 HandleAbortedReadOrDecodeError(PIPELINE_ERROR_DECODE);
832 return;
833 }
834
835 DCHECK_EQ(status, AudioDecoderStream::OK);
836 DCHECK(buffer);
837
838 if (state_ == kFlushing) {
839 ChangeState_Locked(kFlushed);
840 DoFlush_Locked();
841 return;
842 }
843
844 bool need_another_buffer = true;
845
846 if (expecting_config_changes_) {
847 if (!buffer->end_of_stream()) {
848 if (last_decoded_sample_rate_ &&
849 buffer->sample_rate() != last_decoded_sample_rate_) {
850 DVLOG(1) << __func__ << " Updating audio sample_rate."
851 << " ts:" << buffer->timestamp().InMicroseconds()
852 << " old:" << last_decoded_sample_rate_
853 << " new:" << buffer->sample_rate();
854 // Send a bogus config to reset timestamp state.
855 OnConfigChange(AudioDecoderConfig());
856 }
857 last_decoded_sample_rate_ = buffer->sample_rate();
858
859 if (last_decoded_channel_layout_ != buffer->channel_layout()) {
860 if (buffer->channel_layout() == CHANNEL_LAYOUT_DISCRETE) {
861 MEDIA_LOG(ERROR, media_log_)
862 << "Unsupported midstream configuration change! Discrete channel"
863 << " layout not allowed by sink.";
864 HandleAbortedReadOrDecodeError(PIPELINE_ERROR_DECODE);
865 return;
866 } else {
867 last_decoded_channel_layout_ = buffer->channel_layout();
868 last_decoded_channels_ = buffer->channel_count();
869 ConfigureChannelMask();
870 }
871 }
872 }
873
874 DCHECK(buffer_converter_);
875 buffer_converter_->AddInput(std::move(buffer));
876
877 while (buffer_converter_->HasNextBuffer()) {
878 need_another_buffer =
879 HandleDecodedBuffer_Locked(buffer_converter_->GetNextBuffer());
880 }
881 } else {
882 // TODO(chcunningham, tguilbert): Figure out if we want to support implicit
883 // config changes during src=. Doing so requires resampling each individual
884 // stream which is inefficient when there are many tags in a page.
885 //
886 // Check if the buffer we received matches the expected configuration.
887 // Note: We explicitly do not check channel layout here to avoid breaking
888 // weird behavior with multichannel wav files: http://crbug.com/600538.
889 if (!buffer->end_of_stream() &&
890 (buffer->sample_rate() != audio_parameters_.sample_rate() ||
891 buffer->channel_count() != audio_parameters_.channels())) {
892 MEDIA_LOG(ERROR, media_log_)
893 << "Unsupported midstream configuration change!"
894 << " Sample Rate: " << buffer->sample_rate() << " vs "
895 << audio_parameters_.sample_rate()
896 << ", Channels: " << buffer->channel_count() << " vs "
897 << audio_parameters_.channels();
898 HandleAbortedReadOrDecodeError(PIPELINE_ERROR_DECODE);
899 return;
900 }
901
902 need_another_buffer = HandleDecodedBuffer_Locked(std::move(buffer));
903 }
904
905 if (!need_another_buffer && !CanRead_Locked())
906 return;
907
908 AttemptRead_Locked();
909 }
910
911 bool AudioRendererImpl::HandleDecodedBuffer_Locked(
912 scoped_refptr<AudioBuffer> buffer) {
913 lock_.AssertAcquired();
914 bool should_render_end_of_stream = false;
915 if (buffer->end_of_stream()) {
916 received_end_of_stream_ = true;
917 algorithm_->MarkEndOfStream();
918
919 // We received no audio to play before EOS, so enter the ended state.
920 if (first_packet_timestamp_ == kNoTimestamp)
921 should_render_end_of_stream = true;
922 } else {
923 if (buffer->IsBitstreamFormat() && state_ == kPlaying) {
924 if (IsBeforeStartTime(*buffer))
925 return true;
926
927 // Adjust the start time since we are unable to trim a compressed audio
928 // buffer.
929 if (buffer->timestamp() < start_timestamp_ &&
930 (buffer->timestamp() + buffer->duration()) > start_timestamp_) {
931 start_timestamp_ = buffer->timestamp();
932 audio_clock_.reset(new AudioClock(buffer->timestamp(),
933 audio_parameters_.sample_rate()));
934 }
935 } else if (state_ == kPlaying) {
936 if (IsBeforeStartTime(*buffer))
937 return true;
938
939 // Trim off any additional time before the start timestamp.
940 const base::TimeDelta trim_time = start_timestamp_ - buffer->timestamp();
941 if (trim_time > base::TimeDelta()) {
942 const int frames_to_trim = AudioTimestampHelper::TimeToFrames(
943 trim_time, buffer->sample_rate());
944 DVLOG(1) << __func__ << ": Trimming first audio buffer by "
945 << frames_to_trim << " frames so it starts at "
946 << start_timestamp_;
947
948 buffer->TrimStart(frames_to_trim);
949 buffer->set_timestamp(start_timestamp_);
950 }
951 // If the entire buffer was trimmed, request a new one.
952 if (!buffer->frame_count())
953 return true;
954 }
955
956 // Store the timestamp of the first packet so we know when to start actual
957 // audio playback.
958 if (first_packet_timestamp_ == kNoTimestamp)
959 first_packet_timestamp_ = buffer->timestamp();
960
961 #if !defined(OS_ANDROID)
962 if (transcribe_audio_callback_ && volume_ > 0)
963 transcribe_audio_callback_.Run(buffer);
964 #endif
965
966 if (state_ != kUninitialized)
967 algorithm_->EnqueueBuffer(std::move(buffer));
968 }
969
970 const size_t memory_usage = algorithm_->GetMemoryUsage();
971 PipelineStatistics stats;
972 stats.audio_memory_usage = memory_usage - last_audio_memory_usage_;
973 last_audio_memory_usage_ = memory_usage;
974 task_runner_->PostTask(FROM_HERE,
975 base::BindOnce(&AudioRendererImpl::OnStatisticsUpdate,
976 weak_factory_.GetWeakPtr(), stats));
977
978 switch (state_) {
979 case kUninitialized:
980 case kInitializing:
981 case kFlushing:
982 NOTREACHED();
983 return false;
984
985 case kFlushed:
986 DCHECK(!pending_read_);
987 return false;
988
989 case kPlaying:
990 if (received_end_of_stream_ || algorithm_->IsQueueAdequateForPlayback()) {
991 if (buffering_state_ == BUFFERING_HAVE_NOTHING)
992 SetBufferingState_Locked(BUFFERING_HAVE_ENOUGH);
993 // This must be done after SetBufferingState_Locked() to ensure the
994 // proper state transitions for higher levels.
995 if (should_render_end_of_stream) {
996 task_runner_->PostTask(
997 FROM_HERE, base::BindOnce(&AudioRendererImpl::OnPlaybackEnded,
998 weak_factory_.GetWeakPtr()));
999 }
1000 return false;
1001 }
1002 return true;
1003 }
1004 return false;
1005 }
1006
1007 void AudioRendererImpl::AttemptRead() {
1008 base::AutoLock auto_lock(lock_);
1009 AttemptRead_Locked();
1010 }
1011
1012 void AudioRendererImpl::AttemptRead_Locked() {
1013 DCHECK(task_runner_->BelongsToCurrentThread());
1014 lock_.AssertAcquired();
1015
1016 if (!CanRead_Locked())
1017 return;
1018
1019 pending_read_ = true;
1020
1021 // Don't hold the lock while calling Read(), if the demuxer is busy this will
1022 // block audio rendering for an extended period of time.
1023 // |audio_decoder_stream_| is only accessed on |task_runner_| so this is safe.
1024 base::AutoUnlock auto_unlock(lock_);
1025 audio_decoder_stream_->Read(base::BindOnce(
1026 &AudioRendererImpl::DecodedAudioReady, weak_factory_.GetWeakPtr()));
1027 }
1028
1029 bool AudioRendererImpl::CanRead_Locked() {
1030 lock_.AssertAcquired();
1031
1032 switch (state_) {
1033 case kUninitialized:
1034 case kInitializing:
1035 case kFlushing:
1036 case kFlushed:
1037 return false;
1038
1039 case kPlaying:
1040 break;
1041 }
1042
1043 return !pending_read_ && !received_end_of_stream_ &&
1044 !algorithm_->IsQueueFull();
1045 }
1046
1047 void AudioRendererImpl::SetPlaybackRate(double playback_rate) {
1048 DVLOG(1) << __func__ << "(" << playback_rate << ")";
1049 DCHECK(task_runner_->BelongsToCurrentThread());
1050 DCHECK_GE(playback_rate, 0);
1051 DCHECK(sink_);
1052
1053 base::AutoLock auto_lock(lock_);
1054
1055 if (is_passthrough_ && playback_rate != 0 && playback_rate != 1) {
1056 MEDIA_LOG(INFO, media_log_) << "Playback rate changes are not supported "
1057 "when output compressed bitstream."
1058 << " Playback Rate: " << playback_rate;
1059 return;
1060 }
1061
1062 // We have two cases here:
1063 // Play: current_playback_rate == 0 && playback_rate != 0
1064 // Pause: current_playback_rate != 0 && playback_rate == 0
1065 double current_playback_rate = playback_rate_;
1066 playback_rate_ = playback_rate;
1067
1068 if (!rendering_)
1069 return;
1070
1071 if (current_playback_rate == 0 && playback_rate != 0) {
1072 StartRendering_Locked();
1073 return;
1074 }
1075
1076 if (current_playback_rate != 0 && playback_rate == 0) {
1077 StopRendering_Locked();
1078 return;
1079 }
1080 }
1081
1082 bool AudioRendererImpl::IsBeforeStartTime(const AudioBuffer& buffer) {
1083 DCHECK_EQ(state_, kPlaying);
1084 return !buffer.end_of_stream() &&
1085 (buffer.timestamp() + buffer.duration()) < start_timestamp_;
1086 }
1087
1088 int AudioRendererImpl::Render(base::TimeDelta delay,
1089 base::TimeTicks delay_timestamp,
1090 int prior_frames_skipped,
1091 AudioBus* audio_bus) {
1092 TRACE_EVENT1("media", "AudioRendererImpl::Render", "id", media_log_->id());
1093 int frames_requested = audio_bus->frames();
1094 DVLOG(4) << __func__ << " delay:" << delay
1095 << " prior_frames_skipped:" << prior_frames_skipped
1096 << " frames_requested:" << frames_requested;
1097
1098 // Since this information is coming from the OS or potentially a fake stream,
1099 // it may end up with spurious values.
1100 if (delay < base::TimeDelta())
1101 delay = base::TimeDelta();
1102
1103 int frames_written = 0;
1104 {
1105 base::AutoLock auto_lock(lock_);
1106 last_render_time_ = tick_clock_->NowTicks();
1107
1108 int64_t frames_delayed = AudioTimestampHelper::TimeToFrames(
1109 delay, audio_parameters_.sample_rate());
1110
1111 if (!stop_rendering_time_.is_null()) {
1112 audio_clock_->CompensateForSuspendedWrites(
1113 last_render_time_ - stop_rendering_time_, frames_delayed);
1114 stop_rendering_time_ = base::TimeTicks();
1115 }
1116
1117 // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
1118 if (!algorithm_) {
1119 audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
1120 playback_rate_);
1121 return 0;
1122 }
1123
1124 if (playback_rate_ == 0 || is_suspending_) {
1125 audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
1126 playback_rate_);
1127 return 0;
1128 }
1129
1130 // Mute audio by returning 0 when not playing.
1131 if (state_ != kPlaying) {
1132 audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
1133 playback_rate_);
1134 return 0;
1135 }
1136
1137 if (is_passthrough_ && algorithm_->BufferedFrames() > 0) {
1138 // TODO(tsunghung): For compressed bitstream formats, play zeroed buffer
1139 // won't generate delay. It could be discarded immediately. Need another
1140 // way to generate audio delay.
1141 const base::TimeDelta play_delay =
1142 first_packet_timestamp_ - audio_clock_->back_timestamp();
1143 if (play_delay > base::TimeDelta()) {
1144 MEDIA_LOG(ERROR, media_log_)
1145 << "Cannot add delay for compressed audio bitstream foramt."
1146 << " Requested delay: " << play_delay;
1147 }
1148
1149 frames_written += algorithm_->FillBuffer(audio_bus, 0, frames_requested,
1150 playback_rate_);
1151
1152 // See Initialize(), the |audio_bus| should be bigger than we need in
1153 // bitstream cases. Fix |frames_requested| to avoid incorrent time
1154 // calculation of |audio_clock_| below.
1155 frames_requested = frames_written;
1156 } else if (algorithm_->BufferedFrames() > 0) {
1157 // Delay playback by writing silence if we haven't reached the first
1158 // timestamp yet; this can occur if the video starts before the audio.
1159 CHECK_NE(first_packet_timestamp_, kNoTimestamp);
1160 CHECK_GE(first_packet_timestamp_, base::TimeDelta());
1161 const base::TimeDelta play_delay =
1162 first_packet_timestamp_ - audio_clock_->back_timestamp();
1163 if (play_delay > base::TimeDelta()) {
1164 DCHECK_EQ(frames_written, 0);
1165
1166 if (!play_delay_cb_for_testing_.is_null())
1167 play_delay_cb_for_testing_.Run(play_delay);
1168
1169 // Don't multiply |play_delay| out since it can be a huge value on
1170 // poorly encoded media and multiplying by the sample rate could cause
1171 // the value to overflow.
1172 if (play_delay.InSecondsF() > static_cast<double>(frames_requested) /
1173 audio_parameters_.sample_rate()) {
1174 frames_written = frames_requested;
1175 } else {
1176 frames_written =
1177 play_delay.InSecondsF() * audio_parameters_.sample_rate();
1178 }
1179
1180 audio_bus->ZeroFramesPartial(0, frames_written);
1181 }
1182
1183 // If there's any space left, actually render the audio; this is where the
1184 // aural magic happens.
1185 if (frames_written < frames_requested) {
1186 frames_written += algorithm_->FillBuffer(
1187 audio_bus, frames_written, frames_requested - frames_written,
1188 playback_rate_);
1189 }
1190 }
1191
1192 // We use the following conditions to determine end of playback:
1193 // 1) Algorithm can not fill the audio callback buffer
1194 // 2) We received an end of stream buffer
1195 // 3) We haven't already signalled that we've ended
1196 // 4) We've played all known audio data sent to hardware
1197 //
1198 // We use the following conditions to determine underflow:
1199 // 1) Algorithm can not fill the audio callback buffer
1200 // 2) We have NOT received an end of stream buffer
1201 // 3) We are in the kPlaying state
1202 //
1203 // Otherwise the buffer has data we can send to the device.
1204 //
1205 // Per the TimeSource API the media time should always increase even after
1206 // we've rendered all known audio data. Doing so simplifies scenarios where
1207 // we have other sources of media data that need to be scheduled after audio
1208 // data has ended.
1209 //
1210 // That being said, we don't want to advance time when underflowed as we
1211 // know more decoded frames will eventually arrive. If we did, we would
1212 // throw things out of sync when said decoded frames arrive.
1213 int frames_after_end_of_stream = 0;
1214 if (frames_written == 0) {
1215 if (received_end_of_stream_) {
1216 if (ended_timestamp_ == kInfiniteDuration)
1217 ended_timestamp_ = audio_clock_->back_timestamp();
1218 frames_after_end_of_stream = frames_requested;
1219 } else if (state_ == kPlaying &&
1220 buffering_state_ != BUFFERING_HAVE_NOTHING) {
1221 // Don't increase queue capacity if the queue latency is explicitly
1222 // specified.
1223 if (!latency_hint_)
1224 algorithm_->IncreasePlaybackThreshold();
1225
1226 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
1227 }
1228 } else if (frames_written < frames_requested && !received_end_of_stream_ &&
1229 state_ == kPlaying &&
1230 buffering_state_ != BUFFERING_HAVE_NOTHING) {
1231 // If we only partially filled the request and should have more data, go
1232 // ahead and increase queue capacity to try and meet the next request.
1233 // Trigger underflow to give us a chance to refill up to the new cap.
1234 // When a latency hint is present, don't override the user's preference
1235 // with a queue increase, but still signal HAVE_NOTHING for them to take
1236 // action if they choose.
1237
1238 if (!latency_hint_)
1239 algorithm_->IncreasePlaybackThreshold();
1240
1241 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
1242 }
1243
1244 audio_clock_->WroteAudio(frames_written + frames_after_end_of_stream,
1245 frames_requested, frames_delayed, playback_rate_);
1246
1247 if (CanRead_Locked()) {
1248 task_runner_->PostTask(FROM_HERE,
1249 base::BindOnce(&AudioRendererImpl::AttemptRead,
1250 weak_factory_.GetWeakPtr()));
1251 }
1252
1253 if (audio_clock_->front_timestamp() >= ended_timestamp_ &&
1254 !rendered_end_of_stream_) {
1255 rendered_end_of_stream_ = true;
1256 task_runner_->PostTask(FROM_HERE,
1257 base::BindOnce(&AudioRendererImpl::OnPlaybackEnded,
1258 weak_factory_.GetWeakPtr()));
1259 }
1260 }
1261
1262 DCHECK_LE(frames_written, frames_requested);
1263 return frames_written;
1264 }
1265
1266 void AudioRendererImpl::OnRenderError() {
1267 MEDIA_LOG(ERROR, media_log_) << "audio render error";
1268
1269 // Post to |task_runner_| as this is called on the audio callback thread.
1270 task_runner_->PostTask(
1271 FROM_HERE,
1272 base::BindOnce(&AudioRendererImpl::OnPlaybackError,
1273 weak_factory_.GetWeakPtr(), AUDIO_RENDERER_ERROR));
1274 }
1275
1276 void AudioRendererImpl::HandleAbortedReadOrDecodeError(PipelineStatus status) {
1277 DCHECK(task_runner_->BelongsToCurrentThread());
1278 lock_.AssertAcquired();
1279
1280 switch (state_) {
1281 case kUninitialized:
1282 case kInitializing:
1283 NOTREACHED();
1284 return;
1285 case kFlushing:
1286 ChangeState_Locked(kFlushed);
1287 if (status == PIPELINE_OK) {
1288 DoFlush_Locked();
1289 return;
1290 }
1291
1292 MEDIA_LOG(ERROR, media_log_)
1293 << "audio error during flushing, status: " << status;
1294 client_->OnError(status);
1295 FinishFlush();
1296 return;
1297
1298 case kFlushed:
1299 case kPlaying:
1300 if (status != PIPELINE_OK) {
1301 MEDIA_LOG(ERROR, media_log_)
1302 << "audio error during playing, status: " << status;
1303 client_->OnError(status);
1304 }
1305 return;
1306 }
1307 }
1308
1309 void AudioRendererImpl::ChangeState_Locked(State new_state) {
1310 DVLOG(1) << __func__ << " : " << state_ << " -> " << new_state;
1311 lock_.AssertAcquired();
1312 state_ = new_state;
1313 }
1314
1315 void AudioRendererImpl::OnConfigChange(const AudioDecoderConfig& config) {
1316 DCHECK(task_runner_->BelongsToCurrentThread());
1317 DCHECK(expecting_config_changes_);
1318 buffer_converter_->ResetTimestampState();
1319
1320 // An invalid config may be supplied by callers who simply want to reset
1321 // internal state outside of detecting a new config from the demuxer stream.
1322 // RendererClient only cares to know about config changes that differ from
1323 // previous configs.
1324 if (config.IsValidConfig() && !current_decoder_config_.Matches(config)) {
1325 current_decoder_config_ = config;
1326 client_->OnAudioConfigChange(config);
1327 }
1328 }
1329
1330 void AudioRendererImpl::SetBufferingState_Locked(
1331 BufferingState buffering_state) {
1332 DVLOG(1) << __func__ << " : " << buffering_state_ << " -> "
1333 << buffering_state;
1334 DCHECK_NE(buffering_state_, buffering_state);
1335 lock_.AssertAcquired();
1336 buffering_state_ = buffering_state;
1337
1338 task_runner_->PostTask(
1339 FROM_HERE, base::BindOnce(&AudioRendererImpl::OnBufferingStateChange,
1340 weak_factory_.GetWeakPtr(), buffering_state_));
1341 }
1342
1343 void AudioRendererImpl::ConfigureChannelMask() {
1344 DCHECK(algorithm_);
1345 DCHECK(audio_parameters_.IsValid());
1346 DCHECK_NE(last_decoded_channel_layout_, CHANNEL_LAYOUT_NONE);
1347 DCHECK_NE(last_decoded_channel_layout_, CHANNEL_LAYOUT_UNSUPPORTED);
1348
1349 // If we're actually downmixing the signal, no mask is necessary, but ensure
1350 // we clear any existing mask if present.
1351 if (last_decoded_channels_ >= audio_parameters_.channels()) {
1352 algorithm_->SetChannelMask(
1353 std::vector<bool>(audio_parameters_.channels(), true));
1354 return;
1355 }
1356
1357 // Determine the matrix used to upmix the channels.
1358 std::vector<std::vector<float>> matrix;
1359 ChannelMixingMatrix(last_decoded_channel_layout_, last_decoded_channels_,
1360 audio_parameters_.channel_layout(),
1361 audio_parameters_.channels())
1362 .CreateTransformationMatrix(&matrix);
1363
1364 // All channels with a zero mix are muted and can be ignored.
1365 std::vector<bool> channel_mask(audio_parameters_.channels(), false);
1366 for (size_t ch = 0; ch < matrix.size(); ++ch) {
1367 channel_mask[ch] = std::any_of(matrix[ch].begin(), matrix[ch].end(),
1368 [](float mix) { return !!mix; });
1369 }
1370 algorithm_->SetChannelMask(std::move(channel_mask));
1371 }
1372
1373 void AudioRendererImpl::EnableSpeechRecognition() {
1374 #if !defined(OS_ANDROID)
1375 DCHECK(task_runner_->BelongsToCurrentThread());
1376 transcribe_audio_callback_ = base::BindRepeating(
1377 &AudioRendererImpl::TranscribeAudio, weak_factory_.GetWeakPtr());
1378 #endif
1379 }
1380
1381 void AudioRendererImpl::TranscribeAudio(
1382 scoped_refptr<media::AudioBuffer> buffer) {
1383 #if !defined(OS_ANDROID)
1384 DCHECK(task_runner_->BelongsToCurrentThread());
1385 if (speech_recognition_client_)
1386 speech_recognition_client_->AddAudio(std::move(buffer));
1387 #endif
1388 }
1389 } // namespace media
1390