1 // Copyright 2017 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "third_party/blink/renderer/modules/mediarecorder/audio_track_opus_encoder.h"
6
7 #include "base/stl_util.h"
8 #include "media/base/audio_sample_types.h"
9 #include "media/base/audio_timestamp_helper.h"
10
11 namespace {
12
13 enum : int {
14 // Recommended value for opus_encode_float(), according to documentation in
15 // third_party/opus/src/include/opus.h, so that the Opus encoder does not
16 // degrade the audio due to memory constraints, and is independent of the
17 // duration of the encoded buffer.
18 kOpusMaxDataBytes = 4000,
19
20 // Opus preferred sampling rate for encoding. This is also the one WebM likes
21 // to have: https://wiki.xiph.org/MatroskaOpus.
22 kOpusPreferredSamplingRate = 48000,
23
24 // For Opus, we try to encode 60ms, the maximum Opus buffer, for quality
25 // reasons.
26 kOpusPreferredBufferDurationMs = 60,
27
28 // Maximum buffer multiplier for the AudioEncoders' AudioFifo. Recording is
29 // not real time, hence a certain buffering is allowed.
30 kMaxNumberOfFifoBuffers = 3,
31 };
32
33 // The amount of Frames in a 60 ms buffer @ 48000 samples/second.
34 const int kOpusPreferredFramesPerBuffer = kOpusPreferredSamplingRate *
35 kOpusPreferredBufferDurationMs /
36 base::Time::kMillisecondsPerSecond;
37
38 // Tries to encode |data_in|'s |num_samples| into |data_out|.
DoEncode(OpusEncoder * opus_encoder,float * data_in,int num_samples,std::string * data_out)39 bool DoEncode(OpusEncoder* opus_encoder,
40 float* data_in,
41 int num_samples,
42 std::string* data_out) {
43 DCHECK_EQ(kOpusPreferredFramesPerBuffer, num_samples);
44
45 data_out->resize(kOpusMaxDataBytes);
46 const opus_int32 result = opus_encode_float(
47 opus_encoder, data_in, num_samples,
48 reinterpret_cast<uint8_t*>(base::data(*data_out)), kOpusMaxDataBytes);
49
50 if (result > 1) {
51 // TODO(ajose): Investigate improving this. http://crbug.com/547918
52 data_out->resize(result);
53 return true;
54 }
55 // If |result| in {0,1}, do nothing; the documentation says that a return
56 // value of zero or one means the packet does not need to be transmitted.
57 // Otherwise, we have an error.
58 DLOG_IF(ERROR, result < 0) << " encode failed: " << opus_strerror(result);
59 return false;
60 }
61
62 } // anonymous namespace
63
64 namespace blink {
65
AudioTrackOpusEncoder(OnEncodedAudioCB on_encoded_audio_cb,int32_t bits_per_second)66 AudioTrackOpusEncoder::AudioTrackOpusEncoder(
67 OnEncodedAudioCB on_encoded_audio_cb,
68 int32_t bits_per_second)
69 : AudioTrackEncoder(std::move(on_encoded_audio_cb)),
70 bits_per_second_(bits_per_second),
71 opus_encoder_(nullptr) {}
72
~AudioTrackOpusEncoder()73 AudioTrackOpusEncoder::~AudioTrackOpusEncoder() {
74 // We don't DCHECK that we're on the encoder thread here, as it should have
75 // already been deleted at this point.
76 DestroyExistingOpusEncoder();
77 }
78
ProvideInput(media::AudioBus * audio_bus,uint32_t frames_delayed)79 double AudioTrackOpusEncoder::ProvideInput(media::AudioBus* audio_bus,
80 uint32_t frames_delayed) {
81 fifo_->Consume(audio_bus, 0, audio_bus->frames());
82 return 1.0;
83 }
84
OnSetFormat(const media::AudioParameters & input_params)85 void AudioTrackOpusEncoder::OnSetFormat(
86 const media::AudioParameters& input_params) {
87 DVLOG(1) << __func__;
88 DCHECK_CALLED_ON_VALID_THREAD(encoder_thread_checker_);
89 if (input_params_.Equals(input_params))
90 return;
91
92 DestroyExistingOpusEncoder();
93
94 if (!input_params.IsValid()) {
95 DLOG(ERROR) << "Invalid params: " << input_params.AsHumanReadableString();
96 return;
97 }
98 input_params_ = input_params;
99 input_params_.set_frames_per_buffer(input_params_.sample_rate() *
100 kOpusPreferredBufferDurationMs /
101 base::Time::kMillisecondsPerSecond);
102
103 // third_party/libopus supports up to 2 channels (see implementation of
104 // opus_encoder_create()): force |converted_params_| to at most those.
105 converted_params_ = media::AudioParameters(
106 media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
107 media::GuessChannelLayout(std::min(input_params_.channels(), 2)),
108 kOpusPreferredSamplingRate, kOpusPreferredFramesPerBuffer);
109 DVLOG(1) << "|input_params_|:" << input_params_.AsHumanReadableString()
110 << " -->|converted_params_|:"
111 << converted_params_.AsHumanReadableString();
112
113 converter_.reset(new media::AudioConverter(input_params_, converted_params_,
114 false /* disable_fifo */));
115 converter_->AddInput(this);
116 converter_->PrimeWithSilence();
117
118 fifo_.reset(new media::AudioFifo(
119 input_params_.channels(),
120 kMaxNumberOfFifoBuffers * input_params_.frames_per_buffer()));
121
122 buffer_.reset(new float[converted_params_.channels() *
123 converted_params_.frames_per_buffer()]);
124
125 // Initialize OpusEncoder.
126 int opus_result;
127 opus_encoder_ = opus_encoder_create(converted_params_.sample_rate(),
128 converted_params_.channels(),
129 OPUS_APPLICATION_AUDIO, &opus_result);
130 if (opus_result < 0) {
131 DLOG(ERROR) << "Couldn't init Opus encoder: " << opus_strerror(opus_result)
132 << ", sample rate: " << converted_params_.sample_rate()
133 << ", channels: " << converted_params_.channels();
134 return;
135 }
136
137 // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a
138 // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms
139 // buffer duration. The Opus library authors may, of course, adjust this in
140 // later versions.
141 const opus_int32 bitrate =
142 (bits_per_second_ > 0) ? bits_per_second_ : OPUS_AUTO;
143 if (opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate)) != OPUS_OK) {
144 DLOG(ERROR) << "Failed to set Opus bitrate: " << bitrate;
145 return;
146 }
147 }
148
EncodeAudio(std::unique_ptr<media::AudioBus> input_bus,base::TimeTicks capture_time)149 void AudioTrackOpusEncoder::EncodeAudio(
150 std::unique_ptr<media::AudioBus> input_bus,
151 base::TimeTicks capture_time) {
152 DVLOG(3) << __func__ << ", #frames " << input_bus->frames();
153 DCHECK_CALLED_ON_VALID_THREAD(encoder_thread_checker_);
154 DCHECK_EQ(input_bus->channels(), input_params_.channels());
155 DCHECK(!capture_time.is_null());
156 DCHECK(converter_);
157
158 if (!is_initialized() || paused_)
159 return;
160
161 // TODO(mcasas): Consider using a
162 // base::circular_deque<std::unique_ptr<AudioBus>> instead of an AudioFifo,
163 // to avoid copying data needlessly since we know the sizes of both input and
164 // output and they are multiples.
165 fifo_->Push(input_bus.get());
166
167 // Wait to have enough |input_bus|s to guarantee a satisfactory conversion,
168 // accounting for multiple calls to ProvideInput().
169 while (fifo_->frames() >= converter_->GetMaxInputFramesRequested(
170 kOpusPreferredFramesPerBuffer)) {
171 std::unique_ptr<media::AudioBus> audio_bus = media::AudioBus::Create(
172 converted_params_.channels(), kOpusPreferredFramesPerBuffer);
173 converter_->Convert(audio_bus.get());
174 audio_bus->ToInterleaved<media::Float32SampleTypeTraits>(
175 audio_bus->frames(), buffer_.get());
176
177 std::string encoded_data;
178 if (DoEncode(opus_encoder_, buffer_.get(), kOpusPreferredFramesPerBuffer,
179 &encoded_data)) {
180 const base::TimeTicks capture_time_of_first_sample =
181 capture_time - media::AudioTimestampHelper::FramesToTime(
182 input_bus->frames(), input_params_.sample_rate());
183 on_encoded_audio_cb_.Run(converted_params_, std::move(encoded_data),
184 capture_time_of_first_sample);
185 }
186 }
187 }
188
DestroyExistingOpusEncoder()189 void AudioTrackOpusEncoder::DestroyExistingOpusEncoder() {
190 // We don't DCHECK that we're on the encoder thread here, as this could be
191 // called from the dtor (main thread) or from OnSetFormat() (encoder thread).
192 if (opus_encoder_) {
193 opus_encoder_destroy(opus_encoder_);
194 opus_encoder_ = nullptr;
195 }
196 }
197
198 } // namespace blink
199