1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/muxers/webm_muxer.h"
6
7 #include <algorithm>
8 #include <memory>
9
10 #include "base/bind.h"
11 #include "media/base/audio_parameters.h"
12 #include "media/base/limits.h"
13 #include "media/base/video_frame.h"
14 #include "media/formats/common/opus_constants.h"
15
16 namespace media {
17
18 namespace {
19
WriteOpusHeader(const media::AudioParameters & params,uint8_t * header)20 void WriteOpusHeader(const media::AudioParameters& params, uint8_t* header) {
21 // See https://wiki.xiph.org/OggOpus#ID_Header.
22 // Set magic signature.
23 std::string label = "OpusHead";
24 memcpy(header + OPUS_EXTRADATA_LABEL_OFFSET, label.c_str(), label.size());
25 // Set Opus version.
26 header[OPUS_EXTRADATA_VERSION_OFFSET] = 1;
27 // Set channel count.
28 DCHECK_LE(params.channels(), 2);
29 header[OPUS_EXTRADATA_CHANNELS_OFFSET] = params.channels();
30 // Set pre-skip
31 uint16_t skip = 0;
32 memcpy(header + OPUS_EXTRADATA_SKIP_SAMPLES_OFFSET, &skip, sizeof(uint16_t));
33 // Set original input sample rate in Hz.
34 uint32_t sample_rate = params.sample_rate();
35 memcpy(header + OPUS_EXTRADATA_SAMPLE_RATE_OFFSET, &sample_rate,
36 sizeof(uint32_t));
37 // Set output gain in dB.
38 uint16_t gain = 0;
39 memcpy(header + OPUS_EXTRADATA_GAIN_OFFSET, &gain, 2);
40
41 header[OPUS_EXTRADATA_CHANNEL_MAPPING_OFFSET] = 0;
42 }
43
GetFrameRate(const WebmMuxer::VideoParameters & params)44 static double GetFrameRate(const WebmMuxer::VideoParameters& params) {
45 const double kZeroFrameRate = 0.0;
46 const double kDefaultFrameRate = 30.0;
47
48 double frame_rate = params.frame_rate;
49 if (frame_rate <= kZeroFrameRate ||
50 frame_rate > media::limits::kMaxFramesPerSecond) {
51 frame_rate = kDefaultFrameRate;
52 }
53 return frame_rate;
54 }
55
56 static const char kH264CodecId[] = "V_MPEG4/ISO/AVC";
57 static const char kPcmCodecId[] = "A_PCM/FLOAT/IEEE";
58
MkvCodeIcForMediaVideoCodecId(VideoCodec video_codec)59 static const char* MkvCodeIcForMediaVideoCodecId(VideoCodec video_codec) {
60 switch (video_codec) {
61 case kCodecVP8:
62 return mkvmuxer::Tracks::kVp8CodecId;
63 case kCodecVP9:
64 return mkvmuxer::Tracks::kVp9CodecId;
65 case kCodecH264:
66 return kH264CodecId;
67 default:
68 NOTREACHED() << "Unsupported codec " << GetCodecName(video_codec);
69 return "";
70 }
71 }
72
ColorFromColorSpace(const gfx::ColorSpace & color)73 base::Optional<mkvmuxer::Colour> ColorFromColorSpace(
74 const gfx::ColorSpace& color) {
75 using mkvmuxer::Colour;
76 using MatrixID = gfx::ColorSpace::MatrixID;
77 using RangeID = gfx::ColorSpace::RangeID;
78 using TransferID = gfx::ColorSpace::TransferID;
79 using PrimaryID = gfx::ColorSpace::PrimaryID;
80 Colour colour;
81 int matrix_coefficients;
82 switch (color.GetMatrixID()) {
83 case MatrixID::BT709:
84 matrix_coefficients = Colour::kBt709;
85 break;
86 case MatrixID::BT2020_NCL:
87 matrix_coefficients = Colour::kBt2020NonConstantLuminance;
88 break;
89 default:
90 return base::nullopt;
91 }
92 colour.set_matrix_coefficients(matrix_coefficients);
93 int range;
94 switch (color.GetRangeID()) {
95 case RangeID::LIMITED:
96 range = Colour::kBroadcastRange;
97 break;
98 case RangeID::FULL:
99 range = Colour::kFullRange;
100 break;
101 default:
102 return base::nullopt;
103 }
104 colour.set_range(range);
105 int transfer_characteristics;
106 switch (color.GetTransferID()) {
107 case TransferID::BT709:
108 transfer_characteristics = Colour::kIturBt709Tc;
109 break;
110 case TransferID::IEC61966_2_1:
111 transfer_characteristics = Colour::kIec6196621;
112 break;
113 case TransferID::SMPTEST2084:
114 transfer_characteristics = Colour::kSmpteSt2084;
115 break;
116 default:
117 return base::nullopt;
118 }
119 colour.set_transfer_characteristics(transfer_characteristics);
120 int primaries;
121 switch (color.GetPrimaryID()) {
122 case PrimaryID::BT709:
123 primaries = Colour::kIturBt709P;
124 break;
125 case PrimaryID::BT2020:
126 primaries = Colour::kIturBt2020;
127 break;
128 default:
129 return base::nullopt;
130 }
131 colour.set_primaries(primaries);
132 return colour;
133 }
134
135 } // anonymous namespace
136
VideoParameters(scoped_refptr<media::VideoFrame> frame)137 WebmMuxer::VideoParameters::VideoParameters(
138 scoped_refptr<media::VideoFrame> frame)
139 : visible_rect_size(frame->visible_rect().size()),
140 frame_rate(0.0),
141 codec(kUnknownVideoCodec),
142 color_space(frame->ColorSpace()) {
143 ignore_result(frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
144 &frame_rate));
145 }
146
VideoParameters(gfx::Size visible_rect_size,double frame_rate,VideoCodec codec,base::Optional<gfx::ColorSpace> color_space)147 WebmMuxer::VideoParameters::VideoParameters(
148 gfx::Size visible_rect_size,
149 double frame_rate,
150 VideoCodec codec,
151 base::Optional<gfx::ColorSpace> color_space)
152 : visible_rect_size(visible_rect_size),
153 frame_rate(frame_rate),
154 codec(codec),
155 color_space(color_space) {}
156
157 WebmMuxer::VideoParameters::VideoParameters(const VideoParameters&) = default;
158
159 WebmMuxer::VideoParameters::~VideoParameters() = default;
160
WebmMuxer(AudioCodec audio_codec,bool has_video,bool has_audio,const WriteDataCB & write_data_callback)161 WebmMuxer::WebmMuxer(AudioCodec audio_codec,
162 bool has_video,
163 bool has_audio,
164 const WriteDataCB& write_data_callback)
165 : audio_codec_(audio_codec),
166 video_codec_(kUnknownVideoCodec),
167 video_track_index_(0),
168 audio_track_index_(0),
169 has_video_(has_video),
170 has_audio_(has_audio),
171 write_data_callback_(write_data_callback),
172 position_(0),
173 force_one_libwebm_error_(false) {
174 DCHECK(has_video_ || has_audio_);
175 DCHECK(!write_data_callback_.is_null());
176 DCHECK(audio_codec == kCodecOpus || audio_codec == kCodecPCM)
177 << " Unsupported audio codec: " << GetCodecName(audio_codec);
178
179 segment_.Init(this);
180 segment_.set_mode(mkvmuxer::Segment::kLive);
181 segment_.OutputCues(false);
182
183 mkvmuxer::SegmentInfo* const info = segment_.GetSegmentInfo();
184 info->set_writing_app("Chrome");
185 info->set_muxing_app("Chrome");
186
187 // Creation is done on a different thread than main activities.
188 thread_checker_.DetachFromThread();
189 }
190
~WebmMuxer()191 WebmMuxer::~WebmMuxer() {
192 // No need to segment_.Finalize() since is not Seekable(), i.e. a live
193 // stream, but is a good practice.
194 DCHECK(thread_checker_.CalledOnValidThread());
195 FlushQueues();
196 segment_.Finalize();
197 }
198
OnEncodedVideo(const VideoParameters & params,std::string encoded_data,std::string encoded_alpha,base::TimeTicks timestamp,bool is_key_frame)199 bool WebmMuxer::OnEncodedVideo(const VideoParameters& params,
200 std::string encoded_data,
201 std::string encoded_alpha,
202 base::TimeTicks timestamp,
203 bool is_key_frame) {
204 DVLOG(1) << __func__ << " - " << encoded_data.size() << "B";
205 DCHECK(thread_checker_.CalledOnValidThread());
206 DCHECK(params.codec == kCodecVP8 || params.codec == kCodecVP9 ||
207 params.codec == kCodecH264)
208 << " Unsupported video codec: " << GetCodecName(params.codec);
209 DCHECK(video_codec_ == kUnknownVideoCodec || video_codec_ == params.codec)
210 << "Unsupported: codec switched, to: " << GetCodecName(params.codec);
211
212 if (encoded_data.size() == 0u) {
213 DLOG(WARNING) << __func__ << ": zero size encoded frame, skipping";
214 // Some encoders give sporadic zero-size data, see https://crbug.com/716451.
215 return true;
216 }
217
218 if (!video_track_index_) {
219 // |track_index_|, cannot be zero (!), initialize WebmMuxer in that case.
220 // http://www.matroska.org/technical/specs/index.html#Tracks
221 video_codec_ = params.codec;
222 AddVideoTrack(params.visible_rect_size, GetFrameRate(params),
223 params.color_space);
224 if (first_frame_timestamp_video_.is_null()) {
225 // Compensate for time in pause spent before the first frame.
226 first_frame_timestamp_video_ = timestamp - total_time_in_pause_;
227 }
228 }
229
230 // TODO(ajose): Support multiple tracks: http://crbug.com/528523
231 if (has_audio_ && !audio_track_index_) {
232 DVLOG(1) << __func__ << ": delaying until audio track ready.";
233 if (is_key_frame) // Upon Key frame reception, empty the encoded queue.
234 video_frames_.clear();
235 }
236 const base::TimeTicks recorded_timestamp =
237 UpdateLastTimestampMonotonically(timestamp, &last_frame_timestamp_video_);
238 video_frames_.push_back(EncodedFrame{
239 std::move(encoded_data), std::move(encoded_alpha),
240 recorded_timestamp - first_frame_timestamp_video_, is_key_frame});
241 return PartiallyFlushQueues();
242 }
243
OnEncodedAudio(const media::AudioParameters & params,std::string encoded_data,base::TimeTicks timestamp)244 bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
245 std::string encoded_data,
246 base::TimeTicks timestamp) {
247 DVLOG(2) << __func__ << " - " << encoded_data.size() << "B";
248 DCHECK(thread_checker_.CalledOnValidThread());
249
250 if (!audio_track_index_) {
251 AddAudioTrack(params);
252 if (first_frame_timestamp_audio_.is_null()) {
253 // Compensate for time in pause spent before the first frame.
254 first_frame_timestamp_audio_ = timestamp - total_time_in_pause_;
255 }
256 }
257
258 const base::TimeTicks recorded_timestamp =
259 UpdateLastTimestampMonotonically(timestamp, &last_frame_timestamp_audio_);
260 audio_frames_.push_back(
261 EncodedFrame{encoded_data, std::string(),
262 recorded_timestamp - first_frame_timestamp_audio_,
263 /*is_keyframe=*/true});
264 return PartiallyFlushQueues();
265 }
266
Pause()267 void WebmMuxer::Pause() {
268 DVLOG(1) << __func__;
269 DCHECK(thread_checker_.CalledOnValidThread());
270 if (!elapsed_time_in_pause_)
271 elapsed_time_in_pause_.reset(new base::ElapsedTimer());
272 }
273
Resume()274 void WebmMuxer::Resume() {
275 DVLOG(1) << __func__;
276 DCHECK(thread_checker_.CalledOnValidThread());
277 if (elapsed_time_in_pause_) {
278 total_time_in_pause_ += elapsed_time_in_pause_->Elapsed();
279 elapsed_time_in_pause_.reset();
280 }
281 }
282
AddVideoTrack(const gfx::Size & frame_size,double frame_rate,const base::Optional<gfx::ColorSpace> & color_space)283 void WebmMuxer::AddVideoTrack(
284 const gfx::Size& frame_size,
285 double frame_rate,
286 const base::Optional<gfx::ColorSpace>& color_space) {
287 DCHECK(thread_checker_.CalledOnValidThread());
288 DCHECK_EQ(0u, video_track_index_)
289 << "WebmMuxer can only be initialized once.";
290
291 video_track_index_ =
292 segment_.AddVideoTrack(frame_size.width(), frame_size.height(), 0);
293 if (video_track_index_ <= 0) { // See https://crbug.com/616391.
294 NOTREACHED() << "Error adding video track";
295 return;
296 }
297
298 mkvmuxer::VideoTrack* const video_track =
299 reinterpret_cast<mkvmuxer::VideoTrack*>(
300 segment_.GetTrackByNumber(video_track_index_));
301 if (color_space) {
302 auto colour = ColorFromColorSpace(*color_space);
303 if (colour)
304 video_track->SetColour(*colour);
305 }
306 DCHECK(video_track);
307 video_track->set_codec_id(MkvCodeIcForMediaVideoCodecId(video_codec_));
308 DCHECK_EQ(0ull, video_track->crop_right());
309 DCHECK_EQ(0ull, video_track->crop_left());
310 DCHECK_EQ(0ull, video_track->crop_top());
311 DCHECK_EQ(0ull, video_track->crop_bottom());
312 DCHECK_EQ(0.0f, video_track->frame_rate());
313
314 // Segment's timestamps should be in milliseconds, DCHECK it. See
315 // http://www.webmproject.org/docs/container/#muxer-guidelines
316 DCHECK_EQ(1000000ull, segment_.GetSegmentInfo()->timecode_scale());
317
318 // Set alpha channel parameters for only VPX (crbug.com/711825).
319 if (video_codec_ == kCodecH264)
320 return;
321 video_track->SetAlphaMode(mkvmuxer::VideoTrack::kAlpha);
322 // Alpha channel, if present, is stored in a BlockAdditional next to the
323 // associated opaque Block, see
324 // https://matroska.org/technical/specs/index.html#BlockAdditional.
325 // This follows Method 1 for VP8 encoding of A-channel described on
326 // http://wiki.webmproject.org/alpha-channel.
327 video_track->set_max_block_additional_id(1);
328 }
329
AddAudioTrack(const media::AudioParameters & params)330 void WebmMuxer::AddAudioTrack(const media::AudioParameters& params) {
331 DVLOG(1) << __func__ << " " << params.AsHumanReadableString();
332 DCHECK(thread_checker_.CalledOnValidThread());
333 DCHECK_EQ(0u, audio_track_index_)
334 << "WebmMuxer audio can only be initialised once.";
335
336 audio_track_index_ =
337 segment_.AddAudioTrack(params.sample_rate(), params.channels(), 0);
338 if (audio_track_index_ <= 0) { // See https://crbug.com/616391.
339 NOTREACHED() << "Error adding audio track";
340 return;
341 }
342
343 mkvmuxer::AudioTrack* const audio_track =
344 reinterpret_cast<mkvmuxer::AudioTrack*>(
345 segment_.GetTrackByNumber(audio_track_index_));
346 DCHECK(audio_track);
347 DCHECK_EQ(params.sample_rate(), audio_track->sample_rate());
348 DCHECK_EQ(params.channels(), static_cast<int>(audio_track->channels()));
349 DCHECK_LE(params.channels(), 2)
350 << "Only 1 or 2 channels supported, requested " << params.channels();
351
352 // Audio data is always pcm_f32le.
353 audio_track->set_bit_depth(32u);
354
355 if (audio_codec_ == kCodecOpus) {
356 audio_track->set_codec_id(mkvmuxer::Tracks::kOpusCodecId);
357
358 uint8_t opus_header[OPUS_EXTRADATA_SIZE];
359 WriteOpusHeader(params, opus_header);
360
361 if (!audio_track->SetCodecPrivate(opus_header, OPUS_EXTRADATA_SIZE))
362 LOG(ERROR) << __func__ << ": failed to set opus header.";
363
364 // Segment's timestamps should be in milliseconds, DCHECK it. See
365 // http://www.webmproject.org/docs/container/#muxer-guidelines
366 DCHECK_EQ(1000000ull, segment_.GetSegmentInfo()->timecode_scale());
367 } else if (audio_codec_ == kCodecPCM) {
368 audio_track->set_codec_id(kPcmCodecId);
369 }
370 }
371
Write(const void * buf,mkvmuxer::uint32 len)372 mkvmuxer::int32 WebmMuxer::Write(const void* buf, mkvmuxer::uint32 len) {
373 DCHECK(thread_checker_.CalledOnValidThread());
374 DCHECK(buf);
375 write_data_callback_.Run(
376 base::StringPiece(reinterpret_cast<const char*>(buf), len));
377 position_ += len;
378 return 0;
379 }
380
Position() const381 mkvmuxer::int64 WebmMuxer::Position() const {
382 return position_.ValueOrDie();
383 }
384
Position(mkvmuxer::int64 position)385 mkvmuxer::int32 WebmMuxer::Position(mkvmuxer::int64 position) {
386 // The stream is not Seekable() so indicate we cannot set the position.
387 return -1;
388 }
389
Seekable() const390 bool WebmMuxer::Seekable() const {
391 return false;
392 }
393
ElementStartNotify(mkvmuxer::uint64 element_id,mkvmuxer::int64 position)394 void WebmMuxer::ElementStartNotify(mkvmuxer::uint64 element_id,
395 mkvmuxer::int64 position) {
396 // This method gets pinged before items are sent to |write_data_callback_|.
397 DCHECK_GE(position, position_.ValueOrDefault(0))
398 << "Can't go back in a live WebM stream.";
399 }
400
FlushQueues()401 void WebmMuxer::FlushQueues() {
402 DCHECK(thread_checker_.CalledOnValidThread());
403 while ((!video_frames_.empty() || !audio_frames_.empty()) &&
404 FlushNextFrame()) {
405 }
406 }
407
PartiallyFlushQueues()408 bool WebmMuxer::PartiallyFlushQueues() {
409 DCHECK(thread_checker_.CalledOnValidThread());
410 bool result = true;
411 while (!(has_video_ && video_frames_.empty()) &&
412 !(has_audio_ && audio_frames_.empty()) && result) {
413 result = FlushNextFrame();
414 }
415 return result;
416 }
417
FlushNextFrame()418 bool WebmMuxer::FlushNextFrame() {
419 DCHECK(thread_checker_.CalledOnValidThread());
420 base::TimeDelta min_timestamp = base::TimeDelta::Max();
421 base::circular_deque<EncodedFrame>* queue = &video_frames_;
422 uint8_t track_index = video_track_index_;
423 if (!video_frames_.empty())
424 min_timestamp = video_frames_.front().relative_timestamp;
425
426 if (!audio_frames_.empty() &&
427 audio_frames_.front().relative_timestamp < min_timestamp) {
428 queue = &audio_frames_;
429 track_index = audio_track_index_;
430 }
431
432 EncodedFrame frame = std::move(queue->front());
433 queue->pop_front();
434 auto recorded_timestamp = frame.relative_timestamp.InMicroseconds() *
435 base::Time::kNanosecondsPerMicrosecond;
436
437 if (force_one_libwebm_error_) {
438 DVLOG(1) << "Forcing a libwebm error";
439 force_one_libwebm_error_ = false;
440 return false;
441 }
442
443 DCHECK(frame.data.data());
444 bool result =
445 frame.alpha_data.empty()
446 ? segment_.AddFrame(
447 reinterpret_cast<const uint8_t*>(frame.data.data()),
448 frame.data.size(), track_index, recorded_timestamp,
449 frame.is_keyframe)
450 : segment_.AddFrameWithAdditional(
451 reinterpret_cast<const uint8_t*>(frame.data.data()),
452 frame.data.size(),
453 reinterpret_cast<const uint8_t*>(frame.alpha_data.data()),
454 frame.alpha_data.size(), 1 /* add_id */, track_index,
455 recorded_timestamp, frame.is_keyframe);
456 return result;
457 }
458
UpdateLastTimestampMonotonically(base::TimeTicks timestamp,base::TimeTicks * last_timestamp)459 base::TimeTicks WebmMuxer::UpdateLastTimestampMonotonically(
460 base::TimeTicks timestamp,
461 base::TimeTicks* last_timestamp) {
462 base::TimeTicks compensated_timestamp = timestamp - total_time_in_pause_;
463 // In theory, time increases monotonically. In practice, it does not.
464 // See http://crbug/618407.
465 DLOG_IF(WARNING, compensated_timestamp < *last_timestamp)
466 << "Encountered a non-monotonically increasing timestamp. Was: "
467 << *last_timestamp << ", compensated: " << compensated_timestamp
468 << ", uncompensated: " << timestamp;
469 *last_timestamp = std::max(*last_timestamp, compensated_timestamp);
470 return *last_timestamp;
471 }
472
473 } // namespace media
474