1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/cast/sender/external_video_encoder.h"
6
7 #include <cmath>
8 #include <utility>
9
10 #include "base/bind.h"
11 #include "base/command_line.h"
12 #include "base/logging.h"
13 #include "base/memory/shared_memory_mapping.h"
14 #include "base/memory/unsafe_shared_memory_region.h"
15 #include "base/metrics/histogram_macros.h"
16 #include "base/stl_util.h"
17 #include "base/strings/string_number_conversions.h"
18 #include "base/strings/string_util.h"
19 #include "build/build_config.h"
20 #include "media/base/bind_to_current_loop.h"
21 #include "media/base/media_switches.h"
22 #include "media/base/video_frame.h"
23 #include "media/base/video_types.h"
24 #include "media/base/video_util.h"
25 #include "media/cast/cast_config.h"
26 #include "media/cast/common/rtp_time.h"
27 #include "media/cast/logging/logging_defines.h"
28 #include "media/cast/net/cast_transport_config.h"
29 #include "media/cast/sender/vp8_quantizer_parser.h"
30 #include "media/video/h264_parser.h"
31
32 namespace {
33
34 enum { MAX_H264_QUANTIZER = 51 };
35
36 // Number of buffers for encoded bit stream.
37 constexpr size_t kOutputBufferCount = 3;
38
39 // Maximum number of extra input buffers for encoder. The input buffers are only
40 // used when copy is needed to match the required coded size.
41 constexpr size_t kExtraInputBufferCount = 2;
42
43 // This value is used to calculate the encoder utilization. The encoder is
44 // assumed to be in full usage when the number of frames in progress reaches it.
45 constexpr int kBacklogRedlineThreshold = 4;
46
47 } // namespace
48
49 namespace media {
50 namespace cast {
51
52 // Container for the associated data of a video frame being processed.
53 struct InProgressExternalVideoFrameEncode {
54 // The source content to encode.
55 const scoped_refptr<VideoFrame> video_frame;
56
57 // The reference time for this frame.
58 const base::TimeTicks reference_time;
59
60 // The callback to run when the result is ready.
61 VideoEncoder::FrameEncodedCallback frame_encoded_callback;
62
63 // The target encode bit rate.
64 const int target_bit_rate;
65
66 // The real-world encode start time. This is used to compute the encoded
67 // frame's |encoder_utilization| and so it uses the real-world clock instead
68 // of the CastEnvironment clock, the latter of which might be simulated.
69 const base::TimeTicks start_time;
70
InProgressExternalVideoFrameEncodemedia::cast::InProgressExternalVideoFrameEncode71 InProgressExternalVideoFrameEncode(
72 scoped_refptr<VideoFrame> v_frame,
73 base::TimeTicks r_time,
74 VideoEncoder::FrameEncodedCallback callback,
75 int bit_rate)
76 : video_frame(std::move(v_frame)),
77 reference_time(r_time),
78 frame_encoded_callback(std::move(callback)),
79 target_bit_rate(bit_rate),
80 start_time(base::TimeTicks::Now()) {}
81 };
82
83 // Owns a VideoEncoderAccelerator instance and provides the necessary adapters
84 // to encode media::VideoFrames and emit media::cast::EncodedFrames. All
85 // methods must be called on the thread associated with the given
86 // SingleThreadTaskRunner, except for the task_runner() accessor.
87 class ExternalVideoEncoder::VEAClientImpl
88 : public VideoEncodeAccelerator::Client,
89 public base::RefCountedThreadSafe<VEAClientImpl> {
90 public:
VEAClientImpl(const scoped_refptr<CastEnvironment> & cast_environment,const scoped_refptr<base::SingleThreadTaskRunner> & encoder_task_runner,std::unique_ptr<media::VideoEncodeAccelerator> vea,double max_frame_rate,StatusChangeCallback status_change_cb,const CreateVideoEncodeMemoryCallback & create_video_encode_memory_cb)91 VEAClientImpl(
92 const scoped_refptr<CastEnvironment>& cast_environment,
93 const scoped_refptr<base::SingleThreadTaskRunner>& encoder_task_runner,
94 std::unique_ptr<media::VideoEncodeAccelerator> vea,
95 double max_frame_rate,
96 StatusChangeCallback status_change_cb,
97 const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
98 : cast_environment_(cast_environment),
99 task_runner_(encoder_task_runner),
100 max_frame_rate_(max_frame_rate),
101 status_change_cb_(std::move(status_change_cb)),
102 create_video_encode_memory_cb_(create_video_encode_memory_cb),
103 video_encode_accelerator_(std::move(vea)),
104 encoder_active_(false),
105 next_frame_id_(FrameId::first()),
106 key_frame_encountered_(false),
107 codec_profile_(media::VIDEO_CODEC_PROFILE_UNKNOWN),
108 key_frame_quantizer_parsable_(false),
109 requested_bit_rate_(-1),
110 max_allowed_input_buffers_(0),
111 allocate_input_buffer_in_progress_(false) {}
112
task_runner() const113 base::SingleThreadTaskRunner* task_runner() const {
114 return task_runner_.get();
115 }
116
Initialize(const gfx::Size & frame_size,VideoCodecProfile codec_profile,int start_bit_rate,FrameId first_frame_id)117 void Initialize(const gfx::Size& frame_size,
118 VideoCodecProfile codec_profile,
119 int start_bit_rate,
120 FrameId first_frame_id) {
121 DCHECK(task_runner_->RunsTasksInCurrentSequence());
122
123 requested_bit_rate_ = start_bit_rate;
124 const media::VideoEncodeAccelerator::Config config(
125 media::PIXEL_FORMAT_I420, frame_size, codec_profile, start_bit_rate);
126 encoder_active_ = video_encode_accelerator_->Initialize(config, this);
127 next_frame_id_ = first_frame_id;
128 codec_profile_ = codec_profile;
129
130 UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
131 encoder_active_);
132
133 cast_environment_->PostTask(
134 CastEnvironment::MAIN, FROM_HERE,
135 base::BindOnce(status_change_cb_, encoder_active_
136 ? STATUS_INITIALIZED
137 : STATUS_CODEC_INIT_FAILED));
138 }
139
SetBitRate(int bit_rate)140 void SetBitRate(int bit_rate) {
141 DCHECK(task_runner_->RunsTasksInCurrentSequence());
142
143 requested_bit_rate_ = bit_rate;
144 if (encoder_active_) {
145 video_encode_accelerator_->RequestEncodingParametersChange(
146 bit_rate, static_cast<uint32_t>(max_frame_rate_ + 0.5));
147 }
148 }
149
150 // The destruction call back of the copied video frame to free its use of
151 // the input buffer.
ReturnInputBufferToPool(int index)152 void ReturnInputBufferToPool(int index) {
153 DCHECK(task_runner_->RunsTasksInCurrentSequence());
154 DCHECK_GE(index, 0);
155 DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
156 free_input_buffer_index_.push_back(index);
157 }
158
EncodeVideoFrame(scoped_refptr<media::VideoFrame> video_frame,base::TimeTicks reference_time,bool key_frame_requested,VideoEncoder::FrameEncodedCallback frame_encoded_callback)159 void EncodeVideoFrame(
160 scoped_refptr<media::VideoFrame> video_frame,
161 base::TimeTicks reference_time,
162 bool key_frame_requested,
163 VideoEncoder::FrameEncodedCallback frame_encoded_callback) {
164 DCHECK(task_runner_->RunsTasksInCurrentSequence());
165
166 in_progress_frame_encodes_.push_back(InProgressExternalVideoFrameEncode(
167 video_frame, reference_time, std::move(frame_encoded_callback),
168 requested_bit_rate_));
169
170 if (!encoder_active_) {
171 AbortLatestEncodeAttemptDueToErrors();
172 return;
173 }
174
175 // If there are no free input buffers in the pool, request allocation of
176 // another one. Since that's an asynchronous process, simply abort encoding
177 // this frame and hope that the input buffer is ready for the next frame(s).
178 if (free_input_buffer_index_.empty()) {
179 if (!allocate_input_buffer_in_progress_ &&
180 input_buffers_.size() < max_allowed_input_buffers_) {
181 allocate_input_buffer_in_progress_ = true;
182 create_video_encode_memory_cb_.Run(
183 media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420,
184 frame_coded_size_),
185 base::Bind(&VEAClientImpl::OnCreateInputSharedMemory, this));
186 }
187 AbortLatestEncodeAttemptDueToErrors();
188 return;
189 }
190
191 // Copy the |video_frame| into the input buffer provided by the VEA
192 // implementation, and with the exact row stride required. Note that, even
193 // if |video_frame|'s stride matches VEA's requirement, |video_frame|'s
194 // memory backing (heap, base::UnsafeSharedMemoryRegion, etc.) could be
195 // something VEA can't handle (as of this writing, it expects an unsafe
196 // region).
197 //
198 // TODO(crbug.com/888829): Revisit whether we can remove this memcpy, if VEA
199 // can accept other "memory backing" methods.
200 scoped_refptr<media::VideoFrame> frame = video_frame;
201 if (video_frame->coded_size() != frame_coded_size_ ||
202 video_frame->storage_type() !=
203 media::VideoFrame::StorageType::STORAGE_SHMEM) {
204 const int index = free_input_buffer_index_.back();
205 std::pair<base::UnsafeSharedMemoryRegion,
206 base::WritableSharedMemoryMapping>* input_buffer =
207 input_buffers_[index].get();
208 DCHECK(input_buffer->first.IsValid());
209 DCHECK(input_buffer->second.IsValid());
210 frame = VideoFrame::WrapExternalData(
211 video_frame->format(), frame_coded_size_, video_frame->visible_rect(),
212 video_frame->visible_rect().size(),
213 input_buffer->second.GetMemoryAsSpan<uint8_t>().data(),
214 input_buffer->second.size(), video_frame->timestamp());
215 if (!frame || !media::I420CopyWithPadding(*video_frame, frame.get())) {
216 LOG(DFATAL) << "Error: ExternalVideoEncoder: copy failed.";
217 AbortLatestEncodeAttemptDueToErrors();
218 return;
219 }
220 frame->BackWithSharedMemory(&input_buffer->first);
221
222 frame->AddDestructionObserver(media::BindToCurrentLoop(base::Bind(
223 &ExternalVideoEncoder::VEAClientImpl::ReturnInputBufferToPool, this,
224 index)));
225 free_input_buffer_index_.pop_back();
226 }
227 // BitstreamBufferReady will be called once the encoder is done.
228 video_encode_accelerator_->Encode(std::move(frame), key_frame_requested);
229 }
230
231 protected:
NotifyError(VideoEncodeAccelerator::Error error)232 void NotifyError(VideoEncodeAccelerator::Error error) final {
233 DCHECK(task_runner_->RunsTasksInCurrentSequence());
234
235 DCHECK(error != VideoEncodeAccelerator::kInvalidArgumentError &&
236 error != VideoEncodeAccelerator::kIllegalStateError);
237
238 encoder_active_ = false;
239
240 cast_environment_->PostTask(
241 CastEnvironment::MAIN, FROM_HERE,
242 base::BindOnce(status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
243
244 // TODO(miu): Force-flush all |in_progress_frame_encodes_| immediately so
245 // pending frames do not become stuck, freezing VideoSender.
246 }
247
248 // Called to allocate the input and output buffers.
RequireBitstreamBuffers(unsigned int input_count,const gfx::Size & input_coded_size,size_t output_buffer_size)249 void RequireBitstreamBuffers(unsigned int input_count,
250 const gfx::Size& input_coded_size,
251 size_t output_buffer_size) final {
252 DCHECK(task_runner_->RunsTasksInCurrentSequence());
253
254 frame_coded_size_ = input_coded_size;
255
256 max_allowed_input_buffers_ = input_count + kExtraInputBufferCount;
257
258 for (size_t j = 0; j < kOutputBufferCount; ++j) {
259 create_video_encode_memory_cb_.Run(
260 output_buffer_size,
261 base::Bind(&VEAClientImpl::OnCreateSharedMemory, this));
262 }
263 }
264
265 // Encoder has encoded a frame and it's available in one of the output
266 // buffers. Package the result in a media::cast::EncodedFrame and post it
267 // to the Cast MAIN thread via the supplied callback.
BitstreamBufferReady(int32_t bitstream_buffer_id,const BitstreamBufferMetadata & metadata)268 void BitstreamBufferReady(int32_t bitstream_buffer_id,
269 const BitstreamBufferMetadata& metadata) final {
270 DCHECK(task_runner_->RunsTasksInCurrentSequence());
271 if (bitstream_buffer_id < 0 ||
272 bitstream_buffer_id >= static_cast<int32_t>(output_buffers_.size())) {
273 NOTREACHED();
274 VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
275 << bitstream_buffer_id;
276 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
277 return;
278 }
279 const char* output_buffer_memory = output_buffers_[bitstream_buffer_id]
280 .second.GetMemoryAsSpan<char>()
281 .data();
282 if (metadata.payload_size_bytes >
283 output_buffers_[bitstream_buffer_id].second.size()) {
284 NOTREACHED();
285 VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
286 << metadata.payload_size_bytes;
287 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
288 return;
289 }
290 if (metadata.key_frame) {
291 key_frame_encountered_ = true;
292 }
293 if (!key_frame_encountered_) {
294 // Do not send video until we have encountered the first key frame.
295 // Save the bitstream buffer in |stream_header_| to be sent later along
296 // with the first key frame.
297 //
298 // TODO(miu): Should |stream_header_| be an std::ostringstream for
299 // performance reasons?
300 stream_header_.append(output_buffer_memory, metadata.payload_size_bytes);
301 } else if (!in_progress_frame_encodes_.empty()) {
302 InProgressExternalVideoFrameEncode& request =
303 in_progress_frame_encodes_.front();
304
305 std::unique_ptr<SenderEncodedFrame> encoded_frame(
306 new SenderEncodedFrame());
307 encoded_frame->dependency =
308 metadata.key_frame ? EncodedFrame::KEY : EncodedFrame::DEPENDENT;
309 encoded_frame->frame_id = next_frame_id_++;
310 if (metadata.key_frame) {
311 encoded_frame->referenced_frame_id = encoded_frame->frame_id;
312 } else {
313 encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
314 }
315 encoded_frame->rtp_timestamp = RtpTimeTicks::FromTimeDelta(
316 request.video_frame->timestamp(), kVideoFrequency);
317 encoded_frame->reference_time = request.reference_time;
318 if (!stream_header_.empty()) {
319 encoded_frame->data = stream_header_;
320 stream_header_.clear();
321 }
322 encoded_frame->data.append(output_buffer_memory,
323 metadata.payload_size_bytes);
324 DCHECK(!encoded_frame->data.empty()) << "BUG: Encoder must provide data.";
325
326 // If FRAME_DURATION metadata was provided in the source VideoFrame,
327 // compute the utilization metrics.
328 base::TimeDelta frame_duration;
329 if (request.video_frame->metadata()->GetTimeDelta(
330 media::VideoFrameMetadata::FRAME_DURATION, &frame_duration) &&
331 frame_duration > base::TimeDelta()) {
332 // Compute encoder utilization in terms of the number of frames in
333 // backlog, including the current frame encode that is finishing
334 // here. This "backlog" model works as follows: First, assume that all
335 // frames utilize the encoder by the same amount. This is actually a
336 // false assumption, but it still works well because any frame that
337 // takes longer to encode will naturally cause the backlog to
338 // increase, and this will result in a higher computed utilization for
339 // the offending frame. If the backlog continues to increase, because
340 // the following frames are also taking too long to encode, the
341 // computed utilization for each successive frame will be higher. At
342 // some point, upstream control logic will decide that the data volume
343 // must be reduced.
344 encoded_frame->encoder_utilization =
345 static_cast<double>(in_progress_frame_encodes_.size()) /
346 kBacklogRedlineThreshold;
347
348 const double actual_bit_rate =
349 encoded_frame->data.size() * 8.0 / frame_duration.InSecondsF();
350 DCHECK_GT(request.target_bit_rate, 0);
351 const double bitrate_utilization =
352 actual_bit_rate / request.target_bit_rate;
353 double quantizer = QuantizerEstimator::NO_RESULT;
354 // If the quantizer can be parsed from the key frame, try to parse
355 // the following delta frames as well.
356 // Otherwise, switch back to entropy estimation for the key frame
357 // and all the following delta frames.
358 if (metadata.key_frame || key_frame_quantizer_parsable_) {
359 if (codec_profile_ == media::VP8PROFILE_ANY) {
360 quantizer = ParseVp8HeaderQuantizer(
361 reinterpret_cast<const uint8_t*>(encoded_frame->data.data()),
362 encoded_frame->data.size());
363 } else if (codec_profile_ == media::H264PROFILE_MAIN) {
364 quantizer = GetH264FrameQuantizer(
365 reinterpret_cast<const uint8_t*>(encoded_frame->data.data()),
366 encoded_frame->data.size());
367 } else {
368 NOTIMPLEMENTED();
369 }
370 if (quantizer < 0) {
371 LOG(ERROR) << "Unable to parse quantizer from encoded "
372 << (metadata.key_frame ? "key" : "delta")
373 << " frame, id=" << encoded_frame->frame_id;
374 if (metadata.key_frame) {
375 key_frame_quantizer_parsable_ = false;
376 quantizer = quantizer_estimator_.EstimateForKeyFrame(
377 *request.video_frame);
378 }
379 } else {
380 if (metadata.key_frame) {
381 key_frame_quantizer_parsable_ = true;
382 }
383 }
384 } else {
385 quantizer =
386 quantizer_estimator_.EstimateForDeltaFrame(*request.video_frame);
387 }
388 if (quantizer >= 0) {
389 const double max_quantizer =
390 codec_profile_ == media::VP8PROFILE_ANY
391 ? static_cast<int>(QuantizerEstimator::MAX_VP8_QUANTIZER)
392 : static_cast<int>(MAX_H264_QUANTIZER);
393 encoded_frame->lossy_utilization =
394 bitrate_utilization * (quantizer / max_quantizer);
395 }
396 } else {
397 quantizer_estimator_.Reset();
398 }
399
400 encoded_frame->encode_completion_time =
401 cast_environment_->Clock()->NowTicks();
402 cast_environment_->PostTask(
403 CastEnvironment::MAIN, FROM_HERE,
404 base::BindOnce(std::move(request.frame_encoded_callback),
405 std::move(encoded_frame)));
406
407 in_progress_frame_encodes_.pop_front();
408 } else {
409 VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
410 }
411
412 // We need to re-add the output buffer to the encoder after we are done
413 // with it.
414 if (encoder_active_) {
415 video_encode_accelerator_->UseOutputBitstreamBuffer(
416 media::BitstreamBuffer(
417 bitstream_buffer_id,
418 output_buffers_[bitstream_buffer_id].first.Duplicate(),
419 output_buffers_[bitstream_buffer_id].first.GetSize()));
420 }
421 }
422
423 private:
424 friend class base::RefCountedThreadSafe<VEAClientImpl>;
425
~VEAClientImpl()426 ~VEAClientImpl() final {
427 DCHECK(task_runner_->RunsTasksInCurrentSequence());
428
429 while (!in_progress_frame_encodes_.empty())
430 AbortLatestEncodeAttemptDueToErrors();
431
432 // According to the media::VideoEncodeAccelerator interface, Destroy()
433 // should be called instead of invoking its private destructor.
434 if (video_encode_accelerator_) {
435 video_encode_accelerator_.release()->Destroy();
436 }
437 }
438
439 // Note: This method can be called on any thread.
OnCreateSharedMemory(base::UnsafeSharedMemoryRegion memory)440 void OnCreateSharedMemory(base::UnsafeSharedMemoryRegion memory) {
441 task_runner_->PostTask(
442 FROM_HERE, base::BindOnce(&VEAClientImpl::OnReceivedSharedMemory, this,
443 std::move(memory)));
444 }
445
OnCreateInputSharedMemory(base::UnsafeSharedMemoryRegion memory)446 void OnCreateInputSharedMemory(base::UnsafeSharedMemoryRegion memory) {
447 task_runner_->PostTask(
448 FROM_HERE, base::BindOnce(&VEAClientImpl::OnReceivedInputSharedMemory,
449 this, std::move(memory)));
450 }
451
OnReceivedSharedMemory(base::UnsafeSharedMemoryRegion memory)452 void OnReceivedSharedMemory(base::UnsafeSharedMemoryRegion memory) {
453 DCHECK(task_runner_->RunsTasksInCurrentSequence());
454
455 base::WritableSharedMemoryMapping mapping = memory.Map();
456 DCHECK(mapping.IsValid());
457 output_buffers_.push_back(
458 std::make_pair(std::move(memory), std::move(mapping)));
459
460 // Wait until all requested buffers are received.
461 if (output_buffers_.size() < kOutputBufferCount)
462 return;
463
464 // Immediately provide all output buffers to the VEA.
465 for (size_t i = 0; i < output_buffers_.size(); ++i) {
466 video_encode_accelerator_->UseOutputBitstreamBuffer(
467 media::BitstreamBuffer(static_cast<int32_t>(i),
468 output_buffers_[i].first.Duplicate(),
469 output_buffers_[i].first.GetSize()));
470 }
471 }
472
OnReceivedInputSharedMemory(base::UnsafeSharedMemoryRegion region)473 void OnReceivedInputSharedMemory(base::UnsafeSharedMemoryRegion region) {
474 DCHECK(task_runner_->RunsTasksInCurrentSequence());
475
476 if (region.IsValid()) {
477 base::WritableSharedMemoryMapping mapping = region.Map();
478 DCHECK(mapping.IsValid());
479 input_buffers_.push_back(
480 std::make_unique<std::pair<base::UnsafeSharedMemoryRegion,
481 base::WritableSharedMemoryMapping>>(
482 std::move(region), std::move(mapping)));
483 free_input_buffer_index_.push_back(input_buffers_.size() - 1);
484 }
485 allocate_input_buffer_in_progress_ = false;
486 }
487
488 // This is called when an error occurs while preparing a VideoFrame for
489 // encode, or to abort a frame encode when shutting down.
AbortLatestEncodeAttemptDueToErrors()490 void AbortLatestEncodeAttemptDueToErrors() {
491 DCHECK(task_runner_->RunsTasksInCurrentSequence());
492
493 std::unique_ptr<SenderEncodedFrame> no_result(nullptr);
494 cast_environment_->PostTask(
495 CastEnvironment::MAIN, FROM_HERE,
496 base::BindOnce(
497 std::move(in_progress_frame_encodes_.back().frame_encoded_callback),
498 std::move(no_result)));
499 in_progress_frame_encodes_.pop_back();
500 }
501
502 // Parse H264 SPS, PPS, and Slice header, and return the averaged frame
503 // quantizer in the range of [0, 51], or -1 on parse error.
GetH264FrameQuantizer(const uint8_t * encoded_data,off_t size)504 double GetH264FrameQuantizer(const uint8_t* encoded_data, off_t size) {
505 DCHECK(task_runner_->RunsTasksInCurrentSequence());
506 DCHECK(encoded_data);
507
508 if (!size) {
509 return -1;
510 }
511 h264_parser_.SetStream(encoded_data, size);
512 double total_quantizer = 0;
513 int num_slices = 0;
514
515 while (true) {
516 H264NALU nalu;
517 H264Parser::Result res = h264_parser_.AdvanceToNextNALU(&nalu);
518 if (res == H264Parser::kEOStream) {
519 break;
520 }
521 if (res != H264Parser::kOk) {
522 return -1;
523 }
524 switch (nalu.nal_unit_type) {
525 case H264NALU::kIDRSlice:
526 case H264NALU::kNonIDRSlice: {
527 H264SliceHeader slice_header;
528 if (h264_parser_.ParseSliceHeader(nalu, &slice_header) !=
529 H264Parser::kOk)
530 return -1;
531 const H264PPS* pps =
532 h264_parser_.GetPPS(slice_header.pic_parameter_set_id);
533 if (!pps) {
534 return -1;
535 }
536 ++num_slices;
537 int slice_quantizer =
538 26 +
539 ((slice_header.IsSPSlice() || slice_header.IsSISlice())
540 ? pps->pic_init_qs_minus26 + slice_header.slice_qs_delta
541 : pps->pic_init_qp_minus26 + slice_header.slice_qp_delta);
542 DCHECK_GE(slice_quantizer, 0);
543 DCHECK_LE(slice_quantizer, MAX_H264_QUANTIZER);
544 total_quantizer += slice_quantizer;
545 break;
546 }
547 case H264NALU::kSPS: {
548 int id;
549 if (h264_parser_.ParseSPS(&id) != H264Parser::kOk) {
550 return -1;
551 }
552 break;
553 }
554 case H264NALU::kPPS: {
555 int id;
556 if (h264_parser_.ParsePPS(&id) != H264Parser::kOk) {
557 return -1;
558 }
559 break;
560 }
561 default:
562 // Skip other NALUs.
563 break;
564 }
565 }
566 return (num_slices == 0) ? -1 : (total_quantizer / num_slices);
567 }
568
569 const scoped_refptr<CastEnvironment> cast_environment_;
570 const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
571 const double max_frame_rate_;
572 const StatusChangeCallback status_change_cb_; // Must be run on MAIN thread.
573 const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
574 std::unique_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
575 bool encoder_active_;
576 FrameId next_frame_id_;
577 bool key_frame_encountered_;
578 std::string stream_header_;
579 VideoCodecProfile codec_profile_;
580 bool key_frame_quantizer_parsable_;
581 H264Parser h264_parser_;
582
583 // Shared memory buffers for output with the VideoAccelerator.
584 std::vector<std::pair<base::UnsafeSharedMemoryRegion,
585 base::WritableSharedMemoryMapping>>
586 output_buffers_;
587
588 // Shared memory buffers for input video frames with the VideoAccelerator.
589 // These buffers will be allocated only when copy is needed to match the
590 // required coded size for encoder. They are allocated on-demand, up to
591 // |max_allowed_input_buffers_|. A VideoFrame wrapping the region will point
592 // to it, so std::unique_ptr is used to ensure the region has a stable address
593 // even if the vector grows or shrinks.
594 std::vector<std::unique_ptr<std::pair<base::UnsafeSharedMemoryRegion,
595 base::WritableSharedMemoryMapping>>>
596 input_buffers_;
597
598 // Available input buffer index. These buffers are used in FILO order.
599 std::vector<int> free_input_buffer_index_;
600
601 // FIFO list.
602 std::list<InProgressExternalVideoFrameEncode> in_progress_frame_encodes_;
603
604 // The requested encode bit rate for the next frame.
605 int requested_bit_rate_;
606
607 // Used to compute utilization metrics for each frame.
608 QuantizerEstimator quantizer_estimator_;
609
610 // The coded size of the video frame required by Encoder. This size is
611 // obtained from VEA through |RequireBitstreamBuffers()|.
612 gfx::Size frame_coded_size_;
613
614 // The maximum number of input buffers. These buffers are used to copy
615 // VideoFrames in order to match the required coded size for encoder.
616 size_t max_allowed_input_buffers_;
617
618 // Set to true when the allocation of an input buffer is in progress, and
619 // reset to false after the allocated buffer is received.
620 bool allocate_input_buffer_in_progress_;
621
622 DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
623 };
624
625 // static
IsSupported(const FrameSenderConfig & video_config)626 bool ExternalVideoEncoder::IsSupported(const FrameSenderConfig& video_config) {
627 if (video_config.codec != CODEC_VIDEO_VP8 &&
628 video_config.codec != CODEC_VIDEO_H264)
629 return false;
630
631 // TODO(miu): "Layering hooks" are needed to be able to query outside of
632 // libmedia, to determine whether the system provides a hardware encoder. For
633 // now, assume that this was already checked by this point.
634 // http://crbug.com/454029
635 return video_config.use_external_encoder;
636 }
637
ExternalVideoEncoder(const scoped_refptr<CastEnvironment> & cast_environment,const FrameSenderConfig & video_config,const gfx::Size & frame_size,FrameId first_frame_id,StatusChangeCallback status_change_cb,const CreateVideoEncodeAcceleratorCallback & create_vea_cb,const CreateVideoEncodeMemoryCallback & create_video_encode_memory_cb)638 ExternalVideoEncoder::ExternalVideoEncoder(
639 const scoped_refptr<CastEnvironment>& cast_environment,
640 const FrameSenderConfig& video_config,
641 const gfx::Size& frame_size,
642 FrameId first_frame_id,
643 StatusChangeCallback status_change_cb,
644 const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
645 const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
646 : cast_environment_(cast_environment),
647 create_video_encode_memory_cb_(create_video_encode_memory_cb),
648 frame_size_(frame_size),
649 bit_rate_(video_config.start_bitrate),
650 key_frame_requested_(false) {
651 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
652 DCHECK_GT(video_config.max_frame_rate, 0);
653 DCHECK(!frame_size_.IsEmpty());
654 DCHECK(status_change_cb);
655 DCHECK(create_vea_cb);
656 DCHECK(create_video_encode_memory_cb_);
657 DCHECK_GT(bit_rate_, 0);
658
659 create_vea_cb.Run(
660 base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
661 weak_factory_.GetWeakPtr(), video_config, first_frame_id,
662 std::move(status_change_cb)));
663 }
664
~ExternalVideoEncoder()665 ExternalVideoEncoder::~ExternalVideoEncoder() {
666 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
667 DestroyClientSoon();
668 }
669
DestroyClientSoon()670 void ExternalVideoEncoder::DestroyClientSoon() {
671 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
672 // Ensure |client_| is destroyed from the encoder task runner by dropping the
673 // reference to it within an encoder task.
674 if (client_) {
675 client_->task_runner()->PostTask(
676 FROM_HERE, base::BindOnce([](scoped_refptr<VEAClientImpl> client) {},
677 std::move(client_)));
678 }
679 }
680
EncodeVideoFrame(scoped_refptr<media::VideoFrame> video_frame,base::TimeTicks reference_time,FrameEncodedCallback frame_encoded_callback)681 bool ExternalVideoEncoder::EncodeVideoFrame(
682 scoped_refptr<media::VideoFrame> video_frame,
683 base::TimeTicks reference_time,
684 FrameEncodedCallback frame_encoded_callback) {
685 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
686 DCHECK(!frame_encoded_callback.is_null());
687
688 if (!client_ || video_frame->visible_rect().size() != frame_size_) {
689 return false;
690 }
691
692 client_->task_runner()->PostTask(
693 FROM_HERE,
694 base::BindOnce(&VEAClientImpl::EncodeVideoFrame, client_,
695 std::move(video_frame), reference_time,
696 key_frame_requested_, std::move(frame_encoded_callback)));
697 key_frame_requested_ = false;
698 return true;
699 }
700
SetBitRate(int new_bit_rate)701 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
702 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
703 DCHECK_GT(new_bit_rate, 0);
704
705 bit_rate_ = new_bit_rate;
706 if (!client_) {
707 return;
708 }
709 client_->task_runner()->PostTask(
710 FROM_HERE,
711 base::BindOnce(&VEAClientImpl::SetBitRate, client_, bit_rate_));
712 }
713
GenerateKeyFrame()714 void ExternalVideoEncoder::GenerateKeyFrame() {
715 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
716 key_frame_requested_ = true;
717 }
718
OnCreateVideoEncodeAccelerator(const FrameSenderConfig & video_config,FrameId first_frame_id,const StatusChangeCallback & status_change_cb,scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,std::unique_ptr<media::VideoEncodeAccelerator> vea)719 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
720 const FrameSenderConfig& video_config,
721 FrameId first_frame_id,
722 const StatusChangeCallback& status_change_cb,
723 scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
724 std::unique_ptr<media::VideoEncodeAccelerator> vea) {
725 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
726
727 // The callback will be invoked with null pointers in the case where the
728 // system does not support or lacks the resources to provide GPU-accelerated
729 // video encoding.
730 if (!encoder_task_runner || !vea) {
731 cast_environment_->PostTask(
732 CastEnvironment::MAIN, FROM_HERE,
733 base::BindOnce(status_change_cb, STATUS_CODEC_INIT_FAILED));
734 return;
735 }
736
737 VideoCodecProfile codec_profile;
738 switch (video_config.codec) {
739 case CODEC_VIDEO_VP8:
740 codec_profile = media::VP8PROFILE_ANY;
741 break;
742 case CODEC_VIDEO_H264:
743 codec_profile = media::H264PROFILE_MAIN;
744 break;
745 case CODEC_VIDEO_FAKE:
746 NOTREACHED() << "Fake software video encoder cannot be external";
747 FALLTHROUGH;
748 default:
749 cast_environment_->PostTask(
750 CastEnvironment::MAIN, FROM_HERE,
751 base::BindOnce(status_change_cb, STATUS_UNSUPPORTED_CODEC));
752 return;
753 }
754
755 // Create a callback that wraps the StatusChangeCallback. It monitors when a
756 // fatal error occurs and schedules destruction of the VEAClientImpl.
757 StatusChangeCallback wrapped_status_change_cb = base::Bind(
758 [](base::WeakPtr<ExternalVideoEncoder> self,
759 const StatusChangeCallback& status_change_cb,
760 OperationalStatus status) {
761 if (self.get()) {
762 switch (status) {
763 case STATUS_UNINITIALIZED:
764 case STATUS_INITIALIZED:
765 case STATUS_CODEC_REINIT_PENDING:
766 break;
767
768 case STATUS_INVALID_CONFIGURATION:
769 case STATUS_UNSUPPORTED_CODEC:
770 case STATUS_CODEC_INIT_FAILED:
771 case STATUS_CODEC_RUNTIME_ERROR:
772 // Something bad happened. Destroy the client to: 1) fail-out any
773 // currently in-progress frame encodes; and 2) prevent future
774 // EncodeVideoFrame() calls from queuing frames indefinitely.
775 self->DestroyClientSoon();
776 break;
777 }
778 }
779 status_change_cb.Run(status);
780 },
781 weak_factory_.GetWeakPtr(), status_change_cb);
782
783 DCHECK(!client_);
784 client_ = new VEAClientImpl(cast_environment_, encoder_task_runner,
785 std::move(vea), video_config.max_frame_rate,
786 std::move(wrapped_status_change_cb),
787 create_video_encode_memory_cb_);
788 client_->task_runner()->PostTask(
789 FROM_HERE,
790 base::BindOnce(&VEAClientImpl::Initialize, client_, frame_size_,
791 codec_profile, bit_rate_, first_frame_id));
792 }
793
SizeAdaptableExternalVideoEncoder(const scoped_refptr<CastEnvironment> & cast_environment,const FrameSenderConfig & video_config,StatusChangeCallback status_change_cb,const CreateVideoEncodeAcceleratorCallback & create_vea_cb,const CreateVideoEncodeMemoryCallback & create_video_encode_memory_cb)794 SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder(
795 const scoped_refptr<CastEnvironment>& cast_environment,
796 const FrameSenderConfig& video_config,
797 StatusChangeCallback status_change_cb,
798 const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
799 const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
800 : SizeAdaptableVideoEncoderBase(cast_environment,
801 video_config,
802 std::move(status_change_cb)),
803 create_vea_cb_(create_vea_cb),
804 create_video_encode_memory_cb_(create_video_encode_memory_cb) {}
805
806 SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() =
807 default;
808
809 std::unique_ptr<VideoEncoder>
CreateEncoder()810 SizeAdaptableExternalVideoEncoder::CreateEncoder() {
811 return std::make_unique<ExternalVideoEncoder>(
812 cast_environment(), video_config(), frame_size(), next_frame_id(),
813 CreateEncoderStatusChangeCallback(), create_vea_cb_,
814 create_video_encode_memory_cb_);
815 }
816
817 QuantizerEstimator::QuantizerEstimator() = default;
818
819 QuantizerEstimator::~QuantizerEstimator() = default;
820
Reset()821 void QuantizerEstimator::Reset() {
822 last_frame_pixel_buffer_.reset();
823 }
824
EstimateForKeyFrame(const VideoFrame & frame)825 double QuantizerEstimator::EstimateForKeyFrame(const VideoFrame& frame) {
826 if (!CanExamineFrame(frame)) {
827 return NO_RESULT;
828 }
829
830 // If the size of the frame is different from the last frame, allocate a new
831 // buffer. The buffer only needs to be a fraction of the size of the entire
832 // frame, since the entropy analysis only examines a subset of each frame.
833 const gfx::Size size = frame.visible_rect().size();
834 const int rows_in_subset =
835 std::max(1, size.height() * FRAME_SAMPLING_PERCENT / 100);
836 if (last_frame_size_ != size || !last_frame_pixel_buffer_) {
837 last_frame_pixel_buffer_.reset(new uint8_t[size.width() * rows_in_subset]);
838 last_frame_size_ = size;
839 }
840
841 // Compute a histogram where each bucket represents the number of times two
842 // neighboring pixels were different by a specific amount. 511 buckets are
843 // needed, one for each integer in the range [-255,255].
844 int histogram[511];
845 memset(histogram, 0, sizeof(histogram));
846 const int row_skip = size.height() / rows_in_subset;
847 int y = 0;
848 for (int i = 0; i < rows_in_subset; ++i, y += row_skip) {
849 const uint8_t* const row_begin = frame.visible_data(VideoFrame::kYPlane) +
850 y * frame.stride(VideoFrame::kYPlane);
851 const uint8_t* const row_end = row_begin + size.width();
852 int left_hand_pixel_value = static_cast<int>(*row_begin);
853 for (const uint8_t* p = row_begin + 1; p < row_end; ++p) {
854 const int right_hand_pixel_value = static_cast<int>(*p);
855 const int difference = right_hand_pixel_value - left_hand_pixel_value;
856 const int histogram_index = difference + 255;
857 ++histogram[histogram_index];
858 left_hand_pixel_value = right_hand_pixel_value; // For next iteration.
859 }
860
861 // Copy the row of pixels into the buffer. This will be used when
862 // generating histograms for future delta frames.
863 memcpy(last_frame_pixel_buffer_.get() + i * size.width(),
864 row_begin,
865 size.width());
866 }
867
868 // Estimate a quantizer value depending on the difference data in the
869 // histogram and return it.
870 const int num_samples = (size.width() - 1) * rows_in_subset;
871 return ToQuantizerEstimate(ComputeEntropyFromHistogram(
872 histogram, base::size(histogram), num_samples));
873 }
874
EstimateForDeltaFrame(const VideoFrame & frame)875 double QuantizerEstimator::EstimateForDeltaFrame(const VideoFrame& frame) {
876 if (!CanExamineFrame(frame)) {
877 return NO_RESULT;
878 }
879
880 // If the size of the |frame| has changed, no difference can be examined.
881 // In this case, process this frame as if it were a key frame.
882 const gfx::Size size = frame.visible_rect().size();
883 if (last_frame_size_ != size || !last_frame_pixel_buffer_) {
884 return EstimateForKeyFrame(frame);
885 }
886 const int rows_in_subset =
887 std::max(1, size.height() * FRAME_SAMPLING_PERCENT / 100);
888
889 // Compute a histogram where each bucket represents the number of times the
890 // same pixel in this frame versus the last frame was different by a specific
891 // amount. 511 buckets are needed, one for each integer in the range
892 // [-255,255].
893 int histogram[511];
894 memset(histogram, 0, sizeof(histogram));
895 const int row_skip = size.height() / rows_in_subset;
896 int y = 0;
897 for (int i = 0; i < rows_in_subset; ++i, y += row_skip) {
898 const uint8_t* const row_begin = frame.visible_data(VideoFrame::kYPlane) +
899 y * frame.stride(VideoFrame::kYPlane);
900 const uint8_t* const row_end = row_begin + size.width();
901 uint8_t* const last_frame_row_begin =
902 last_frame_pixel_buffer_.get() + i * size.width();
903 for (const uint8_t *p = row_begin, *q = last_frame_row_begin; p < row_end;
904 ++p, ++q) {
905 const int difference = static_cast<int>(*p) - static_cast<int>(*q);
906 const int histogram_index = difference + 255;
907 ++histogram[histogram_index];
908 }
909
910 // Copy the row of pixels into the buffer. This will be used when
911 // generating histograms for future delta frames.
912 memcpy(last_frame_row_begin, row_begin, size.width());
913 }
914
915 // Estimate a quantizer value depending on the difference data in the
916 // histogram and return it.
917 const int num_samples = size.width() * rows_in_subset;
918 return ToQuantizerEstimate(ComputeEntropyFromHistogram(
919 histogram, base::size(histogram), num_samples));
920 }
921
922 // static
CanExamineFrame(const VideoFrame & frame)923 bool QuantizerEstimator::CanExamineFrame(const VideoFrame& frame) {
924 DCHECK_EQ(8, VideoFrame::PlaneHorizontalBitsPerPixel(frame.format(),
925 VideoFrame::kYPlane));
926 return media::IsYuvPlanar(frame.format()) &&
927 !frame.visible_rect().IsEmpty();
928 }
929
930 // static
ComputeEntropyFromHistogram(const int * histogram,size_t num_buckets,int num_samples)931 double QuantizerEstimator::ComputeEntropyFromHistogram(const int* histogram,
932 size_t num_buckets,
933 int num_samples) {
934 #if defined(OS_ANDROID)
935 // Android does not currently provide a log2() function in their C++ standard
936 // library. This is a substitute.
937 const auto log2 = [](double num) -> double {
938 return log(num) / 0.69314718055994528622676398299518041312694549560546875;
939 };
940 #endif
941
942 DCHECK_LT(0, num_samples);
943 double entropy = 0.0;
944 for (size_t i = 0; i < num_buckets; ++i) {
945 const double probability = static_cast<double>(histogram[i]) / num_samples;
946 if (probability > 0.0) {
947 entropy = entropy - probability * log2(probability);
948 }
949 }
950 return entropy;
951 }
952
953 // static
ToQuantizerEstimate(double shannon_entropy)954 double QuantizerEstimator::ToQuantizerEstimate(double shannon_entropy) {
955 DCHECK_GE(shannon_entropy, 0.0);
956
957 // This math is based on an analysis of data produced by running a wide range
958 // of mirroring content in a Cast streaming session on a Chromebook Pixel
959 // (2013 edition). The output from the Pixel's built-in hardware encoder was
960 // compared to an identically-configured software implementation (libvpx)
961 // running alongside. Based on an analysis of the data, the following linear
962 // mapping seems to produce reasonable VP8 quantizer values from the
963 // |shannon_entropy| values.
964 //
965 // TODO(miu): Confirm whether this model and value work well on other
966 // platforms.
967 const double kEntropyAtMaxQuantizer = 7.5;
968 const double slope =
969 (MAX_VP8_QUANTIZER - MIN_VP8_QUANTIZER) / kEntropyAtMaxQuantizer;
970 const double quantizer = std::min<double>(
971 MAX_VP8_QUANTIZER, MIN_VP8_QUANTIZER + slope * shannon_entropy);
972 return quantizer;
973 }
974
975 } // namespace cast
976 } // namespace media
977