1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/gpu/android/media_codec_video_decoder.h"
6
7 #include <memory>
8
9 #include "base/android/build_info.h"
10 #include "base/bind.h"
11 #include "base/callback.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/logging.h"
15 #include "base/memory/ptr_util.h"
16 #include "base/memory/weak_ptr.h"
17 #include "base/metrics/histogram_macros.h"
18 #include "base/threading/sequenced_task_runner_handle.h"
19 #include "base/trace_event/trace_event.h"
20 #include "media/base/android/media_codec_bridge_impl.h"
21 #include "media/base/android/media_codec_util.h"
22 #include "media/base/async_destroy_video_decoder.h"
23 #include "media/base/bind_to_current_loop.h"
24 #include "media/base/decoder_buffer.h"
25 #include "media/base/media_log.h"
26 #include "media/base/media_switches.h"
27 #include "media/base/scoped_async_trace.h"
28 #include "media/base/status.h"
29 #include "media/base/video_codecs.h"
30 #include "media/base/video_decoder_config.h"
31 #include "media/base/video_frame.h"
32 #include "media/base/video_util.h"
33 #include "media/gpu/android/android_video_surface_chooser.h"
34 #include "media/gpu/android/codec_allocator.h"
35 #include "media/media_buildflags.h"
36 #include "media/video/supported_video_decoder_config.h"
37
38 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
39 #include "media/base/android/extract_sps_and_pps.h"
40 #endif
41
42 namespace media {
43 namespace {
44
OutputBufferReleased(bool using_async_api,base::RepeatingClosure pump_cb,bool is_drained_or_draining)45 void OutputBufferReleased(bool using_async_api,
46 base::RepeatingClosure pump_cb,
47 bool is_drained_or_draining) {
48 // The asynchronous API doesn't need pumping upon calls to ReleaseOutputBuffer
49 // unless we're draining or drained.
50 if (using_async_api && !is_drained_or_draining)
51 return;
52 pump_cb.Run();
53 }
54
IsSurfaceControlEnabled(const gpu::GpuFeatureInfo & info)55 bool IsSurfaceControlEnabled(const gpu::GpuFeatureInfo& info) {
56 return info.status_values[gpu::GPU_FEATURE_TYPE_ANDROID_SURFACE_CONTROL] ==
57 gpu::kGpuFeatureStatusEnabled;
58 }
59
GetSupportedConfigsInternal(DeviceInfo * device_info)60 std::vector<SupportedVideoDecoderConfig> GetSupportedConfigsInternal(
61 DeviceInfo* device_info) {
62 std::vector<SupportedVideoDecoderConfig> supported_configs;
63
64 if (device_info->IsVp8DecoderAvailable()) {
65 // For unencrypted content, require that the size is at least 360p and that
66 // the MediaCodec implementation is hardware; otherwise fall back to libvpx.
67 if (!device_info->IsDecoderKnownUnaccelerated(kCodecVP8)) {
68 supported_configs.emplace_back(VP8PROFILE_ANY, VP8PROFILE_ANY,
69 gfx::Size(480, 360), gfx::Size(3840, 2160),
70 false, // allow_encrypted
71 false); // require_encrypted
72 }
73
74 // Encrypted content must be decoded by MediaCodec.
75 supported_configs.emplace_back(VP8PROFILE_ANY, VP8PROFILE_ANY,
76 gfx::Size(0, 0), gfx::Size(3840, 2160),
77 true, // allow_encrypted
78 true); // require_encrypted
79 }
80
81 // TODO(dalecurtis): This needs to actually check the profiles available. This
82 // can be done by calling MediaCodecUtil::AddSupportedCodecProfileLevels.
83 if (device_info->IsVp9DecoderAvailable()) {
84 const bool is_sw = device_info->IsDecoderKnownUnaccelerated(kCodecVP9);
85
86 std::vector<CodecProfileLevel> profiles;
87
88 // Support for VP9.2, VP9.3 was not added until Nougat.
89 if (device_info->SdkVersion() >= base::android::SDK_VERSION_NOUGAT)
90 device_info->AddSupportedCodecProfileLevels(&profiles);
91
92 // If we think a VP9 decoder is available, but we didn't get any profiles
93 // returned, just assume support for vp9.0 only.
94 if (profiles.empty())
95 profiles.push_back({kCodecVP9, VP9PROFILE_PROFILE0, 0});
96
97 for (const auto& p : profiles) {
98 if (p.codec != kCodecVP9)
99 continue;
100
101 // We don't compile support into libvpx for these profiles, so allow them
102 // for all resolutions. See notes on H264 profiles below for more detail.
103 if (p.profile > VP9PROFILE_PROFILE1) {
104 supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
105 gfx::Size(3840, 2160),
106 true, // allow_encrypted
107 false); // require_encrypted
108 supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
109 gfx::Size(2160, 3840),
110 true, // allow_encrypted
111 false); // require_encrypted
112 continue;
113 }
114
115 // For unencrypted vp9.0 and vp9.1 content, require that the size is at
116 // least 360p and that the MediaCodec implementation is hardware;
117 // otherwise fall back to libvpx.
118 if (!is_sw) {
119 supported_configs.emplace_back(
120 p.profile, p.profile, gfx::Size(480, 360), gfx::Size(3840, 2160),
121 false, // allow_encrypted
122 false); // require_encrypted
123 supported_configs.emplace_back(
124 p.profile, p.profile, gfx::Size(360, 480), gfx::Size(2160, 3840),
125 false, // allow_encrypted
126 false); // require_encrypted
127 }
128
129 // Encrypted content must be decoded by MediaCodec.
130 supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
131 gfx::Size(3840, 2160),
132 true, // allow_encrypted
133 true); // require_encrypted
134 supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
135 gfx::Size(2160, 3840),
136 true, // allow_encrypted
137 true); // require_encrypted
138 }
139 }
140
141 if (device_info->IsAv1DecoderAvailable()) {
142 // Technically we should check which profiles are supported, but since we
143 // don't have an AV1 SW decoder, just allow them all. See notes below for
144 // H264 profiles on the reasons why.
145 supported_configs.emplace_back(AV1PROFILE_MIN, AV1PROFILE_MAX,
146 gfx::Size(0, 0), gfx::Size(3840, 2160),
147 true, // allow_encrypted
148 false); // require_encrypted
149 supported_configs.emplace_back(AV1PROFILE_MIN, AV1PROFILE_MAX,
150 gfx::Size(0, 0), gfx::Size(2160, 3840),
151 true, // allow_encrypted
152 false); // require_encrypted
153 }
154
155 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
156 // MediaCodec is only guaranteed to support baseline, but some devices may
157 // support others. Advertise support for all H.264 profiles and let the
158 // MediaCodec fail when decoding if it's not actually supported. It's assumed
159 // that there is not software fallback for H.264 on Android.
160 supported_configs.emplace_back(H264PROFILE_MIN, H264PROFILE_MAX,
161 gfx::Size(0, 0), gfx::Size(3840, 2160),
162 true, // allow_encrypted
163 false); // require_encrypted
164 supported_configs.emplace_back(H264PROFILE_MIN, H264PROFILE_MAX,
165 gfx::Size(0, 0), gfx::Size(2160, 3840),
166 true, // allow_encrypted
167 false); // require_encrypted
168
169 #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
170 supported_configs.emplace_back(HEVCPROFILE_MIN, HEVCPROFILE_MAX,
171 gfx::Size(0, 0), gfx::Size(3840, 2160),
172 true, // allow_encrypted
173 false); // require_encrypted
174 #endif
175 #endif
176
177 return supported_configs;
178 }
179
180 } // namespace
181
182 // When re-initializing the codec changes the resolution to be more than
183 // |kReallocateThreshold| times the old one, force a codec reallocation to
184 // update the hints that we provide to MediaCodec. crbug.com/989182 .
185 constexpr static float kReallocateThreshold = 4;
186
187 // static
CreateEos()188 PendingDecode PendingDecode::CreateEos() {
189 return {DecoderBuffer::CreateEOSBuffer(), base::DoNothing()};
190 }
191
PendingDecode(scoped_refptr<DecoderBuffer> buffer,VideoDecoder::DecodeCB decode_cb)192 PendingDecode::PendingDecode(scoped_refptr<DecoderBuffer> buffer,
193 VideoDecoder::DecodeCB decode_cb)
194 : buffer(std::move(buffer)), decode_cb(std::move(decode_cb)) {}
195 PendingDecode::PendingDecode(PendingDecode&& other) = default;
196 PendingDecode::~PendingDecode() = default;
197
198 // static
199 std::vector<SupportedVideoDecoderConfig>
GetSupportedConfigs()200 MediaCodecVideoDecoder::GetSupportedConfigs() {
201 static const auto configs =
202 GetSupportedConfigsInternal(DeviceInfo::GetInstance());
203 return configs;
204 }
205
MediaCodecVideoDecoder(const gpu::GpuPreferences & gpu_preferences,const gpu::GpuFeatureInfo & gpu_feature_info,std::unique_ptr<MediaLog> media_log,DeviceInfo * device_info,CodecAllocator * codec_allocator,std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,AndroidOverlayMojoFactoryCB overlay_factory_cb,RequestOverlayInfoCB request_overlay_info_cb,std::unique_ptr<VideoFrameFactory> video_frame_factory)206 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
207 const gpu::GpuPreferences& gpu_preferences,
208 const gpu::GpuFeatureInfo& gpu_feature_info,
209 std::unique_ptr<MediaLog> media_log,
210 DeviceInfo* device_info,
211 CodecAllocator* codec_allocator,
212 std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
213 AndroidOverlayMojoFactoryCB overlay_factory_cb,
214 RequestOverlayInfoCB request_overlay_info_cb,
215 std::unique_ptr<VideoFrameFactory> video_frame_factory)
216 : media_log_(std::move(media_log)),
217 codec_allocator_(codec_allocator),
218 request_overlay_info_cb_(std::move(request_overlay_info_cb)),
219 is_surface_control_enabled_(IsSurfaceControlEnabled(gpu_feature_info)),
220 surface_chooser_helper_(
221 std::move(surface_chooser),
222 base::CommandLine::ForCurrentProcess()->HasSwitch(
223 switches::kForceVideoOverlays),
224 base::FeatureList::IsEnabled(media::kUseAndroidOverlayAggressively),
225 is_surface_control_enabled_),
226 video_frame_factory_(std::move(video_frame_factory)),
227 overlay_factory_cb_(std::move(overlay_factory_cb)),
228 device_info_(device_info),
229 enable_threaded_texture_mailboxes_(
230 gpu_preferences.enable_threaded_texture_mailboxes),
231 allow_nonsecure_overlays_(
232 base::FeatureList::IsEnabled(media::kAllowNonSecureOverlays)) {
233 DVLOG(2) << __func__;
234 surface_chooser_helper_.chooser()->SetClientCallbacks(
235 base::BindRepeating(&MediaCodecVideoDecoder::OnSurfaceChosen,
236 weak_factory_.GetWeakPtr()),
237 base::BindRepeating(&MediaCodecVideoDecoder::OnSurfaceChosen,
238 weak_factory_.GetWeakPtr(), nullptr));
239 }
240
Create(const gpu::GpuPreferences & gpu_preferences,const gpu::GpuFeatureInfo & gpu_feature_info,std::unique_ptr<MediaLog> media_log,DeviceInfo * device_info,CodecAllocator * codec_allocator,std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,AndroidOverlayMojoFactoryCB overlay_factory_cb,RequestOverlayInfoCB request_overlay_info_cb,std::unique_ptr<VideoFrameFactory> video_frame_factory)241 std::unique_ptr<VideoDecoder> MediaCodecVideoDecoder::Create(
242 const gpu::GpuPreferences& gpu_preferences,
243 const gpu::GpuFeatureInfo& gpu_feature_info,
244 std::unique_ptr<MediaLog> media_log,
245 DeviceInfo* device_info,
246 CodecAllocator* codec_allocator,
247 std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
248 AndroidOverlayMojoFactoryCB overlay_factory_cb,
249 RequestOverlayInfoCB request_overlay_info_cb,
250 std::unique_ptr<VideoFrameFactory> video_frame_factory) {
251 auto* decoder = new MediaCodecVideoDecoder(
252 gpu_preferences, gpu_feature_info, std::move(media_log), device_info,
253 codec_allocator, std::move(surface_chooser),
254 std::move(overlay_factory_cb), std::move(request_overlay_info_cb),
255 std::move(video_frame_factory));
256 return std::make_unique<AsyncDestroyVideoDecoder<MediaCodecVideoDecoder>>(
257 base::WrapUnique(decoder));
258 }
259
~MediaCodecVideoDecoder()260 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
261 DVLOG(2) << __func__;
262 TRACE_EVENT0("media", "MediaCodecVideoDecoder::~MediaCodecVideoDecoder");
263 ReleaseCodec();
264 }
265
DestroyAsync(std::unique_ptr<MediaCodecVideoDecoder> decoder)266 void MediaCodecVideoDecoder::DestroyAsync(
267 std::unique_ptr<MediaCodecVideoDecoder> decoder) {
268 DVLOG(1) << __func__;
269 TRACE_EVENT0("media", "MediaCodecVideoDecoder::Destroy");
270 DCHECK(decoder);
271
272 // This will be destroyed by a call to |DeleteSoon|
273 // in |OnCodecDrained|.
274 auto* self = decoder.release();
275
276 // Cancel pending callbacks.
277 //
278 // WARNING: This will lose the callback we've given to MediaCodecBridge for
279 // asynchronous notifications; so we must not leave this function with any
280 // work necessary from StartTimerOrPumpCodec().
281 self->weak_factory_.InvalidateWeakPtrs();
282
283 if (self->media_crypto_context_) {
284 // Cancel previously registered callback (if any).
285 self->event_cb_registration_.reset();
286 self->media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback());
287 self->media_crypto_context_ = nullptr;
288 }
289
290 // Mojo callbacks require that they're run before destruction.
291 if (self->reset_cb_)
292 std::move(self->reset_cb_).Run();
293
294 // Cancel callbacks we no longer want.
295 self->codec_allocator_weak_factory_.InvalidateWeakPtrs();
296 self->CancelPendingDecodes(DecodeStatus::ABORTED);
297 self->StartDrainingCodec(DrainType::kForDestroy);
298
299 // Per the WARNING above. Validate that no draining work remains.
300 if (self->using_async_api_)
301 DCHECK(!self->drain_type_.has_value());
302 }
303
Initialize(const VideoDecoderConfig & config,bool low_delay,CdmContext * cdm_context,InitCB init_cb,const OutputCB & output_cb,const WaitingCB & waiting_cb)304 void MediaCodecVideoDecoder::Initialize(const VideoDecoderConfig& config,
305 bool low_delay,
306 CdmContext* cdm_context,
307 InitCB init_cb,
308 const OutputCB& output_cb,
309 const WaitingCB& waiting_cb) {
310 DCHECK(output_cb);
311 DCHECK(waiting_cb);
312
313 const bool first_init = !decoder_config_.IsValidConfig();
314 DVLOG(1) << (first_init ? "Initializing" : "Reinitializing")
315 << " MCVD with config: " << config.AsHumanReadableString()
316 << ", cdm_context = " << cdm_context;
317
318 if (!config.IsValidConfig()) {
319 MEDIA_LOG(INFO, media_log_) << "Video configuration is not valid: "
320 << config.AsHumanReadableString();
321 DVLOG(1) << "Invalid configuration.";
322 BindToCurrentLoop(std::move(init_cb))
323 .Run(StatusCode::kDecoderUnsupportedConfig);
324 return;
325 }
326
327 // Tests override the DeviceInfo, so if an override is provided query the
328 // configs as they look under that DeviceInfo. If not, use the default method
329 // which is statically cached for faster Initialize().
330 const auto configs = device_info_ == DeviceInfo::GetInstance()
331 ? GetSupportedConfigs()
332 : GetSupportedConfigsInternal(device_info_);
333 if (!IsVideoDecoderConfigSupported(configs, config)) {
334 DVLOG(1) << "Unsupported configuration.";
335 MEDIA_LOG(INFO, media_log_) << "Video configuration is not valid: "
336 << config.AsHumanReadableString();
337 BindToCurrentLoop(std::move(init_cb))
338 .Run(StatusCode::kDecoderUnsupportedConfig);
339 return;
340 }
341
342 // Disallow codec changes when reinitializing.
343 if (!first_init && decoder_config_.codec() != config.codec()) {
344 DVLOG(1) << "Codec changed: cannot reinitialize";
345 MEDIA_LOG(INFO, media_log_) << "Cannot change codec during re-init: "
346 << decoder_config_.AsHumanReadableString()
347 << " -> " << config.AsHumanReadableString();
348 BindToCurrentLoop(std::move(init_cb))
349 .Run(StatusCode::kDecoderCantChangeCodec);
350 return;
351 }
352 decoder_config_ = config;
353
354 surface_chooser_helper_.SetVideoRotation(
355 decoder_config_.video_transformation().rotation);
356
357 output_cb_ = output_cb;
358 waiting_cb_ = waiting_cb;
359
360 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
361 if (config.codec() == kCodecH264)
362 ExtractSpsAndPps(config.extra_data(), &csd0_, &csd1_);
363 #endif
364
365 // We only support setting CDM at first initialization. Even if the initial
366 // config is clear, we'll still try to set CDM since we may switch to an
367 // encrypted config later.
368 if (first_init && cdm_context && cdm_context->GetMediaCryptoContext()) {
369 DCHECK(media_crypto_.is_null());
370 SetCdm(cdm_context, std::move(init_cb));
371 return;
372 }
373
374 if (config.is_encrypted() && media_crypto_.is_null()) {
375 DVLOG(1) << "No MediaCrypto to handle encrypted config";
376 MEDIA_LOG(INFO, media_log_) << "No MediaCrypto to handle encrypted config";
377 BindToCurrentLoop(std::move(init_cb))
378 .Run(StatusCode::kDecoderMissingCdmForEncryptedContent);
379 return;
380 }
381
382 // Do the rest of the initialization lazily on the first decode.
383 BindToCurrentLoop(std::move(init_cb)).Run(OkStatus());
384
385 const int width = config.coded_size().width();
386 // On re-init, reallocate the codec if the size has changed too much.
387 // Restrict this behavior to Q, where the behavior changed.
388 if (first_init) {
389 last_width_ = width;
390 } else if (width > last_width_ * kReallocateThreshold && device_info_ &&
391 device_info_->SdkVersion() > base::android::SDK_VERSION_P) {
392 // Reallocate the codec the next time we queue input, once there are no
393 // outstanding output buffers. Note that |deferred_flush_pending_| might
394 // already be set, which is fine. We're just upgrading the flush.
395 //
396 // If the codec IsDrained(), then we'll flush anyway. However, just to be
397 // sure, request a deferred flush.
398 deferred_flush_pending_ = true;
399 deferred_reallocation_pending_ = true;
400 last_width_ = width;
401 } // else leave |last_width_| unmodified, since we're re-using the codec.
402 }
403
SetCdm(CdmContext * cdm_context,InitCB init_cb)404 void MediaCodecVideoDecoder::SetCdm(CdmContext* cdm_context, InitCB init_cb) {
405 DVLOG(1) << __func__;
406 DCHECK(cdm_context) << "No CDM provided";
407 DCHECK(cdm_context->GetMediaCryptoContext());
408
409 media_crypto_context_ = cdm_context->GetMediaCryptoContext();
410
411 // CdmContext will always post the registered callback back to this thread.
412 event_cb_registration_ = cdm_context->RegisterEventCB(base::BindRepeating(
413 &MediaCodecVideoDecoder::OnCdmContextEvent, weak_factory_.GetWeakPtr()));
414
415 // The callback will be posted back to this thread via BindToCurrentLoop.
416 media_crypto_context_->SetMediaCryptoReadyCB(media::BindToCurrentLoop(
417 base::BindOnce(&MediaCodecVideoDecoder::OnMediaCryptoReady,
418 weak_factory_.GetWeakPtr(), std::move(init_cb))));
419 }
420
OnMediaCryptoReady(InitCB init_cb,JavaObjectPtr media_crypto,bool requires_secure_video_codec)421 void MediaCodecVideoDecoder::OnMediaCryptoReady(
422 InitCB init_cb,
423 JavaObjectPtr media_crypto,
424 bool requires_secure_video_codec) {
425 DVLOG(1) << __func__
426 << ": requires_secure_video_codec = " << requires_secure_video_codec;
427
428 DCHECK(state_ == State::kInitializing);
429 DCHECK(media_crypto);
430
431 if (media_crypto->is_null()) {
432 media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback());
433 media_crypto_context_ = nullptr;
434
435 if (decoder_config_.is_encrypted()) {
436 LOG(ERROR) << "MediaCrypto is not available";
437 EnterTerminalState(State::kError, "MediaCrypto is not available");
438 std::move(init_cb).Run(StatusCode::kDecoderMissingCdmForEncryptedContent);
439 return;
440 }
441
442 // MediaCrypto is not available, but the stream is clear. So we can still
443 // play the current stream. But if we switch to an encrypted stream playback
444 // will fail.
445 std::move(init_cb).Run(OkStatus());
446 return;
447 }
448
449 media_crypto_ = *media_crypto;
450 requires_secure_codec_ = requires_secure_video_codec;
451
452 // Request a secure surface in all cases. For L3, it's okay if we fall back
453 // to TextureOwner rather than fail composition. For L1, it's required.
454 surface_chooser_helper_.SetSecureSurfaceMode(
455 requires_secure_video_codec
456 ? SurfaceChooserHelper::SecureSurfaceMode::kRequired
457 : SurfaceChooserHelper::SecureSurfaceMode::kRequested);
458
459 // Signal success, and create the codec lazily on the first decode.
460 std::move(init_cb).Run(OkStatus());
461 }
462
OnCdmContextEvent(CdmContext::Event event)463 void MediaCodecVideoDecoder::OnCdmContextEvent(CdmContext::Event event) {
464 DVLOG(2) << __func__;
465
466 if (event != CdmContext::Event::kHasAdditionalUsableKey)
467 return;
468
469 waiting_for_key_ = false;
470 StartTimerOrPumpCodec();
471 }
472
StartLazyInit()473 void MediaCodecVideoDecoder::StartLazyInit() {
474 DVLOG(2) << __func__;
475 TRACE_EVENT0("media", "MediaCodecVideoDecoder::StartLazyInit");
476 lazy_init_pending_ = false;
477
478 // Only ask for promotion hints if we can actually switch surfaces, since we
479 // wouldn't be able to do anything with them. Also, if threaded texture
480 // mailboxes are enabled, then we turn off overlays anyway.
481 const bool want_promotion_hints =
482 device_info_->IsSetOutputSurfaceSupported() &&
483 !enable_threaded_texture_mailboxes_;
484
485 VideoFrameFactory::OverlayMode overlay_mode =
486 VideoFrameFactory::OverlayMode::kDontRequestPromotionHints;
487 if (is_surface_control_enabled_) {
488 overlay_mode =
489 requires_secure_codec_
490 ? VideoFrameFactory::OverlayMode::kSurfaceControlSecure
491 : VideoFrameFactory::OverlayMode::kSurfaceControlInsecure;
492 } else if (want_promotion_hints) {
493 overlay_mode = VideoFrameFactory::OverlayMode::kRequestPromotionHints;
494 }
495
496 // Regardless of whether we're using SurfaceControl or Dialog overlays, don't
497 // allow any overlays in A/B power testing mode, unless this requires a
498 // secure surface. Don't fail the playback for power testing.
499 if (!requires_secure_codec_ && !allow_nonsecure_overlays_)
500 overlay_mode = VideoFrameFactory::OverlayMode::kDontRequestPromotionHints;
501
502 video_frame_factory_->Initialize(
503 overlay_mode, base::BindRepeating(
504 &MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized,
505 weak_factory_.GetWeakPtr()));
506 }
507
OnVideoFrameFactoryInitialized(scoped_refptr<gpu::TextureOwner> texture_owner)508 void MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized(
509 scoped_refptr<gpu::TextureOwner> texture_owner) {
510 DVLOG(2) << __func__;
511 TRACE_EVENT0("media",
512 "MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized");
513 if (!texture_owner) {
514 EnterTerminalState(State::kError, "Could not allocated TextureOwner");
515 return;
516 }
517 texture_owner_bundle_ = new CodecSurfaceBundle(std::move(texture_owner));
518
519 // This is for A/B power testing only. Turn off Dialog-based overlays in
520 // power testing mode, unless we need them for L1 content.
521 // See https://crbug.com/1081346 .
522 const bool allowed_for_experiment =
523 requires_secure_codec_ || allow_nonsecure_overlays_;
524
525 // Overlays are disabled when |enable_threaded_texture_mailboxes| is true
526 // (http://crbug.com/582170).
527 if (enable_threaded_texture_mailboxes_ ||
528 !device_info_->SupportsOverlaySurfaces() || !allowed_for_experiment) {
529 OnSurfaceChosen(nullptr);
530 return;
531 }
532
533 // Request OverlayInfo updates. Initialization continues on the first one.
534 bool restart_for_transitions = !device_info_->IsSetOutputSurfaceSupported();
535 std::move(request_overlay_info_cb_)
536 .Run(restart_for_transitions,
537 base::BindRepeating(&MediaCodecVideoDecoder::OnOverlayInfoChanged,
538 weak_factory_.GetWeakPtr()));
539 }
540
OnOverlayInfoChanged(const OverlayInfo & overlay_info)541 void MediaCodecVideoDecoder::OnOverlayInfoChanged(
542 const OverlayInfo& overlay_info) {
543 DVLOG(2) << __func__;
544 DCHECK(device_info_->SupportsOverlaySurfaces());
545 DCHECK(!enable_threaded_texture_mailboxes_);
546 if (InTerminalState())
547 return;
548
549 bool overlay_changed = !overlay_info_.RefersToSameOverlayAs(overlay_info);
550 overlay_info_ = overlay_info;
551 surface_chooser_helper_.SetIsFullscreen(overlay_info_.is_fullscreen);
552 surface_chooser_helper_.SetIsPersistentVideo(
553 overlay_info_.is_persistent_video);
554 surface_chooser_helper_.UpdateChooserState(
555 overlay_changed ? base::make_optional(CreateOverlayFactoryCb())
556 : base::nullopt);
557 }
558
OnSurfaceChosen(std::unique_ptr<AndroidOverlay> overlay)559 void MediaCodecVideoDecoder::OnSurfaceChosen(
560 std::unique_ptr<AndroidOverlay> overlay) {
561 DVLOG(2) << __func__;
562 DCHECK(state_ == State::kInitializing ||
563 device_info_->IsSetOutputSurfaceSupported());
564 TRACE_EVENT1("media", "MediaCodecVideoDecoder::OnSurfaceChosen", "overlay",
565 overlay ? "yes" : "no");
566
567 if (overlay) {
568 overlay->AddSurfaceDestroyedCallback(
569 base::BindOnce(&MediaCodecVideoDecoder::OnSurfaceDestroyed,
570 weak_factory_.GetWeakPtr()));
571 target_surface_bundle_ = new CodecSurfaceBundle(std::move(overlay));
572 } else {
573 target_surface_bundle_ = texture_owner_bundle_;
574 }
575
576 // If we were waiting for our first surface during initialization, then
577 // proceed to create a codec.
578 if (state_ == State::kInitializing) {
579 state_ = State::kRunning;
580 CreateCodec();
581 }
582 }
583
OnSurfaceDestroyed(AndroidOverlay * overlay)584 void MediaCodecVideoDecoder::OnSurfaceDestroyed(AndroidOverlay* overlay) {
585 DVLOG(2) << __func__;
586 DCHECK_NE(state_, State::kInitializing);
587 TRACE_EVENT0("media", "MediaCodecVideoDecoder::OnSurfaceDestroyed");
588
589 // If SetOutputSurface() is not supported we only ever observe destruction of
590 // a single overlay so this must be the one we're using. In this case it's
591 // the responsibility of our consumer to destroy us for surface transitions.
592 // TODO(liberato): This might not be true for L1 / L3, since our caller has
593 // no idea that this has happened. We should unback the frames here. This
594 // might work now that we have CodecImageGroup -- verify this.
595 if (!device_info_->IsSetOutputSurfaceSupported()) {
596 EnterTerminalState(State::kSurfaceDestroyed, "Surface destroyed");
597 return;
598 }
599
600 // Reset the target bundle if it is the one being destroyed.
601 if (target_surface_bundle_ && target_surface_bundle_->overlay() == overlay)
602 target_surface_bundle_ = texture_owner_bundle_;
603
604 // Transition the codec away from the overlay if necessary.
605 if (SurfaceTransitionPending())
606 TransitionToTargetSurface();
607 }
608
SurfaceTransitionPending()609 bool MediaCodecVideoDecoder::SurfaceTransitionPending() {
610 return codec_ && codec_->SurfaceBundle() != target_surface_bundle_;
611 }
612
TransitionToTargetSurface()613 void MediaCodecVideoDecoder::TransitionToTargetSurface() {
614 DVLOG(2) << __func__;
615 DCHECK(SurfaceTransitionPending());
616 DCHECK(device_info_->IsSetOutputSurfaceSupported());
617
618 if (!codec_->SetSurface(target_surface_bundle_)) {
619 video_frame_factory_->SetSurfaceBundle(nullptr);
620 EnterTerminalState(State::kError, "Could not switch codec output surface");
621 return;
622 }
623
624 video_frame_factory_->SetSurfaceBundle(target_surface_bundle_);
625 CacheFrameInformation();
626 }
627
CreateCodec()628 void MediaCodecVideoDecoder::CreateCodec() {
629 DCHECK(!codec_);
630 DCHECK(target_surface_bundle_);
631 DCHECK_EQ(state_, State::kRunning);
632
633 auto config = std::make_unique<VideoCodecConfig>();
634 if (requires_secure_codec_)
635 config->codec_type = CodecType::kSecure;
636 config->codec = decoder_config_.codec();
637 config->csd0 = csd0_;
638 config->csd1 = csd1_;
639 config->surface = target_surface_bundle_->GetJavaSurface();
640 config->media_crypto = media_crypto_;
641 config->initial_expected_coded_size = decoder_config_.coded_size();
642 config->container_color_space = decoder_config_.color_space_info();
643 config->hdr_metadata = decoder_config_.hdr_metadata();
644
645 // Use the asynchronous API if we can.
646 if (device_info_->IsAsyncApiSupported()) {
647 using_async_api_ = true;
648 config->on_buffers_available_cb = BindToCurrentLoop(
649 base::BindRepeating(&MediaCodecVideoDecoder::StartTimerOrPumpCodec,
650 weak_factory_.GetWeakPtr()));
651 }
652
653 // Note that this might be the same surface bundle that we've been using, if
654 // we're reinitializing the codec without changing surfaces. That's fine.
655 video_frame_factory_->SetSurfaceBundle(target_surface_bundle_);
656 codec_allocator_->CreateMediaCodecAsync(
657 base::BindOnce(&MediaCodecVideoDecoder::OnCodecConfiguredInternal,
658 codec_allocator_weak_factory_.GetWeakPtr(),
659 codec_allocator_, target_surface_bundle_),
660 std::move(config));
661 }
662
663 // static
OnCodecConfiguredInternal(base::WeakPtr<MediaCodecVideoDecoder> weak_this,CodecAllocator * codec_allocator,scoped_refptr<CodecSurfaceBundle> surface_bundle,std::unique_ptr<MediaCodecBridge> codec)664 void MediaCodecVideoDecoder::OnCodecConfiguredInternal(
665 base::WeakPtr<MediaCodecVideoDecoder> weak_this,
666 CodecAllocator* codec_allocator,
667 scoped_refptr<CodecSurfaceBundle> surface_bundle,
668 std::unique_ptr<MediaCodecBridge> codec) {
669 if (!weak_this) {
670 if (codec) {
671 codec_allocator->ReleaseMediaCodec(
672 std::move(codec),
673 base::BindOnce(
674 &base::SequencedTaskRunner::ReleaseSoon<CodecSurfaceBundle>,
675 base::SequencedTaskRunnerHandle::Get(), FROM_HERE,
676 std::move(surface_bundle)));
677 }
678 return;
679 }
680 weak_this->OnCodecConfigured(std::move(surface_bundle), std::move(codec));
681 }
682
OnCodecConfigured(scoped_refptr<CodecSurfaceBundle> surface_bundle,std::unique_ptr<MediaCodecBridge> codec)683 void MediaCodecVideoDecoder::OnCodecConfigured(
684 scoped_refptr<CodecSurfaceBundle> surface_bundle,
685 std::unique_ptr<MediaCodecBridge> codec) {
686 DCHECK(!codec_);
687 DCHECK_EQ(state_, State::kRunning);
688
689 if (!codec) {
690 EnterTerminalState(State::kError, "Unable to allocate codec");
691 return;
692 }
693
694 codec_ = std::make_unique<CodecWrapper>(
695 CodecSurfacePair(std::move(codec), std::move(surface_bundle)),
696 base::BindRepeating(&OutputBufferReleased, using_async_api_,
697 BindToCurrentLoop(base::BindRepeating(
698 &MediaCodecVideoDecoder::StartTimerOrPumpCodec,
699 weak_factory_.GetWeakPtr()))),
700 base::SequencedTaskRunnerHandle::Get());
701
702 // If the target surface changed while codec creation was in progress,
703 // transition to it immediately.
704 // Note: this can only happen if we support SetOutputSurface() because if we
705 // don't OnSurfaceDestroyed() cancels codec creations, and
706 // |surface_chooser_| doesn't change the target surface.
707 if (SurfaceTransitionPending())
708 TransitionToTargetSurface();
709
710 // Cache the frame information that goes with this codec.
711 CacheFrameInformation();
712
713 StartTimerOrPumpCodec();
714 }
715
Decode(scoped_refptr<DecoderBuffer> buffer,DecodeCB decode_cb)716 void MediaCodecVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
717 DecodeCB decode_cb) {
718 DVLOG(3) << __func__ << ": " << buffer->AsHumanReadableString();
719 if (state_ == State::kError) {
720 std::move(decode_cb).Run(DecodeStatus::DECODE_ERROR);
721 return;
722 }
723 pending_decodes_.emplace_back(std::move(buffer), std::move(decode_cb));
724
725 if (state_ == State::kInitializing) {
726 if (lazy_init_pending_)
727 StartLazyInit();
728 return;
729 }
730 PumpCodec(true);
731 }
732
FlushCodec()733 void MediaCodecVideoDecoder::FlushCodec() {
734 DVLOG(2) << __func__;
735
736 // If a deferred flush was pending, then it isn't anymore.
737 deferred_flush_pending_ = false;
738
739 // Release and re-allocate the codec, if needed, for a resolution change.
740 // This also counts as a flush. Note that we could also stop / configure /
741 // start the codec, but there's a fair bit of complexity in that. Timing
742 // tests didn't show any big advantage. During a resolution change, the time
743 // between the next time we queue an input buffer and the next time we get an
744 // output buffer were:
745 //
746 // flush only: 0.04 s
747 // stop / configure / start: 0.026 s
748 // release / create: 0.03 s
749 //
750 // So, it seems that flushing the codec defers some work (buffer reallocation
751 // or similar) that ends up on the critical path. I didn't verify what
752 // happens when we're flushing without a resolution change, nor can I quite
753 // explain how anything can be done off the critical path when a flush is
754 // deferred to the first queued input.
755 if (deferred_reallocation_pending_) {
756 deferred_reallocation_pending_ = false;
757 ReleaseCodec();
758 CreateCodec();
759 }
760
761 if (!codec_ || codec_->IsFlushed())
762 return;
763
764 if (codec_->SupportsFlush(device_info_)) {
765 DVLOG(2) << "Flushing codec";
766 if (!codec_->Flush())
767 EnterTerminalState(State::kError, "Codec flush failed");
768 } else {
769 DVLOG(2) << "flush() workaround: creating a new codec";
770 // Release the codec and create a new one.
771 // Note: we may end up with two codecs attached to the same surface if the
772 // release hangs on one thread and create proceeds on another. This will
773 // result in an error, letting the user retry the playback. The alternative
774 // of waiting for the release risks hanging the playback forever.
775 ReleaseCodec();
776 CreateCodec();
777 }
778 }
779
PumpCodec(bool force_start_timer)780 void MediaCodecVideoDecoder::PumpCodec(bool force_start_timer) {
781 DVLOG(4) << __func__;
782 bool did_work = false, did_input = false, did_output = false;
783 do {
784 did_input = QueueInput();
785 did_output = DequeueOutput();
786 if (did_input || did_output)
787 did_work = true;
788 } while (did_input || did_output);
789
790 if (using_async_api_)
791 return;
792
793 if (did_work || force_start_timer)
794 StartTimerOrPumpCodec();
795 else
796 StopTimerIfIdle();
797 }
798
StartTimerOrPumpCodec()799 void MediaCodecVideoDecoder::StartTimerOrPumpCodec() {
800 DVLOG(4) << __func__;
801 if (state_ != State::kRunning)
802 return;
803
804 if (using_async_api_) {
805 PumpCodec(false);
806 return;
807 }
808
809 idle_timer_ = base::ElapsedTimer();
810
811 // Poll at 10ms somewhat arbitrarily.
812 // TODO: Don't poll on new devices; use the callback API.
813 // TODO: Experiment with this number to save power. Since we already pump the
814 // codec in response to receiving a decode and output buffer release, polling
815 // at this frequency is likely overkill in the steady state.
816 const auto kPollingPeriod = base::TimeDelta::FromMilliseconds(10);
817 if (!pump_codec_timer_.IsRunning()) {
818 pump_codec_timer_.Start(
819 FROM_HERE, kPollingPeriod,
820 base::BindRepeating(&MediaCodecVideoDecoder::PumpCodec,
821 base::Unretained(this), false));
822 }
823 }
824
StopTimerIfIdle()825 void MediaCodecVideoDecoder::StopTimerIfIdle() {
826 DVLOG(4) << __func__;
827 DCHECK(!using_async_api_);
828
829 // Stop the timer if we've been idle for one second. Chosen arbitrarily.
830 const auto kTimeout = base::TimeDelta::FromSeconds(1);
831 if (idle_timer_.Elapsed() > kTimeout) {
832 DVLOG(2) << "Stopping timer; idle timeout hit";
833 pump_codec_timer_.Stop();
834 // Draining for destroy can no longer proceed if the timer is stopping,
835 // because no more Decode() calls can be made, so complete it now to avoid
836 // leaking |this|.
837 if (drain_type_ == DrainType::kForDestroy)
838 OnCodecDrained();
839 }
840 }
841
QueueInput()842 bool MediaCodecVideoDecoder::QueueInput() {
843 DVLOG(4) << __func__;
844 if (!codec_ || waiting_for_key_)
845 return false;
846
847 // If the codec is drained, flush it when there is a pending decode and no
848 // unreleased output buffers. This lets us avoid both unbacking frames when we
849 // flush, and flushing unnecessarily, like at EOS.
850 //
851 // Often, we'll elide the eos to drain the codec, but we want to pretend that
852 // we did. In this case, we should also flush.
853 if (codec_->IsDrained() || deferred_flush_pending_) {
854 if (!codec_->HasUnreleasedOutputBuffers() && !pending_decodes_.empty()) {
855 FlushCodec();
856 return true;
857 }
858 return false;
859 }
860
861 if (pending_decodes_.empty())
862 return false;
863
864 PendingDecode& pending_decode = pending_decodes_.front();
865 auto status = codec_->QueueInputBuffer(*pending_decode.buffer);
866 DVLOG((status == CodecWrapper::QueueStatus::kTryAgainLater ||
867 status == CodecWrapper::QueueStatus::kOk
868 ? 3
869 : 2))
870 << "QueueInput(" << pending_decode.buffer->AsHumanReadableString()
871 << ") status=" << static_cast<int>(status);
872
873 switch (status) {
874 case CodecWrapper::QueueStatus::kOk:
875 break;
876 case CodecWrapper::QueueStatus::kTryAgainLater:
877 return false;
878 case CodecWrapper::QueueStatus::kNoKey:
879 // Retry when a key is added.
880 waiting_for_key_ = true;
881 waiting_cb_.Run(WaitingReason::kNoDecryptionKey);
882 return false;
883 case CodecWrapper::QueueStatus::kError:
884 EnterTerminalState(State::kError, "QueueInputBuffer failed");
885 return false;
886 }
887
888 if (pending_decode.buffer->end_of_stream()) {
889 // The VideoDecoder interface requires that the EOS DecodeCB is called after
890 // all decodes before it are delivered, so we have to save it and call it
891 // when the EOS is dequeued.
892 DCHECK(!eos_decode_cb_);
893 eos_decode_cb_ = std::move(pending_decode.decode_cb);
894 } else {
895 std::move(pending_decode.decode_cb).Run(DecodeStatus::OK);
896 }
897 pending_decodes_.pop_front();
898 return true;
899 }
900
DequeueOutput()901 bool MediaCodecVideoDecoder::DequeueOutput() {
902 DVLOG(4) << __func__;
903 if (!codec_ || codec_->IsDrained() || waiting_for_key_)
904 return false;
905
906 // If a surface transition is pending, wait for all outstanding buffers to be
907 // released before doing the transition. This is necessary because the
908 // VideoFrames corresponding to these buffers have metadata flags specific to
909 // the surface type, and changing the surface before they're rendered would
910 // invalidate them.
911 if (SurfaceTransitionPending()) {
912 if (!codec_->HasUnreleasedOutputBuffers()) {
913 TransitionToTargetSurface();
914 return true;
915 }
916 return false;
917 }
918
919 base::TimeDelta presentation_time;
920 bool eos = false;
921 std::unique_ptr<CodecOutputBuffer> output_buffer;
922 auto status =
923 codec_->DequeueOutputBuffer(&presentation_time, &eos, &output_buffer);
924 switch (status) {
925 case CodecWrapper::DequeueStatus::kOk:
926 break;
927 case CodecWrapper::DequeueStatus::kTryAgainLater:
928 return false;
929 case CodecWrapper::DequeueStatus::kError:
930 DVLOG(1) << "DequeueOutputBuffer() error";
931 EnterTerminalState(State::kError, "DequeueOutputBuffer failed");
932 return false;
933 }
934 DVLOG(3) << "DequeueOutputBuffer(): pts="
935 << (eos ? "EOS"
936 : std::to_string(presentation_time.InMilliseconds()));
937
938 if (eos) {
939 if (eos_decode_cb_) {
940 // Schedule the EOS DecodeCB to run after all previous frames.
941 video_frame_factory_->RunAfterPendingVideoFrames(
942 base::BindOnce(&MediaCodecVideoDecoder::RunEosDecodeCb,
943 weak_factory_.GetWeakPtr(), reset_generation_));
944 }
945 if (drain_type_)
946 OnCodecDrained();
947 // We don't flush the drained codec immediately because it might be
948 // backing unrendered frames near EOS. It's flushed lazily in QueueInput().
949 return false;
950 }
951
952 // If we're draining for reset or destroy we can discard |output_buffer|
953 // without rendering it. This is also true if we elided the drain itself,
954 // and deferred a flush that would have happened when the drain completed.
955 if (drain_type_ || deferred_flush_pending_)
956 return true;
957
958 // Record the frame type that we're sending and some information about why.
959 UMA_HISTOGRAM_ENUMERATION(
960 "Media.AVDA.FrameInformation", cached_frame_information_,
961 static_cast<int>(
962 SurfaceChooserHelper::FrameInformation::FRAME_INFORMATION_MAX) +
963 1); // PRESUBMIT_IGNORE_UMA_MAX
964
965 gfx::Rect visible_rect(output_buffer->size());
966 std::unique_ptr<ScopedAsyncTrace> async_trace =
967 ScopedAsyncTrace::CreateIfEnabled(
968 "MediaCodecVideoDecoder::CreateVideoFrame");
969 video_frame_factory_->CreateVideoFrame(
970 std::move(output_buffer), presentation_time,
971 GetNaturalSize(visible_rect, decoder_config_.GetPixelAspectRatio()),
972 CreatePromotionHintCB(),
973 base::BindOnce(&MediaCodecVideoDecoder::ForwardVideoFrame,
974 weak_factory_.GetWeakPtr(), reset_generation_,
975 std::move(async_trace), base::TimeTicks::Now()));
976 return true;
977 }
978
RunEosDecodeCb(int reset_generation)979 void MediaCodecVideoDecoder::RunEosDecodeCb(int reset_generation) {
980 // Both of the following conditions are necessary because:
981 // * In an error state, the reset generations will match but |eos_decode_cb_|
982 // will be aborted.
983 // * After a Reset(), the reset generations won't match, but we might already
984 // have a new |eos_decode_cb_| for the new generation.
985 if (reset_generation == reset_generation_ && eos_decode_cb_)
986 std::move(eos_decode_cb_).Run(DecodeStatus::OK);
987 }
988
ForwardVideoFrame(int reset_generation,std::unique_ptr<ScopedAsyncTrace> async_trace,base::TimeTicks started_at,scoped_refptr<VideoFrame> frame)989 void MediaCodecVideoDecoder::ForwardVideoFrame(
990 int reset_generation,
991 std::unique_ptr<ScopedAsyncTrace> async_trace,
992 base::TimeTicks started_at,
993 scoped_refptr<VideoFrame> frame) {
994 DVLOG(3) << __func__ << " : "
995 << (frame ? frame->AsHumanReadableString() : "null");
996
997 // Record how long this frame was pending.
998 const base::TimeDelta duration = base::TimeTicks::Now() - started_at;
999 UMA_HISTOGRAM_CUSTOM_TIMES("Media.MCVD.ForwardVideoFrameTiming", duration,
1000 base::TimeDelta::FromMilliseconds(1),
1001 base::TimeDelta::FromMilliseconds(100), 25);
1002
1003 // No |frame| indicates an error creating it.
1004 if (!frame) {
1005 DLOG(ERROR) << __func__ << " |frame| is null";
1006 EnterTerminalState(State::kError, "Could not create VideoFrame");
1007 return;
1008 }
1009
1010 if (reset_generation == reset_generation_) {
1011 // TODO(liberato): We might actually have a SW decoder. Consider setting
1012 // this to false if so, especially for higher bitrates.
1013 frame->metadata()->power_efficient = true;
1014 output_cb_.Run(std::move(frame));
1015 }
1016 }
1017
1018 // Our Reset() provides a slightly stronger guarantee than VideoDecoder does.
1019 // After |closure| runs:
1020 // 1) no VideoFrames from before the Reset() will be output, and
1021 // 2) no DecodeCBs (including EOS) from before the Reset() will be run.
Reset(base::OnceClosure closure)1022 void MediaCodecVideoDecoder::Reset(base::OnceClosure closure) {
1023 DVLOG(2) << __func__;
1024 DCHECK(!reset_cb_);
1025 reset_generation_++;
1026 reset_cb_ = std::move(closure);
1027 CancelPendingDecodes(DecodeStatus::ABORTED);
1028 StartDrainingCodec(DrainType::kForReset);
1029 }
1030
StartDrainingCodec(DrainType drain_type)1031 void MediaCodecVideoDecoder::StartDrainingCodec(DrainType drain_type) {
1032 DVLOG(2) << __func__;
1033 TRACE_EVENT0("media", "MediaCodecVideoDecoder::StartDrainingCodec");
1034 DCHECK(pending_decodes_.empty());
1035 // It's okay if there's already a drain ongoing. We'll only enqueue an EOS if
1036 // the codec isn't already draining.
1037 drain_type_ = drain_type;
1038
1039 // We can safely invalidate outstanding buffers for both types of drain, and
1040 // doing so can only make the drain complete quicker. Note that we do this
1041 // even if we're eliding the drain, since we're either going to flush the
1042 // codec or destroy it. While we're not required to do this, it might affect
1043 // stability if we don't (https://crbug.com/869365). AVDA, in particular,
1044 // dropped all pending codec output buffers when starting a reset (seek) or
1045 // a destroy.
1046 if (codec_)
1047 codec_->DiscardOutputBuffers();
1048
1049 // Skip the drain if possible. Only VP8 codecs need draining because
1050 // they can hang in release() or flush() otherwise
1051 // (http://crbug.com/598963).
1052 // TODO(watk): Strongly consider blocking VP8 (or specific MediaCodecs)
1053 // instead. Draining is responsible for a lot of complexity.
1054 if (decoder_config_.codec() != kCodecVP8 || !codec_ || codec_->IsFlushed() ||
1055 codec_->IsDrained() || using_async_api_) {
1056 // If the codec isn't already drained or flushed, then we have to remember
1057 // that we owe it a flush. We also have to remember not to deliver any
1058 // output buffers that might still be in progress in the codec.
1059 deferred_flush_pending_ =
1060 codec_ && !codec_->IsDrained() && !codec_->IsFlushed();
1061 OnCodecDrained();
1062 return;
1063 }
1064
1065 // Queue EOS if the codec isn't already processing one.
1066 if (!codec_->IsDraining())
1067 pending_decodes_.push_back(PendingDecode::CreateEos());
1068
1069 PumpCodec(true);
1070 }
1071
OnCodecDrained()1072 void MediaCodecVideoDecoder::OnCodecDrained() {
1073 DVLOG(2) << __func__;
1074 TRACE_EVENT0("media", "MediaCodecVideoDecoder::OnCodecDrained");
1075 DrainType drain_type = *drain_type_;
1076 drain_type_.reset();
1077
1078 if (drain_type == DrainType::kForDestroy) {
1079 // Post the delete in case the caller uses |this| after we return.
1080 base::SequencedTaskRunnerHandle::Get()->DeleteSoon(FROM_HERE, this);
1081 return;
1082 }
1083
1084 std::move(reset_cb_).Run();
1085
1086 // Flush the codec unless (a) it's already flushed, (b) it's drained and the
1087 // flush will be handled automatically on the next decode, or (c) we've
1088 // elided the eos and want to defer the flush.
1089 if (codec_ && !codec_->IsFlushed() && !codec_->IsDrained() &&
1090 !deferred_flush_pending_) {
1091 FlushCodec();
1092 }
1093 }
1094
EnterTerminalState(State state,const char * reason)1095 void MediaCodecVideoDecoder::EnterTerminalState(State state,
1096 const char* reason) {
1097 DVLOG(2) << __func__ << " " << static_cast<int>(state) << " " << reason;
1098 MEDIA_LOG(INFO, media_log_) << "Entering Terminal State: " << reason;
1099
1100 state_ = state;
1101 DCHECK(InTerminalState());
1102
1103 // Cancel pending codec creation.
1104 codec_allocator_weak_factory_.InvalidateWeakPtrs();
1105 pump_codec_timer_.Stop();
1106 ReleaseCodec();
1107 target_surface_bundle_ = nullptr;
1108 texture_owner_bundle_ = nullptr;
1109 if (state == State::kError)
1110 CancelPendingDecodes(DecodeStatus::DECODE_ERROR);
1111 if (drain_type_)
1112 OnCodecDrained();
1113 }
1114
InTerminalState()1115 bool MediaCodecVideoDecoder::InTerminalState() {
1116 return state_ == State::kSurfaceDestroyed || state_ == State::kError;
1117 }
1118
CancelPendingDecodes(DecodeStatus status)1119 void MediaCodecVideoDecoder::CancelPendingDecodes(DecodeStatus status) {
1120 for (auto& pending_decode : pending_decodes_)
1121 std::move(pending_decode.decode_cb).Run(status);
1122 pending_decodes_.clear();
1123 if (eos_decode_cb_)
1124 std::move(eos_decode_cb_).Run(status);
1125 }
1126
ReleaseCodec()1127 void MediaCodecVideoDecoder::ReleaseCodec() {
1128 if (!codec_)
1129 return;
1130 auto pair = codec_->TakeCodecSurfacePair();
1131 codec_ = nullptr;
1132 codec_allocator_->ReleaseMediaCodec(
1133 std::move(pair.first),
1134 base::BindOnce(
1135 &base::SequencedTaskRunner::ReleaseSoon<CodecSurfaceBundle>,
1136 base::SequencedTaskRunnerHandle::Get(), FROM_HERE,
1137 std::move(pair.second)));
1138 }
1139
CreateOverlayFactoryCb()1140 AndroidOverlayFactoryCB MediaCodecVideoDecoder::CreateOverlayFactoryCb() {
1141 if (!overlay_factory_cb_ || !overlay_info_.HasValidRoutingToken())
1142 return AndroidOverlayFactoryCB();
1143
1144 return base::BindRepeating(overlay_factory_cb_, *overlay_info_.routing_token);
1145 }
1146
GetDisplayName() const1147 std::string MediaCodecVideoDecoder::GetDisplayName() const {
1148 return "MediaCodecVideoDecoder";
1149 }
1150
NeedsBitstreamConversion() const1151 bool MediaCodecVideoDecoder::NeedsBitstreamConversion() const {
1152 return true;
1153 }
1154
CanReadWithoutStalling() const1155 bool MediaCodecVideoDecoder::CanReadWithoutStalling() const {
1156 // MediaCodec gives us no indication that it will stop producing outputs
1157 // until we provide more inputs or release output buffers back to it, so
1158 // we have to always return false.
1159 // TODO(watk): This puts all MCVD playbacks into low delay mode (i.e., the
1160 // renderer won't try to preroll). Ideally we'd be smarter about
1161 // this and attempt preroll but be able to give up if we can't produce
1162 // enough frames.
1163 return false;
1164 }
1165
GetMaxDecodeRequests() const1166 int MediaCodecVideoDecoder::GetMaxDecodeRequests() const {
1167 // We indicate that we're done decoding a frame as soon as we submit it to
1168 // MediaCodec so the number of parallel decode requests just sets the upper
1169 // limit of the size of our pending decode queue.
1170 return 2;
1171 }
1172
1173 PromotionHintAggregator::NotifyPromotionHintCB
CreatePromotionHintCB()1174 MediaCodecVideoDecoder::CreatePromotionHintCB() {
1175 // Right now, we don't request promotion hints. This is only used by SOP.
1176 // While we could simplify it a bit, this is the general form that we'll use
1177 // when handling promotion hints.
1178
1179 // Note that this keeps only a wp to the surface bundle via |layout_cb|. It
1180 // also continues to work even if |this| is destroyed; images might want to
1181 // move an overlay around even after MCVD has been torn down. For example
1182 // inline L1 content will fall into this case.
1183 return BindToCurrentLoop(base::BindRepeating(
1184 [](base::WeakPtr<MediaCodecVideoDecoder> mcvd,
1185 CodecSurfaceBundle::ScheduleLayoutCB layout_cb,
1186 PromotionHintAggregator::Hint hint) {
1187 // If we're promotable, and we have a surface bundle, then also
1188 // position the overlay. We could do this even if the overlay is
1189 // not promotable, but it wouldn't have any visible effect.
1190 if (hint.is_promotable)
1191 layout_cb.Run(hint.screen_rect);
1192
1193 // Notify MCVD about the promotion hint, so that it can decide if it
1194 // wants to switch to / from an overlay.
1195 if (mcvd)
1196 mcvd->NotifyPromotionHint(hint);
1197 },
1198 weak_factory_.GetWeakPtr(),
1199 codec_->SurfaceBundle()->GetScheduleLayoutCB()));
1200 }
1201
IsUsingOverlay() const1202 bool MediaCodecVideoDecoder::IsUsingOverlay() const {
1203 return codec_ && codec_->SurfaceBundle() &&
1204 codec_->SurfaceBundle()->overlay();
1205 }
1206
NotifyPromotionHint(PromotionHintAggregator::Hint hint)1207 void MediaCodecVideoDecoder::NotifyPromotionHint(
1208 PromotionHintAggregator::Hint hint) {
1209 surface_chooser_helper_.NotifyPromotionHintAndUpdateChooser(hint,
1210 IsUsingOverlay());
1211 }
1212
CacheFrameInformation()1213 void MediaCodecVideoDecoder::CacheFrameInformation() {
1214 cached_frame_information_ =
1215 surface_chooser_helper_.ComputeFrameInformation(IsUsingOverlay());
1216 }
1217
1218 } // namespace media
1219