1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "media/gpu/android/media_codec_video_decoder.h"
6 
7 #include <memory>
8 
9 #include "base/android/build_info.h"
10 #include "base/bind.h"
11 #include "base/callback.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/logging.h"
15 #include "base/memory/ptr_util.h"
16 #include "base/memory/weak_ptr.h"
17 #include "base/metrics/histogram_macros.h"
18 #include "base/threading/sequenced_task_runner_handle.h"
19 #include "base/trace_event/trace_event.h"
20 #include "media/base/android/media_codec_bridge_impl.h"
21 #include "media/base/android/media_codec_util.h"
22 #include "media/base/async_destroy_video_decoder.h"
23 #include "media/base/bind_to_current_loop.h"
24 #include "media/base/decoder_buffer.h"
25 #include "media/base/media_log.h"
26 #include "media/base/media_switches.h"
27 #include "media/base/scoped_async_trace.h"
28 #include "media/base/status.h"
29 #include "media/base/video_codecs.h"
30 #include "media/base/video_decoder_config.h"
31 #include "media/base/video_frame.h"
32 #include "media/base/video_util.h"
33 #include "media/gpu/android/android_video_surface_chooser.h"
34 #include "media/gpu/android/codec_allocator.h"
35 #include "media/media_buildflags.h"
36 #include "media/video/supported_video_decoder_config.h"
37 
38 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
39 #include "media/base/android/extract_sps_and_pps.h"
40 #endif
41 
42 namespace media {
43 namespace {
44 
OutputBufferReleased(bool using_async_api,base::RepeatingClosure pump_cb,bool is_drained_or_draining)45 void OutputBufferReleased(bool using_async_api,
46                           base::RepeatingClosure pump_cb,
47                           bool is_drained_or_draining) {
48   // The asynchronous API doesn't need pumping upon calls to ReleaseOutputBuffer
49   // unless we're draining or drained.
50   if (using_async_api && !is_drained_or_draining)
51     return;
52   pump_cb.Run();
53 }
54 
IsSurfaceControlEnabled(const gpu::GpuFeatureInfo & info)55 bool IsSurfaceControlEnabled(const gpu::GpuFeatureInfo& info) {
56   return info.status_values[gpu::GPU_FEATURE_TYPE_ANDROID_SURFACE_CONTROL] ==
57          gpu::kGpuFeatureStatusEnabled;
58 }
59 
GetSupportedConfigsInternal(DeviceInfo * device_info)60 std::vector<SupportedVideoDecoderConfig> GetSupportedConfigsInternal(
61     DeviceInfo* device_info) {
62   std::vector<SupportedVideoDecoderConfig> supported_configs;
63 
64   if (device_info->IsVp8DecoderAvailable()) {
65     // For unencrypted content, require that the size is at least 360p and that
66     // the MediaCodec implementation is hardware; otherwise fall back to libvpx.
67     if (!device_info->IsDecoderKnownUnaccelerated(kCodecVP8)) {
68       supported_configs.emplace_back(VP8PROFILE_ANY, VP8PROFILE_ANY,
69                                      gfx::Size(480, 360), gfx::Size(3840, 2160),
70                                      false,   // allow_encrypted
71                                      false);  // require_encrypted
72     }
73 
74     // Encrypted content must be decoded by MediaCodec.
75     supported_configs.emplace_back(VP8PROFILE_ANY, VP8PROFILE_ANY,
76                                    gfx::Size(0, 0), gfx::Size(3840, 2160),
77                                    true,   // allow_encrypted
78                                    true);  // require_encrypted
79   }
80 
81   // TODO(dalecurtis): This needs to actually check the profiles available. This
82   // can be done by calling MediaCodecUtil::AddSupportedCodecProfileLevels.
83   if (device_info->IsVp9DecoderAvailable()) {
84     const bool is_sw = device_info->IsDecoderKnownUnaccelerated(kCodecVP9);
85 
86     std::vector<CodecProfileLevel> profiles;
87 
88     // Support for VP9.2, VP9.3 was not added until Nougat.
89     if (device_info->SdkVersion() >= base::android::SDK_VERSION_NOUGAT)
90       device_info->AddSupportedCodecProfileLevels(&profiles);
91 
92     // If we think a VP9 decoder is available, but we didn't get any profiles
93     // returned, just assume support for vp9.0 only.
94     if (profiles.empty())
95       profiles.push_back({kCodecVP9, VP9PROFILE_PROFILE0, 0});
96 
97     for (const auto& p : profiles) {
98       if (p.codec != kCodecVP9)
99         continue;
100 
101       // We don't compile support into libvpx for these profiles, so allow them
102       // for all resolutions. See notes on H264 profiles below for more detail.
103       if (p.profile > VP9PROFILE_PROFILE1) {
104         supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
105                                        gfx::Size(3840, 2160),
106                                        true,    // allow_encrypted
107                                        false);  // require_encrypted
108         supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
109                                        gfx::Size(2160, 3840),
110                                        true,    // allow_encrypted
111                                        false);  // require_encrypted
112         continue;
113       }
114 
115       // For unencrypted vp9.0 and vp9.1 content, require that the size is at
116       // least 360p and that the MediaCodec implementation is hardware;
117       // otherwise fall back to libvpx.
118       if (!is_sw) {
119         supported_configs.emplace_back(
120             p.profile, p.profile, gfx::Size(480, 360), gfx::Size(3840, 2160),
121             false,   // allow_encrypted
122             false);  // require_encrypted
123         supported_configs.emplace_back(
124             p.profile, p.profile, gfx::Size(360, 480), gfx::Size(2160, 3840),
125             false,   // allow_encrypted
126             false);  // require_encrypted
127       }
128 
129       // Encrypted content must be decoded by MediaCodec.
130       supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
131                                      gfx::Size(3840, 2160),
132                                      true,   // allow_encrypted
133                                      true);  // require_encrypted
134       supported_configs.emplace_back(p.profile, p.profile, gfx::Size(0, 0),
135                                      gfx::Size(2160, 3840),
136                                      true,   // allow_encrypted
137                                      true);  // require_encrypted
138     }
139   }
140 
141   if (device_info->IsAv1DecoderAvailable()) {
142     // Technically we should check which profiles are supported, but since we
143     // don't have an AV1 SW decoder, just allow them all. See notes below for
144     // H264 profiles on the reasons why.
145     supported_configs.emplace_back(AV1PROFILE_MIN, AV1PROFILE_MAX,
146                                    gfx::Size(0, 0), gfx::Size(3840, 2160),
147                                    true,    // allow_encrypted
148                                    false);  // require_encrypted
149     supported_configs.emplace_back(AV1PROFILE_MIN, AV1PROFILE_MAX,
150                                    gfx::Size(0, 0), gfx::Size(2160, 3840),
151                                    true,    // allow_encrypted
152                                    false);  // require_encrypted
153   }
154 
155 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
156   // MediaCodec is only guaranteed to support baseline, but some devices may
157   // support others. Advertise support for all H.264 profiles and let the
158   // MediaCodec fail when decoding if it's not actually supported. It's assumed
159   // that there is not software fallback for H.264 on Android.
160   supported_configs.emplace_back(H264PROFILE_MIN, H264PROFILE_MAX,
161                                  gfx::Size(0, 0), gfx::Size(3840, 2160),
162                                  true,    // allow_encrypted
163                                  false);  // require_encrypted
164   supported_configs.emplace_back(H264PROFILE_MIN, H264PROFILE_MAX,
165                                  gfx::Size(0, 0), gfx::Size(2160, 3840),
166                                  true,    // allow_encrypted
167                                  false);  // require_encrypted
168 
169 #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
170   supported_configs.emplace_back(HEVCPROFILE_MIN, HEVCPROFILE_MAX,
171                                  gfx::Size(0, 0), gfx::Size(3840, 2160),
172                                  true,    // allow_encrypted
173                                  false);  // require_encrypted
174 #endif
175 #endif
176 
177   return supported_configs;
178 }
179 
180 }  // namespace
181 
182 // When re-initializing the codec changes the resolution to be more than
183 // |kReallocateThreshold| times the old one, force a codec reallocation to
184 // update the hints that we provide to MediaCodec.  crbug.com/989182 .
185 constexpr static float kReallocateThreshold = 4;
186 
187 // static
CreateEos()188 PendingDecode PendingDecode::CreateEos() {
189   return {DecoderBuffer::CreateEOSBuffer(), base::DoNothing()};
190 }
191 
PendingDecode(scoped_refptr<DecoderBuffer> buffer,VideoDecoder::DecodeCB decode_cb)192 PendingDecode::PendingDecode(scoped_refptr<DecoderBuffer> buffer,
193                              VideoDecoder::DecodeCB decode_cb)
194     : buffer(std::move(buffer)), decode_cb(std::move(decode_cb)) {}
195 PendingDecode::PendingDecode(PendingDecode&& other) = default;
196 PendingDecode::~PendingDecode() = default;
197 
198 // static
199 std::vector<SupportedVideoDecoderConfig>
GetSupportedConfigs()200 MediaCodecVideoDecoder::GetSupportedConfigs() {
201   static const auto configs =
202       GetSupportedConfigsInternal(DeviceInfo::GetInstance());
203   return configs;
204 }
205 
MediaCodecVideoDecoder(const gpu::GpuPreferences & gpu_preferences,const gpu::GpuFeatureInfo & gpu_feature_info,std::unique_ptr<MediaLog> media_log,DeviceInfo * device_info,CodecAllocator * codec_allocator,std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,AndroidOverlayMojoFactoryCB overlay_factory_cb,RequestOverlayInfoCB request_overlay_info_cb,std::unique_ptr<VideoFrameFactory> video_frame_factory)206 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
207     const gpu::GpuPreferences& gpu_preferences,
208     const gpu::GpuFeatureInfo& gpu_feature_info,
209     std::unique_ptr<MediaLog> media_log,
210     DeviceInfo* device_info,
211     CodecAllocator* codec_allocator,
212     std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
213     AndroidOverlayMojoFactoryCB overlay_factory_cb,
214     RequestOverlayInfoCB request_overlay_info_cb,
215     std::unique_ptr<VideoFrameFactory> video_frame_factory)
216     : media_log_(std::move(media_log)),
217       codec_allocator_(codec_allocator),
218       request_overlay_info_cb_(std::move(request_overlay_info_cb)),
219       is_surface_control_enabled_(IsSurfaceControlEnabled(gpu_feature_info)),
220       surface_chooser_helper_(
221           std::move(surface_chooser),
222           base::CommandLine::ForCurrentProcess()->HasSwitch(
223               switches::kForceVideoOverlays),
224           base::FeatureList::IsEnabled(media::kUseAndroidOverlayAggressively),
225           is_surface_control_enabled_),
226       video_frame_factory_(std::move(video_frame_factory)),
227       overlay_factory_cb_(std::move(overlay_factory_cb)),
228       device_info_(device_info),
229       enable_threaded_texture_mailboxes_(
230           gpu_preferences.enable_threaded_texture_mailboxes),
231       allow_nonsecure_overlays_(
232           base::FeatureList::IsEnabled(media::kAllowNonSecureOverlays)) {
233   DVLOG(2) << __func__;
234   surface_chooser_helper_.chooser()->SetClientCallbacks(
235       base::BindRepeating(&MediaCodecVideoDecoder::OnSurfaceChosen,
236                           weak_factory_.GetWeakPtr()),
237       base::BindRepeating(&MediaCodecVideoDecoder::OnSurfaceChosen,
238                           weak_factory_.GetWeakPtr(), nullptr));
239 }
240 
Create(const gpu::GpuPreferences & gpu_preferences,const gpu::GpuFeatureInfo & gpu_feature_info,std::unique_ptr<MediaLog> media_log,DeviceInfo * device_info,CodecAllocator * codec_allocator,std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,AndroidOverlayMojoFactoryCB overlay_factory_cb,RequestOverlayInfoCB request_overlay_info_cb,std::unique_ptr<VideoFrameFactory> video_frame_factory)241 std::unique_ptr<VideoDecoder> MediaCodecVideoDecoder::Create(
242     const gpu::GpuPreferences& gpu_preferences,
243     const gpu::GpuFeatureInfo& gpu_feature_info,
244     std::unique_ptr<MediaLog> media_log,
245     DeviceInfo* device_info,
246     CodecAllocator* codec_allocator,
247     std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
248     AndroidOverlayMojoFactoryCB overlay_factory_cb,
249     RequestOverlayInfoCB request_overlay_info_cb,
250     std::unique_ptr<VideoFrameFactory> video_frame_factory) {
251   auto* decoder = new MediaCodecVideoDecoder(
252       gpu_preferences, gpu_feature_info, std::move(media_log), device_info,
253       codec_allocator, std::move(surface_chooser),
254       std::move(overlay_factory_cb), std::move(request_overlay_info_cb),
255       std::move(video_frame_factory));
256   return std::make_unique<AsyncDestroyVideoDecoder<MediaCodecVideoDecoder>>(
257       base::WrapUnique(decoder));
258 }
259 
~MediaCodecVideoDecoder()260 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
261   DVLOG(2) << __func__;
262   TRACE_EVENT0("media", "MediaCodecVideoDecoder::~MediaCodecVideoDecoder");
263   ReleaseCodec();
264 }
265 
DestroyAsync(std::unique_ptr<MediaCodecVideoDecoder> decoder)266 void MediaCodecVideoDecoder::DestroyAsync(
267     std::unique_ptr<MediaCodecVideoDecoder> decoder) {
268   DVLOG(1) << __func__;
269   TRACE_EVENT0("media", "MediaCodecVideoDecoder::Destroy");
270   DCHECK(decoder);
271 
272   // This will be destroyed by a call to |DeleteSoon|
273   // in |OnCodecDrained|.
274   auto* self = decoder.release();
275 
276   // Cancel pending callbacks.
277   //
278   // WARNING: This will lose the callback we've given to MediaCodecBridge for
279   // asynchronous notifications; so we must not leave this function with any
280   // work necessary from StartTimerOrPumpCodec().
281   self->weak_factory_.InvalidateWeakPtrs();
282 
283   if (self->media_crypto_context_) {
284     // Cancel previously registered callback (if any).
285     self->event_cb_registration_.reset();
286     self->media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback());
287     self->media_crypto_context_ = nullptr;
288   }
289 
290   // Mojo callbacks require that they're run before destruction.
291   if (self->reset_cb_)
292     std::move(self->reset_cb_).Run();
293 
294   // Cancel callbacks we no longer want.
295   self->codec_allocator_weak_factory_.InvalidateWeakPtrs();
296   self->CancelPendingDecodes(DecodeStatus::ABORTED);
297   self->StartDrainingCodec(DrainType::kForDestroy);
298 
299   // Per the WARNING above. Validate that no draining work remains.
300   if (self->using_async_api_)
301     DCHECK(!self->drain_type_.has_value());
302 }
303 
Initialize(const VideoDecoderConfig & config,bool low_delay,CdmContext * cdm_context,InitCB init_cb,const OutputCB & output_cb,const WaitingCB & waiting_cb)304 void MediaCodecVideoDecoder::Initialize(const VideoDecoderConfig& config,
305                                         bool low_delay,
306                                         CdmContext* cdm_context,
307                                         InitCB init_cb,
308                                         const OutputCB& output_cb,
309                                         const WaitingCB& waiting_cb) {
310   DCHECK(output_cb);
311   DCHECK(waiting_cb);
312 
313   const bool first_init = !decoder_config_.IsValidConfig();
314   DVLOG(1) << (first_init ? "Initializing" : "Reinitializing")
315            << " MCVD with config: " << config.AsHumanReadableString()
316            << ", cdm_context = " << cdm_context;
317 
318   if (!config.IsValidConfig()) {
319     MEDIA_LOG(INFO, media_log_) << "Video configuration is not valid: "
320                                 << config.AsHumanReadableString();
321     DVLOG(1) << "Invalid configuration.";
322     BindToCurrentLoop(std::move(init_cb))
323         .Run(StatusCode::kDecoderUnsupportedConfig);
324     return;
325   }
326 
327   // Tests override the DeviceInfo, so if an override is provided query the
328   // configs as they look under that DeviceInfo. If not, use the default method
329   // which is statically cached for faster Initialize().
330   const auto configs = device_info_ == DeviceInfo::GetInstance()
331                            ? GetSupportedConfigs()
332                            : GetSupportedConfigsInternal(device_info_);
333   if (!IsVideoDecoderConfigSupported(configs, config)) {
334     DVLOG(1) << "Unsupported configuration.";
335     MEDIA_LOG(INFO, media_log_) << "Video configuration is not valid: "
336                                 << config.AsHumanReadableString();
337     BindToCurrentLoop(std::move(init_cb))
338         .Run(StatusCode::kDecoderUnsupportedConfig);
339     return;
340   }
341 
342   // Disallow codec changes when reinitializing.
343   if (!first_init && decoder_config_.codec() != config.codec()) {
344     DVLOG(1) << "Codec changed: cannot reinitialize";
345     MEDIA_LOG(INFO, media_log_) << "Cannot change codec during re-init: "
346                                 << decoder_config_.AsHumanReadableString()
347                                 << " -> " << config.AsHumanReadableString();
348     BindToCurrentLoop(std::move(init_cb))
349         .Run(StatusCode::kDecoderCantChangeCodec);
350     return;
351   }
352   decoder_config_ = config;
353 
354   surface_chooser_helper_.SetVideoRotation(
355       decoder_config_.video_transformation().rotation);
356 
357   output_cb_ = output_cb;
358   waiting_cb_ = waiting_cb;
359 
360 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
361   if (config.codec() == kCodecH264)
362     ExtractSpsAndPps(config.extra_data(), &csd0_, &csd1_);
363 #endif
364 
365   // We only support setting CDM at first initialization. Even if the initial
366   // config is clear, we'll still try to set CDM since we may switch to an
367   // encrypted config later.
368   if (first_init && cdm_context && cdm_context->GetMediaCryptoContext()) {
369     DCHECK(media_crypto_.is_null());
370     SetCdm(cdm_context, std::move(init_cb));
371     return;
372   }
373 
374   if (config.is_encrypted() && media_crypto_.is_null()) {
375     DVLOG(1) << "No MediaCrypto to handle encrypted config";
376     MEDIA_LOG(INFO, media_log_) << "No MediaCrypto to handle encrypted config";
377     BindToCurrentLoop(std::move(init_cb))
378         .Run(StatusCode::kDecoderMissingCdmForEncryptedContent);
379     return;
380   }
381 
382   // Do the rest of the initialization lazily on the first decode.
383   BindToCurrentLoop(std::move(init_cb)).Run(OkStatus());
384 
385   const int width = config.coded_size().width();
386   // On re-init, reallocate the codec if the size has changed too much.
387   // Restrict this behavior to Q, where the behavior changed.
388   if (first_init) {
389     last_width_ = width;
390   } else if (width > last_width_ * kReallocateThreshold && device_info_ &&
391              device_info_->SdkVersion() > base::android::SDK_VERSION_P) {
392     // Reallocate the codec the next time we queue input, once there are no
393     // outstanding output buffers.  Note that |deferred_flush_pending_| might
394     // already be set, which is fine.  We're just upgrading the flush.
395     //
396     // If the codec IsDrained(), then we'll flush anyway.  However, just to be
397     // sure, request a deferred flush.
398     deferred_flush_pending_ = true;
399     deferred_reallocation_pending_ = true;
400     last_width_ = width;
401   }  // else leave |last_width_| unmodified, since we're re-using the codec.
402 }
403 
SetCdm(CdmContext * cdm_context,InitCB init_cb)404 void MediaCodecVideoDecoder::SetCdm(CdmContext* cdm_context, InitCB init_cb) {
405   DVLOG(1) << __func__;
406   DCHECK(cdm_context) << "No CDM provided";
407   DCHECK(cdm_context->GetMediaCryptoContext());
408 
409   media_crypto_context_ = cdm_context->GetMediaCryptoContext();
410 
411   // CdmContext will always post the registered callback back to this thread.
412   event_cb_registration_ = cdm_context->RegisterEventCB(base::BindRepeating(
413       &MediaCodecVideoDecoder::OnCdmContextEvent, weak_factory_.GetWeakPtr()));
414 
415   // The callback will be posted back to this thread via BindToCurrentLoop.
416   media_crypto_context_->SetMediaCryptoReadyCB(media::BindToCurrentLoop(
417       base::BindOnce(&MediaCodecVideoDecoder::OnMediaCryptoReady,
418                      weak_factory_.GetWeakPtr(), std::move(init_cb))));
419 }
420 
OnMediaCryptoReady(InitCB init_cb,JavaObjectPtr media_crypto,bool requires_secure_video_codec)421 void MediaCodecVideoDecoder::OnMediaCryptoReady(
422     InitCB init_cb,
423     JavaObjectPtr media_crypto,
424     bool requires_secure_video_codec) {
425   DVLOG(1) << __func__
426            << ": requires_secure_video_codec = " << requires_secure_video_codec;
427 
428   DCHECK(state_ == State::kInitializing);
429   DCHECK(media_crypto);
430 
431   if (media_crypto->is_null()) {
432     media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback());
433     media_crypto_context_ = nullptr;
434 
435     if (decoder_config_.is_encrypted()) {
436       LOG(ERROR) << "MediaCrypto is not available";
437       EnterTerminalState(State::kError, "MediaCrypto is not available");
438       std::move(init_cb).Run(StatusCode::kDecoderMissingCdmForEncryptedContent);
439       return;
440     }
441 
442     // MediaCrypto is not available, but the stream is clear. So we can still
443     // play the current stream. But if we switch to an encrypted stream playback
444     // will fail.
445     std::move(init_cb).Run(OkStatus());
446     return;
447   }
448 
449   media_crypto_ = *media_crypto;
450   requires_secure_codec_ = requires_secure_video_codec;
451 
452   // Request a secure surface in all cases.  For L3, it's okay if we fall back
453   // to TextureOwner rather than fail composition.  For L1, it's required.
454   surface_chooser_helper_.SetSecureSurfaceMode(
455       requires_secure_video_codec
456           ? SurfaceChooserHelper::SecureSurfaceMode::kRequired
457           : SurfaceChooserHelper::SecureSurfaceMode::kRequested);
458 
459   // Signal success, and create the codec lazily on the first decode.
460   std::move(init_cb).Run(OkStatus());
461 }
462 
OnCdmContextEvent(CdmContext::Event event)463 void MediaCodecVideoDecoder::OnCdmContextEvent(CdmContext::Event event) {
464   DVLOG(2) << __func__;
465 
466   if (event != CdmContext::Event::kHasAdditionalUsableKey)
467     return;
468 
469   waiting_for_key_ = false;
470   StartTimerOrPumpCodec();
471 }
472 
StartLazyInit()473 void MediaCodecVideoDecoder::StartLazyInit() {
474   DVLOG(2) << __func__;
475   TRACE_EVENT0("media", "MediaCodecVideoDecoder::StartLazyInit");
476   lazy_init_pending_ = false;
477 
478   // Only ask for promotion hints if we can actually switch surfaces, since we
479   // wouldn't be able to do anything with them. Also, if threaded texture
480   // mailboxes are enabled, then we turn off overlays anyway.
481   const bool want_promotion_hints =
482       device_info_->IsSetOutputSurfaceSupported() &&
483       !enable_threaded_texture_mailboxes_;
484 
485   VideoFrameFactory::OverlayMode overlay_mode =
486       VideoFrameFactory::OverlayMode::kDontRequestPromotionHints;
487   if (is_surface_control_enabled_) {
488     overlay_mode =
489         requires_secure_codec_
490             ? VideoFrameFactory::OverlayMode::kSurfaceControlSecure
491             : VideoFrameFactory::OverlayMode::kSurfaceControlInsecure;
492   } else if (want_promotion_hints) {
493     overlay_mode = VideoFrameFactory::OverlayMode::kRequestPromotionHints;
494   }
495 
496   // Regardless of whether we're using SurfaceControl or Dialog overlays, don't
497   // allow any overlays in A/B power testing mode, unless this requires a
498   // secure surface.  Don't fail the playback for power testing.
499   if (!requires_secure_codec_ && !allow_nonsecure_overlays_)
500     overlay_mode = VideoFrameFactory::OverlayMode::kDontRequestPromotionHints;
501 
502   video_frame_factory_->Initialize(
503       overlay_mode, base::BindRepeating(
504                         &MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized,
505                         weak_factory_.GetWeakPtr()));
506 }
507 
OnVideoFrameFactoryInitialized(scoped_refptr<gpu::TextureOwner> texture_owner)508 void MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized(
509     scoped_refptr<gpu::TextureOwner> texture_owner) {
510   DVLOG(2) << __func__;
511   TRACE_EVENT0("media",
512                "MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized");
513   if (!texture_owner) {
514     EnterTerminalState(State::kError, "Could not allocated TextureOwner");
515     return;
516   }
517   texture_owner_bundle_ = new CodecSurfaceBundle(std::move(texture_owner));
518 
519   // This is for A/B power testing only.  Turn off Dialog-based overlays in
520   // power testing mode, unless we need them for L1 content.
521   // See https://crbug.com/1081346 .
522   const bool allowed_for_experiment =
523       requires_secure_codec_ || allow_nonsecure_overlays_;
524 
525   // Overlays are disabled when |enable_threaded_texture_mailboxes| is true
526   // (http://crbug.com/582170).
527   if (enable_threaded_texture_mailboxes_ ||
528       !device_info_->SupportsOverlaySurfaces() || !allowed_for_experiment) {
529     OnSurfaceChosen(nullptr);
530     return;
531   }
532 
533   // Request OverlayInfo updates. Initialization continues on the first one.
534   bool restart_for_transitions = !device_info_->IsSetOutputSurfaceSupported();
535   std::move(request_overlay_info_cb_)
536       .Run(restart_for_transitions,
537            base::BindRepeating(&MediaCodecVideoDecoder::OnOverlayInfoChanged,
538                                weak_factory_.GetWeakPtr()));
539 }
540 
OnOverlayInfoChanged(const OverlayInfo & overlay_info)541 void MediaCodecVideoDecoder::OnOverlayInfoChanged(
542     const OverlayInfo& overlay_info) {
543   DVLOG(2) << __func__;
544   DCHECK(device_info_->SupportsOverlaySurfaces());
545   DCHECK(!enable_threaded_texture_mailboxes_);
546   if (InTerminalState())
547     return;
548 
549   bool overlay_changed = !overlay_info_.RefersToSameOverlayAs(overlay_info);
550   overlay_info_ = overlay_info;
551   surface_chooser_helper_.SetIsFullscreen(overlay_info_.is_fullscreen);
552   surface_chooser_helper_.SetIsPersistentVideo(
553       overlay_info_.is_persistent_video);
554   surface_chooser_helper_.UpdateChooserState(
555       overlay_changed ? base::make_optional(CreateOverlayFactoryCb())
556                       : base::nullopt);
557 }
558 
OnSurfaceChosen(std::unique_ptr<AndroidOverlay> overlay)559 void MediaCodecVideoDecoder::OnSurfaceChosen(
560     std::unique_ptr<AndroidOverlay> overlay) {
561   DVLOG(2) << __func__;
562   DCHECK(state_ == State::kInitializing ||
563          device_info_->IsSetOutputSurfaceSupported());
564   TRACE_EVENT1("media", "MediaCodecVideoDecoder::OnSurfaceChosen", "overlay",
565                overlay ? "yes" : "no");
566 
567   if (overlay) {
568     overlay->AddSurfaceDestroyedCallback(
569         base::BindOnce(&MediaCodecVideoDecoder::OnSurfaceDestroyed,
570                        weak_factory_.GetWeakPtr()));
571     target_surface_bundle_ = new CodecSurfaceBundle(std::move(overlay));
572   } else {
573     target_surface_bundle_ = texture_owner_bundle_;
574   }
575 
576   // If we were waiting for our first surface during initialization, then
577   // proceed to create a codec.
578   if (state_ == State::kInitializing) {
579     state_ = State::kRunning;
580     CreateCodec();
581   }
582 }
583 
OnSurfaceDestroyed(AndroidOverlay * overlay)584 void MediaCodecVideoDecoder::OnSurfaceDestroyed(AndroidOverlay* overlay) {
585   DVLOG(2) << __func__;
586   DCHECK_NE(state_, State::kInitializing);
587   TRACE_EVENT0("media", "MediaCodecVideoDecoder::OnSurfaceDestroyed");
588 
589   // If SetOutputSurface() is not supported we only ever observe destruction of
590   // a single overlay so this must be the one we're using. In this case it's
591   // the responsibility of our consumer to destroy us for surface transitions.
592   // TODO(liberato): This might not be true for L1 / L3, since our caller has
593   // no idea that this has happened.  We should unback the frames here.  This
594   // might work now that we have CodecImageGroup -- verify this.
595   if (!device_info_->IsSetOutputSurfaceSupported()) {
596     EnterTerminalState(State::kSurfaceDestroyed, "Surface destroyed");
597     return;
598   }
599 
600   // Reset the target bundle if it is the one being destroyed.
601   if (target_surface_bundle_ && target_surface_bundle_->overlay() == overlay)
602     target_surface_bundle_ = texture_owner_bundle_;
603 
604   // Transition the codec away from the overlay if necessary.
605   if (SurfaceTransitionPending())
606     TransitionToTargetSurface();
607 }
608 
SurfaceTransitionPending()609 bool MediaCodecVideoDecoder::SurfaceTransitionPending() {
610   return codec_ && codec_->SurfaceBundle() != target_surface_bundle_;
611 }
612 
TransitionToTargetSurface()613 void MediaCodecVideoDecoder::TransitionToTargetSurface() {
614   DVLOG(2) << __func__;
615   DCHECK(SurfaceTransitionPending());
616   DCHECK(device_info_->IsSetOutputSurfaceSupported());
617 
618   if (!codec_->SetSurface(target_surface_bundle_)) {
619     video_frame_factory_->SetSurfaceBundle(nullptr);
620     EnterTerminalState(State::kError, "Could not switch codec output surface");
621     return;
622   }
623 
624   video_frame_factory_->SetSurfaceBundle(target_surface_bundle_);
625   CacheFrameInformation();
626 }
627 
CreateCodec()628 void MediaCodecVideoDecoder::CreateCodec() {
629   DCHECK(!codec_);
630   DCHECK(target_surface_bundle_);
631   DCHECK_EQ(state_, State::kRunning);
632 
633   auto config = std::make_unique<VideoCodecConfig>();
634   if (requires_secure_codec_)
635     config->codec_type = CodecType::kSecure;
636   config->codec = decoder_config_.codec();
637   config->csd0 = csd0_;
638   config->csd1 = csd1_;
639   config->surface = target_surface_bundle_->GetJavaSurface();
640   config->media_crypto = media_crypto_;
641   config->initial_expected_coded_size = decoder_config_.coded_size();
642   config->container_color_space = decoder_config_.color_space_info();
643   config->hdr_metadata = decoder_config_.hdr_metadata();
644 
645   // Use the asynchronous API if we can.
646   if (device_info_->IsAsyncApiSupported()) {
647     using_async_api_ = true;
648     config->on_buffers_available_cb = BindToCurrentLoop(
649         base::BindRepeating(&MediaCodecVideoDecoder::StartTimerOrPumpCodec,
650                             weak_factory_.GetWeakPtr()));
651   }
652 
653   // Note that this might be the same surface bundle that we've been using, if
654   // we're reinitializing the codec without changing surfaces.  That's fine.
655   video_frame_factory_->SetSurfaceBundle(target_surface_bundle_);
656   codec_allocator_->CreateMediaCodecAsync(
657       base::BindOnce(&MediaCodecVideoDecoder::OnCodecConfiguredInternal,
658                      codec_allocator_weak_factory_.GetWeakPtr(),
659                      codec_allocator_, target_surface_bundle_),
660       std::move(config));
661 }
662 
663 // static
OnCodecConfiguredInternal(base::WeakPtr<MediaCodecVideoDecoder> weak_this,CodecAllocator * codec_allocator,scoped_refptr<CodecSurfaceBundle> surface_bundle,std::unique_ptr<MediaCodecBridge> codec)664 void MediaCodecVideoDecoder::OnCodecConfiguredInternal(
665     base::WeakPtr<MediaCodecVideoDecoder> weak_this,
666     CodecAllocator* codec_allocator,
667     scoped_refptr<CodecSurfaceBundle> surface_bundle,
668     std::unique_ptr<MediaCodecBridge> codec) {
669   if (!weak_this) {
670     if (codec) {
671       codec_allocator->ReleaseMediaCodec(
672           std::move(codec),
673           base::BindOnce(
674               &base::SequencedTaskRunner::ReleaseSoon<CodecSurfaceBundle>,
675               base::SequencedTaskRunnerHandle::Get(), FROM_HERE,
676               std::move(surface_bundle)));
677     }
678     return;
679   }
680   weak_this->OnCodecConfigured(std::move(surface_bundle), std::move(codec));
681 }
682 
OnCodecConfigured(scoped_refptr<CodecSurfaceBundle> surface_bundle,std::unique_ptr<MediaCodecBridge> codec)683 void MediaCodecVideoDecoder::OnCodecConfigured(
684     scoped_refptr<CodecSurfaceBundle> surface_bundle,
685     std::unique_ptr<MediaCodecBridge> codec) {
686   DCHECK(!codec_);
687   DCHECK_EQ(state_, State::kRunning);
688 
689   if (!codec) {
690     EnterTerminalState(State::kError, "Unable to allocate codec");
691     return;
692   }
693 
694   codec_ = std::make_unique<CodecWrapper>(
695       CodecSurfacePair(std::move(codec), std::move(surface_bundle)),
696       base::BindRepeating(&OutputBufferReleased, using_async_api_,
697                           BindToCurrentLoop(base::BindRepeating(
698                               &MediaCodecVideoDecoder::StartTimerOrPumpCodec,
699                               weak_factory_.GetWeakPtr()))),
700       base::SequencedTaskRunnerHandle::Get());
701 
702   // If the target surface changed while codec creation was in progress,
703   // transition to it immediately.
704   // Note: this can only happen if we support SetOutputSurface() because if we
705   // don't OnSurfaceDestroyed() cancels codec creations, and
706   // |surface_chooser_| doesn't change the target surface.
707   if (SurfaceTransitionPending())
708     TransitionToTargetSurface();
709 
710   // Cache the frame information that goes with this codec.
711   CacheFrameInformation();
712 
713   StartTimerOrPumpCodec();
714 }
715 
Decode(scoped_refptr<DecoderBuffer> buffer,DecodeCB decode_cb)716 void MediaCodecVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
717                                     DecodeCB decode_cb) {
718   DVLOG(3) << __func__ << ": " << buffer->AsHumanReadableString();
719   if (state_ == State::kError) {
720     std::move(decode_cb).Run(DecodeStatus::DECODE_ERROR);
721     return;
722   }
723   pending_decodes_.emplace_back(std::move(buffer), std::move(decode_cb));
724 
725   if (state_ == State::kInitializing) {
726     if (lazy_init_pending_)
727       StartLazyInit();
728     return;
729   }
730   PumpCodec(true);
731 }
732 
FlushCodec()733 void MediaCodecVideoDecoder::FlushCodec() {
734   DVLOG(2) << __func__;
735 
736   // If a deferred flush was pending, then it isn't anymore.
737   deferred_flush_pending_ = false;
738 
739   // Release and re-allocate the codec, if needed, for a resolution change.
740   // This also counts as a flush.  Note that we could also stop / configure /
741   // start the codec, but there's a fair bit of complexity in that.  Timing
742   // tests didn't show any big advantage.  During a resolution change, the time
743   // between the next time we queue an input buffer and the next time we get an
744   // output buffer were:
745   //
746   //  flush only:               0.04 s
747   //  stop / configure / start: 0.026 s
748   //  release / create:         0.03 s
749   //
750   // So, it seems that flushing the codec defers some work (buffer reallocation
751   // or similar) that ends up on the critical path.  I didn't verify what
752   // happens when we're flushing without a resolution change, nor can I quite
753   // explain how anything can be done off the critical path when a flush is
754   // deferred to the first queued input.
755   if (deferred_reallocation_pending_) {
756     deferred_reallocation_pending_ = false;
757     ReleaseCodec();
758     CreateCodec();
759   }
760 
761   if (!codec_ || codec_->IsFlushed())
762     return;
763 
764   if (codec_->SupportsFlush(device_info_)) {
765     DVLOG(2) << "Flushing codec";
766     if (!codec_->Flush())
767       EnterTerminalState(State::kError, "Codec flush failed");
768   } else {
769     DVLOG(2) << "flush() workaround: creating a new codec";
770     // Release the codec and create a new one.
771     // Note: we may end up with two codecs attached to the same surface if the
772     // release hangs on one thread and create proceeds on another. This will
773     // result in an error, letting the user retry the playback. The alternative
774     // of waiting for the release risks hanging the playback forever.
775     ReleaseCodec();
776     CreateCodec();
777   }
778 }
779 
PumpCodec(bool force_start_timer)780 void MediaCodecVideoDecoder::PumpCodec(bool force_start_timer) {
781   DVLOG(4) << __func__;
782   bool did_work = false, did_input = false, did_output = false;
783   do {
784     did_input = QueueInput();
785     did_output = DequeueOutput();
786     if (did_input || did_output)
787       did_work = true;
788   } while (did_input || did_output);
789 
790   if (using_async_api_)
791     return;
792 
793   if (did_work || force_start_timer)
794     StartTimerOrPumpCodec();
795   else
796     StopTimerIfIdle();
797 }
798 
StartTimerOrPumpCodec()799 void MediaCodecVideoDecoder::StartTimerOrPumpCodec() {
800   DVLOG(4) << __func__;
801   if (state_ != State::kRunning)
802     return;
803 
804   if (using_async_api_) {
805     PumpCodec(false);
806     return;
807   }
808 
809   idle_timer_ = base::ElapsedTimer();
810 
811   // Poll at 10ms somewhat arbitrarily.
812   // TODO: Don't poll on new devices; use the callback API.
813   // TODO: Experiment with this number to save power. Since we already pump the
814   // codec in response to receiving a decode and output buffer release, polling
815   // at this frequency is likely overkill in the steady state.
816   const auto kPollingPeriod = base::TimeDelta::FromMilliseconds(10);
817   if (!pump_codec_timer_.IsRunning()) {
818     pump_codec_timer_.Start(
819         FROM_HERE, kPollingPeriod,
820         base::BindRepeating(&MediaCodecVideoDecoder::PumpCodec,
821                             base::Unretained(this), false));
822   }
823 }
824 
StopTimerIfIdle()825 void MediaCodecVideoDecoder::StopTimerIfIdle() {
826   DVLOG(4) << __func__;
827   DCHECK(!using_async_api_);
828 
829   // Stop the timer if we've been idle for one second. Chosen arbitrarily.
830   const auto kTimeout = base::TimeDelta::FromSeconds(1);
831   if (idle_timer_.Elapsed() > kTimeout) {
832     DVLOG(2) << "Stopping timer; idle timeout hit";
833     pump_codec_timer_.Stop();
834     // Draining for destroy can no longer proceed if the timer is stopping,
835     // because no more Decode() calls can be made, so complete it now to avoid
836     // leaking |this|.
837     if (drain_type_ == DrainType::kForDestroy)
838       OnCodecDrained();
839   }
840 }
841 
QueueInput()842 bool MediaCodecVideoDecoder::QueueInput() {
843   DVLOG(4) << __func__;
844   if (!codec_ || waiting_for_key_)
845     return false;
846 
847   // If the codec is drained, flush it when there is a pending decode and no
848   // unreleased output buffers. This lets us avoid both unbacking frames when we
849   // flush, and flushing unnecessarily, like at EOS.
850   //
851   // Often, we'll elide the eos to drain the codec, but we want to pretend that
852   // we did.  In this case, we should also flush.
853   if (codec_->IsDrained() || deferred_flush_pending_) {
854     if (!codec_->HasUnreleasedOutputBuffers() && !pending_decodes_.empty()) {
855       FlushCodec();
856       return true;
857     }
858     return false;
859   }
860 
861   if (pending_decodes_.empty())
862     return false;
863 
864   PendingDecode& pending_decode = pending_decodes_.front();
865   auto status = codec_->QueueInputBuffer(*pending_decode.buffer);
866   DVLOG((status == CodecWrapper::QueueStatus::kTryAgainLater ||
867                  status == CodecWrapper::QueueStatus::kOk
868              ? 3
869              : 2))
870       << "QueueInput(" << pending_decode.buffer->AsHumanReadableString()
871       << ") status=" << static_cast<int>(status);
872 
873   switch (status) {
874     case CodecWrapper::QueueStatus::kOk:
875       break;
876     case CodecWrapper::QueueStatus::kTryAgainLater:
877       return false;
878     case CodecWrapper::QueueStatus::kNoKey:
879       // Retry when a key is added.
880       waiting_for_key_ = true;
881       waiting_cb_.Run(WaitingReason::kNoDecryptionKey);
882       return false;
883     case CodecWrapper::QueueStatus::kError:
884       EnterTerminalState(State::kError, "QueueInputBuffer failed");
885       return false;
886   }
887 
888   if (pending_decode.buffer->end_of_stream()) {
889     // The VideoDecoder interface requires that the EOS DecodeCB is called after
890     // all decodes before it are delivered, so we have to save it and call it
891     // when the EOS is dequeued.
892     DCHECK(!eos_decode_cb_);
893     eos_decode_cb_ = std::move(pending_decode.decode_cb);
894   } else {
895     std::move(pending_decode.decode_cb).Run(DecodeStatus::OK);
896   }
897   pending_decodes_.pop_front();
898   return true;
899 }
900 
DequeueOutput()901 bool MediaCodecVideoDecoder::DequeueOutput() {
902   DVLOG(4) << __func__;
903   if (!codec_ || codec_->IsDrained() || waiting_for_key_)
904     return false;
905 
906   // If a surface transition is pending, wait for all outstanding buffers to be
907   // released before doing the transition. This is necessary because the
908   // VideoFrames corresponding to these buffers have metadata flags specific to
909   // the surface type, and changing the surface before they're rendered would
910   // invalidate them.
911   if (SurfaceTransitionPending()) {
912     if (!codec_->HasUnreleasedOutputBuffers()) {
913       TransitionToTargetSurface();
914       return true;
915     }
916     return false;
917   }
918 
919   base::TimeDelta presentation_time;
920   bool eos = false;
921   std::unique_ptr<CodecOutputBuffer> output_buffer;
922   auto status =
923       codec_->DequeueOutputBuffer(&presentation_time, &eos, &output_buffer);
924   switch (status) {
925     case CodecWrapper::DequeueStatus::kOk:
926       break;
927     case CodecWrapper::DequeueStatus::kTryAgainLater:
928       return false;
929     case CodecWrapper::DequeueStatus::kError:
930       DVLOG(1) << "DequeueOutputBuffer() error";
931       EnterTerminalState(State::kError, "DequeueOutputBuffer failed");
932       return false;
933   }
934   DVLOG(3) << "DequeueOutputBuffer(): pts="
935            << (eos ? "EOS"
936                    : std::to_string(presentation_time.InMilliseconds()));
937 
938   if (eos) {
939     if (eos_decode_cb_) {
940       // Schedule the EOS DecodeCB to run after all previous frames.
941       video_frame_factory_->RunAfterPendingVideoFrames(
942           base::BindOnce(&MediaCodecVideoDecoder::RunEosDecodeCb,
943                          weak_factory_.GetWeakPtr(), reset_generation_));
944     }
945     if (drain_type_)
946       OnCodecDrained();
947     // We don't flush the drained codec immediately because it might be
948     // backing unrendered frames near EOS. It's flushed lazily in QueueInput().
949     return false;
950   }
951 
952   // If we're draining for reset or destroy we can discard |output_buffer|
953   // without rendering it.  This is also true if we elided the drain itself,
954   // and deferred a flush that would have happened when the drain completed.
955   if (drain_type_ || deferred_flush_pending_)
956     return true;
957 
958   // Record the frame type that we're sending and some information about why.
959   UMA_HISTOGRAM_ENUMERATION(
960       "Media.AVDA.FrameInformation", cached_frame_information_,
961       static_cast<int>(
962           SurfaceChooserHelper::FrameInformation::FRAME_INFORMATION_MAX) +
963           1);  // PRESUBMIT_IGNORE_UMA_MAX
964 
965   gfx::Rect visible_rect(output_buffer->size());
966   std::unique_ptr<ScopedAsyncTrace> async_trace =
967       ScopedAsyncTrace::CreateIfEnabled(
968           "MediaCodecVideoDecoder::CreateVideoFrame");
969   video_frame_factory_->CreateVideoFrame(
970       std::move(output_buffer), presentation_time,
971       GetNaturalSize(visible_rect, decoder_config_.GetPixelAspectRatio()),
972       CreatePromotionHintCB(),
973       base::BindOnce(&MediaCodecVideoDecoder::ForwardVideoFrame,
974                      weak_factory_.GetWeakPtr(), reset_generation_,
975                      std::move(async_trace), base::TimeTicks::Now()));
976   return true;
977 }
978 
RunEosDecodeCb(int reset_generation)979 void MediaCodecVideoDecoder::RunEosDecodeCb(int reset_generation) {
980   // Both of the following conditions are necessary because:
981   //  * In an error state, the reset generations will match but |eos_decode_cb_|
982   //    will be aborted.
983   //  * After a Reset(), the reset generations won't match, but we might already
984   //    have a new |eos_decode_cb_| for the new generation.
985   if (reset_generation == reset_generation_ && eos_decode_cb_)
986     std::move(eos_decode_cb_).Run(DecodeStatus::OK);
987 }
988 
ForwardVideoFrame(int reset_generation,std::unique_ptr<ScopedAsyncTrace> async_trace,base::TimeTicks started_at,scoped_refptr<VideoFrame> frame)989 void MediaCodecVideoDecoder::ForwardVideoFrame(
990     int reset_generation,
991     std::unique_ptr<ScopedAsyncTrace> async_trace,
992     base::TimeTicks started_at,
993     scoped_refptr<VideoFrame> frame) {
994   DVLOG(3) << __func__ << " : "
995            << (frame ? frame->AsHumanReadableString() : "null");
996 
997   // Record how long this frame was pending.
998   const base::TimeDelta duration = base::TimeTicks::Now() - started_at;
999   UMA_HISTOGRAM_CUSTOM_TIMES("Media.MCVD.ForwardVideoFrameTiming", duration,
1000                              base::TimeDelta::FromMilliseconds(1),
1001                              base::TimeDelta::FromMilliseconds(100), 25);
1002 
1003   // No |frame| indicates an error creating it.
1004   if (!frame) {
1005     DLOG(ERROR) << __func__ << " |frame| is null";
1006     EnterTerminalState(State::kError, "Could not create VideoFrame");
1007     return;
1008   }
1009 
1010   if (reset_generation == reset_generation_) {
1011     // TODO(liberato): We might actually have a SW decoder.  Consider setting
1012     // this to false if so, especially for higher bitrates.
1013     frame->metadata()->power_efficient = true;
1014     output_cb_.Run(std::move(frame));
1015   }
1016 }
1017 
1018 // Our Reset() provides a slightly stronger guarantee than VideoDecoder does.
1019 // After |closure| runs:
1020 // 1) no VideoFrames from before the Reset() will be output, and
1021 // 2) no DecodeCBs (including EOS) from before the Reset() will be run.
Reset(base::OnceClosure closure)1022 void MediaCodecVideoDecoder::Reset(base::OnceClosure closure) {
1023   DVLOG(2) << __func__;
1024   DCHECK(!reset_cb_);
1025   reset_generation_++;
1026   reset_cb_ = std::move(closure);
1027   CancelPendingDecodes(DecodeStatus::ABORTED);
1028   StartDrainingCodec(DrainType::kForReset);
1029 }
1030 
StartDrainingCodec(DrainType drain_type)1031 void MediaCodecVideoDecoder::StartDrainingCodec(DrainType drain_type) {
1032   DVLOG(2) << __func__;
1033   TRACE_EVENT0("media", "MediaCodecVideoDecoder::StartDrainingCodec");
1034   DCHECK(pending_decodes_.empty());
1035   // It's okay if there's already a drain ongoing. We'll only enqueue an EOS if
1036   // the codec isn't already draining.
1037   drain_type_ = drain_type;
1038 
1039   // We can safely invalidate outstanding buffers for both types of drain, and
1040   // doing so can only make the drain complete quicker.  Note that we do this
1041   // even if we're eliding the drain, since we're either going to flush the
1042   // codec or destroy it.  While we're not required to do this, it might affect
1043   // stability if we don't (https://crbug.com/869365).  AVDA, in particular,
1044   // dropped all pending codec output buffers when starting a reset (seek) or
1045   // a destroy.
1046   if (codec_)
1047     codec_->DiscardOutputBuffers();
1048 
1049   // Skip the drain if possible. Only VP8 codecs need draining because
1050   // they can hang in release() or flush() otherwise
1051   // (http://crbug.com/598963).
1052   // TODO(watk): Strongly consider blocking VP8 (or specific MediaCodecs)
1053   // instead. Draining is responsible for a lot of complexity.
1054   if (decoder_config_.codec() != kCodecVP8 || !codec_ || codec_->IsFlushed() ||
1055       codec_->IsDrained() || using_async_api_) {
1056     // If the codec isn't already drained or flushed, then we have to remember
1057     // that we owe it a flush.  We also have to remember not to deliver any
1058     // output buffers that might still be in progress in the codec.
1059     deferred_flush_pending_ =
1060         codec_ && !codec_->IsDrained() && !codec_->IsFlushed();
1061     OnCodecDrained();
1062     return;
1063   }
1064 
1065   // Queue EOS if the codec isn't already processing one.
1066   if (!codec_->IsDraining())
1067     pending_decodes_.push_back(PendingDecode::CreateEos());
1068 
1069   PumpCodec(true);
1070 }
1071 
OnCodecDrained()1072 void MediaCodecVideoDecoder::OnCodecDrained() {
1073   DVLOG(2) << __func__;
1074   TRACE_EVENT0("media", "MediaCodecVideoDecoder::OnCodecDrained");
1075   DrainType drain_type = *drain_type_;
1076   drain_type_.reset();
1077 
1078   if (drain_type == DrainType::kForDestroy) {
1079     // Post the delete in case the caller uses |this| after we return.
1080     base::SequencedTaskRunnerHandle::Get()->DeleteSoon(FROM_HERE, this);
1081     return;
1082   }
1083 
1084   std::move(reset_cb_).Run();
1085 
1086   // Flush the codec unless (a) it's already flushed, (b) it's drained and the
1087   // flush will be handled automatically on the next decode, or (c) we've
1088   // elided the eos and want to defer the flush.
1089   if (codec_ && !codec_->IsFlushed() && !codec_->IsDrained() &&
1090       !deferred_flush_pending_) {
1091     FlushCodec();
1092   }
1093 }
1094 
EnterTerminalState(State state,const char * reason)1095 void MediaCodecVideoDecoder::EnterTerminalState(State state,
1096                                                 const char* reason) {
1097   DVLOG(2) << __func__ << " " << static_cast<int>(state) << " " << reason;
1098   MEDIA_LOG(INFO, media_log_) << "Entering Terminal State: " << reason;
1099 
1100   state_ = state;
1101   DCHECK(InTerminalState());
1102 
1103   // Cancel pending codec creation.
1104   codec_allocator_weak_factory_.InvalidateWeakPtrs();
1105   pump_codec_timer_.Stop();
1106   ReleaseCodec();
1107   target_surface_bundle_ = nullptr;
1108   texture_owner_bundle_ = nullptr;
1109   if (state == State::kError)
1110     CancelPendingDecodes(DecodeStatus::DECODE_ERROR);
1111   if (drain_type_)
1112     OnCodecDrained();
1113 }
1114 
InTerminalState()1115 bool MediaCodecVideoDecoder::InTerminalState() {
1116   return state_ == State::kSurfaceDestroyed || state_ == State::kError;
1117 }
1118 
CancelPendingDecodes(DecodeStatus status)1119 void MediaCodecVideoDecoder::CancelPendingDecodes(DecodeStatus status) {
1120   for (auto& pending_decode : pending_decodes_)
1121     std::move(pending_decode.decode_cb).Run(status);
1122   pending_decodes_.clear();
1123   if (eos_decode_cb_)
1124     std::move(eos_decode_cb_).Run(status);
1125 }
1126 
ReleaseCodec()1127 void MediaCodecVideoDecoder::ReleaseCodec() {
1128   if (!codec_)
1129     return;
1130   auto pair = codec_->TakeCodecSurfacePair();
1131   codec_ = nullptr;
1132   codec_allocator_->ReleaseMediaCodec(
1133       std::move(pair.first),
1134       base::BindOnce(
1135           &base::SequencedTaskRunner::ReleaseSoon<CodecSurfaceBundle>,
1136           base::SequencedTaskRunnerHandle::Get(), FROM_HERE,
1137           std::move(pair.second)));
1138 }
1139 
CreateOverlayFactoryCb()1140 AndroidOverlayFactoryCB MediaCodecVideoDecoder::CreateOverlayFactoryCb() {
1141   if (!overlay_factory_cb_ || !overlay_info_.HasValidRoutingToken())
1142     return AndroidOverlayFactoryCB();
1143 
1144   return base::BindRepeating(overlay_factory_cb_, *overlay_info_.routing_token);
1145 }
1146 
GetDisplayName() const1147 std::string MediaCodecVideoDecoder::GetDisplayName() const {
1148   return "MediaCodecVideoDecoder";
1149 }
1150 
NeedsBitstreamConversion() const1151 bool MediaCodecVideoDecoder::NeedsBitstreamConversion() const {
1152   return true;
1153 }
1154 
CanReadWithoutStalling() const1155 bool MediaCodecVideoDecoder::CanReadWithoutStalling() const {
1156   // MediaCodec gives us no indication that it will stop producing outputs
1157   // until we provide more inputs or release output buffers back to it, so
1158   // we have to always return false.
1159   // TODO(watk): This puts all MCVD playbacks into low delay mode (i.e., the
1160   // renderer won't try to preroll). Ideally we'd be smarter about
1161   // this and attempt preroll but be able to give up if we can't produce
1162   // enough frames.
1163   return false;
1164 }
1165 
GetMaxDecodeRequests() const1166 int MediaCodecVideoDecoder::GetMaxDecodeRequests() const {
1167   // We indicate that we're done decoding a frame as soon as we submit it to
1168   // MediaCodec so the number of parallel decode requests just sets the upper
1169   // limit of the size of our pending decode queue.
1170   return 2;
1171 }
1172 
1173 PromotionHintAggregator::NotifyPromotionHintCB
CreatePromotionHintCB()1174 MediaCodecVideoDecoder::CreatePromotionHintCB() {
1175   // Right now, we don't request promotion hints.  This is only used by SOP.
1176   // While we could simplify it a bit, this is the general form that we'll use
1177   // when handling promotion hints.
1178 
1179   // Note that this keeps only a wp to the surface bundle via |layout_cb|.  It
1180   // also continues to work even if |this| is destroyed; images might want to
1181   // move an overlay around even after MCVD has been torn down.  For example
1182   // inline L1 content will fall into this case.
1183   return BindToCurrentLoop(base::BindRepeating(
1184       [](base::WeakPtr<MediaCodecVideoDecoder> mcvd,
1185          CodecSurfaceBundle::ScheduleLayoutCB layout_cb,
1186          PromotionHintAggregator::Hint hint) {
1187         // If we're promotable, and we have a surface bundle, then also
1188         // position the overlay.  We could do this even if the overlay is
1189         // not promotable, but it wouldn't have any visible effect.
1190         if (hint.is_promotable)
1191           layout_cb.Run(hint.screen_rect);
1192 
1193         // Notify MCVD about the promotion hint, so that it can decide if it
1194         // wants to switch to / from an overlay.
1195         if (mcvd)
1196           mcvd->NotifyPromotionHint(hint);
1197       },
1198       weak_factory_.GetWeakPtr(),
1199       codec_->SurfaceBundle()->GetScheduleLayoutCB()));
1200 }
1201 
IsUsingOverlay() const1202 bool MediaCodecVideoDecoder::IsUsingOverlay() const {
1203   return codec_ && codec_->SurfaceBundle() &&
1204          codec_->SurfaceBundle()->overlay();
1205 }
1206 
NotifyPromotionHint(PromotionHintAggregator::Hint hint)1207 void MediaCodecVideoDecoder::NotifyPromotionHint(
1208     PromotionHintAggregator::Hint hint) {
1209   surface_chooser_helper_.NotifyPromotionHintAndUpdateChooser(hint,
1210                                                               IsUsingOverlay());
1211 }
1212 
CacheFrameInformation()1213 void MediaCodecVideoDecoder::CacheFrameInformation() {
1214   cached_frame_information_ =
1215       surface_chooser_helper_.ComputeFrameInformation(IsUsingOverlay());
1216 }
1217 
1218 }  // namespace media
1219