1 // Copyright 2017 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "third_party/blink/renderer/platform/graphics/video_frame_submitter.h"
6
7 #include <utility>
8
9 #include "base/bind.h"
10 #include "base/metrics/histogram_macros.h"
11 #include "base/threading/sequenced_task_runner_handle.h"
12 #include "base/trace_event/trace_event.h"
13 #include "cc/metrics/video_playback_roughness_reporter.h"
14 #include "components/viz/common/resources/resource_id.h"
15 #include "components/viz/common/resources/returned_resource.h"
16 #include "media/base/video_frame.h"
17 #include "media/base/video_types.h"
18 #include "mojo/public/cpp/bindings/remote.h"
19 #include "services/viz/public/cpp/gpu/context_provider_command_buffer.h"
20 #include "services/viz/public/mojom/compositing/compositor_frame_sink.mojom-blink.h"
21 #include "services/viz/public/mojom/hit_test/hit_test_region_list.mojom-blink.h"
22 #include "third_party/blink/public/common/thread_safe_browser_interface_broker_proxy.h"
23 #include "third_party/blink/public/mojom/frame_sinks/embedded_frame_sink.mojom-blink.h"
24 #include "third_party/blink/public/platform/platform.h"
25 #include "third_party/blink/renderer/platform/graphics/canvas_resource.h"
26 #include "ui/gfx/presentation_feedback.h"
27
28 namespace blink {
29
VideoFrameSubmitter(WebContextProviderCallback context_provider_callback,cc::PlaybackRoughnessReportingCallback roughness_reporting_callback,std::unique_ptr<VideoFrameResourceProvider> resource_provider)30 VideoFrameSubmitter::VideoFrameSubmitter(
31 WebContextProviderCallback context_provider_callback,
32 cc::PlaybackRoughnessReportingCallback roughness_reporting_callback,
33 std::unique_ptr<VideoFrameResourceProvider> resource_provider)
34 : context_provider_callback_(context_provider_callback),
35 resource_provider_(std::move(resource_provider)),
36 rotation_(media::VIDEO_ROTATION_0),
37 roughness_reporter_(std::make_unique<cc::VideoPlaybackRoughnessReporter>(
38 std::move(roughness_reporting_callback))),
39 frame_trackers_(false, nullptr) {
40 DETACH_FROM_THREAD(thread_checker_);
41 }
42
~VideoFrameSubmitter()43 VideoFrameSubmitter::~VideoFrameSubmitter() {
44 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
45 if (context_provider_)
46 context_provider_->RemoveObserver(this);
47
48 // Release VideoFrameResourceProvider early since its destruction will make
49 // calls back into this class via the viz::SharedBitmapReporter interface.
50 resource_provider_.reset();
51 }
52
StopUsingProvider()53 void VideoFrameSubmitter::StopUsingProvider() {
54 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
55 if (is_rendering_)
56 StopRendering();
57 video_frame_provider_ = nullptr;
58 }
59
StartRendering()60 void VideoFrameSubmitter::StartRendering() {
61 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
62 DCHECK(!is_rendering_);
63 is_rendering_ = true;
64
65 if (compositor_frame_sink_)
66 compositor_frame_sink_->SetNeedsBeginFrame(is_rendering_ && ShouldSubmit());
67
68 frame_trackers_.StartSequence(cc::FrameSequenceTrackerType::kVideo);
69 }
70
StopRendering()71 void VideoFrameSubmitter::StopRendering() {
72 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
73 DCHECK(is_rendering_);
74 DCHECK(video_frame_provider_);
75
76 is_rendering_ = false;
77
78 frame_trackers_.StopSequence(cc::FrameSequenceTrackerType::kVideo);
79 roughness_reporter_->Reset();
80
81 UpdateSubmissionState();
82 }
83
DidReceiveFrame()84 void VideoFrameSubmitter::DidReceiveFrame() {
85 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
86 DCHECK(video_frame_provider_);
87 SubmitSingleFrame();
88 }
89
IsDrivingFrameUpdates() const90 bool VideoFrameSubmitter::IsDrivingFrameUpdates() const {
91 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
92
93 // We drive frame updates only when we believe that something is consuming
94 // them. This is different than VideoLayer, which drives updates any time
95 // they're in the layer tree.
96 return is_rendering_ && ShouldSubmit();
97 }
98
Initialize(cc::VideoFrameProvider * provider)99 void VideoFrameSubmitter::Initialize(cc::VideoFrameProvider* provider) {
100 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
101 if (!provider)
102 return;
103
104 DCHECK(!video_frame_provider_);
105 video_frame_provider_ = provider;
106 context_provider_callback_.Run(
107 nullptr, base::BindOnce(&VideoFrameSubmitter::OnReceivedContextProvider,
108 weak_ptr_factory_.GetWeakPtr()));
109 }
110
SetRotation(media::VideoRotation rotation)111 void VideoFrameSubmitter::SetRotation(media::VideoRotation rotation) {
112 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
113 rotation_ = rotation;
114 }
115
EnableSubmission(viz::SurfaceId surface_id,base::TimeTicks local_surface_id_allocation_time)116 void VideoFrameSubmitter::EnableSubmission(
117 viz::SurfaceId surface_id,
118 base::TimeTicks local_surface_id_allocation_time) {
119 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
120
121 // TODO(lethalantidote): Set these fields earlier in the constructor. Will
122 // need to construct VideoFrameSubmitter later in order to do this.
123 frame_sink_id_ = surface_id.frame_sink_id();
124 child_local_surface_id_allocator_.UpdateFromParent(
125 viz::LocalSurfaceIdAllocation(surface_id.local_surface_id(),
126 local_surface_id_allocation_time));
127 if (resource_provider_->IsInitialized())
128 StartSubmitting();
129 }
130
SetIsSurfaceVisible(bool is_visible)131 void VideoFrameSubmitter::SetIsSurfaceVisible(bool is_visible) {
132 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
133 is_surface_visible_ = is_visible;
134 UpdateSubmissionState();
135 }
136
SetIsPageVisible(bool is_visible)137 void VideoFrameSubmitter::SetIsPageVisible(bool is_visible) {
138 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
139 is_page_visible_ = is_visible;
140 UpdateSubmissionState();
141 }
142
SetForceSubmit(bool force_submit)143 void VideoFrameSubmitter::SetForceSubmit(bool force_submit) {
144 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
145 force_submit_ = force_submit;
146 UpdateSubmissionState();
147 }
148
OnContextLost()149 void VideoFrameSubmitter::OnContextLost() {
150 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
151 receiver_.reset();
152
153 if (context_provider_)
154 context_provider_->RemoveObserver(this);
155
156 waiting_for_compositor_ack_ = false;
157 last_frame_id_.reset();
158
159 resource_provider_->OnContextLost();
160
161 // |compositor_frame_sink_| should be reset last.
162 compositor_frame_sink_.reset();
163
164 context_provider_callback_.Run(
165 context_provider_,
166 base::BindOnce(&VideoFrameSubmitter::OnReceivedContextProvider,
167 weak_ptr_factory_.GetWeakPtr()));
168 }
169
DidReceiveCompositorFrameAck(const WTF::Vector<viz::ReturnedResource> & resources)170 void VideoFrameSubmitter::DidReceiveCompositorFrameAck(
171 const WTF::Vector<viz::ReturnedResource>& resources) {
172 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
173 ReclaimResources(resources);
174 waiting_for_compositor_ack_ = false;
175 }
176
OnBeginFrame(const viz::BeginFrameArgs & args,WTF::HashMap<uint32_t,::viz::mojom::blink::FrameTimingDetailsPtr> timing_details)177 void VideoFrameSubmitter::OnBeginFrame(
178 const viz::BeginFrameArgs& args,
179 WTF::HashMap<uint32_t, ::viz::mojom::blink::FrameTimingDetailsPtr>
180 timing_details) {
181 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
182 TRACE_EVENT0("media", "VideoFrameSubmitter::OnBeginFrame");
183
184 last_begin_frame_args_ = args;
185
186 for (const auto& pair : timing_details) {
187 if (viz::FrameTokenGT(pair.key, *next_frame_token_))
188 continue;
189
190 #if defined(OS_LINUX) || defined(OS_BSD)
191 // TODO: On Linux failure flag is unreliable, and perfectly rendered frames
192 // are reported as failures all the time.
193 bool presentation_failure = false;
194 #else
195 bool presentation_failure = !!(pair.value->presentation_feedback->flags &
196 gfx::PresentationFeedback::kFailure);
197 #endif
198 if (!presentation_failure &&
199 !ignorable_submitted_frames_.contains(pair.key)) {
200 frame_trackers_.NotifyFramePresented(
201 pair.key, gfx::PresentationFeedback(
202 pair.value->presentation_feedback->timestamp,
203 pair.value->presentation_feedback->interval,
204 pair.value->presentation_feedback->flags));
205 roughness_reporter_->FramePresented(
206 pair.key, pair.value->presentation_feedback->timestamp);
207 }
208
209 ignorable_submitted_frames_.erase(pair.key);
210 TRACE_EVENT_NESTABLE_ASYNC_END_WITH_TIMESTAMP0(
211 "media", "VideoFrameSubmitter", TRACE_ID_LOCAL(pair.key),
212 pair.value->presentation_feedback->timestamp);
213 }
214 frame_trackers_.NotifyBeginImplFrame(args);
215
216 base::ScopedClosureRunner end_frame(
217 base::BindOnce(&cc::FrameSequenceTrackerCollection::NotifyFrameEnd,
218 base::Unretained(&frame_trackers_), args, args));
219 base::ScopedClosureRunner roughness_processing(
220 base::BindOnce(&cc::VideoPlaybackRoughnessReporter::ProcessFrameWindow,
221 base::Unretained(roughness_reporter_.get())));
222
223 // Don't call UpdateCurrentFrame() for MISSED BeginFrames. Also don't call it
224 // after StopRendering() has been called (forbidden by API contract).
225 viz::BeginFrameAck current_begin_frame_ack(args, false);
226 if (args.type == viz::BeginFrameArgs::MISSED || !is_rendering_) {
227 compositor_frame_sink_->DidNotProduceFrame(current_begin_frame_ack);
228 frame_trackers_.NotifyImplFrameCausedNoDamage(current_begin_frame_ack);
229 return;
230 }
231
232 // Update the current frame, even if we haven't gotten an ack for a previous
233 // frame yet. That probably signals a dropped frame, and this will let the
234 // provider know that it happened, since we won't PutCurrentFrame this one.
235 // Note that we should DidNotProduceFrame with or without the ack.
236 if (!video_frame_provider_ || !video_frame_provider_->UpdateCurrentFrame(
237 args.frame_time + args.interval,
238 args.frame_time + 2 * args.interval)) {
239 compositor_frame_sink_->DidNotProduceFrame(current_begin_frame_ack);
240 frame_trackers_.NotifyImplFrameCausedNoDamage(current_begin_frame_ack);
241 return;
242 }
243
244 // We do have a new frame that we could display. See if we're supposed to
245 // actually submit a frame or not, and try to submit one.
246 auto video_frame = video_frame_provider_->GetCurrentFrame();
247 if (!SubmitFrame(current_begin_frame_ack, std::move(video_frame))) {
248 compositor_frame_sink_->DidNotProduceFrame(current_begin_frame_ack);
249 frame_trackers_.NotifyImplFrameCausedNoDamage(current_begin_frame_ack);
250 return;
251 }
252
253 // We submitted a frame!
254
255 // We still signal PutCurrentFrame here, rather than on the ack, so that it
256 // lines up with the correct frame. Otherwise, any intervening calls to
257 // OnBeginFrame => UpdateCurrentFrame will cause the put to signal that the
258 // later frame was displayed.
259 video_frame_provider_->PutCurrentFrame();
260 }
261
ReclaimResources(const WTF::Vector<viz::ReturnedResource> & resources)262 void VideoFrameSubmitter::ReclaimResources(
263 const WTF::Vector<viz::ReturnedResource>& resources) {
264 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
265 resource_provider_->ReceiveReturnsFromParent(resources);
266 }
267
DidAllocateSharedBitmap(base::ReadOnlySharedMemoryRegion region,const viz::SharedBitmapId & id)268 void VideoFrameSubmitter::DidAllocateSharedBitmap(
269 base::ReadOnlySharedMemoryRegion region,
270 const viz::SharedBitmapId& id) {
271 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
272 DCHECK(compositor_frame_sink_);
273 compositor_frame_sink_->DidAllocateSharedBitmap(
274 std::move(region), SharedBitmapIdToGpuMailboxPtr(id));
275 }
276
DidDeleteSharedBitmap(const viz::SharedBitmapId & id)277 void VideoFrameSubmitter::DidDeleteSharedBitmap(const viz::SharedBitmapId& id) {
278 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
279 DCHECK(compositor_frame_sink_);
280 compositor_frame_sink_->DidDeleteSharedBitmap(
281 SharedBitmapIdToGpuMailboxPtr(id));
282 }
283
OnReceivedContextProvider(bool use_gpu_compositing,scoped_refptr<viz::RasterContextProvider> context_provider)284 void VideoFrameSubmitter::OnReceivedContextProvider(
285 bool use_gpu_compositing,
286 scoped_refptr<viz::RasterContextProvider> context_provider) {
287 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
288 if (!use_gpu_compositing) {
289 resource_provider_->Initialize(nullptr, this);
290 if (frame_sink_id_.is_valid())
291 StartSubmitting();
292 return;
293 }
294
295 bool has_good_context = false;
296 while (!has_good_context) {
297 if (!context_provider) {
298 // Delay to retry getting the context_provider.
299 constexpr base::TimeDelta kGetContextProviderRetryTimeout =
300 base::TimeDelta::FromMilliseconds(150);
301
302 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
303 FROM_HERE,
304 base::BindOnce(
305 context_provider_callback_, context_provider_,
306 base::BindOnce(&VideoFrameSubmitter::OnReceivedContextProvider,
307 weak_ptr_factory_.GetWeakPtr())),
308 kGetContextProviderRetryTimeout);
309 return;
310 }
311
312 // Note that |context_provider| is now null after the move, such that if we
313 // end up having !|has_good_context|, we will retry to obtain the
314 // context_provider.
315 context_provider_ = std::move(context_provider);
316 auto result = context_provider_->BindToCurrentThread();
317
318 has_good_context =
319 result == gpu::ContextResult::kSuccess &&
320 context_provider_->ContextGL()->GetGraphicsResetStatusKHR() ==
321 GL_NO_ERROR;
322 }
323 context_provider_->AddObserver(this);
324 resource_provider_->Initialize(context_provider_.get(), nullptr);
325
326 if (frame_sink_id_.is_valid())
327 StartSubmitting();
328 }
329
StartSubmitting()330 void VideoFrameSubmitter::StartSubmitting() {
331 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
332 DCHECK(frame_sink_id_.is_valid());
333
334 mojo::Remote<mojom::blink::EmbeddedFrameSinkProvider> provider;
335 Platform::Current()->GetBrowserInterfaceBroker()->GetInterface(
336 provider.BindNewPipeAndPassReceiver());
337
338 provider->CreateCompositorFrameSink(
339 frame_sink_id_, receiver_.BindNewPipeAndPassRemote(),
340 compositor_frame_sink_.BindNewPipeAndPassReceiver());
341 if (!surface_embedder_.is_bound()) {
342 provider->ConnectToEmbedder(frame_sink_id_,
343 surface_embedder_.BindNewPipeAndPassReceiver());
344 } else {
345 GenerateNewSurfaceId();
346 }
347
348 compositor_frame_sink_.set_disconnect_handler(base::BindOnce(
349 &VideoFrameSubmitter::OnContextLost, base::Unretained(this)));
350
351 UpdateSubmissionState();
352 }
353
UpdateSubmissionState()354 void VideoFrameSubmitter::UpdateSubmissionState() {
355 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
356 if (!compositor_frame_sink_)
357 return;
358
359 compositor_frame_sink_->SetNeedsBeginFrame(IsDrivingFrameUpdates());
360
361 // These two calls are very important; they are responsible for significant
362 // memory savings when content is off-screen.
363 //
364 // While off-screen, we do not submit frames (unless |force_submit_| is true),
365 // which prevents GPU resource creation and accumulation on the remote side.
366 // During the transition to off-screen we further send an empty frame with the
367 // intent to evict any resources held for the previous frame. Combined these
368 // optimizations save 30-50% in cc:: resource memory usage.
369 //
370 // See https://crbug.com/829813 and https://crbug.com/829565.
371 if (ShouldSubmit()) {
372 // Submit even if we're rendering, otherwise we may display an empty frame
373 // before the next OnBeginFrame() which can cause a visible flash.
374 SubmitSingleFrame();
375 } else {
376 // Post a delayed task to submit an empty frame. We don't do this here,
377 // since there is a race between when we're notified that the player is not
378 // visible, and when auto-PiP starts. In PiP, we'll be set to force submit,
379 // but we're notified after we find out that the page is hidden. If we
380 // submit an empty frame now, then there will be a flicker in the video
381 // when the empty frame is displayed. By delaying the empty frame, we give
382 // the auto-PiP a chance to start. Note that the empty frame isn't required
383 // for visual correctness; it's just for resource cleanup. We can delay
384 // resource cleanup a little.
385 //
386 // If there are any in-flight empty frame requests, this cancels them. We
387 // want to wait until any group of state changes stabilizes.
388 empty_frame_timer_.Start(
389 FROM_HERE, base::TimeDelta::FromMilliseconds(500),
390 base::BindOnce(&VideoFrameSubmitter::SubmitEmptyFrameIfNeeded,
391 base::Unretained(this)));
392 }
393 }
394
SubmitEmptyFrameIfNeeded()395 void VideoFrameSubmitter::SubmitEmptyFrameIfNeeded() {
396 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
397 if (!compositor_frame_sink_)
398 return;
399
400 // If we are allowed to submit real frames, then don't send a blank frame
401 // since the last real frame might actually be visible.
402 //
403 // We do not actually submit a real frame here, though; that should be done
404 // (if desired) by whatever switched us to ShouldSubmit() mode.
405 if (ShouldSubmit())
406 return;
407
408 // If we don't have a frame size, then we can't send a blank frame.
409 if (frame_size_.IsEmpty())
410 return;
411
412 SubmitEmptyFrame();
413 }
414
SubmitFrame(const viz::BeginFrameAck & begin_frame_ack,scoped_refptr<media::VideoFrame> video_frame)415 bool VideoFrameSubmitter::SubmitFrame(
416 const viz::BeginFrameAck& begin_frame_ack,
417 scoped_refptr<media::VideoFrame> video_frame) {
418 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
419 DCHECK(video_frame);
420 TRACE_EVENT1("media", "VideoFrameSubmitter::SubmitFrame", "frame",
421 video_frame->AsHumanReadableString());
422
423 if (!compositor_frame_sink_ || !ShouldSubmit())
424 return false;
425
426 // Not submitting a frame when waiting for a previous ack saves memory by
427 // not building up unused remote side resources. See https://crbug.com/830828.
428 //
429 // Similarly we don't submit the same frame multiple times.
430 if (waiting_for_compositor_ack_ || last_frame_id_ == video_frame->unique_id())
431 return false;
432
433 last_frame_id_ = video_frame->unique_id();
434
435 gfx::Size frame_size(video_frame->natural_size());
436 if (rotation_ == media::VIDEO_ROTATION_90 ||
437 rotation_ == media::VIDEO_ROTATION_270) {
438 frame_size = gfx::Size(frame_size.height(), frame_size.width());
439 }
440
441 if (frame_size.IsEmpty()) {
442 // We're not supposed to get 0x0 frames. For now, just ignore it until we
443 // track down where they're coming from. Creating a CompositorFrame with an
444 // empty output rectangle isn't allowed.
445 // crbug.com/979564
446 return false;
447 }
448
449 if (frame_size_ != frame_size) {
450 if (!frame_size_.IsEmpty())
451 GenerateNewSurfaceId();
452 frame_size_ = frame_size;
453 }
454
455 auto frame_token = ++next_frame_token_;
456 auto source_id = begin_frame_ack.frame_id.source_id;
457 if (source_id != viz::BeginFrameArgs::kManualSourceId) {
458 // Roughness reporter only cares about true video frames.
459 roughness_reporter_->FrameSubmitted(frame_token, *video_frame.get(),
460 last_begin_frame_args_.interval);
461 }
462 auto compositor_frame = CreateCompositorFrame(frame_token, begin_frame_ack,
463 std::move(video_frame));
464
465 WebVector<viz::ResourceId> resources;
466 const auto& quad_list = compositor_frame.render_pass_list.back()->quad_list;
467 if (!quad_list.empty()) {
468 DCHECK_EQ(quad_list.size(), 1u);
469 resources.Assign(quad_list.front()->resources);
470 }
471
472 WebVector<viz::TransferableResource> resource_list;
473 resource_provider_->PrepareSendToParent(resources, &resource_list);
474 compositor_frame.resource_list = resource_list.ReleaseVector();
475
476 // We can pass nullptr for the HitTestData as the CompositorFram will not
477 // contain any SurfaceDrawQuads.
478 compositor_frame_sink_->SubmitCompositorFrame(
479 child_local_surface_id_allocator_.GetCurrentLocalSurfaceIdAllocation()
480 .local_surface_id(),
481 std::move(compositor_frame), nullptr, 0);
482 frame_trackers_.NotifySubmitFrame(frame_token, false, begin_frame_ack,
483 last_begin_frame_args_);
484 resource_provider_->ReleaseFrameResources();
485
486 waiting_for_compositor_ack_ = true;
487 return true;
488 }
489
SubmitEmptyFrame()490 void VideoFrameSubmitter::SubmitEmptyFrame() {
491 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
492 DCHECK(compositor_frame_sink_ && !ShouldSubmit());
493 DCHECK(!frame_size_.IsEmpty());
494 TRACE_EVENT0("media", "VideoFrameSubmitter::SubmitEmptyFrame");
495
496 // If there's nothing to submit to or we've already submitted an empty frame,
497 // don't submit another one.
498 if (!compositor_frame_sink_ || !last_frame_id_.has_value())
499 return;
500
501 last_frame_id_.reset();
502 auto begin_frame_ack = viz::BeginFrameAck::CreateManualAckWithDamage();
503 auto frame_token = ++next_frame_token_;
504 auto compositor_frame =
505 CreateCompositorFrame(frame_token, begin_frame_ack, nullptr);
506
507 compositor_frame_sink_->SubmitCompositorFrame(
508 child_local_surface_id_allocator_.GetCurrentLocalSurfaceIdAllocation()
509 .local_surface_id(),
510 std::move(compositor_frame), nullptr, 0);
511 frame_trackers_.NotifySubmitFrame(frame_token, false, begin_frame_ack,
512 last_begin_frame_args_);
513
514 // We don't set |waiting_for_compositor_ack_| here since we want to allow a
515 // subsequent real frame to replace it at any time if needed.
516 }
517
SubmitSingleFrame()518 void VideoFrameSubmitter::SubmitSingleFrame() {
519 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
520
521 // If we haven't gotten a valid result yet from |context_provider_callback_|
522 // |resource_provider_| will remain uninitialized.
523 // |video_frame_provider_| may be null if StopUsingProvider has been called,
524 // which could happen if the |video_frame_provider_| is destructing while we
525 // are waiting for the ContextProvider.
526 if (!resource_provider_->IsInitialized() || !video_frame_provider_)
527 return;
528
529 auto video_frame = video_frame_provider_->GetCurrentFrame();
530 if (!video_frame)
531 return;
532
533 if (SubmitFrame(viz::BeginFrameAck::CreateManualAckWithDamage(),
534 std::move(video_frame))) {
535 video_frame_provider_->PutCurrentFrame();
536 }
537 }
538
ShouldSubmit() const539 bool VideoFrameSubmitter::ShouldSubmit() const {
540 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
541 return (is_surface_visible_ && is_page_visible_) || force_submit_;
542 }
543
CreateCompositorFrame(uint32_t frame_token,const viz::BeginFrameAck & begin_frame_ack,scoped_refptr<media::VideoFrame> video_frame)544 viz::CompositorFrame VideoFrameSubmitter::CreateCompositorFrame(
545 uint32_t frame_token,
546 const viz::BeginFrameAck& begin_frame_ack,
547 scoped_refptr<media::VideoFrame> video_frame) {
548 DCHECK(!frame_size_.IsEmpty());
549
550 viz::CompositorFrame compositor_frame;
551 compositor_frame.metadata.begin_frame_ack = begin_frame_ack;
552 compositor_frame.metadata.frame_token = frame_token;
553 compositor_frame.metadata.preferred_frame_interval =
554 video_frame_provider_
555 ? video_frame_provider_->GetPreferredRenderInterval()
556 : viz::BeginFrameArgs::MinInterval();
557
558 base::TimeTicks value;
559 if (video_frame && video_frame->metadata()->GetTimeTicks(
560 media::VideoFrameMetadata::DECODE_END_TIME, &value)) {
561 TRACE_EVENT_NESTABLE_ASYNC_BEGIN_WITH_TIMESTAMP0(
562 "media", "VideoFrameSubmitter", TRACE_ID_LOCAL(frame_token), value);
563 TRACE_EVENT_NESTABLE_ASYNC_BEGIN_WITH_TIMESTAMP0(
564 "media", "Pre-submit buffering", TRACE_ID_LOCAL(frame_token), value);
565 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "Pre-submit buffering",
566 TRACE_ID_LOCAL(frame_token));
567
568 if (begin_frame_ack.frame_id.source_id ==
569 viz::BeginFrameArgs::kManualSourceId)
570 ignorable_submitted_frames_.insert(frame_token);
571
572 UMA_HISTOGRAM_TIMES("Media.VideoFrameSubmitter.PreSubmitBuffering",
573 base::TimeTicks::Now() - value);
574 } else {
575 TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "VideoFrameSubmitter",
576 TRACE_ID_LOCAL(frame_token),
577 "empty video frame?", !video_frame);
578 }
579
580 // We don't assume that the ack is marked as having damage. However, we're
581 // definitely emitting a CompositorFrame that damages the entire surface.
582 compositor_frame.metadata.begin_frame_ack.has_damage = true;
583 compositor_frame.metadata.device_scale_factor = 1;
584 compositor_frame.metadata.may_contain_video = true;
585 compositor_frame.metadata.local_surface_id_allocation_time =
586 child_local_surface_id_allocator_.GetCurrentLocalSurfaceIdAllocation()
587 .allocation_time();
588
589 auto render_pass = viz::RenderPass::Create();
590 render_pass->SetNew(1, gfx::Rect(frame_size_), gfx::Rect(frame_size_),
591 gfx::Transform());
592
593 if (video_frame) {
594 compositor_frame.metadata.content_color_usage =
595 video_frame->ColorSpace().GetContentColorUsage();
596 const bool is_opaque = media::IsOpaque(video_frame->format());
597 resource_provider_->AppendQuads(render_pass.get(), std::move(video_frame),
598 rotation_, is_opaque);
599 }
600
601 compositor_frame.render_pass_list.emplace_back(std::move(render_pass));
602 return compositor_frame;
603 }
604
GenerateNewSurfaceId()605 void VideoFrameSubmitter::GenerateNewSurfaceId() {
606 last_frame_id_.reset();
607
608 // We need a new id in the event of context loss.
609 child_local_surface_id_allocator_.GenerateId();
610
611 surface_embedder_->SetLocalSurfaceId(
612 child_local_surface_id_allocator_.GetCurrentLocalSurfaceIdAllocation()
613 .local_surface_id());
614 }
615
616 } // namespace blink
617