1 // Copyright 2017 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_GRAPHICS_VIDEO_FRAME_SUBMITTER_H_ 6 #define THIRD_PARTY_BLINK_RENDERER_PLATFORM_GRAPHICS_VIDEO_FRAME_SUBMITTER_H_ 7 8 #include <memory> 9 10 #include "base/memory/read_only_shared_memory_region.h" 11 #include "base/memory/weak_ptr.h" 12 #include "base/optional.h" 13 #include "base/threading/thread_checker.h" 14 #include "base/time/time.h" 15 #include "base/timer/timer.h" 16 #include "cc/metrics/frame_sequence_tracker.h" 17 #include "cc/metrics/video_playback_roughness_reporter.h" 18 #include "components/viz/client/shared_bitmap_reporter.h" 19 #include "components/viz/common/gpu/context_provider.h" 20 #include "components/viz/common/resources/shared_bitmap.h" 21 #include "components/viz/common/surfaces/child_local_surface_id_allocator.h" 22 #include "mojo/public/cpp/bindings/receiver.h" 23 #include "mojo/public/cpp/bindings/remote.h" 24 #include "mojo/public/cpp/system/buffer.h" 25 #include "services/viz/public/mojom/compositing/compositor_frame_sink.mojom-blink.h" 26 #include "services/viz/public/mojom/compositing/frame_timing_details.mojom-blink.h" 27 #include "third_party/blink/public/mojom/frame_sinks/embedded_frame_sink.mojom-blink.h" 28 #include "third_party/blink/public/platform/web_video_frame_submitter.h" 29 #include "third_party/blink/renderer/platform/graphics/video_frame_resource_provider.h" 30 #include "third_party/blink/renderer/platform/platform_export.h" 31 #include "third_party/blink/renderer/platform/wtf/functional.h" 32 33 namespace blink { 34 35 // This single-threaded class facilitates the communication between the media 36 // stack and browser renderer, providing compositor frames containing video 37 // frames and corresponding resources to the |compositor_frame_sink_|. 38 // 39 // This class requires and uses a viz::ContextProvider, and thus, besides 40 // construction, must be consistently accessed from the same thread. 41 class PLATFORM_EXPORT VideoFrameSubmitter 42 : public WebVideoFrameSubmitter, 43 public viz::ContextLostObserver, 44 public viz::SharedBitmapReporter, 45 public viz::mojom::blink::CompositorFrameSinkClient { 46 public: 47 VideoFrameSubmitter(WebContextProviderCallback, 48 cc::PlaybackRoughnessReportingCallback, 49 std::unique_ptr<VideoFrameResourceProvider>); 50 ~VideoFrameSubmitter() override; 51 52 // cc::VideoFrameProvider::Client implementation. 53 void StopUsingProvider() override; 54 void StartRendering() override; 55 void StopRendering() override; 56 void DidReceiveFrame() override; 57 bool IsDrivingFrameUpdates() const override; 58 59 // WebVideoFrameSubmitter implementation. 60 void Initialize(cc::VideoFrameProvider*) override; 61 void SetRotation(media::VideoRotation) override; 62 void EnableSubmission( 63 viz::SurfaceId, 64 base::TimeTicks local_surface_id_allocation_time) override; 65 void SetIsSurfaceVisible(bool is_visible) override; 66 void SetIsPageVisible(bool is_visible) override; 67 void SetForceSubmit(bool) override; 68 69 // viz::ContextLostObserver implementation. 70 void OnContextLost() override; 71 72 // cc::mojom::CompositorFrameSinkClient implementation. 73 void DidReceiveCompositorFrameAck( 74 const WTF::Vector<viz::ReturnedResource>& resources) override; 75 void OnBeginFrame( 76 const viz::BeginFrameArgs&, 77 WTF::HashMap<uint32_t, ::viz::mojom::blink::FrameTimingDetailsPtr>) 78 override; OnBeginFramePausedChanged(bool paused)79 void OnBeginFramePausedChanged(bool paused) override {} 80 void ReclaimResources( 81 const WTF::Vector<viz::ReturnedResource>& resources) override; 82 83 // viz::SharedBitmapReporter implementation. 84 void DidAllocateSharedBitmap(base::ReadOnlySharedMemoryRegion, 85 const viz::SharedBitmapId&) override; 86 void DidDeleteSharedBitmap(const viz::SharedBitmapId&) override; 87 88 private: 89 friend class VideoFrameSubmitterTest; 90 91 // Called during Initialize() and OnContextLost() after a new ContextGL is 92 // requested. 93 void OnReceivedContextProvider( 94 bool use_gpu_compositing, 95 scoped_refptr<viz::RasterContextProvider> context_provider); 96 97 // Starts submission and calls UpdateSubmissionState(); which may submit. 98 void StartSubmitting(); 99 100 // Sets CompositorFrameSink::SetNeedsBeginFrame() state and submits a frame if 101 // visible or an empty frame if not. 102 void UpdateSubmissionState(); 103 104 // Will submit an empty frame to clear resource usage if it's safe. 105 void SubmitEmptyFrameIfNeeded(); 106 107 // Returns whether a frame was submitted. 108 bool SubmitFrame(const viz::BeginFrameAck&, scoped_refptr<media::VideoFrame>); 109 110 // SubmitEmptyFrame() is used to force the remote CompositorFrameSink to 111 // release resources for the last submission; saving a significant amount of 112 // memory (~30%) when content goes off-screen. See https://crbug.com/829813. 113 void SubmitEmptyFrame(); 114 115 // Pulls frame and submits it to compositor. Used in cases like 116 // DidReceiveFrame(), which occurs before video rendering has started to post 117 // the first frame or to submit a final frame before ending rendering. 118 void SubmitSingleFrame(); 119 120 // Return whether the submitter should submit frames based on its current 121 // state. It's important to only submit when this is true to save memory. See 122 // comments above and in UpdateSubmissionState(). 123 bool ShouldSubmit() const; 124 125 // Generates a new surface ID using using |child_local_surface_id_allocator_|. 126 // Called during context loss or during a frame size change. 127 void GenerateNewSurfaceId(); 128 129 // Helper method for creating viz::CompositorFrame. If |video_frame| is null 130 // then the frame will be empty. 131 viz::CompositorFrame CreateCompositorFrame( 132 uint32_t frame_token, 133 const viz::BeginFrameAck& begin_frame_ack, 134 scoped_refptr<media::VideoFrame> video_frame); 135 136 cc::VideoFrameProvider* video_frame_provider_ = nullptr; 137 scoped_refptr<viz::RasterContextProvider> context_provider_; 138 mojo::Remote<viz::mojom::blink::CompositorFrameSink> compositor_frame_sink_; 139 mojo::Remote<mojom::blink::SurfaceEmbedder> surface_embedder_; 140 mojo::Receiver<viz::mojom::blink::CompositorFrameSinkClient> receiver_{this}; 141 WebContextProviderCallback context_provider_callback_; 142 std::unique_ptr<VideoFrameResourceProvider> resource_provider_; 143 bool waiting_for_compositor_ack_ = false; 144 145 // Current rendering state. Set by StartRendering() and StopRendering(). 146 bool is_rendering_ = false; 147 148 // If the surface is not visible within in the current view port, we should 149 // not submit. Not submitting when off-screen saves significant memory. 150 bool is_surface_visible_ = false; 151 152 // Likewise, if the entire page is not visible, we should not submit. Not 153 // submitting in the background causes the VideoFrameProvider to enter a 154 // background rendering mode using lower frequency artificial BeginFrames. 155 bool is_page_visible_ = true; 156 157 // Whether frames should always be submitted, even if we're not visible. Used 158 // by Picture-in-Picture mode to ensure submission occurs even off-screen. 159 bool force_submit_ = false; 160 161 // Needs to be initialized in implementation because media isn't a public_dep 162 // of blink/platform. 163 media::VideoRotation rotation_; 164 165 viz::FrameSinkId frame_sink_id_; 166 167 // Size of the video frame being submitted. It is set the first time a frame 168 // is submitted. Every time there is a change in the video frame size, the 169 // child component of the LocalSurfaceId will be updated. 170 gfx::Size frame_size_; 171 172 // Used to updated the LocalSurfaceId when detecting a change in video frame 173 // size. 174 viz::ChildLocalSurfaceIdAllocator child_local_surface_id_allocator_; 175 176 viz::FrameTokenGenerator next_frame_token_; 177 178 std::unique_ptr<cc::VideoPlaybackRoughnessReporter> roughness_reporter_; 179 180 base::OneShotTimer empty_frame_timer_; 181 182 base::Optional<int> last_frame_id_; 183 184 cc::FrameSequenceTrackerCollection frame_trackers_; 185 186 // The BeginFrameArgs passed to the most recent call of OnBeginFrame(). 187 // Required for FrameSequenceTrackerCollection::NotifySubmitFrame 188 viz::BeginFrameArgs last_begin_frame_args_; 189 190 // The token of the frames that are submitted outside OnBeginFrame(). These 191 // frames should be ignored by the video tracker even if they are reported as 192 // presented. 193 base::flat_set<uint32_t> ignorable_submitted_frames_; 194 195 THREAD_CHECKER(thread_checker_); 196 197 base::WeakPtrFactory<VideoFrameSubmitter> weak_ptr_factory_{this}; 198 199 DISALLOW_COPY_AND_ASSIGN(VideoFrameSubmitter); 200 }; 201 202 } // namespace blink 203 204 #endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_GRAPHICS_VIDEO_FRAME_SUBMITTER_H_ 205