1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "content/browser/media/capture/frame_sink_video_capture_device.h"
6 
7 #include "base/bind.h"
8 #include "base/bind_helpers.h"
9 #include "base/callback.h"
10 #include "base/callback_helpers.h"
11 #include "base/location.h"
12 #include "base/logging.h"
13 #include "base/macros.h"
14 #include "base/memory/ref_counted.h"
15 #include "base/memory/weak_ptr.h"
16 #include "base/numerics/safe_conversions.h"
17 #include "base/task/post_task.h"
18 #include "base/threading/thread_task_runner_handle.h"
19 #include "base/time/time.h"
20 #include "components/viz/host/host_frame_sink_manager.h"
21 #include "content/browser/compositor/surface_utils.h"
22 #include "content/browser/media/capture/mouse_cursor_overlay_controller.h"
23 #include "content/public/browser/browser_task_traits.h"
24 #include "content/public/browser/device_service.h"
25 #include "media/base/bind_to_current_loop.h"
26 #include "media/capture/mojom/video_capture_types.mojom.h"
27 #include "services/device/public/mojom/wake_lock_provider.mojom.h"
28 
29 namespace content {
30 
31 namespace {
32 
33 constexpr int32_t kMouseCursorStackingIndex = 1;
34 
35 // Transfers ownership of an object to a std::unique_ptr with a custom deleter
36 // that ensures the object is destroyed on the UI BrowserThread.
37 template <typename T>
RescopeToUIThread(std::unique_ptr<T> && ptr)38 std::unique_ptr<T, BrowserThread::DeleteOnUIThread> RescopeToUIThread(
39     std::unique_ptr<T>&& ptr) {
40   return std::unique_ptr<T, BrowserThread::DeleteOnUIThread>(ptr.release());
41 }
42 
43 // Adapter for a VideoFrameReceiver to notify once frame consumption is
44 // complete. VideoFrameReceiver requires owning an object that it will destroy
45 // once consumption is complete. This class adapts between that scheme and
46 // running a "done callback" to notify that consumption is complete.
47 class ScopedFrameDoneHelper
48     : public base::ScopedClosureRunner,
49       public media::VideoCaptureDevice::Client::Buffer::ScopedAccessPermission {
50  public:
ScopedFrameDoneHelper(base::OnceClosure done_callback)51   explicit ScopedFrameDoneHelper(base::OnceClosure done_callback)
52       : base::ScopedClosureRunner(std::move(done_callback)) {}
53   ~ScopedFrameDoneHelper() final = default;
54 };
55 
BindWakeLockProvider(mojo::PendingReceiver<device::mojom::WakeLockProvider> receiver)56 void BindWakeLockProvider(
57     mojo::PendingReceiver<device::mojom::WakeLockProvider> receiver) {
58   DCHECK_CURRENTLY_ON(BrowserThread::UI);
59   GetDeviceService().BindWakeLockProvider(std::move(receiver));
60 }
61 
62 }  // namespace
63 
FrameSinkVideoCaptureDevice()64 FrameSinkVideoCaptureDevice::FrameSinkVideoCaptureDevice()
65     : cursor_controller_(
66           RescopeToUIThread(std::make_unique<MouseCursorOverlayController>())) {
67   DCHECK(cursor_controller_);
68 }
69 
~FrameSinkVideoCaptureDevice()70 FrameSinkVideoCaptureDevice::~FrameSinkVideoCaptureDevice() {
71   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
72   DCHECK(!receiver_) << "StopAndDeAllocate() was never called after start.";
73 }
74 
AllocateAndStartWithReceiver(const media::VideoCaptureParams & params,std::unique_ptr<media::VideoFrameReceiver> receiver)75 void FrameSinkVideoCaptureDevice::AllocateAndStartWithReceiver(
76     const media::VideoCaptureParams& params,
77     std::unique_ptr<media::VideoFrameReceiver> receiver) {
78   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
79   DCHECK(params.IsValid());
80   DCHECK(receiver);
81 
82   // If the device has already ended on a fatal error, abort immediately.
83   if (fatal_error_message_) {
84     receiver->OnLog(*fatal_error_message_);
85     receiver->OnError(media::VideoCaptureError::
86                           kFrameSinkVideoCaptureDeviceAleradyEndedOnFatalError);
87     return;
88   }
89 
90   capture_params_ = params;
91   WillStart();
92   DCHECK(!receiver_);
93   receiver_ = std::move(receiver);
94 
95   // Shutdown the prior capturer, if any.
96   MaybeStopConsuming();
97 
98   capturer_ = std::make_unique<viz::ClientFrameSinkVideoCapturer>(
99       base::BindRepeating(&FrameSinkVideoCaptureDevice::CreateCapturer,
100                           base::Unretained(this)));
101 
102   capturer_->SetFormat(capture_params_.requested_format.pixel_format,
103                        gfx::ColorSpace::CreateREC709());
104   capturer_->SetMinCapturePeriod(
105       base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>(
106           base::Time::kMicrosecondsPerSecond /
107           capture_params_.requested_format.frame_rate)));
108   const auto& constraints = capture_params_.SuggestConstraints();
109   capturer_->SetResolutionConstraints(constraints.min_frame_size,
110                                       constraints.max_frame_size,
111                                       constraints.fixed_aspect_ratio);
112 
113   if (target_.is_valid()) {
114     capturer_->ChangeTarget(target_);
115   }
116 
117   base::PostTask(
118       FROM_HERE, {BrowserThread::UI},
119       base::BindOnce(&MouseCursorOverlayController::Start,
120                      cursor_controller_->GetWeakPtr(),
121                      capturer_->CreateOverlay(kMouseCursorStackingIndex),
122                      base::ThreadTaskRunnerHandle::Get()));
123 
124   receiver_->OnStarted();
125 
126   if (!suspend_requested_) {
127     MaybeStartConsuming();
128   }
129 
130   DCHECK(!wake_lock_);
131   RequestWakeLock();
132 }
133 
AllocateAndStart(const media::VideoCaptureParams & params,std::unique_ptr<media::VideoCaptureDevice::Client> client)134 void FrameSinkVideoCaptureDevice::AllocateAndStart(
135     const media::VideoCaptureParams& params,
136     std::unique_ptr<media::VideoCaptureDevice::Client> client) {
137   // FrameSinkVideoCaptureDevice does not use a
138   // VideoCaptureDevice::Client. Instead, it provides frames to a
139   // VideoFrameReceiver directly.
140   NOTREACHED();
141 }
142 
RequestRefreshFrame()143 void FrameSinkVideoCaptureDevice::RequestRefreshFrame() {
144   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
145 
146   if (capturer_ && !suspend_requested_) {
147     capturer_->RequestRefreshFrame();
148   }
149 }
150 
MaybeSuspend()151 void FrameSinkVideoCaptureDevice::MaybeSuspend() {
152   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
153 
154   suspend_requested_ = true;
155   MaybeStopConsuming();
156 }
157 
Resume()158 void FrameSinkVideoCaptureDevice::Resume() {
159   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
160 
161   suspend_requested_ = false;
162   MaybeStartConsuming();
163 }
164 
StopAndDeAllocate()165 void FrameSinkVideoCaptureDevice::StopAndDeAllocate() {
166   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
167 
168   if (wake_lock_) {
169     wake_lock_->CancelWakeLock();
170     wake_lock_.reset();
171   }
172 
173   base::PostTask(FROM_HERE, {BrowserThread::UI},
174                  base::BindOnce(&MouseCursorOverlayController::Stop,
175                                 cursor_controller_->GetWeakPtr()));
176 
177   MaybeStopConsuming();
178   capturer_.reset();
179   if (receiver_) {
180     receiver_.reset();
181     DidStop();
182   }
183 }
184 
OnUtilizationReport(int frame_feedback_id,double utilization)185 void FrameSinkVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id,
186                                                       double utilization) {
187   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
188 
189   // Assumption: The mojo InterfacePtr in |frame_callbacks_| should be valid at
190   // this point because this method will always be called before the
191   // VideoFrameReceiver signals it is done consuming the frame.
192   const auto index = static_cast<size_t>(frame_feedback_id);
193   DCHECK_LT(index, frame_callbacks_.size());
194   frame_callbacks_[index]->ProvideFeedback(utilization);
195 }
196 
OnFrameCaptured(base::ReadOnlySharedMemoryRegion data,media::mojom::VideoFrameInfoPtr info,const gfx::Rect & content_rect,mojo::PendingRemote<viz::mojom::FrameSinkVideoConsumerFrameCallbacks> callbacks)197 void FrameSinkVideoCaptureDevice::OnFrameCaptured(
198     base::ReadOnlySharedMemoryRegion data,
199     media::mojom::VideoFrameInfoPtr info,
200     const gfx::Rect& content_rect,
201     mojo::PendingRemote<viz::mojom::FrameSinkVideoConsumerFrameCallbacks>
202         callbacks) {
203   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
204   DCHECK(callbacks);
205 
206   mojo::Remote<viz::mojom::FrameSinkVideoConsumerFrameCallbacks>
207       callbacks_remote(std::move(callbacks));
208 
209   if (!receiver_ || !data.IsValid()) {
210     callbacks_remote->Done();
211     return;
212   }
213 
214   // Search for the next available element in |frame_callbacks_| and bind
215   // |callbacks| there.
216   size_t index = 0;
217   for (;; ++index) {
218     if (index == frame_callbacks_.size()) {
219       // The growth of |frame_callbacks_| should be bounded because the
220       // viz::mojom::FrameSinkVideoCapturer should enforce an upper-bound on the
221       // number of frames in-flight.
222       constexpr size_t kMaxInFlightFrames = 32;  // Arbitrarily-chosen limit.
223       DCHECK_LT(frame_callbacks_.size(), kMaxInFlightFrames);
224       frame_callbacks_.emplace_back(std::move(callbacks_remote));
225       break;
226     }
227     if (!frame_callbacks_[index].is_bound()) {
228       frame_callbacks_[index] = std::move(callbacks_remote);
229       break;
230     }
231   }
232   const BufferId buffer_id = static_cast<BufferId>(index);
233 
234   // Set the INTERACTIVE_CONTENT frame metadata.
235   media::VideoFrameMetadata modified_metadata;
236   modified_metadata.MergeInternalValuesFrom(info->metadata);
237   modified_metadata.SetBoolean(media::VideoFrameMetadata::INTERACTIVE_CONTENT,
238                                cursor_controller_->IsUserInteractingWithView());
239   info->metadata = modified_metadata.GetInternalValues().Clone();
240 
241   // Pass the video frame to the VideoFrameReceiver. This is done by first
242   // passing the shared memory buffer handle and then notifying it that a new
243   // frame is ready to be read from the buffer.
244   receiver_->OnNewBuffer(
245       buffer_id,
246       media::mojom::VideoBufferHandle::NewReadOnlyShmemRegion(std::move(data)));
247   receiver_->OnFrameReadyInBuffer(
248       buffer_id, buffer_id,
249       std::make_unique<ScopedFrameDoneHelper>(
250           media::BindToCurrentLoop(base::BindOnce(
251               &FrameSinkVideoCaptureDevice::OnFramePropagationComplete,
252               weak_factory_.GetWeakPtr(), buffer_id))),
253       std::move(info));
254 }
255 
OnStopped()256 void FrameSinkVideoCaptureDevice::OnStopped() {
257   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
258 
259   // This method would never be called if FrameSinkVideoCaptureDevice explicitly
260   // called capturer_->StopAndResetConsumer(), because the binding is closed at
261   // that time. Therefore, a call to this method means that the capturer cannot
262   // continue; and that's a permanent failure.
263   OnFatalError("Capturer service cannot continue.");
264 }
265 
OnLog(const std::string & message)266 void FrameSinkVideoCaptureDevice::OnLog(const std::string& message) {
267   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
268 
269   if (receiver_) {
270     if (BrowserThread::CurrentlyOn(BrowserThread::IO)) {
271       receiver_->OnLog(message);
272     } else {
273       base::PostTask(
274           FROM_HERE, {BrowserThread::IO},
275           base::BindOnce(&media::VideoFrameReceiver::OnLog,
276                          base::Unretained(receiver_.get()), message));
277     }
278   }
279 }
280 
OnTargetChanged(const viz::FrameSinkId & frame_sink_id)281 void FrameSinkVideoCaptureDevice::OnTargetChanged(
282     const viz::FrameSinkId& frame_sink_id) {
283   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
284 
285   target_ = frame_sink_id;
286   if (capturer_) {
287     if (target_.is_valid()) {
288       capturer_->ChangeTarget(target_);
289     } else {
290       capturer_->ChangeTarget(base::nullopt);
291     }
292   }
293 }
294 
OnTargetPermanentlyLost()295 void FrameSinkVideoCaptureDevice::OnTargetPermanentlyLost() {
296   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
297 
298   OnTargetChanged(viz::FrameSinkId());
299   OnFatalError("Capture target has been permanently lost.");
300 }
301 
WillStart()302 void FrameSinkVideoCaptureDevice::WillStart() {}
303 
DidStop()304 void FrameSinkVideoCaptureDevice::DidStop() {}
305 
CreateCapturer(mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver)306 void FrameSinkVideoCaptureDevice::CreateCapturer(
307     mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver) {
308   CreateCapturerViaGlobalManager(std::move(receiver));
309 }
310 
311 // static
CreateCapturerViaGlobalManager(mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver)312 void FrameSinkVideoCaptureDevice::CreateCapturerViaGlobalManager(
313     mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver) {
314   // Send the receiver to UI thread because that's where HostFrameSinkManager
315   // lives.
316   base::PostTask(
317       FROM_HERE, {BrowserThread::UI},
318       base::BindOnce(
319           [](mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer>
320                  receiver) {
321             viz::HostFrameSinkManager* const manager =
322                 GetHostFrameSinkManager();
323             DCHECK(manager);
324             manager->CreateVideoCapturer(std::move(receiver));
325           },
326           std::move(receiver)));
327 }
328 
MaybeStartConsuming()329 void FrameSinkVideoCaptureDevice::MaybeStartConsuming() {
330   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
331 
332   if (!receiver_ || !capturer_) {
333     return;
334   }
335 
336   capturer_->Start(this);
337 }
338 
MaybeStopConsuming()339 void FrameSinkVideoCaptureDevice::MaybeStopConsuming() {
340   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
341 
342   if (capturer_)
343     capturer_->StopAndResetConsumer();
344 }
345 
OnFramePropagationComplete(BufferId buffer_id)346 void FrameSinkVideoCaptureDevice::OnFramePropagationComplete(
347     BufferId buffer_id) {
348   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
349 
350   // Notify the VideoFrameReceiver that the buffer is no longer valid.
351   if (receiver_) {
352     receiver_->OnBufferRetired(buffer_id);
353   }
354 
355   // Notify the capturer that consumption of the frame is complete.
356   const size_t index = static_cast<size_t>(buffer_id);
357   DCHECK_LT(index, frame_callbacks_.size());
358   auto& callbacks_ptr = frame_callbacks_[index];
359   callbacks_ptr->Done();
360   callbacks_ptr.reset();
361 }
362 
OnFatalError(std::string message)363 void FrameSinkVideoCaptureDevice::OnFatalError(std::string message) {
364   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
365 
366   fatal_error_message_ = std::move(message);
367   if (receiver_) {
368     receiver_->OnLog(*fatal_error_message_);
369     receiver_->OnError(media::VideoCaptureError::
370                            kFrameSinkVideoCaptureDeviceEncounteredFatalError);
371   }
372 
373   StopAndDeAllocate();
374 }
375 
RequestWakeLock()376 void FrameSinkVideoCaptureDevice::RequestWakeLock() {
377   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
378 
379   mojo::Remote<device::mojom::WakeLockProvider> wake_lock_provider;
380   auto receiver = wake_lock_provider.BindNewPipeAndPassReceiver();
381   base::PostTask(FROM_HERE, {BrowserThread::UI},
382                  base::BindOnce(&BindWakeLockProvider, std::move(receiver)));
383   wake_lock_provider->GetWakeLockWithoutContext(
384       device::mojom::WakeLockType::kPreventDisplaySleep,
385       device::mojom::WakeLockReason::kOther, "screen capture",
386       wake_lock_.BindNewPipeAndPassReceiver());
387 
388   wake_lock_->RequestWakeLock();
389 }
390 
391 }  // namespace content
392