1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/browser/media/capture/frame_sink_video_capture_device.h"
6
7 #include "base/bind.h"
8 #include "base/bind_helpers.h"
9 #include "base/callback.h"
10 #include "base/callback_helpers.h"
11 #include "base/location.h"
12 #include "base/logging.h"
13 #include "base/macros.h"
14 #include "base/memory/ref_counted.h"
15 #include "base/memory/weak_ptr.h"
16 #include "base/numerics/safe_conversions.h"
17 #include "base/task/post_task.h"
18 #include "base/threading/thread_task_runner_handle.h"
19 #include "base/time/time.h"
20 #include "components/viz/host/host_frame_sink_manager.h"
21 #include "content/browser/compositor/surface_utils.h"
22 #include "content/browser/media/capture/mouse_cursor_overlay_controller.h"
23 #include "content/public/browser/browser_task_traits.h"
24 #include "content/public/browser/device_service.h"
25 #include "media/base/bind_to_current_loop.h"
26 #include "media/capture/mojom/video_capture_types.mojom.h"
27 #include "services/device/public/mojom/wake_lock_provider.mojom.h"
28
29 namespace content {
30
31 namespace {
32
33 constexpr int32_t kMouseCursorStackingIndex = 1;
34
35 // Transfers ownership of an object to a std::unique_ptr with a custom deleter
36 // that ensures the object is destroyed on the UI BrowserThread.
37 template <typename T>
RescopeToUIThread(std::unique_ptr<T> && ptr)38 std::unique_ptr<T, BrowserThread::DeleteOnUIThread> RescopeToUIThread(
39 std::unique_ptr<T>&& ptr) {
40 return std::unique_ptr<T, BrowserThread::DeleteOnUIThread>(ptr.release());
41 }
42
43 // Adapter for a VideoFrameReceiver to notify once frame consumption is
44 // complete. VideoFrameReceiver requires owning an object that it will destroy
45 // once consumption is complete. This class adapts between that scheme and
46 // running a "done callback" to notify that consumption is complete.
47 class ScopedFrameDoneHelper
48 : public base::ScopedClosureRunner,
49 public media::VideoCaptureDevice::Client::Buffer::ScopedAccessPermission {
50 public:
ScopedFrameDoneHelper(base::OnceClosure done_callback)51 explicit ScopedFrameDoneHelper(base::OnceClosure done_callback)
52 : base::ScopedClosureRunner(std::move(done_callback)) {}
53 ~ScopedFrameDoneHelper() final = default;
54 };
55
BindWakeLockProvider(mojo::PendingReceiver<device::mojom::WakeLockProvider> receiver)56 void BindWakeLockProvider(
57 mojo::PendingReceiver<device::mojom::WakeLockProvider> receiver) {
58 DCHECK_CURRENTLY_ON(BrowserThread::UI);
59 GetDeviceService().BindWakeLockProvider(std::move(receiver));
60 }
61
62 } // namespace
63
FrameSinkVideoCaptureDevice()64 FrameSinkVideoCaptureDevice::FrameSinkVideoCaptureDevice()
65 : cursor_controller_(
66 RescopeToUIThread(std::make_unique<MouseCursorOverlayController>())) {
67 DCHECK(cursor_controller_);
68 }
69
~FrameSinkVideoCaptureDevice()70 FrameSinkVideoCaptureDevice::~FrameSinkVideoCaptureDevice() {
71 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
72 DCHECK(!receiver_) << "StopAndDeAllocate() was never called after start.";
73 }
74
AllocateAndStartWithReceiver(const media::VideoCaptureParams & params,std::unique_ptr<media::VideoFrameReceiver> receiver)75 void FrameSinkVideoCaptureDevice::AllocateAndStartWithReceiver(
76 const media::VideoCaptureParams& params,
77 std::unique_ptr<media::VideoFrameReceiver> receiver) {
78 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
79 DCHECK(params.IsValid());
80 DCHECK(receiver);
81
82 // If the device has already ended on a fatal error, abort immediately.
83 if (fatal_error_message_) {
84 receiver->OnLog(*fatal_error_message_);
85 receiver->OnError(media::VideoCaptureError::
86 kFrameSinkVideoCaptureDeviceAleradyEndedOnFatalError);
87 return;
88 }
89
90 capture_params_ = params;
91 WillStart();
92 DCHECK(!receiver_);
93 receiver_ = std::move(receiver);
94
95 // Shutdown the prior capturer, if any.
96 MaybeStopConsuming();
97
98 capturer_ = std::make_unique<viz::ClientFrameSinkVideoCapturer>(
99 base::BindRepeating(&FrameSinkVideoCaptureDevice::CreateCapturer,
100 base::Unretained(this)));
101
102 capturer_->SetFormat(capture_params_.requested_format.pixel_format,
103 gfx::ColorSpace::CreateREC709());
104 capturer_->SetMinCapturePeriod(
105 base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>(
106 base::Time::kMicrosecondsPerSecond /
107 capture_params_.requested_format.frame_rate)));
108 const auto& constraints = capture_params_.SuggestConstraints();
109 capturer_->SetResolutionConstraints(constraints.min_frame_size,
110 constraints.max_frame_size,
111 constraints.fixed_aspect_ratio);
112
113 if (target_.is_valid()) {
114 capturer_->ChangeTarget(target_);
115 }
116
117 base::PostTask(
118 FROM_HERE, {BrowserThread::UI},
119 base::BindOnce(&MouseCursorOverlayController::Start,
120 cursor_controller_->GetWeakPtr(),
121 capturer_->CreateOverlay(kMouseCursorStackingIndex),
122 base::ThreadTaskRunnerHandle::Get()));
123
124 receiver_->OnStarted();
125
126 if (!suspend_requested_) {
127 MaybeStartConsuming();
128 }
129
130 DCHECK(!wake_lock_);
131 RequestWakeLock();
132 }
133
AllocateAndStart(const media::VideoCaptureParams & params,std::unique_ptr<media::VideoCaptureDevice::Client> client)134 void FrameSinkVideoCaptureDevice::AllocateAndStart(
135 const media::VideoCaptureParams& params,
136 std::unique_ptr<media::VideoCaptureDevice::Client> client) {
137 // FrameSinkVideoCaptureDevice does not use a
138 // VideoCaptureDevice::Client. Instead, it provides frames to a
139 // VideoFrameReceiver directly.
140 NOTREACHED();
141 }
142
RequestRefreshFrame()143 void FrameSinkVideoCaptureDevice::RequestRefreshFrame() {
144 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
145
146 if (capturer_ && !suspend_requested_) {
147 capturer_->RequestRefreshFrame();
148 }
149 }
150
MaybeSuspend()151 void FrameSinkVideoCaptureDevice::MaybeSuspend() {
152 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
153
154 suspend_requested_ = true;
155 MaybeStopConsuming();
156 }
157
Resume()158 void FrameSinkVideoCaptureDevice::Resume() {
159 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
160
161 suspend_requested_ = false;
162 MaybeStartConsuming();
163 }
164
StopAndDeAllocate()165 void FrameSinkVideoCaptureDevice::StopAndDeAllocate() {
166 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
167
168 if (wake_lock_) {
169 wake_lock_->CancelWakeLock();
170 wake_lock_.reset();
171 }
172
173 base::PostTask(FROM_HERE, {BrowserThread::UI},
174 base::BindOnce(&MouseCursorOverlayController::Stop,
175 cursor_controller_->GetWeakPtr()));
176
177 MaybeStopConsuming();
178 capturer_.reset();
179 if (receiver_) {
180 receiver_.reset();
181 DidStop();
182 }
183 }
184
OnUtilizationReport(int frame_feedback_id,double utilization)185 void FrameSinkVideoCaptureDevice::OnUtilizationReport(int frame_feedback_id,
186 double utilization) {
187 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
188
189 // Assumption: The mojo InterfacePtr in |frame_callbacks_| should be valid at
190 // this point because this method will always be called before the
191 // VideoFrameReceiver signals it is done consuming the frame.
192 const auto index = static_cast<size_t>(frame_feedback_id);
193 DCHECK_LT(index, frame_callbacks_.size());
194 frame_callbacks_[index]->ProvideFeedback(utilization);
195 }
196
OnFrameCaptured(base::ReadOnlySharedMemoryRegion data,media::mojom::VideoFrameInfoPtr info,const gfx::Rect & content_rect,mojo::PendingRemote<viz::mojom::FrameSinkVideoConsumerFrameCallbacks> callbacks)197 void FrameSinkVideoCaptureDevice::OnFrameCaptured(
198 base::ReadOnlySharedMemoryRegion data,
199 media::mojom::VideoFrameInfoPtr info,
200 const gfx::Rect& content_rect,
201 mojo::PendingRemote<viz::mojom::FrameSinkVideoConsumerFrameCallbacks>
202 callbacks) {
203 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
204 DCHECK(callbacks);
205
206 mojo::Remote<viz::mojom::FrameSinkVideoConsumerFrameCallbacks>
207 callbacks_remote(std::move(callbacks));
208
209 if (!receiver_ || !data.IsValid()) {
210 callbacks_remote->Done();
211 return;
212 }
213
214 // Search for the next available element in |frame_callbacks_| and bind
215 // |callbacks| there.
216 size_t index = 0;
217 for (;; ++index) {
218 if (index == frame_callbacks_.size()) {
219 // The growth of |frame_callbacks_| should be bounded because the
220 // viz::mojom::FrameSinkVideoCapturer should enforce an upper-bound on the
221 // number of frames in-flight.
222 constexpr size_t kMaxInFlightFrames = 32; // Arbitrarily-chosen limit.
223 DCHECK_LT(frame_callbacks_.size(), kMaxInFlightFrames);
224 frame_callbacks_.emplace_back(std::move(callbacks_remote));
225 break;
226 }
227 if (!frame_callbacks_[index].is_bound()) {
228 frame_callbacks_[index] = std::move(callbacks_remote);
229 break;
230 }
231 }
232 const BufferId buffer_id = static_cast<BufferId>(index);
233
234 // Set the INTERACTIVE_CONTENT frame metadata.
235 media::VideoFrameMetadata modified_metadata;
236 modified_metadata.MergeInternalValuesFrom(info->metadata);
237 modified_metadata.SetBoolean(media::VideoFrameMetadata::INTERACTIVE_CONTENT,
238 cursor_controller_->IsUserInteractingWithView());
239 info->metadata = modified_metadata.GetInternalValues().Clone();
240
241 // Pass the video frame to the VideoFrameReceiver. This is done by first
242 // passing the shared memory buffer handle and then notifying it that a new
243 // frame is ready to be read from the buffer.
244 receiver_->OnNewBuffer(
245 buffer_id,
246 media::mojom::VideoBufferHandle::NewReadOnlyShmemRegion(std::move(data)));
247 receiver_->OnFrameReadyInBuffer(
248 buffer_id, buffer_id,
249 std::make_unique<ScopedFrameDoneHelper>(
250 media::BindToCurrentLoop(base::BindOnce(
251 &FrameSinkVideoCaptureDevice::OnFramePropagationComplete,
252 weak_factory_.GetWeakPtr(), buffer_id))),
253 std::move(info));
254 }
255
OnStopped()256 void FrameSinkVideoCaptureDevice::OnStopped() {
257 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
258
259 // This method would never be called if FrameSinkVideoCaptureDevice explicitly
260 // called capturer_->StopAndResetConsumer(), because the binding is closed at
261 // that time. Therefore, a call to this method means that the capturer cannot
262 // continue; and that's a permanent failure.
263 OnFatalError("Capturer service cannot continue.");
264 }
265
OnLog(const std::string & message)266 void FrameSinkVideoCaptureDevice::OnLog(const std::string& message) {
267 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
268
269 if (receiver_) {
270 if (BrowserThread::CurrentlyOn(BrowserThread::IO)) {
271 receiver_->OnLog(message);
272 } else {
273 base::PostTask(
274 FROM_HERE, {BrowserThread::IO},
275 base::BindOnce(&media::VideoFrameReceiver::OnLog,
276 base::Unretained(receiver_.get()), message));
277 }
278 }
279 }
280
OnTargetChanged(const viz::FrameSinkId & frame_sink_id)281 void FrameSinkVideoCaptureDevice::OnTargetChanged(
282 const viz::FrameSinkId& frame_sink_id) {
283 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
284
285 target_ = frame_sink_id;
286 if (capturer_) {
287 if (target_.is_valid()) {
288 capturer_->ChangeTarget(target_);
289 } else {
290 capturer_->ChangeTarget(base::nullopt);
291 }
292 }
293 }
294
OnTargetPermanentlyLost()295 void FrameSinkVideoCaptureDevice::OnTargetPermanentlyLost() {
296 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
297
298 OnTargetChanged(viz::FrameSinkId());
299 OnFatalError("Capture target has been permanently lost.");
300 }
301
WillStart()302 void FrameSinkVideoCaptureDevice::WillStart() {}
303
DidStop()304 void FrameSinkVideoCaptureDevice::DidStop() {}
305
CreateCapturer(mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver)306 void FrameSinkVideoCaptureDevice::CreateCapturer(
307 mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver) {
308 CreateCapturerViaGlobalManager(std::move(receiver));
309 }
310
311 // static
CreateCapturerViaGlobalManager(mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver)312 void FrameSinkVideoCaptureDevice::CreateCapturerViaGlobalManager(
313 mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver) {
314 // Send the receiver to UI thread because that's where HostFrameSinkManager
315 // lives.
316 base::PostTask(
317 FROM_HERE, {BrowserThread::UI},
318 base::BindOnce(
319 [](mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer>
320 receiver) {
321 viz::HostFrameSinkManager* const manager =
322 GetHostFrameSinkManager();
323 DCHECK(manager);
324 manager->CreateVideoCapturer(std::move(receiver));
325 },
326 std::move(receiver)));
327 }
328
MaybeStartConsuming()329 void FrameSinkVideoCaptureDevice::MaybeStartConsuming() {
330 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
331
332 if (!receiver_ || !capturer_) {
333 return;
334 }
335
336 capturer_->Start(this);
337 }
338
MaybeStopConsuming()339 void FrameSinkVideoCaptureDevice::MaybeStopConsuming() {
340 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
341
342 if (capturer_)
343 capturer_->StopAndResetConsumer();
344 }
345
OnFramePropagationComplete(BufferId buffer_id)346 void FrameSinkVideoCaptureDevice::OnFramePropagationComplete(
347 BufferId buffer_id) {
348 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
349
350 // Notify the VideoFrameReceiver that the buffer is no longer valid.
351 if (receiver_) {
352 receiver_->OnBufferRetired(buffer_id);
353 }
354
355 // Notify the capturer that consumption of the frame is complete.
356 const size_t index = static_cast<size_t>(buffer_id);
357 DCHECK_LT(index, frame_callbacks_.size());
358 auto& callbacks_ptr = frame_callbacks_[index];
359 callbacks_ptr->Done();
360 callbacks_ptr.reset();
361 }
362
OnFatalError(std::string message)363 void FrameSinkVideoCaptureDevice::OnFatalError(std::string message) {
364 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
365
366 fatal_error_message_ = std::move(message);
367 if (receiver_) {
368 receiver_->OnLog(*fatal_error_message_);
369 receiver_->OnError(media::VideoCaptureError::
370 kFrameSinkVideoCaptureDeviceEncounteredFatalError);
371 }
372
373 StopAndDeAllocate();
374 }
375
RequestWakeLock()376 void FrameSinkVideoCaptureDevice::RequestWakeLock() {
377 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
378
379 mojo::Remote<device::mojom::WakeLockProvider> wake_lock_provider;
380 auto receiver = wake_lock_provider.BindNewPipeAndPassReceiver();
381 base::PostTask(FROM_HERE, {BrowserThread::UI},
382 base::BindOnce(&BindWakeLockProvider, std::move(receiver)));
383 wake_lock_provider->GetWakeLockWithoutContext(
384 device::mojom::WakeLockType::kPreventDisplaySleep,
385 device::mojom::WakeLockReason::kOther, "screen capture",
386 wake_lock_.BindNewPipeAndPassReceiver());
387
388 wake_lock_->RequestWakeLock();
389 }
390
391 } // namespace content
392