1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "chrome/browser/media/webrtc/native_desktop_media_list.h"
6
7 #include <utility>
8
9 #include "base/bind.h"
10 #include "base/hash/hash.h"
11 #include "base/message_loop/message_pump_type.h"
12 #include "base/single_thread_task_runner.h"
13 #include "base/strings/utf_string_conversions.h"
14 #include "base/threading/thread_restrictions.h"
15 #include "base/threading/thread_task_runner_handle.h"
16 #include "build/build_config.h"
17 #include "chrome/browser/media/webrtc/desktop_media_list.h"
18 #include "chrome/grit/generated_resources.h"
19 #include "content/public/browser/browser_task_traits.h"
20 #include "content/public/browser/browser_thread.h"
21 #include "media/base/video_util.h"
22 #include "third_party/libyuv/include/libyuv/scale_argb.h"
23 #include "third_party/skia/include/core/SkBitmap.h"
24 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
25 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
26 #include "ui/base/l10n/l10n_util.h"
27 #include "ui/gfx/native_widget_types.h"
28 #include "ui/snapshot/snapshot.h"
29
30 #if defined(USE_AURA)
31 #include "ui/aura/window.h"
32 #include "ui/aura/window_tree_host.h"
33 #include "ui/snapshot/snapshot_aura.h"
34 #endif
35
36 using content::DesktopMediaID;
37
38 namespace {
39
40 // Update the list every second.
41 const int kDefaultNativeDesktopMediaListUpdatePeriod = 1000;
42
IsFrameValid(webrtc::DesktopFrame * frame)43 bool IsFrameValid(webrtc::DesktopFrame* frame) {
44 // These checks ensure invalid data isn't passed along, potentially leading to
45 // crashes, e.g. when we calculate the hash which assumes a positive height
46 // and stride.
47 // TODO(crbug.com/1085230): figure out why the height is sometimes negative.
48 return frame && frame->data() && frame->stride() >= 0 &&
49 frame->size().height() >= 0;
50 }
51
52 // Returns a hash of a DesktopFrame content to detect when image for a desktop
53 // media source has changed.
GetFrameHash(webrtc::DesktopFrame * frame)54 uint32_t GetFrameHash(webrtc::DesktopFrame* frame) {
55 // TODO(dcheng): Is this vulnerable to overflow??
56 int data_size = frame->stride() * frame->size().height();
57 return base::FastHash(base::make_span(frame->data(), data_size));
58 }
59
ScaleDesktopFrame(std::unique_ptr<webrtc::DesktopFrame> frame,gfx::Size size)60 gfx::ImageSkia ScaleDesktopFrame(std::unique_ptr<webrtc::DesktopFrame> frame,
61 gfx::Size size) {
62 gfx::Rect scaled_rect = media::ComputeLetterboxRegion(
63 gfx::Rect(0, 0, size.width(), size.height()),
64 gfx::Size(frame->size().width(), frame->size().height()));
65
66 SkBitmap result;
67 result.allocN32Pixels(scaled_rect.width(), scaled_rect.height(), true);
68
69 uint8_t* pixels_data = reinterpret_cast<uint8_t*>(result.getPixels());
70 libyuv::ARGBScale(frame->data(), frame->stride(), frame->size().width(),
71 frame->size().height(), pixels_data, result.rowBytes(),
72 scaled_rect.width(), scaled_rect.height(),
73 libyuv::kFilterBilinear);
74
75 // Set alpha channel values to 255 for all pixels.
76 // TODO(sergeyu): Fix screen/window capturers to capture alpha channel and
77 // remove this code. Currently screen/window capturers (at least some
78 // implementations) only capture R, G and B channels and set Alpha to 0.
79 // crbug.com/264424
80 for (int y = 0; y < result.height(); ++y) {
81 for (int x = 0; x < result.width(); ++x) {
82 pixels_data[result.rowBytes() * y + x * result.bytesPerPixel() + 3] =
83 0xff;
84 }
85 }
86
87 return gfx::ImageSkia::CreateFrom1xBitmap(result);
88 }
89
90 } // namespace
91
92 class NativeDesktopMediaList::Worker
93 : public webrtc::DesktopCapturer::Callback {
94 public:
95 Worker(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
96 base::WeakPtr<NativeDesktopMediaList> media_list,
97 DesktopMediaID::Type type,
98 std::unique_ptr<webrtc::DesktopCapturer> capturer);
99 ~Worker() override;
100
101 void Start();
102 void Refresh(const DesktopMediaID::Id& view_dialog_id, bool update_thumnails);
103
104 void RefreshThumbnails(std::vector<DesktopMediaID> native_ids,
105 const gfx::Size& thumbnail_size);
106
107 private:
108 typedef std::map<DesktopMediaID, uint32_t> ImageHashesMap;
109
110 // Used to hold state associated with a call to RefreshThumbnails.
111 struct RefreshThumbnailsState {
112 std::vector<DesktopMediaID> source_ids;
113 gfx::Size thumbnail_size;
114 ImageHashesMap new_image_hashes;
115 size_t next_source_index = 0;
116 };
117
118 void RefreshNextThumbnail();
119
120 // webrtc::DesktopCapturer::Callback interface.
121 void OnCaptureResult(webrtc::DesktopCapturer::Result result,
122 std::unique_ptr<webrtc::DesktopFrame> frame) override;
123
124 // Task runner used for capturing operations.
125 scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
126
127 base::WeakPtr<NativeDesktopMediaList> media_list_;
128
129 DesktopMediaID::Type type_;
130 std::unique_ptr<webrtc::DesktopCapturer> capturer_;
131
132 // Stores hashes of snapshots previously captured.
133 ImageHashesMap image_hashes_;
134
135 // Non-null when RefreshThumbnails hasn't yet completed.
136 std::unique_ptr<RefreshThumbnailsState> refresh_thumbnails_state_;
137
138 base::WeakPtrFactory<Worker> weak_factory_{this};
139
140 DISALLOW_COPY_AND_ASSIGN(Worker);
141 };
142
Worker(scoped_refptr<base::SingleThreadTaskRunner> task_runner,base::WeakPtr<NativeDesktopMediaList> media_list,DesktopMediaID::Type type,std::unique_ptr<webrtc::DesktopCapturer> capturer)143 NativeDesktopMediaList::Worker::Worker(
144 scoped_refptr<base::SingleThreadTaskRunner> task_runner,
145 base::WeakPtr<NativeDesktopMediaList> media_list,
146 DesktopMediaID::Type type,
147 std::unique_ptr<webrtc::DesktopCapturer> capturer)
148 : task_runner_(task_runner),
149 media_list_(media_list),
150 type_(type),
151 capturer_(std::move(capturer)) {
152 DCHECK(capturer_);
153 }
154
~Worker()155 NativeDesktopMediaList::Worker::~Worker() {
156 DCHECK(task_runner_->BelongsToCurrentThread());
157 }
158
Start()159 void NativeDesktopMediaList::Worker::Start() {
160 DCHECK(task_runner_->BelongsToCurrentThread());
161 capturer_->Start(this);
162 }
163
Refresh(const DesktopMediaID::Id & view_dialog_id,bool update_thumnails)164 void NativeDesktopMediaList::Worker::Refresh(
165 const DesktopMediaID::Id& view_dialog_id,
166 bool update_thumnails) {
167 DCHECK(task_runner_->BelongsToCurrentThread());
168 std::vector<SourceDescription> result;
169
170 webrtc::DesktopCapturer::SourceList sources;
171 if (!capturer_->GetSourceList(&sources)) {
172 // Will pass empty results list to RefreshForAuraWindows().
173 sources.clear();
174 }
175
176 bool mutiple_sources = sources.size() > 1;
177 base::string16 title;
178 for (size_t i = 0; i < sources.size(); ++i) {
179 switch (type_) {
180 case DesktopMediaID::TYPE_SCREEN:
181 // Just in case 'Screen' is inflected depending on the screen number,
182 // use plural formatter.
183 title = mutiple_sources
184 ? l10n_util::GetPluralStringFUTF16(
185 IDS_DESKTOP_MEDIA_PICKER_MULTIPLE_SCREEN_NAME,
186 static_cast<int>(i + 1))
187 : l10n_util::GetStringUTF16(
188 IDS_DESKTOP_MEDIA_PICKER_SINGLE_SCREEN_NAME);
189 break;
190
191 case DesktopMediaID::TYPE_WINDOW:
192 // Skip the picker dialog window.
193 if (sources[i].id == view_dialog_id)
194 continue;
195 title = base::UTF8ToUTF16(sources[i].title);
196 break;
197
198 default:
199 NOTREACHED();
200 }
201 result.push_back(
202 SourceDescription(DesktopMediaID(type_, sources[i].id), title));
203 }
204
205 content::GetUIThreadTaskRunner({})->PostTask(
206 FROM_HERE, base::BindOnce(&NativeDesktopMediaList::RefreshForAuraWindows,
207 media_list_, result, update_thumnails));
208 }
209
RefreshThumbnails(std::vector<DesktopMediaID> native_ids,const gfx::Size & thumbnail_size)210 void NativeDesktopMediaList::Worker::RefreshThumbnails(
211 std::vector<DesktopMediaID> native_ids,
212 const gfx::Size& thumbnail_size) {
213 DCHECK(task_runner_->BelongsToCurrentThread());
214
215 // Ignore if refresh is already in progress.
216 if (refresh_thumbnails_state_)
217 return;
218
219 // To refresh thumbnails, a snapshot of each window is captured and scaled
220 // down to the specified size. Snapshotting can be asynchronous, and so
221 // the process looks like the following steps:
222 //
223 // 1) RefreshNextThumbnail
224 // 2) OnCaptureResult
225 // 3) UpdateSourceThumbnail (if the snapshot changed)
226 // [repeat 1, 2 and 3 until all thumbnails are refreshed]
227 // 4) RefreshNextThumbnail
228 // 5) UpdateNativeThumbnailsFinished
229 //
230 // |image_hashes_| is used to help avoid updating thumbnails that haven't
231 // changed since the last refresh.
232
233 refresh_thumbnails_state_ = std::make_unique<RefreshThumbnailsState>();
234 refresh_thumbnails_state_->source_ids = std::move(native_ids);
235 refresh_thumbnails_state_->thumbnail_size = thumbnail_size;
236 RefreshNextThumbnail();
237 }
238
RefreshNextThumbnail()239 void NativeDesktopMediaList::Worker::RefreshNextThumbnail() {
240 DCHECK(refresh_thumbnails_state_);
241
242 for (size_t index = refresh_thumbnails_state_->next_source_index;
243 index < refresh_thumbnails_state_->source_ids.size(); ++index) {
244 refresh_thumbnails_state_->next_source_index = index + 1;
245 DesktopMediaID source_id = refresh_thumbnails_state_->source_ids[index];
246 if (capturer_->SelectSource(source_id.id)) {
247 capturer_->CaptureFrame(); // Completes with OnCaptureResult.
248 return;
249 }
250 }
251
252 // Done capturing thumbnails.
253 image_hashes_.swap(refresh_thumbnails_state_->new_image_hashes);
254 refresh_thumbnails_state_.reset();
255
256 content::GetUIThreadTaskRunner({})->PostTask(
257 FROM_HERE,
258 base::BindOnce(&NativeDesktopMediaList::UpdateNativeThumbnailsFinished,
259 media_list_));
260 }
261
OnCaptureResult(webrtc::DesktopCapturer::Result result,std::unique_ptr<webrtc::DesktopFrame> frame)262 void NativeDesktopMediaList::Worker::OnCaptureResult(
263 webrtc::DesktopCapturer::Result result,
264 std::unique_ptr<webrtc::DesktopFrame> frame) {
265 auto index = refresh_thumbnails_state_->next_source_index - 1;
266 DCHECK(index < refresh_thumbnails_state_->source_ids.size());
267 DesktopMediaID id = refresh_thumbnails_state_->source_ids[index];
268
269 // |frame| may be null if capture failed (e.g. because window has been
270 // closed).
271 if (IsFrameValid(frame.get())) {
272 uint32_t frame_hash = GetFrameHash(frame.get());
273 refresh_thumbnails_state_->new_image_hashes[id] = frame_hash;
274
275 // Scale the image only if it has changed.
276 auto it = image_hashes_.find(id);
277 if (it == image_hashes_.end() || it->second != frame_hash) {
278 gfx::ImageSkia thumbnail = ScaleDesktopFrame(
279 std::move(frame), refresh_thumbnails_state_->thumbnail_size);
280 content::GetUIThreadTaskRunner({})->PostTask(
281 FROM_HERE,
282 base::BindOnce(&NativeDesktopMediaList::UpdateSourceThumbnail,
283 media_list_, id, thumbnail));
284 }
285 }
286
287 // Protect against possible re-entrancy since OnCaptureResult can be invoked
288 // from within the call to CaptureFrame.
289 base::ThreadTaskRunnerHandle::Get()->PostTask(
290 FROM_HERE, base::BindOnce(&Worker::RefreshNextThumbnail,
291 weak_factory_.GetWeakPtr()));
292 }
293
NativeDesktopMediaList(DesktopMediaID::Type type,std::unique_ptr<webrtc::DesktopCapturer> capturer)294 NativeDesktopMediaList::NativeDesktopMediaList(
295 DesktopMediaID::Type type,
296 std::unique_ptr<webrtc::DesktopCapturer> capturer)
297 : DesktopMediaListBase(base::TimeDelta::FromMilliseconds(
298 kDefaultNativeDesktopMediaListUpdatePeriod)),
299 thread_("DesktopMediaListCaptureThread") {
300 type_ = type;
301
302 #if defined(OS_WIN) || defined(OS_MAC)
303 // On Windows/OSX the thread must be a UI thread.
304 base::MessagePumpType thread_type = base::MessagePumpType::UI;
305 #else
306 base::MessagePumpType thread_type = base::MessagePumpType::DEFAULT;
307 #endif
308 thread_.StartWithOptions(base::Thread::Options(thread_type, 0));
309
310 worker_.reset(new Worker(thread_.task_runner(), weak_factory_.GetWeakPtr(),
311 type, std::move(capturer)));
312
313 thread_.task_runner()->PostTask(
314 FROM_HERE,
315 base::BindOnce(&Worker::Start, base::Unretained(worker_.get())));
316 }
317
~NativeDesktopMediaList()318 NativeDesktopMediaList::~NativeDesktopMediaList() {
319 // This thread should mostly be an idle observer. Stopping it should be fast.
320 base::ScopedAllowBaseSyncPrimitivesOutsideBlockingScope allow_thread_join;
321 thread_.task_runner()->DeleteSoon(FROM_HERE, worker_.release());
322 thread_.Stop();
323 }
324
Refresh(bool update_thumnails)325 void NativeDesktopMediaList::Refresh(bool update_thumnails) {
326 DCHECK(can_refresh());
327
328 #if defined(USE_AURA)
329 DCHECK_EQ(pending_aura_capture_requests_, 0);
330 DCHECK(!pending_native_thumbnail_capture_);
331 new_aura_thumbnail_hashes_.clear();
332 #endif
333
334 thread_.task_runner()->PostTask(
335 FROM_HERE,
336 base::BindOnce(&Worker::Refresh, base::Unretained(worker_.get()),
337 view_dialog_id_.id, update_thumnails));
338 }
339
RefreshForAuraWindows(std::vector<SourceDescription> sources,bool update_thumnails)340 void NativeDesktopMediaList::RefreshForAuraWindows(
341 std::vector<SourceDescription> sources,
342 bool update_thumnails) {
343 DCHECK(can_refresh());
344
345 #if defined(USE_AURA)
346 // Associate aura id with native id.
347 for (auto& source : sources) {
348 if (source.id.type != DesktopMediaID::TYPE_WINDOW)
349 continue;
350
351 aura::WindowTreeHost* const host =
352 aura::WindowTreeHost::GetForAcceleratedWidget(
353 *reinterpret_cast<gfx::AcceleratedWidget*>(&source.id.id));
354 aura::Window* const aura_window = host ? host->window() : nullptr;
355 if (aura_window) {
356 DesktopMediaID aura_id = DesktopMediaID::RegisterNativeWindow(
357 DesktopMediaID::TYPE_WINDOW, aura_window);
358 source.id.window_id = aura_id.window_id;
359 }
360 }
361 #endif // defined(USE_AURA)
362
363 UpdateSourcesList(sources);
364
365 if (!update_thumnails) {
366 OnRefreshComplete();
367 return;
368 }
369
370 if (thumbnail_size_.IsEmpty()) {
371 #if defined(USE_AURA)
372 pending_native_thumbnail_capture_ = true;
373 #endif
374 content::GetUIThreadTaskRunner({})->PostTask(
375 FROM_HERE,
376 base::BindOnce(&NativeDesktopMediaList::UpdateNativeThumbnailsFinished,
377 weak_factory_.GetWeakPtr()));
378 return;
379 }
380
381 // OnAuraThumbnailCaptured() and UpdateNativeThumbnailsFinished() are
382 // guaranteed to be executed after RefreshForAuraWindows() and
383 // CaptureAuraWindowThumbnail() in the browser UI thread.
384 // Therefore pending_aura_capture_requests_ will be set the number of aura
385 // windows to be captured and pending_native_thumbnail_capture_ will be set
386 // true if native thumbnail capture is needed before OnAuraThumbnailCaptured()
387 // or UpdateNativeThumbnailsFinished() are called.
388 std::vector<DesktopMediaID> native_ids;
389 for (const auto& source : sources) {
390 #if defined(USE_AURA)
391 if (source.id.window_id > DesktopMediaID::kNullId) {
392 CaptureAuraWindowThumbnail(source.id);
393 continue;
394 }
395 #endif // defined(USE_AURA)
396 native_ids.push_back(source.id);
397 }
398
399 if (!native_ids.empty()) {
400 #if defined(USE_AURA)
401 pending_native_thumbnail_capture_ = true;
402 #endif
403 thread_.task_runner()->PostTask(
404 FROM_HERE, base::BindOnce(&Worker::RefreshThumbnails,
405 base::Unretained(worker_.get()),
406 std::move(native_ids), thumbnail_size_));
407 }
408 }
409
UpdateNativeThumbnailsFinished()410 void NativeDesktopMediaList::UpdateNativeThumbnailsFinished() {
411 #if defined(USE_AURA)
412 DCHECK(pending_native_thumbnail_capture_);
413 pending_native_thumbnail_capture_ = false;
414 // If native thumbnail captures finished after aura thumbnail captures,
415 // execute |done_callback| to let the caller know the update process is
416 // finished. If necessary, this will schedule the next refresh.
417 if (pending_aura_capture_requests_ == 0)
418 OnRefreshComplete();
419 #else
420 OnRefreshComplete();
421 #endif // defined(USE_AURA)
422 }
423
424 #if defined(USE_AURA)
425
CaptureAuraWindowThumbnail(const DesktopMediaID & id)426 void NativeDesktopMediaList::CaptureAuraWindowThumbnail(
427 const DesktopMediaID& id) {
428 DCHECK(can_refresh());
429
430 gfx::NativeWindow window = DesktopMediaID::GetNativeWindowById(id);
431 if (!window)
432 return;
433
434 gfx::Rect window_rect(window->bounds().width(), window->bounds().height());
435 gfx::Rect scaled_rect = media::ComputeLetterboxRegion(
436 gfx::Rect(thumbnail_size_), window_rect.size());
437
438 pending_aura_capture_requests_++;
439 ui::GrabWindowSnapshotAndScaleAsyncAura(
440 window, window_rect, scaled_rect.size(),
441 base::BindOnce(&NativeDesktopMediaList::OnAuraThumbnailCaptured,
442 weak_factory_.GetWeakPtr(), id));
443 }
444
OnAuraThumbnailCaptured(const DesktopMediaID & id,gfx::Image image)445 void NativeDesktopMediaList::OnAuraThumbnailCaptured(const DesktopMediaID& id,
446 gfx::Image image) {
447 DCHECK(can_refresh());
448
449 if (!image.IsEmpty()) {
450 // Only new or changed thumbnail need update.
451 new_aura_thumbnail_hashes_[id] = GetImageHash(image);
452 if (!previous_aura_thumbnail_hashes_.count(id) ||
453 previous_aura_thumbnail_hashes_[id] != new_aura_thumbnail_hashes_[id]) {
454 UpdateSourceThumbnail(id, image.AsImageSkia());
455 }
456 }
457
458 // After all aura windows are processed, schedule next refresh;
459 pending_aura_capture_requests_--;
460 DCHECK_GE(pending_aura_capture_requests_, 0);
461 if (pending_aura_capture_requests_ == 0) {
462 previous_aura_thumbnail_hashes_ = std::move(new_aura_thumbnail_hashes_);
463 // Schedule next refresh if aura thumbnail captures finished after native
464 // thumbnail captures.
465 if (!pending_native_thumbnail_capture_)
466 OnRefreshComplete();
467 }
468 }
469
470 #endif // defined(USE_AURA)
471