1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h"
6
7 #include "media/base/bind_to_current_loop.h"
8 #include "media/base/video_frame.h"
9 #include "media/base/video_types.h"
10 #include "media/base/video_util.h"
11 #include "skia/ext/platform_canvas.h"
12 #include "third_party/blink/public/platform/web_media_stream_source.h"
13 #include "third_party/blink/public/platform/web_media_stream_track.h"
14 #include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h"
15 #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
16 #include "third_party/blink/renderer/platform/wtf/functional.h"
17 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
18 #include "third_party/libyuv/include/libyuv.h"
19 #include "third_party/skia/include/core/SkImage.h"
20 #include "third_party/skia/include/core/SkSurface.h"
21 #include "ui/gfx/gpu_memory_buffer.h"
22
23 namespace WTF {
24 // Template specialization of [1], needed to be able to pass callbacks
25 // that have ScopedWebCallbacks paramaters across threads.
26 //
27 // [1] third_party/blink/renderer/platform/wtf/cross_thread_copier.h.
28 template <typename T>
29 struct CrossThreadCopier<blink::ScopedWebCallbacks<T>>
30 : public CrossThreadCopierPassThrough<blink::ScopedWebCallbacks<T>> {
31 STATIC_ONLY(CrossThreadCopier);
32 using Type = blink::ScopedWebCallbacks<T>;
CopyWTF::CrossThreadCopier33 static blink::ScopedWebCallbacks<T> Copy(
34 blink::ScopedWebCallbacks<T> pointer) {
35 return pointer;
36 }
37 };
38
39 } // namespace WTF
40
41 namespace blink {
42
43 namespace {
44
OnError(std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks)45 void OnError(std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks) {
46 callbacks->OnError();
47 }
48
49 } // anonymous namespace
50
51 // Ref-counted class to receive a single VideoFrame on IO thread, convert it and
52 // send it to |main_task_runner_|, where this class is created and destroyed.
53 class ImageCaptureFrameGrabber::SingleShotFrameHandler
54 : public WTF::ThreadSafeRefCounted<SingleShotFrameHandler> {
55 public:
SingleShotFrameHandler()56 SingleShotFrameHandler() : first_frame_received_(false) {}
57
58 // Receives a |frame| and converts its pixels into a SkImage via an internal
59 // PaintSurface and SkPixmap. Alpha channel, if any, is copied.
60 using SkImageDeliverCB = WTF::CrossThreadFunction<void(sk_sp<SkImage>)>;
61 void OnVideoFrameOnIOThread(
62 SkImageDeliverCB callback,
63 scoped_refptr<base::SingleThreadTaskRunner> task_runner,
64 scoped_refptr<media::VideoFrame> frame,
65 base::TimeTicks current_time);
66
67 private:
68 friend class WTF::ThreadSafeRefCounted<SingleShotFrameHandler>;
69
70 // Flag to indicate that the first frames has been processed, and subsequent
71 // ones can be safely discarded.
72 bool first_frame_received_;
73
74 DISALLOW_COPY_AND_ASSIGN(SingleShotFrameHandler);
75 };
76
OnVideoFrameOnIOThread(SkImageDeliverCB callback,scoped_refptr<base::SingleThreadTaskRunner> task_runner,scoped_refptr<media::VideoFrame> frame,base::TimeTicks)77 void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
78 SkImageDeliverCB callback,
79 scoped_refptr<base::SingleThreadTaskRunner> task_runner,
80 scoped_refptr<media::VideoFrame> frame,
81 base::TimeTicks /* current_time */) {
82 DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
83 frame->format() == media::PIXEL_FORMAT_I420A ||
84 frame->format() == media::PIXEL_FORMAT_NV12);
85
86 if (first_frame_received_)
87 return;
88 first_frame_received_ = true;
89
90 const SkAlphaType alpha = media::IsOpaque(frame->format())
91 ? kOpaque_SkAlphaType
92 : kPremul_SkAlphaType;
93 const SkImageInfo info = SkImageInfo::MakeN32(
94 frame->visible_rect().width(), frame->visible_rect().height(), alpha);
95
96 sk_sp<SkSurface> surface = SkSurface::MakeRaster(info);
97 DCHECK(surface);
98
99 auto wrapper_callback =
100 media::BindToLoop(std::move(task_runner),
101 ConvertToBaseRepeatingCallback(std::move(callback)));
102
103 SkPixmap pixmap;
104 if (!skia::GetWritablePixels(surface->getCanvas(), &pixmap)) {
105 DLOG(ERROR) << "Error trying to map SkSurface's pixels";
106 std::move(wrapper_callback).Run(sk_sp<SkImage>());
107 return;
108 }
109
110 #if SK_PMCOLOR_BYTE_ORDER(R, G, B, A)
111 const uint32_t destination_pixel_format = libyuv::FOURCC_ABGR;
112 #else
113 const uint32_t destination_pixel_format = libyuv::FOURCC_ARGB;
114 #endif
115 uint8_t* destination_plane = static_cast<uint8_t*>(pixmap.writable_addr());
116 int destination_stride = pixmap.width() * 4;
117 int destination_width = pixmap.width();
118 int destination_height = pixmap.height();
119
120 if (frame->storage_type() == media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
121 auto* gmb = frame->GetGpuMemoryBuffer();
122 if (!gmb->Map()) {
123 DLOG(ERROR) << "Error mapping GpuMemoryBuffer video frame";
124 std::move(wrapper_callback).Run(sk_sp<SkImage>());
125 return;
126 }
127
128 // NV12 is the only supported pixel format at the moment.
129 DCHECK_EQ(frame->format(), media::PIXEL_FORMAT_NV12);
130 int y_stride = gmb->stride(0);
131 int uv_stride = gmb->stride(1);
132 const uint8_t* y_plane =
133 (static_cast<uint8_t*>(gmb->memory(0)) + frame->visible_rect().x() +
134 (frame->visible_rect().y() * y_stride));
135 // UV plane of NV12 has 2-byte pixel width, with half chroma subsampling
136 // both horizontally and vertically.
137 const uint8_t* uv_plane = (static_cast<uint8_t*>(gmb->memory(1)) +
138 ((frame->visible_rect().x() * 2) / 2) +
139 ((frame->visible_rect().y() / 2) * uv_stride));
140
141 switch (destination_pixel_format) {
142 case libyuv::FOURCC_ABGR:
143 libyuv::NV12ToABGR(y_plane, y_stride, uv_plane, uv_stride,
144 destination_plane, destination_stride,
145 destination_width, destination_height);
146 break;
147 case libyuv::FOURCC_ARGB:
148 libyuv::NV12ToARGB(y_plane, y_stride, uv_plane, uv_stride,
149 destination_plane, destination_stride,
150 destination_width, destination_height);
151 break;
152 default:
153 NOTREACHED();
154 }
155 gmb->Unmap();
156 } else {
157 DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
158 frame->format() == media::PIXEL_FORMAT_I420A);
159 libyuv::ConvertFromI420(frame->visible_data(media::VideoFrame::kYPlane),
160 frame->stride(media::VideoFrame::kYPlane),
161 frame->visible_data(media::VideoFrame::kUPlane),
162 frame->stride(media::VideoFrame::kUPlane),
163 frame->visible_data(media::VideoFrame::kVPlane),
164 frame->stride(media::VideoFrame::kVPlane),
165 destination_plane, destination_stride,
166 destination_width, destination_height,
167 destination_pixel_format);
168
169 if (frame->format() == media::PIXEL_FORMAT_I420A) {
170 DCHECK(!info.isOpaque());
171 // This function copies any plane into the alpha channel of an ARGB image.
172 libyuv::ARGBCopyYToAlpha(frame->visible_data(media::VideoFrame::kAPlane),
173 frame->stride(media::VideoFrame::kAPlane),
174 destination_plane, destination_stride,
175 destination_width, destination_height);
176 }
177 }
178
179 std::move(wrapper_callback).Run(surface->makeImageSnapshot());
180 }
181
ImageCaptureFrameGrabber()182 ImageCaptureFrameGrabber::ImageCaptureFrameGrabber()
183 : frame_grab_in_progress_(false) {}
184
~ImageCaptureFrameGrabber()185 ImageCaptureFrameGrabber::~ImageCaptureFrameGrabber() {
186 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
187 }
188
GrabFrame(WebMediaStreamTrack * track,std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks,scoped_refptr<base::SingleThreadTaskRunner> task_runner)189 void ImageCaptureFrameGrabber::GrabFrame(
190 WebMediaStreamTrack* track,
191 std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks,
192 scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
193 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
194 DCHECK(!!callbacks);
195
196 DCHECK(track && !track->IsNull() && track->GetPlatformTrack());
197 DCHECK_EQ(WebMediaStreamSource::kTypeVideo, track->Source().GetType());
198
199 if (frame_grab_in_progress_) {
200 // Reject grabFrame()s too close back to back.
201 callbacks->OnError();
202 return;
203 }
204
205 auto scoped_callbacks =
206 MakeScopedWebCallbacks(std::move(callbacks), WTF::Bind(&OnError));
207
208 // A SingleShotFrameHandler is bound and given to the Track to guarantee that
209 // only one VideoFrame is converted and delivered to OnSkImage(), otherwise
210 // SKImages might be sent to resolved |callbacks| while DisconnectFromTrack()
211 // is being processed, which might be further held up if UI is busy, see
212 // https://crbug.com/623042.
213 frame_grab_in_progress_ = true;
214 MediaStreamVideoSink::ConnectToTrack(
215 *track,
216 ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
217 &SingleShotFrameHandler::OnVideoFrameOnIOThread,
218 base::MakeRefCounted<SingleShotFrameHandler>(),
219 WTF::Passed(CrossThreadBindRepeating(
220 &ImageCaptureFrameGrabber::OnSkImage, weak_factory_.GetWeakPtr(),
221 WTF::Passed(std::move(scoped_callbacks)))),
222 WTF::Passed(std::move(task_runner)))),
223 false);
224 }
225
OnSkImage(ScopedWebCallbacks<ImageCaptureGrabFrameCallbacks> callbacks,sk_sp<SkImage> image)226 void ImageCaptureFrameGrabber::OnSkImage(
227 ScopedWebCallbacks<ImageCaptureGrabFrameCallbacks> callbacks,
228 sk_sp<SkImage> image) {
229 DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
230
231 MediaStreamVideoSink::DisconnectFromTrack();
232 frame_grab_in_progress_ = false;
233 if (image)
234 callbacks.PassCallbacks()->OnSuccess(image);
235 else
236 callbacks.PassCallbacks()->OnError();
237 }
238
239 } // namespace blink
240