1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h"
6 
7 #include "media/base/bind_to_current_loop.h"
8 #include "media/base/video_frame.h"
9 #include "media/base/video_types.h"
10 #include "media/base/video_util.h"
11 #include "skia/ext/platform_canvas.h"
12 #include "third_party/blink/public/platform/web_media_stream_source.h"
13 #include "third_party/blink/public/platform/web_media_stream_track.h"
14 #include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h"
15 #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
16 #include "third_party/blink/renderer/platform/wtf/functional.h"
17 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
18 #include "third_party/libyuv/include/libyuv.h"
19 #include "third_party/skia/include/core/SkImage.h"
20 #include "third_party/skia/include/core/SkSurface.h"
21 #include "ui/gfx/gpu_memory_buffer.h"
22 
23 namespace WTF {
24 // Template specialization of [1], needed to be able to pass callbacks
25 // that have ScopedWebCallbacks paramaters across threads.
26 //
27 // [1] third_party/blink/renderer/platform/wtf/cross_thread_copier.h.
28 template <typename T>
29 struct CrossThreadCopier<blink::ScopedWebCallbacks<T>>
30     : public CrossThreadCopierPassThrough<blink::ScopedWebCallbacks<T>> {
31   STATIC_ONLY(CrossThreadCopier);
32   using Type = blink::ScopedWebCallbacks<T>;
CopyWTF::CrossThreadCopier33   static blink::ScopedWebCallbacks<T> Copy(
34       blink::ScopedWebCallbacks<T> pointer) {
35     return pointer;
36   }
37 };
38 
39 }  // namespace WTF
40 
41 namespace blink {
42 
43 namespace {
44 
OnError(std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks)45 void OnError(std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks) {
46   callbacks->OnError();
47 }
48 
49 }  // anonymous namespace
50 
51 // Ref-counted class to receive a single VideoFrame on IO thread, convert it and
52 // send it to |main_task_runner_|, where this class is created and destroyed.
53 class ImageCaptureFrameGrabber::SingleShotFrameHandler
54     : public WTF::ThreadSafeRefCounted<SingleShotFrameHandler> {
55  public:
SingleShotFrameHandler()56   SingleShotFrameHandler() : first_frame_received_(false) {}
57 
58   // Receives a |frame| and converts its pixels into a SkImage via an internal
59   // PaintSurface and SkPixmap. Alpha channel, if any, is copied.
60   using SkImageDeliverCB = WTF::CrossThreadFunction<void(sk_sp<SkImage>)>;
61   void OnVideoFrameOnIOThread(
62       SkImageDeliverCB callback,
63       scoped_refptr<base::SingleThreadTaskRunner> task_runner,
64       scoped_refptr<media::VideoFrame> frame,
65       base::TimeTicks current_time);
66 
67  private:
68   friend class WTF::ThreadSafeRefCounted<SingleShotFrameHandler>;
69 
70   // Flag to indicate that the first frames has been processed, and subsequent
71   // ones can be safely discarded.
72   bool first_frame_received_;
73 
74   DISALLOW_COPY_AND_ASSIGN(SingleShotFrameHandler);
75 };
76 
OnVideoFrameOnIOThread(SkImageDeliverCB callback,scoped_refptr<base::SingleThreadTaskRunner> task_runner,scoped_refptr<media::VideoFrame> frame,base::TimeTicks)77 void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
78     SkImageDeliverCB callback,
79     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
80     scoped_refptr<media::VideoFrame> frame,
81     base::TimeTicks /* current_time */) {
82   DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
83          frame->format() == media::PIXEL_FORMAT_I420A ||
84          frame->format() == media::PIXEL_FORMAT_NV12);
85 
86   if (first_frame_received_)
87     return;
88   first_frame_received_ = true;
89 
90   const SkAlphaType alpha = media::IsOpaque(frame->format())
91                                 ? kOpaque_SkAlphaType
92                                 : kPremul_SkAlphaType;
93   const SkImageInfo info = SkImageInfo::MakeN32(
94       frame->visible_rect().width(), frame->visible_rect().height(), alpha);
95 
96   sk_sp<SkSurface> surface = SkSurface::MakeRaster(info);
97   DCHECK(surface);
98 
99   auto wrapper_callback =
100       media::BindToLoop(std::move(task_runner),
101                         ConvertToBaseRepeatingCallback(std::move(callback)));
102 
103   SkPixmap pixmap;
104   if (!skia::GetWritablePixels(surface->getCanvas(), &pixmap)) {
105     DLOG(ERROR) << "Error trying to map SkSurface's pixels";
106     std::move(wrapper_callback).Run(sk_sp<SkImage>());
107     return;
108   }
109 
110 #if SK_PMCOLOR_BYTE_ORDER(R, G, B, A)
111   const uint32_t destination_pixel_format = libyuv::FOURCC_ABGR;
112 #else
113   const uint32_t destination_pixel_format = libyuv::FOURCC_ARGB;
114 #endif
115   uint8_t* destination_plane = static_cast<uint8_t*>(pixmap.writable_addr());
116   int destination_stride = pixmap.width() * 4;
117   int destination_width = pixmap.width();
118   int destination_height = pixmap.height();
119 
120   if (frame->storage_type() == media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
121     auto* gmb = frame->GetGpuMemoryBuffer();
122     if (!gmb->Map()) {
123       DLOG(ERROR) << "Error mapping GpuMemoryBuffer video frame";
124       std::move(wrapper_callback).Run(sk_sp<SkImage>());
125       return;
126     }
127 
128     // NV12 is the only supported pixel format at the moment.
129     DCHECK_EQ(frame->format(), media::PIXEL_FORMAT_NV12);
130     int y_stride = gmb->stride(0);
131     int uv_stride = gmb->stride(1);
132     const uint8_t* y_plane =
133         (static_cast<uint8_t*>(gmb->memory(0)) + frame->visible_rect().x() +
134          (frame->visible_rect().y() * y_stride));
135     // UV plane of NV12 has 2-byte pixel width, with half chroma subsampling
136     // both horizontally and vertically.
137     const uint8_t* uv_plane = (static_cast<uint8_t*>(gmb->memory(1)) +
138                                ((frame->visible_rect().x() * 2) / 2) +
139                                ((frame->visible_rect().y() / 2) * uv_stride));
140 
141     switch (destination_pixel_format) {
142       case libyuv::FOURCC_ABGR:
143         libyuv::NV12ToABGR(y_plane, y_stride, uv_plane, uv_stride,
144                            destination_plane, destination_stride,
145                            destination_width, destination_height);
146         break;
147       case libyuv::FOURCC_ARGB:
148         libyuv::NV12ToARGB(y_plane, y_stride, uv_plane, uv_stride,
149                            destination_plane, destination_stride,
150                            destination_width, destination_height);
151         break;
152       default:
153         NOTREACHED();
154     }
155     gmb->Unmap();
156   } else {
157     DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
158            frame->format() == media::PIXEL_FORMAT_I420A);
159     libyuv::ConvertFromI420(frame->visible_data(media::VideoFrame::kYPlane),
160                             frame->stride(media::VideoFrame::kYPlane),
161                             frame->visible_data(media::VideoFrame::kUPlane),
162                             frame->stride(media::VideoFrame::kUPlane),
163                             frame->visible_data(media::VideoFrame::kVPlane),
164                             frame->stride(media::VideoFrame::kVPlane),
165                             destination_plane, destination_stride,
166                             destination_width, destination_height,
167                             destination_pixel_format);
168 
169     if (frame->format() == media::PIXEL_FORMAT_I420A) {
170       DCHECK(!info.isOpaque());
171       // This function copies any plane into the alpha channel of an ARGB image.
172       libyuv::ARGBCopyYToAlpha(frame->visible_data(media::VideoFrame::kAPlane),
173                                frame->stride(media::VideoFrame::kAPlane),
174                                destination_plane, destination_stride,
175                                destination_width, destination_height);
176     }
177   }
178 
179   std::move(wrapper_callback).Run(surface->makeImageSnapshot());
180 }
181 
ImageCaptureFrameGrabber()182 ImageCaptureFrameGrabber::ImageCaptureFrameGrabber()
183     : frame_grab_in_progress_(false) {}
184 
~ImageCaptureFrameGrabber()185 ImageCaptureFrameGrabber::~ImageCaptureFrameGrabber() {
186   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
187 }
188 
GrabFrame(WebMediaStreamTrack * track,std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks,scoped_refptr<base::SingleThreadTaskRunner> task_runner)189 void ImageCaptureFrameGrabber::GrabFrame(
190     WebMediaStreamTrack* track,
191     std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks,
192     scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
193   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
194   DCHECK(!!callbacks);
195 
196   DCHECK(track && !track->IsNull() && track->GetPlatformTrack());
197   DCHECK_EQ(WebMediaStreamSource::kTypeVideo, track->Source().GetType());
198 
199   if (frame_grab_in_progress_) {
200     // Reject grabFrame()s too close back to back.
201     callbacks->OnError();
202     return;
203   }
204 
205   auto scoped_callbacks =
206       MakeScopedWebCallbacks(std::move(callbacks), WTF::Bind(&OnError));
207 
208   // A SingleShotFrameHandler is bound and given to the Track to guarantee that
209   // only one VideoFrame is converted and delivered to OnSkImage(), otherwise
210   // SKImages might be sent to resolved |callbacks| while DisconnectFromTrack()
211   // is being processed, which might be further held up if UI is busy, see
212   // https://crbug.com/623042.
213   frame_grab_in_progress_ = true;
214   MediaStreamVideoSink::ConnectToTrack(
215       *track,
216       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
217           &SingleShotFrameHandler::OnVideoFrameOnIOThread,
218           base::MakeRefCounted<SingleShotFrameHandler>(),
219           WTF::Passed(CrossThreadBindRepeating(
220               &ImageCaptureFrameGrabber::OnSkImage, weak_factory_.GetWeakPtr(),
221               WTF::Passed(std::move(scoped_callbacks)))),
222           WTF::Passed(std::move(task_runner)))),
223       false);
224 }
225 
OnSkImage(ScopedWebCallbacks<ImageCaptureGrabFrameCallbacks> callbacks,sk_sp<SkImage> image)226 void ImageCaptureFrameGrabber::OnSkImage(
227     ScopedWebCallbacks<ImageCaptureGrabFrameCallbacks> callbacks,
228     sk_sp<SkImage> image) {
229   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
230 
231   MediaStreamVideoSink::DisconnectFromTrack();
232   frame_grab_in_progress_ = false;
233   if (image)
234     callbacks.PassCallbacks()->OnSuccess(image);
235   else
236     callbacks.PassCallbacks()->OnError();
237 }
238 
239 }  // namespace blink
240