1 #include "UwpScreenCapturer.h"
2
3 #include "api/video/i420_buffer.h"
4 #include "api/video/video_frame_buffer.h"
5 #include "api/video/video_rotation.h"
6 #include "modules/video_capture/video_capture_factory.h"
7 #include "rtc_base/checks.h"
8 #include "rtc_base/logging.h"
9
10 #include <stdint.h>
11 #include <memory>
12 #include <algorithm>
13 #include <libyuv.h>
14 #include <StaticThreads.h>
15
16 namespace tgcalls {
17 namespace {
18
19 constexpr auto kPreferredWidth = 640;
20 constexpr auto kPreferredHeight = 480;
21 constexpr auto kPreferredFps = 30;
22
23 // We must use a BGRA pixel format that has 4 bytes per pixel, as required by
24 // the DesktopFrame interface.
25 constexpr auto kPixelFormat = winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized;
26
27 // We only want 1 buffer in our frame pool to reduce latency. If we had more,
28 // they would sit in the pool for longer and be stale by the time we are asked
29 // for a new frame.
30 constexpr int kNumBuffers = 1;
31
32 } // namespace
33
UwpScreenCapturer(std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink,GraphicsCaptureItem item)34 UwpScreenCapturer::UwpScreenCapturer(
35 std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink, GraphicsCaptureItem item)
36 : _sink(sink),
37 item_(item) {
38 }
39
~UwpScreenCapturer()40 UwpScreenCapturer::~UwpScreenCapturer() {
41 destroy();
42 }
43
create()44 void UwpScreenCapturer::create() {
45 winrt::slim_lock_guard const guard(lock_);
46
47 RTC_DCHECK(!is_capture_started_);
48
49 if (item_closed_) {
50 RTC_LOG(LS_ERROR) << "The target source has been closed.";
51 //RecordStartCaptureResult(StartCaptureResult::kSourceClosed);
52 onFatalError();
53 return;
54 }
55
56 HRESULT hr = D3D11CreateDevice(
57 /*adapter=*/nullptr, D3D_DRIVER_TYPE_HARDWARE,
58 /*software_rasterizer=*/nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT,
59 /*feature_levels=*/nullptr, /*feature_levels_size=*/0, D3D11_SDK_VERSION,
60 d3d11_device_.put(), /*feature_level=*/nullptr, /*device_context=*/nullptr);
61 if (hr == DXGI_ERROR_UNSUPPORTED) {
62 // If a hardware device could not be created, use WARP which is a high speed
63 // software device.
64 hr = D3D11CreateDevice(
65 /*adapter=*/nullptr, D3D_DRIVER_TYPE_WARP,
66 /*software_rasterizer=*/nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT,
67 /*feature_levels=*/nullptr, /*feature_levels_size=*/0,
68 D3D11_SDK_VERSION, d3d11_device_.put(), /*feature_level=*/nullptr,
69 /*device_context=*/nullptr);
70 }
71
72 RTC_DCHECK(d3d11_device_);
73 RTC_DCHECK(item_);
74
75 // Listen for the Closed event, to detect if the source we are capturing is
76 // closed (e.g. application window is closed or monitor is disconnected). If
77 // it is, we should abort the capture.
78 item_.Closed({ this, &UwpScreenCapturer::OnClosed });
79
80 winrt::com_ptr<IDXGIDevice> dxgi_device;
81 hr = d3d11_device_->QueryInterface(IID_PPV_ARGS(&dxgi_device));
82 if (FAILED(hr)) {
83 //RecordStartCaptureResult(StartCaptureResult::kDxgiDeviceCastFailed);
84 onFatalError();
85 return;
86 }
87
88 hr = CreateDirect3D11DeviceFromDXGIDevice(dxgi_device.get(), direct3d_device_.put());
89 if (FAILED(hr)) {
90 //RecordStartCaptureResult(StartCaptureResult::kD3dDeviceCreationFailed);
91 onFatalError();
92 return;
93 }
94
95 // Cast to FramePoolStatics2 so we can use CreateFreeThreaded and avoid the
96 // need to have a DispatcherQueue. We don't listen for the FrameArrived event,
97 // so there's no difference.
98
99 previous_size_ = item_.Size();
100 _dimensions = std::make_pair(previous_size_.Width, previous_size_.Height);
101
102 winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice directDevice;
103 direct3d_device_->QueryInterface(winrt::guid_of<winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice>(), winrt::put_abi(directDevice));
104
105 frame_pool_ = Direct3D11CaptureFramePool::CreateFreeThreaded(directDevice, kPixelFormat, kNumBuffers, previous_size_);
106 //frame_pool_.FrameArrived({ this, &UwpScreenCapturer::OnFrameArrived });
107
108 session_ = frame_pool_.CreateCaptureSession(item_);
109 session_.StartCapture();
110
111 is_capture_started_ = true;
112 queueController_ = DispatcherQueueController::CreateOnDedicatedThread();
113 queue_ = queueController_.DispatcherQueue();
114
115 repeatingTimer_ = queue_.CreateTimer();
116 repeatingTimer_.Interval(std::chrono::milliseconds{ 1000 / kPreferredFps });
117 repeatingTimer_.Tick({this, &UwpScreenCapturer::OnFrameArrived});
118 repeatingTimer_.Start();
119 }
120
121 //void UwpScreenCapturer::OnFrameArrived(Direct3D11CaptureFramePool const& sender, winrt::Windows::Foundation::IInspectable const&) {
OnFrameArrived(DispatcherQueueTimer const & sender,winrt::Windows::Foundation::IInspectable const & args)122 void UwpScreenCapturer::OnFrameArrived(DispatcherQueueTimer const& sender, winrt::Windows::Foundation::IInspectable const& args) {
123 winrt::slim_lock_guard const guard(lock_);
124
125 if (item_closed_ || _state != VideoState::Active) {
126 RTC_LOG(LS_ERROR) << "The target source has been closed.";
127 onFatalError();
128 return;
129 }
130
131 RTC_DCHECK(is_capture_started_);
132
133 auto capture_frame = frame_pool_.TryGetNextFrame();
134 if (!capture_frame) {
135 // When resuming the capture after minimizing a window there seems to be a subsequent
136 // frame drop, as I'm lazing to deal with this from here this event is debounced in C# code.
137 if (!_paused) {
138 _paused = true;
139
140 if (_onPause){
141 _onPause(true);
142 }
143 }
144
145 //RecordGetFrameResult(GetFrameResult::kFrameDropped);
146 return /*hr*/;
147 }
148
149 if (_paused) {
150 _paused = false;
151
152 if (_onPause){
153 _onPause(false);
154 }
155 }
156
157 // We need to get this CaptureFrame as an ID3D11Texture2D so that we can get
158 // the raw image data in the format required by the DesktopFrame interface.
159 auto d3d_surface = capture_frame.Surface();
160
161 auto direct3DDxgiInterfaceAccess
162 = d3d_surface.as<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
163
164 winrt::com_ptr<ID3D11Texture2D> texture_2D;
165 auto hr = direct3DDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&texture_2D));
166 if (FAILED(hr)) {
167 //RecordGetFrameResult(GetFrameResult::kTexture2dCastFailed);
168 onFatalError();
169 return;
170 }
171
172 if (!mapped_texture_) {
173 hr = CreateMappedTexture(texture_2D);
174 if (FAILED(hr)) {
175 //RecordGetFrameResult(GetFrameResult::kCreateMappedTextureFailed);
176 onFatalError();
177 return;
178 }
179 }
180
181 //// We need to copy |texture_2D| into |mapped_texture_| as the latter has the
182 //// D3D11_CPU_ACCESS_READ flag set, which lets us access the image data.
183 //// Otherwise it would only be readable by the GPU.
184 winrt::com_ptr<ID3D11DeviceContext> d3d_context;
185 d3d11_device_->GetImmediateContext(d3d_context.put());
186 d3d_context->CopyResource(mapped_texture_.get(), texture_2D.get());
187
188 D3D11_MAPPED_SUBRESOURCE map_info;
189 hr = d3d_context->Map(mapped_texture_.get(), /*subresource_index=*/0,
190 D3D11_MAP_READ, /*D3D11_MAP_FLAG_DO_NOT_WAIT=*/0,
191 &map_info);
192 if (FAILED(hr)) {
193 //RecordGetFrameResult(GetFrameResult::kMapFrameFailed);
194 onFatalError();
195 return;
196 }
197
198 auto new_size = capture_frame.ContentSize();
199
200 // If the size has changed since the last capture, we must be sure to use
201 // the smaller dimensions. Otherwise we might overrun our buffer, or
202 // read stale data from the last frame.
203 int image_height = std::min(previous_size_.Height, new_size.Height);
204 int image_width = std::min(previous_size_.Width, new_size.Width);
205 int row_data_length = image_width * 4;
206
207 // Make a copy of the data pointed to by |map_info.pData| so we are free to
208 // unmap our texture.
209 uint8_t* src_data = static_cast<uint8_t*>(map_info.pData);
210 std::vector<uint8_t> image_data;
211 image_data.reserve(image_height * row_data_length);
212 uint8_t* image_data_ptr = image_data.data();
213 for (int i = 0; i < image_height; i++) {
214 memcpy(image_data_ptr, src_data, row_data_length);
215 image_data_ptr += row_data_length;
216 src_data += map_info.RowPitch;
217 }
218
219 // Transfer ownership of |image_data| to the output_frame.
220 OnFrame(std::move(image_data), image_width, image_height);
221
222 d3d_context->Unmap(mapped_texture_.get(), 0);
223
224 // If the size changed, we must resize the texture and frame pool to fit the
225 // new size.
226 if (previous_size_.Height != new_size.Height ||
227 previous_size_.Width != new_size.Width) {
228 hr = CreateMappedTexture(texture_2D, new_size.Width, new_size.Height);
229 if (FAILED(hr)) {
230 //RecordGetFrameResult(GetFrameResult::kResizeMappedTextureFailed);
231 onFatalError();
232 return;
233 }
234
235 winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice directDevice;
236 direct3d_device_->QueryInterface(winrt::guid_of<winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice>(), winrt::put_abi(directDevice));
237
238 frame_pool_.Recreate(directDevice, kPixelFormat, kNumBuffers, new_size);
239 }
240
241 //RecordGetFrameResult(GetFrameResult::kSuccess);
242
243 previous_size_ = new_size;
244 }
245
OnClosed(GraphicsCaptureItem const & sender,winrt::Windows::Foundation::IInspectable const &)246 void UwpScreenCapturer::OnClosed(GraphicsCaptureItem const& sender, winrt::Windows::Foundation::IInspectable const&)
247 {
248 winrt::slim_lock_guard const guard(lock_);
249
250 RTC_LOG(LS_INFO) << "Capture target has been closed.";
251 item_closed_ = true;
252 is_capture_started_ = false;
253
254 onFatalError();
255 }
256
CreateMappedTexture(winrt::com_ptr<ID3D11Texture2D> src_texture,UINT width,UINT height)257 HRESULT UwpScreenCapturer::CreateMappedTexture(winrt::com_ptr<ID3D11Texture2D> src_texture, UINT width, UINT height) {
258 if (mapped_texture_ != nullptr) {
259 mapped_texture_ = nullptr;
260 }
261
262 D3D11_TEXTURE2D_DESC src_desc;
263 src_texture->GetDesc(&src_desc);
264 D3D11_TEXTURE2D_DESC map_desc;
265 map_desc.Width = width == 0 ? src_desc.Width : width;
266 map_desc.Height = height == 0 ? src_desc.Height : height;
267 map_desc.MipLevels = src_desc.MipLevels;
268 map_desc.ArraySize = src_desc.ArraySize;
269 map_desc.Format = src_desc.Format;
270 map_desc.SampleDesc = src_desc.SampleDesc;
271 map_desc.Usage = D3D11_USAGE_STAGING;
272 map_desc.BindFlags = 0;
273 map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
274 map_desc.MiscFlags = 0;
275 return d3d11_device_->CreateTexture2D(&map_desc, nullptr, mapped_texture_.put());
276 }
277
setState(VideoState state)278 void UwpScreenCapturer::setState(VideoState state) {
279 if (_state == state) {
280 return;
281 }
282 _state = state;
283 if (_state == VideoState::Active) {
284 create();
285 } else {
286 destroy();
287 }
288 }
289
setPreferredCaptureAspectRatio(float aspectRatio)290 void UwpScreenCapturer::setPreferredCaptureAspectRatio(float aspectRatio) {
291 _aspectRatio = aspectRatio;
292 }
293
setOnFatalError(std::function<void ()> error)294 void UwpScreenCapturer::setOnFatalError(std::function<void ()> error) {
295 if (_fatalError) {
296 error();
297 } else {
298 _onFatalError = std::move(error);
299 }
300 }
301
setOnPause(std::function<void (bool)> pause)302 void UwpScreenCapturer::setOnPause(std::function<void(bool)> pause) {
303 if (_paused) {
304 pause(true);
305 }
306
307 _onPause = std::move(pause);
308 }
309
resolution() const310 std::pair<int, int> UwpScreenCapturer::resolution() const {
311 return _dimensions;
312 }
313
onFatalError()314 void UwpScreenCapturer::onFatalError() {
315 if (repeatingTimer_ != nullptr) {
316 repeatingTimer_.Stop();
317 repeatingTimer_ = nullptr;
318 queue_ = nullptr;
319 queueController_ = nullptr;
320 }
321
322 if (session_ != nullptr) {
323 session_.Close();
324 item_ = nullptr;
325 item_closed_ = true;
326 }
327
328 if (frame_pool_ != nullptr) {
329 frame_pool_.Close();
330 }
331
332 mapped_texture_ = nullptr;
333 frame_pool_ = nullptr;
334 session_ = nullptr;
335 direct3d_device_ = nullptr;
336 d3d11_device_ = nullptr;
337
338 _fatalError = true;
339 if (_onFatalError) {
340 _onFatalError();
341 }
342 }
343
destroy()344 void UwpScreenCapturer::destroy() {
345 winrt::slim_lock_guard const guard(lock_);
346
347 _onFatalError = nullptr;
348 onFatalError();
349 }
350
OnFrame(std::vector<uint8_t> bytes,int width,int height)351 void UwpScreenCapturer::OnFrame(std::vector<uint8_t> bytes, int width, int height) {
352 if (_state != VideoState::Active) {
353 return;
354 }
355
356 int dst_width = width & ~1;
357 int dst_height = abs(height) & ~1;
358 int dst_stride_y = dst_width;
359 int dst_stride_uv = (dst_width + 1) / 2;
360
361 uint8_t* plane_y = bytes.data();
362 size_t videoFrameLength = bytes.size();
363 int32_t stride_y = width * 4;
364 uint8_t* plane_uv = plane_y + videoFrameLength;
365 int32_t stride_uv = stride_y / 2;
366
367 rtc::scoped_refptr<webrtc::I420Buffer> buffer = webrtc::I420Buffer::Create(
368 dst_width, dst_height, dst_stride_y, dst_stride_uv, dst_stride_uv);
369
370 const int conversionResult = libyuv::ConvertToI420(
371 plane_y, videoFrameLength, stride_y, plane_uv, stride_uv,
372 buffer.get()->MutableDataY(), buffer.get()->StrideY(),
373 buffer.get()->MutableDataU(), buffer.get()->StrideU(),
374 buffer.get()->MutableDataV(), buffer.get()->StrideV(),
375 0, 0, // No Cropping
376 width, height, dst_width, dst_height, libyuv::kRotate0,
377 libyuv::FOURCC_ARGB);
378 if (conversionResult < 0) {
379 RTC_LOG(LS_ERROR) << "Failed to convert capture frame from type "
380 << static_cast<int>(libyuv::FOURCC_ARGB) << "to I420.";
381 return;
382 }
383
384 webrtc::VideoFrame captureFrame =
385 webrtc::VideoFrame::Builder()
386 .set_video_frame_buffer(buffer)
387 .set_timestamp_rtp(0)
388 .set_timestamp_ms(rtc::TimeMillis())
389 .set_rotation(webrtc::kVideoRotation_0)
390 .build();
391
392 _sink->OnFrame(captureFrame);
393 }
394
395 } // namespace tgcalls
396