1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "remoting/host/client_session.h"
6
7 #include <algorithm>
8 #include <utility>
9
10 #include "base/bind.h"
11 #include "base/command_line.h"
12 #include "base/optional.h"
13 #include "base/single_thread_task_runner.h"
14 #include "base/strings/string_number_conversions.h"
15 #include "base/threading/thread_task_runner_handle.h"
16 #include "build/build_config.h"
17 #include "remoting/base/capabilities.h"
18 #include "remoting/base/constants.h"
19 #include "remoting/base/logging.h"
20 #include "remoting/base/session_options.h"
21 #include "remoting/host/action_executor.h"
22 #include "remoting/host/action_message_handler.h"
23 #include "remoting/host/audio_capturer.h"
24 #include "remoting/host/desktop_environment.h"
25 #include "remoting/host/file_transfer/file_transfer_message_handler.h"
26 #include "remoting/host/file_transfer/rtc_log_file_operations.h"
27 #include "remoting/host/host_extension_session.h"
28 #include "remoting/host/input_injector.h"
29 #include "remoting/host/keyboard_layout_monitor.h"
30 #include "remoting/host/mouse_shape_pump.h"
31 #include "remoting/host/screen_controls.h"
32 #include "remoting/host/screen_resolution.h"
33 #include "remoting/proto/control.pb.h"
34 #include "remoting/proto/event.pb.h"
35 #include "remoting/protocol/audio_stream.h"
36 #include "remoting/protocol/capability_names.h"
37 #include "remoting/protocol/client_stub.h"
38 #include "remoting/protocol/clipboard_thread_proxy.h"
39 #include "remoting/protocol/pairing_registry.h"
40 #include "remoting/protocol/peer_connection_controls.h"
41 #include "remoting/protocol/session.h"
42 #include "remoting/protocol/session_config.h"
43 #include "remoting/protocol/video_frame_pump.h"
44 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
45
46 namespace {
47 constexpr char kRtcLogTransferDataChannelPrefix[] = "rtc-log-transfer-";
48 } // namespace
49
50 namespace remoting {
51
52 using protocol::ActionRequest;
53
ClientSession(EventHandler * event_handler,std::unique_ptr<protocol::ConnectionToClient> connection,DesktopEnvironmentFactory * desktop_environment_factory,const DesktopEnvironmentOptions & desktop_environment_options,const base::TimeDelta & max_duration,scoped_refptr<protocol::PairingRegistry> pairing_registry,const std::vector<HostExtension * > & extensions)54 ClientSession::ClientSession(
55 EventHandler* event_handler,
56 std::unique_ptr<protocol::ConnectionToClient> connection,
57 DesktopEnvironmentFactory* desktop_environment_factory,
58 const DesktopEnvironmentOptions& desktop_environment_options,
59 const base::TimeDelta& max_duration,
60 scoped_refptr<protocol::PairingRegistry> pairing_registry,
61 const std::vector<HostExtension*>& extensions)
62 : event_handler_(event_handler),
63 desktop_environment_factory_(desktop_environment_factory),
64 desktop_environment_options_(desktop_environment_options),
65 input_tracker_(&host_input_filter_),
66 remote_input_filter_(&input_tracker_),
67 mouse_clamping_filter_(&remote_input_filter_),
68 pointer_lock_detector_(&mouse_clamping_filter_, this),
69 disable_input_filter_(&pointer_lock_detector_),
70 disable_clipboard_filter_(clipboard_echo_filter_.host_filter()),
71 client_clipboard_factory_(clipboard_echo_filter_.client_filter()),
72 max_duration_(max_duration),
73 pairing_registry_(pairing_registry),
74 connection_(std::move(connection)),
75 client_jid_(connection_->session()->jid()) {
76 connection_->session()->AddPlugin(&host_experiment_session_plugin_);
77 connection_->SetEventHandler(this);
78
79 // Create a manager for the configured extensions, if any.
80 extension_manager_.reset(new HostExtensionSessionManager(extensions, this));
81
82 #if defined(OS_WIN)
83 // LocalInputMonitorWin filters out an echo of the injected input before it
84 // reaches |remote_input_filter_|.
85 remote_input_filter_.SetExpectLocalEcho(false);
86 #endif // defined(OS_WIN)
87 }
88
~ClientSession()89 ClientSession::~ClientSession() {
90 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
91 DCHECK(!audio_stream_);
92 DCHECK(!desktop_environment_);
93 DCHECK(!input_injector_);
94 DCHECK(!screen_controls_);
95 DCHECK(!video_stream_);
96 }
97
NotifyClientResolution(const protocol::ClientResolution & resolution)98 void ClientSession::NotifyClientResolution(
99 const protocol::ClientResolution& resolution) {
100 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
101 DCHECK(resolution.dips_width() >= 0 && resolution.dips_height() >= 0);
102 VLOG(1) << "Received ClientResolution (dips_width="
103 << resolution.dips_width() << ", dips_height="
104 << resolution.dips_height() << ")";
105
106 if (!screen_controls_)
107 return;
108
109 webrtc::DesktopSize client_size(resolution.dips_width(),
110 resolution.dips_height());
111 if (connection_->session()->config().protocol() ==
112 protocol::SessionConfig::Protocol::WEBRTC) {
113 // When using WebRTC round down the dimensions to multiple of 2. Otherwise
114 // the dimensions will be rounded on the receiver, which will cause blurring
115 // due to scaling. The resulting size is still close to the client size and
116 // will fit on the client's screen without scaling.
117 // TODO(sergeyu): Make WebRTC handle odd dimensions properly.
118 // crbug.com/636071
119 client_size.set(client_size.width() & (~1), client_size.height() & (~1));
120 }
121
122 // Try to match the client's resolution.
123 // TODO(sergeyu): Pass clients DPI to the resizer.
124 screen_controls_->SetScreenResolution(ScreenResolution(
125 client_size, webrtc::DesktopVector(kDefaultDpi, kDefaultDpi)));
126 }
127
ControlVideo(const protocol::VideoControl & video_control)128 void ClientSession::ControlVideo(const protocol::VideoControl& video_control) {
129 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
130
131 // Note that |video_stream_| may be null, depending upon whether
132 // extensions choose to wrap or "steal" the video capturer or encoder.
133 if (video_control.has_enable()) {
134 VLOG(1) << "Received VideoControl (enable="
135 << video_control.enable() << ")";
136 pause_video_ = !video_control.enable();
137 if (video_stream_)
138 video_stream_->Pause(pause_video_);
139 }
140 if (video_control.has_lossless_encode()) {
141 VLOG(1) << "Received VideoControl (lossless_encode="
142 << video_control.lossless_encode() << ")";
143 lossless_video_encode_ = video_control.lossless_encode();
144 if (video_stream_)
145 video_stream_->SetLosslessEncode(lossless_video_encode_);
146 }
147 if (video_control.has_lossless_color()) {
148 VLOG(1) << "Received VideoControl (lossless_color="
149 << video_control.lossless_color() << ")";
150 lossless_video_color_ = video_control.lossless_color();
151 if (video_stream_)
152 video_stream_->SetLosslessColor(lossless_video_color_);
153 }
154 }
155
ControlAudio(const protocol::AudioControl & audio_control)156 void ClientSession::ControlAudio(const protocol::AudioControl& audio_control) {
157 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
158
159 if (audio_control.has_enable()) {
160 VLOG(1) << "Received AudioControl (enable="
161 << audio_control.enable() << ")";
162 if (audio_stream_)
163 audio_stream_->Pause(!audio_control.enable());
164 }
165 }
166
SetCapabilities(const protocol::Capabilities & capabilities)167 void ClientSession::SetCapabilities(
168 const protocol::Capabilities& capabilities) {
169 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
170
171 // Ignore all the messages but the 1st one.
172 if (client_capabilities_) {
173 LOG(WARNING) << "protocol::Capabilities has been received already.";
174 return;
175 }
176
177 // Compute the set of capabilities supported by both client and host.
178 client_capabilities_ = std::make_unique<std::string>();
179 if (capabilities.has_capabilities())
180 *client_capabilities_ = capabilities.capabilities();
181 capabilities_ = IntersectCapabilities(*client_capabilities_,
182 host_capabilities_);
183 extension_manager_->OnNegotiatedCapabilities(
184 connection_->client_stub(), capabilities_);
185
186 if (HasCapability(capabilities_, protocol::kFileTransferCapability)) {
187 data_channel_manager_.RegisterCreateHandlerCallback(
188 kFileTransferDataChannelPrefix,
189 base::BindRepeating(&ClientSession::CreateFileTransferMessageHandler,
190 base::Unretained(this)));
191 }
192
193 if (HasCapability(capabilities_, protocol::kRtcLogTransferCapability)) {
194 data_channel_manager_.RegisterCreateHandlerCallback(
195 kRtcLogTransferDataChannelPrefix,
196 base::BindRepeating(&ClientSession::CreateRtcLogTransferMessageHandler,
197 base::Unretained(this)));
198 }
199
200 std::vector<ActionRequest::Action> supported_actions;
201 if (HasCapability(capabilities_, protocol::kSendAttentionSequenceAction))
202 supported_actions.push_back(ActionRequest::SEND_ATTENTION_SEQUENCE);
203 if (HasCapability(capabilities_, protocol::kLockWorkstationAction))
204 supported_actions.push_back(ActionRequest::LOCK_WORKSTATION);
205
206 if (supported_actions.size() > 0) {
207 // Register the action message handler.
208 data_channel_manager_.RegisterCreateHandlerCallback(
209 kActionDataChannelPrefix,
210 base::BindRepeating(&ClientSession::CreateActionMessageHandler,
211 base::Unretained(this),
212 std::move(supported_actions)));
213 }
214
215 VLOG(1) << "Client capabilities: " << *client_capabilities_;
216
217 desktop_environment_->SetCapabilities(capabilities_);
218 }
219
RequestPairing(const protocol::PairingRequest & pairing_request)220 void ClientSession::RequestPairing(
221 const protocol::PairingRequest& pairing_request) {
222 if (pairing_registry_.get() && pairing_request.has_client_name()) {
223 protocol::PairingRegistry::Pairing pairing =
224 pairing_registry_->CreatePairing(pairing_request.client_name());
225 protocol::PairingResponse pairing_response;
226 pairing_response.set_client_id(pairing.client_id());
227 pairing_response.set_shared_secret(pairing.shared_secret());
228 connection_->client_stub()->SetPairingResponse(pairing_response);
229 }
230 }
231
DeliverClientMessage(const protocol::ExtensionMessage & message)232 void ClientSession::DeliverClientMessage(
233 const protocol::ExtensionMessage& message) {
234 if (message.has_type()) {
235 if (extension_manager_->OnExtensionMessage(message))
236 return;
237
238 DLOG(INFO) << "Unexpected message received: " << message.type() << ": "
239 << message.data();
240 }
241 }
242
SelectDesktopDisplay(const protocol::SelectDesktopDisplayRequest & select_display)243 void ClientSession::SelectDesktopDisplay(
244 const protocol::SelectDesktopDisplayRequest& select_display) {
245 LOG(INFO) << "SelectDesktopDisplay "
246 << "'" << select_display.id() << "'";
247
248 // Parse the string with the selected display.
249 int id = webrtc::kInvalidScreenId;
250 if (select_display.id() == "all") {
251 id = webrtc::kFullDesktopScreenId;
252 } else {
253 if (!base::StringToInt(select_display.id().c_str(), &id)) {
254 LOG(ERROR) << " Unable to parse display id "
255 << "'" << select_display.id() << "'";
256 id = webrtc::kInvalidScreenId;
257 }
258 if (!desktop_display_info_.GetDisplayInfo(id)) {
259 LOG(ERROR) << " Invalid display id "
260 << "'" << select_display.id() << "'";
261 id = webrtc::kInvalidScreenId;
262 }
263 }
264 // Don't allow requests for fullscreen if not supported by the current
265 // display configuration.
266 if (!can_capture_full_desktop_ && id == webrtc::kFullDesktopScreenId) {
267 LOG(ERROR) << " Full desktop not supported";
268 id = webrtc::kInvalidScreenId;
269 }
270 // Fall back to default capture config if invalid request.
271 if (id == webrtc::kInvalidScreenId) {
272 LOG(ERROR) << " Invalid display specification, falling back to default";
273 id = can_capture_full_desktop_ ? webrtc::kFullDesktopScreenId : 0;
274 }
275
276 if (show_display_id_ == id) {
277 LOG(INFO) << " Display " << id << " is already selected. Ignoring";
278 return;
279 }
280
281 video_stream_->SelectSource(id);
282 show_display_id_ = id;
283
284 // If the old and new displays are the different sizes, then SelectSource()
285 // will trigger an OnVideoSizeChanged() message which will update the mouse
286 // filters.
287 // However, if the old and new displays are the exact same size, then the
288 // video size message will not be generated (because the size of the video
289 // has not changed). But we still need to update the mouse clamping filter
290 // with the new display origin, so we update that directly.
291 const DisplayGeometry* oldGeo =
292 desktop_display_info_.GetDisplayInfo(show_display_id_);
293 const DisplayGeometry* newGeo = desktop_display_info_.GetDisplayInfo(id);
294 if (oldGeo != nullptr && newGeo != nullptr) {
295 if (oldGeo->width == newGeo->width && oldGeo->height == newGeo->height) {
296 UpdateMouseClampingFilterOffset();
297 }
298 }
299 }
300
ControlPeerConnection(const protocol::PeerConnectionParameters & parameters)301 void ClientSession::ControlPeerConnection(
302 const protocol::PeerConnectionParameters& parameters) {
303 if (!connection_->peer_connection_controls()) {
304 return;
305 }
306 base::Optional<int> min_bitrate_bps;
307 base::Optional<int> max_bitrate_bps;
308 bool set_preferred_bitrates = false;
309 if (parameters.has_preferred_min_bitrate_bps()) {
310 min_bitrate_bps = parameters.preferred_min_bitrate_bps();
311 set_preferred_bitrates = true;
312 }
313 if (parameters.has_preferred_max_bitrate_bps()) {
314 max_bitrate_bps = parameters.preferred_max_bitrate_bps();
315 set_preferred_bitrates = true;
316 }
317 if (set_preferred_bitrates) {
318 connection_->peer_connection_controls()->SetPreferredBitrates(
319 min_bitrate_bps, max_bitrate_bps);
320 }
321
322 if (parameters.request_ice_restart()) {
323 connection_->peer_connection_controls()->RequestIceRestart();
324 }
325
326 if (parameters.request_sdp_restart()) {
327 connection_->peer_connection_controls()->RequestSdpRestart();
328 }
329 }
330
OnConnectionAuthenticating()331 void ClientSession::OnConnectionAuthenticating() {
332 event_handler_->OnSessionAuthenticating(this);
333 }
334
OnConnectionAuthenticated()335 void ClientSession::OnConnectionAuthenticated() {
336 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
337 DCHECK(!audio_stream_);
338 DCHECK(!desktop_environment_);
339 DCHECK(!input_injector_);
340 DCHECK(!screen_controls_);
341 DCHECK(!video_stream_);
342
343 is_authenticated_ = true;
344
345 desktop_display_info_.Reset();
346
347 if (max_duration_ > base::TimeDelta()) {
348 max_duration_timer_.Start(
349 FROM_HERE, max_duration_,
350 base::BindOnce(&ClientSession::DisconnectSession,
351 base::Unretained(this), protocol::MAX_SESSION_LENGTH));
352 }
353
354 // Notify EventHandler.
355 event_handler_->OnSessionAuthenticated(this);
356
357 const SessionOptions session_options(
358 host_experiment_session_plugin_.configuration());
359
360 connection_->ApplySessionOptions(session_options);
361
362 DesktopEnvironmentOptions options = desktop_environment_options_;
363 options.ApplySessionOptions(session_options);
364 // Create the desktop environment. Drop the connection if it could not be
365 // created for any reason (for instance the curtain could not initialize).
366 desktop_environment_ =
367 desktop_environment_factory_->Create(weak_factory_.GetWeakPtr(), options);
368 if (!desktop_environment_) {
369 DisconnectSession(protocol::HOST_CONFIGURATION_ERROR);
370 return;
371 }
372
373 // Connect host stub.
374 connection_->set_host_stub(this);
375
376 // Collate the set of capabilities to offer the client, if it supports them.
377 host_capabilities_ = desktop_environment_->GetCapabilities();
378 if (!host_capabilities_.empty())
379 host_capabilities_.append(" ");
380 host_capabilities_.append(extension_manager_->GetCapabilities());
381 if (!host_capabilities_.empty())
382 host_capabilities_.append(" ");
383 host_capabilities_.append(protocol::kRtcLogTransferCapability);
384 host_capabilities_.append(" ");
385 host_capabilities_.append(protocol::kWebrtcIceSdpRestartAction);
386
387 // Create the object that controls the screen resolution.
388 screen_controls_ = desktop_environment_->CreateScreenControls();
389
390 // Create the event executor.
391 input_injector_ = desktop_environment_->CreateInputInjector();
392
393 // Connect the host input stubs.
394 connection_->set_input_stub(&disable_input_filter_);
395 host_input_filter_.set_input_stub(input_injector_.get());
396
397 // Connect the clipboard stubs.
398 connection_->set_clipboard_stub(&disable_clipboard_filter_);
399 clipboard_echo_filter_.set_host_stub(input_injector_.get());
400 clipboard_echo_filter_.set_client_stub(connection_->client_stub());
401 }
402
CreateMediaStreams()403 void ClientSession::CreateMediaStreams() {
404 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
405
406 // Create a VideoStream to pump frames from the capturer to the client.
407 auto composer = desktop_environment_->CreateComposingVideoCapturer();
408 if (composer) {
409 desktop_and_cursor_composer_ = composer->GetWeakPtr();
410 video_stream_ = connection_->StartVideoStream(std::move(composer));
411 } else {
412 video_stream_ = connection_->StartVideoStream(
413 desktop_environment_->CreateVideoCapturer());
414 }
415
416 // Create a AudioStream to pump audio from the capturer to the client.
417 std::unique_ptr<protocol::AudioSource> audio_capturer =
418 desktop_environment_->CreateAudioCapturer();
419 if (audio_capturer) {
420 audio_stream_ = connection_->StartAudioStream(std::move(audio_capturer));
421 }
422
423 video_stream_->SetObserver(this);
424
425 // Apply video-control parameters to the new stream.
426 video_stream_->SetLosslessEncode(lossless_video_encode_);
427 video_stream_->SetLosslessColor(lossless_video_color_);
428
429 // Pause capturing if necessary.
430 video_stream_->Pause(pause_video_);
431
432 if (event_timestamp_source_for_tests_)
433 video_stream_->SetEventTimestampsSource(event_timestamp_source_for_tests_);
434 }
435
OnConnectionChannelsConnected()436 void ClientSession::OnConnectionChannelsConnected() {
437 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
438
439 DCHECK(!channels_connected_);
440 channels_connected_ = true;
441
442 // Negotiate capabilities with the client.
443 VLOG(1) << "Host capabilities: " << host_capabilities_;
444 protocol::Capabilities capabilities;
445 capabilities.set_capabilities(host_capabilities_);
446 connection_->client_stub()->SetCapabilities(capabilities);
447
448 // Start the event executor.
449 input_injector_->Start(CreateClipboardProxy());
450 SetDisableInputs(false);
451
452 // Create MouseShapePump to send mouse cursor shape.
453 mouse_shape_pump_.reset(
454 new MouseShapePump(desktop_environment_->CreateMouseCursorMonitor(),
455 connection_->client_stub()));
456 mouse_shape_pump_->SetMouseCursorMonitorCallback(this);
457
458 // Create KeyboardLayoutMonitor to send keyboard layout.
459 // Unretained is sound because callback will never be called after
460 // |keyboard_layout_monitor_| has been destroyed, and |connection_| (which
461 // owns the client stub) is guaranteed to outlive |keyboard_layout_monitor_|.
462 keyboard_layout_monitor_ = desktop_environment_->CreateKeyboardLayoutMonitor(
463 base::BindRepeating(&protocol::KeyboardLayoutStub::SetKeyboardLayout,
464 base::Unretained(connection_->client_stub())));
465 keyboard_layout_monitor_->Start();
466
467 if (pending_video_layout_message_) {
468 connection_->client_stub()->SetVideoLayout(*pending_video_layout_message_);
469 pending_video_layout_message_.reset();
470 }
471
472 // Notify the event handler that all our channels are now connected.
473 event_handler_->OnSessionChannelsConnected(this);
474 }
475
OnConnectionClosed(protocol::ErrorCode error)476 void ClientSession::OnConnectionClosed(protocol::ErrorCode error) {
477 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
478
479 HOST_LOG << "Client disconnected: " << client_jid_ << "; error = " << error;
480
481 // Ignore any further callbacks.
482 weak_factory_.InvalidateWeakPtrs();
483
484 // If the client never authenticated then the session failed.
485 if (!is_authenticated_)
486 event_handler_->OnSessionAuthenticationFailed(this);
487
488 // Ensure that any pressed keys or buttons are released.
489 input_tracker_.ReleaseAll();
490
491 // Stop components access the client, audio or video stubs, which are no
492 // longer valid once ConnectionToClient calls OnConnectionClosed().
493 desktop_and_cursor_composer_.reset();
494 audio_stream_.reset();
495 mouse_shape_pump_.reset();
496 video_stream_.reset();
497 keyboard_layout_monitor_.reset();
498 client_clipboard_factory_.InvalidateWeakPtrs();
499 input_injector_.reset();
500 screen_controls_.reset();
501 desktop_environment_.reset();
502
503 // Notify the ChromotingHost that this client is disconnected.
504 event_handler_->OnSessionClosed(this);
505 }
506
OnTransportProtocolChange(const std::string & protocol)507 void ClientSession::OnTransportProtocolChange(const std::string& protocol) {
508 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
509 HOST_LOG << "Transport protocol: " << protocol;
510 protocol::TransportInfo transport_info;
511 transport_info.set_protocol(protocol);
512 connection_->client_stub()->SetTransportInfo(transport_info);
513 }
514
OnRouteChange(const std::string & channel_name,const protocol::TransportRoute & route)515 void ClientSession::OnRouteChange(const std::string& channel_name,
516 const protocol::TransportRoute& route) {
517 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
518 event_handler_->OnSessionRouteChange(this, channel_name, route);
519 }
520
OnIncomingDataChannel(const std::string & channel_name,std::unique_ptr<protocol::MessagePipe> pipe)521 void ClientSession::OnIncomingDataChannel(
522 const std::string& channel_name,
523 std::unique_ptr<protocol::MessagePipe> pipe) {
524 data_channel_manager_.OnIncomingDataChannel(channel_name, std::move(pipe));
525 }
526
client_jid() const527 const std::string& ClientSession::client_jid() const {
528 return client_jid_;
529 }
530
DisconnectSession(protocol::ErrorCode error)531 void ClientSession::DisconnectSession(protocol::ErrorCode error) {
532 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
533 DCHECK(connection_.get());
534
535 max_duration_timer_.Stop();
536
537 // This triggers OnConnectionClosed(), and the session may be destroyed
538 // as the result, so this call must be the last in this method.
539 connection_->Disconnect(error);
540 }
541
OnLocalKeyPressed(uint32_t usb_keycode)542 void ClientSession::OnLocalKeyPressed(uint32_t usb_keycode) {
543 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
544 bool is_local = remote_input_filter_.LocalKeyPressed(usb_keycode);
545 if (is_local && desktop_environment_options_.terminate_upon_input())
546 DisconnectSession(protocol::OK);
547 }
548
OnLocalPointerMoved(const webrtc::DesktopVector & position,ui::EventType type)549 void ClientSession::OnLocalPointerMoved(const webrtc::DesktopVector& position,
550 ui::EventType type) {
551 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
552 bool is_local = remote_input_filter_.LocalPointerMoved(position, type);
553 if (is_local && desktop_environment_options_.terminate_upon_input())
554 DisconnectSession(protocol::OK);
555 }
556
SetDisableInputs(bool disable_inputs)557 void ClientSession::SetDisableInputs(bool disable_inputs) {
558 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
559
560 if (disable_inputs)
561 input_tracker_.ReleaseAll();
562
563 disable_input_filter_.set_enabled(!disable_inputs);
564 disable_clipboard_filter_.set_enabled(!disable_inputs);
565 }
566
desktop_session_id() const567 uint32_t ClientSession::desktop_session_id() const {
568 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
569 DCHECK(desktop_environment_);
570 return desktop_environment_->GetDesktopSessionId();
571 }
572
session_control()573 ClientSessionControl* ClientSession::session_control() {
574 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
575 return this;
576 }
577
OnPointerLockChanged(bool active)578 void ClientSession::OnPointerLockChanged(bool active) {
579 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
580 if (desktop_and_cursor_composer_)
581 desktop_and_cursor_composer_->SetComposeEnabled(active);
582 }
583
OnMouseCursor(webrtc::MouseCursor * mouse_cursor)584 void ClientSession::OnMouseCursor(webrtc::MouseCursor* mouse_cursor) {
585 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
586 if (desktop_and_cursor_composer_)
587 desktop_and_cursor_composer_->SetMouseCursor(mouse_cursor);
588 }
589
OnMouseCursorPosition(const webrtc::DesktopVector & position)590 void ClientSession::OnMouseCursorPosition(
591 const webrtc::DesktopVector& position) {
592 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
593 if (desktop_and_cursor_composer_)
594 desktop_and_cursor_composer_->SetMouseCursorPosition(position);
595 }
596
RegisterCreateHandlerCallbackForTesting(const std::string & prefix,protocol::DataChannelManager::CreateHandlerCallback constructor)597 void ClientSession::RegisterCreateHandlerCallbackForTesting(
598 const std::string& prefix,
599 protocol::DataChannelManager::CreateHandlerCallback constructor) {
600 data_channel_manager_.RegisterCreateHandlerCallback(
601 prefix, std::move(constructor));
602 }
603
SetEventTimestampsSourceForTests(scoped_refptr<protocol::InputEventTimestampsSource> event_timestamp_source)604 void ClientSession::SetEventTimestampsSourceForTests(
605 scoped_refptr<protocol::InputEventTimestampsSource>
606 event_timestamp_source) {
607 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
608 event_timestamp_source_for_tests_ = event_timestamp_source;
609 if (video_stream_)
610 video_stream_->SetEventTimestampsSource(event_timestamp_source_for_tests_);
611 }
612
CreateClipboardProxy()613 std::unique_ptr<protocol::ClipboardStub> ClientSession::CreateClipboardProxy() {
614 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
615 return std::make_unique<protocol::ClipboardThreadProxy>(
616 client_clipboard_factory_.GetWeakPtr(),
617 base::ThreadTaskRunnerHandle::Get());
618 }
619
SetMouseClampingFilter(const DisplaySize & size)620 void ClientSession::SetMouseClampingFilter(const DisplaySize& size) {
621 UpdateMouseClampingFilterOffset();
622
623 mouse_clamping_filter_.set_output_size(size.WidthAsPixels(),
624 size.HeightAsPixels());
625
626 switch (connection_->session()->config().protocol()) {
627 case protocol::SessionConfig::Protocol::ICE:
628 mouse_clamping_filter_.set_input_size(size.WidthAsPixels(),
629 size.HeightAsPixels());
630 break;
631
632 case protocol::SessionConfig::Protocol::WEBRTC: {
633 #if defined(OS_APPLE)
634 mouse_clamping_filter_.set_input_size(size.WidthAsPixels(),
635 size.HeightAsPixels());
636 #else
637 // When using the WebRTC protocol the client sends mouse coordinates in
638 // DIPs, while InputInjector expects them in physical pixels.
639 // TODO(sergeyu): Fix InputInjector implementations to use DIPs as well.
640 mouse_clamping_filter_.set_input_size(size.WidthAsDips(),
641 size.HeightAsDips());
642 #endif // defined(OS_APPLE)
643 }
644 }
645 }
646
UpdateMouseClampingFilterOffset()647 void ClientSession::UpdateMouseClampingFilterOffset() {
648 if (show_display_id_ == webrtc::kInvalidScreenId)
649 return;
650
651 webrtc::DesktopVector origin;
652 origin = desktop_display_info_.CalcDisplayOffset(show_display_id_);
653 mouse_clamping_filter_.set_output_offset(origin);
654 }
655
OnVideoSizeChanged(protocol::VideoStream * video_stream,const webrtc::DesktopSize & size_px,const webrtc::DesktopVector & dpi)656 void ClientSession::OnVideoSizeChanged(protocol::VideoStream* video_stream,
657 const webrtc::DesktopSize& size_px,
658 const webrtc::DesktopVector& dpi) {
659 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
660 LOG(INFO) << "ClientSession::OnVideoSizeChanged";
661 DisplaySize size =
662 DisplaySize::FromPixels(size_px.width(), size_px.height(), dpi.x());
663 LOG(INFO) << " DisplaySize: " << size;
664
665 // The first video size message that we receive from WebRtc is the full
666 // desktop size (if supported). If full desktop capture is not supported,
667 // then this will be the size of the default display.
668 if (default_webrtc_desktop_size_.IsEmpty()) {
669 default_webrtc_desktop_size_ = size;
670 LOG(INFO) << " display id " << show_display_id_;
671 DCHECK(show_display_id_ == webrtc::kInvalidScreenId);
672 LOG(INFO) << " Recording default webrtc capture size "
673 << default_webrtc_desktop_size_;
674 }
675 webrtc_capture_size_ = size;
676
677 SetMouseClampingFilter(size);
678
679 // Record default DPI in case a display reports 0 for DPI.
680 default_x_dpi_ = dpi.x();
681 default_y_dpi_ = dpi.y();
682 if (dpi.x() != dpi.y()) {
683 LOG(WARNING) << "Mismatch x,y dpi. x=" << dpi.x() << " y=" << dpi.y();
684 }
685
686 if (connection_->session()->config().protocol() !=
687 protocol::SessionConfig::Protocol::WEBRTC) {
688 return;
689 }
690
691 // Generate and send VideoLayout message.
692 protocol::VideoLayout layout;
693 protocol::VideoTrackLayout* video_track = layout.add_video_track();
694 video_track->set_position_x(0);
695 video_track->set_position_y(0);
696 video_track->set_width(size.WidthAsDips());
697 video_track->set_height(size.HeightAsDips());
698 video_track->set_x_dpi(dpi.x());
699 video_track->set_y_dpi(dpi.y());
700
701 // VideoLayout can be sent only after the control channel is connected.
702 // TODO(sergeyu): Change client_stub() implementation to allow queuing
703 // while connection is being established.
704 if (channels_connected_) {
705 connection_->client_stub()->SetVideoLayout(layout);
706 } else {
707 pending_video_layout_message_.reset(new protocol::VideoLayout(layout));
708 }
709 }
710
OnDesktopDisplayChanged(std::unique_ptr<protocol::VideoLayout> displays)711 void ClientSession::OnDesktopDisplayChanged(
712 std::unique_ptr<protocol::VideoLayout> displays) {
713 LOG(INFO) << "ClientSession::OnDesktopDisplayChanged";
714 // Scan display list to calculate the full desktop size.
715 int min_x = 0;
716 int max_x = 0;
717 int min_y = 0;
718 int max_y = 0;
719 int dpi_x = 0;
720 int dpi_y = 0;
721 LOG(INFO) << " Scanning display info... (dips)";
722 for (int display_id = 0; display_id < displays->video_track_size();
723 display_id++) {
724 protocol::VideoTrackLayout track = displays->video_track(display_id);
725 LOG(INFO) << " #" << display_id << " : " << track.position_x() << ","
726 << track.position_y() << " " << track.width() << "x"
727 << track.height() << " [" << track.x_dpi() << "," << track.y_dpi()
728 << "]";
729 if (dpi_x == 0)
730 dpi_x = track.x_dpi();
731 if (dpi_y == 0)
732 dpi_y = track.y_dpi();
733
734 int x = track.position_x();
735 int y = track.position_y();
736 min_x = std::min(x, min_x);
737 min_y = std::min(y, min_y);
738 max_x = std::max(x + track.width(), max_x);
739 max_y = std::max(y + track.height(), max_y);
740 }
741
742 // TODO(garykac): Investigate why these DPI values are 0 for some users.
743 if (dpi_x == 0)
744 dpi_x = default_x_dpi_;
745 if (dpi_y == 0)
746 dpi_y = default_y_dpi_;
747
748 // Calc desktop scaled geometry (in DIPs)
749 // See comment in OnVideoSizeChanged() for details.
750 const webrtc::DesktopSize size(max_x - min_x, max_y - min_y);
751
752 // If this is our first message, then we need to determine if the current
753 // display configuration supports capturing the entire desktop.
754 LOG(INFO) << " Webrtc desktop size " << default_webrtc_desktop_size_;
755 if (show_display_id_ == webrtc::kInvalidScreenId) {
756 #if defined(OS_APPLE)
757 // On MacOS, there are situations where webrtc cannot capture the entire
758 // desktop (e.g, when there are displays with different DPIs). We detect
759 // this situation by comparing the full desktop size (calculated above
760 // from the displays) and the size of the initial webrtc capture (which
761 // defaults to the full desktop if supported).
762 if (size.width() == default_webrtc_desktop_size_.WidthAsDips() &&
763 size.height() == default_webrtc_desktop_size_.HeightAsDips()) {
764 LOG(INFO) << " Full desktop capture supported.";
765 can_capture_full_desktop_ = true;
766 } else {
767 LOG(INFO)
768 << " This configuration does not support full desktop capture.";
769 can_capture_full_desktop_ = false;
770 }
771 #else
772 // Windows/Linux can capture full desktop if multiple displays.
773 can_capture_full_desktop_ = true;
774 #endif // defined(OS_APPLE)
775 }
776
777 // Generate and send VideoLayout message.
778 protocol::VideoLayout layout;
779 layout.set_supports_full_desktop_capture(can_capture_full_desktop_);
780 protocol::VideoTrackLayout* video_track;
781
782 // The first layout must be the current webrtc capture size.
783 // This is required because we reuse the same message for both
784 // VideoSizeChanged (which is used to scale mouse coordinates)
785 // and DisplayDesktopChanged.
786 video_track = layout.add_video_track();
787 video_track->set_position_x(0);
788 video_track->set_position_y(0);
789 video_track->set_width(webrtc_capture_size_.WidthAsDips());
790 video_track->set_height(webrtc_capture_size_.HeightAsDips());
791 video_track->set_x_dpi(dpi_x);
792 video_track->set_y_dpi(dpi_y);
793 LOG(INFO) << " Webrtc capture size (DIPS) = 0,0 "
794 << default_webrtc_desktop_size_;
795
796 // Add raw geometry for entire desktop (in DIPs).
797 video_track = layout.add_video_track();
798 video_track->set_position_x(0);
799 video_track->set_position_y(0);
800 video_track->set_width(size.width());
801 video_track->set_height(size.height());
802 video_track->set_x_dpi(dpi_x);
803 video_track->set_y_dpi(dpi_y);
804 LOG(INFO) << " Full Desktop (DIPS) = 0,0 " << size.width() << "x"
805 << size.height() << " [" << dpi_x << "," << dpi_y << "]";
806
807 // Add a VideoTrackLayout entry for each separate display.
808 desktop_display_info_.Reset();
809 for (int display_id = 0; display_id < displays->video_track_size();
810 display_id++) {
811 protocol::VideoTrackLayout display = displays->video_track(display_id);
812 desktop_display_info_.AddDisplayFrom(display);
813
814 protocol::VideoTrackLayout* video_track = layout.add_video_track();
815 video_track->CopyFrom(display);
816 LOG(INFO) << " Display " << display_id << " = " << display.position_x()
817 << "," << display.position_y() << " " << display.width() << "x"
818 << display.height() << " [" << display.x_dpi() << ","
819 << display.y_dpi() << "]";
820 }
821
822 // Set the display id, if this is the first message being processed.
823 if (show_display_id_ == webrtc::kInvalidScreenId) {
824 if (can_capture_full_desktop_) {
825 show_display_id_ = webrtc::kFullDesktopScreenId;
826 } else {
827 // Select the default display.
828 protocol::SelectDesktopDisplayRequest req;
829 req.set_id("0");
830 SelectDesktopDisplay(req);
831 }
832 }
833
834 // We need to update the input filters whenever the displays change.
835 DisplaySize display_size =
836 DisplaySize::FromPixels(size.width(), size.height(), default_x_dpi_);
837 SetMouseClampingFilter(display_size);
838
839 connection_->client_stub()->SetVideoLayout(layout);
840 }
841
CreateFileTransferMessageHandler(const std::string & channel_name,std::unique_ptr<protocol::MessagePipe> pipe)842 void ClientSession::CreateFileTransferMessageHandler(
843 const std::string& channel_name,
844 std::unique_ptr<protocol::MessagePipe> pipe) {
845 // FileTransferMessageHandler manages its own lifetime and is tied to the
846 // lifetime of |pipe|. Once |pipe| is closed, this instance will be cleaned
847 // up.
848 new FileTransferMessageHandler(channel_name, std::move(pipe),
849 desktop_environment_->CreateFileOperations());
850 }
851
CreateRtcLogTransferMessageHandler(const std::string & channel_name,std::unique_ptr<protocol::MessagePipe> pipe)852 void ClientSession::CreateRtcLogTransferMessageHandler(
853 const std::string& channel_name,
854 std::unique_ptr<protocol::MessagePipe> pipe) {
855 new FileTransferMessageHandler(
856 channel_name, std::move(pipe),
857 std::make_unique<RtcLogFileOperations>(connection_.get()));
858 }
859
CreateActionMessageHandler(std::vector<ActionRequest::Action> capabilities,const std::string & channel_name,std::unique_ptr<protocol::MessagePipe> pipe)860 void ClientSession::CreateActionMessageHandler(
861 std::vector<ActionRequest::Action> capabilities,
862 const std::string& channel_name,
863 std::unique_ptr<protocol::MessagePipe> pipe) {
864 std::unique_ptr<ActionExecutor> action_executor =
865 desktop_environment_->CreateActionExecutor();
866 if (!action_executor)
867 return;
868
869 // ActionMessageHandler manages its own lifetime and is tied to the lifetime
870 // of |pipe|. Once |pipe| is closed, this instance will be cleaned up.
871 new ActionMessageHandler(channel_name, capabilities, std::move(pipe),
872 std::move(action_executor));
873 }
874
875 } // namespace remoting
876