1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "components/exo/keyboard.h"
6
7 #include "ash/keyboard/ui/keyboard_ui_controller.h"
8 #include "ash/keyboard/ui/keyboard_util.h"
9 #include "ash/public/cpp/app_types.h"
10 #include "ash/public/cpp/keyboard/keyboard_controller.h"
11 #include "ash/shell.h"
12 #include "base/bind.h"
13 #include "base/threading/thread_task_runner_handle.h"
14 #include "components/exo/input_trace.h"
15 #include "components/exo/keyboard_delegate.h"
16 #include "components/exo/keyboard_device_configuration_delegate.h"
17 #include "components/exo/keyboard_modifiers.h"
18 #include "components/exo/seat.h"
19 #include "components/exo/shell_surface.h"
20 #include "components/exo/shell_surface_util.h"
21 #include "components/exo/surface.h"
22 #include "components/exo/wm_helper.h"
23 #include "components/exo/xkb_tracker.h"
24 #include "ui/aura/client/aura_constants.h"
25 #include "ui/aura/client/focus_client.h"
26 #include "ui/aura/window.h"
27 #include "ui/base/ime/input_method.h"
28 #include "ui/events/base_event_utils.h"
29 #include "ui/events/event.h"
30 #include "ui/views/widget/widget.h"
31
32 namespace exo {
33 namespace {
34
35 // Delay until a key state change expected to be acknowledged is expired.
36 const int kExpirationDelayForPendingKeyAcksMs = 1000;
37
38 // The accelerator keys reserved to be processed by chrome.
39 const struct {
40 ui::KeyboardCode keycode;
41 int modifiers;
42 } kReservedAccelerators[] = {
43 {ui::VKEY_F13, ui::EF_NONE},
44 {ui::VKEY_I, ui::EF_SHIFT_DOWN | ui::EF_ALT_DOWN},
45 {ui::VKEY_Z, ui::EF_CONTROL_DOWN | ui::EF_ALT_DOWN}};
46
ProcessAccelerator(Surface * surface,const ui::KeyEvent * event)47 bool ProcessAccelerator(Surface* surface, const ui::KeyEvent* event) {
48 views::Widget* widget =
49 views::Widget::GetTopLevelWidgetForNativeView(surface->window());
50 if (widget) {
51 views::FocusManager* focus_manager = widget->GetFocusManager();
52 return focus_manager->ProcessAccelerator(ui::Accelerator(*event));
53 }
54 return false;
55 }
56
ConsumedByIme(Surface * focus,const ui::KeyEvent * event)57 bool ConsumedByIme(Surface* focus, const ui::KeyEvent* event) {
58 // When IME is blocked, Exo can handle any key events.
59 if (WMHelper::GetInstance()->IsImeBlocked(focus->window()))
60 return false;
61
62 // Check if IME consumed the event, to avoid it to be doubly processed.
63 // First let us see whether IME is active and is in text input mode.
64 views::Widget* widget =
65 views::Widget::GetTopLevelWidgetForNativeView(focus->window());
66 ui::InputMethod* ime = widget ? widget->GetInputMethod() : nullptr;
67 if (!ime || ime->GetTextInputType() == ui::TEXT_INPUT_TYPE_NONE ||
68 ime->GetTextInputType() == ui::TEXT_INPUT_TYPE_NULL) {
69 return false;
70 }
71
72 // Case 1:
73 // When IME ate a key event but did not emit character insertion event yet
74 // (e.g., when it is still showing a candidate list UI to the user,) the
75 // consumed key event is re-sent after masked |key_code| by VKEY_PROCESSKEY.
76 if (event->key_code() == ui::VKEY_PROCESSKEY)
77 return true;
78
79 // Except for PROCESSKEY, never discard "key-up" events. A keydown not paired
80 // by a keyup can trigger a never-ending key repeat in the client, which can
81 // never be desirable.
82 if (event->type() == ui::ET_KEY_RELEASED)
83 return false;
84
85 // Case 2:
86 // When IME ate a key event and generated a single character input, it leaves
87 // the key event as-is, and in addition calls the active ui::TextInputClient's
88 // InsertChar() method. (In our case, arc::ArcImeService::InsertChar()).
89 //
90 // In Chrome OS (and Web) convention, the two calls won't cause duplicates,
91 // because key-down events do not mean any character inputs there.
92 // (InsertChar issues a DOM "keypress" event, which is distinct from keydown.)
93 // Unfortunately, this is not necessary the case for our clients that may
94 // treat keydown as a trigger of text inputs. We need suppression for keydown.
95 //
96 // Same condition as components/arc/ime/arc_ime_service.cc#InsertChar.
97 const base::char16 ch = event->GetCharacter();
98 const bool is_control_char =
99 (0x00 <= ch && ch <= 0x1f) || (0x7f <= ch && ch <= 0x9f);
100 if (!is_control_char && !ui::IsSystemKeyModifier(event->flags()))
101 return true;
102
103 // Case 3:
104 // Workaround for apps that doesn't handle hardware keyboard events well.
105 // Keys typically on software keyboard and lack of them are fatal, namely,
106 // unmodified enter and backspace keys, are sent through IME.
107 constexpr int kModifierMask = ui::EF_SHIFT_DOWN | ui::EF_CONTROL_DOWN |
108 ui::EF_ALT_DOWN | ui::EF_COMMAND_DOWN |
109 ui::EF_ALTGR_DOWN | ui::EF_MOD3_DOWN;
110 // Same condition as components/arc/ime/arc_ime_service.cc#InsertChar.
111 if ((event->flags() & kModifierMask) == 0) {
112 if (event->key_code() == ui::VKEY_RETURN ||
113 event->key_code() == ui::VKEY_BACK) {
114 return true;
115 }
116 }
117
118 return false;
119 }
120
IsVirtualKeyboardEnabled()121 bool IsVirtualKeyboardEnabled() {
122 return keyboard::GetAccessibilityKeyboardEnabled() ||
123 keyboard::GetTouchKeyboardEnabled() ||
124 (keyboard::KeyboardUIController::HasInstance() &&
125 keyboard::KeyboardUIController::Get()->IsEnableFlagSet(
126 keyboard::KeyboardEnableFlag::kCommandLineEnabled));
127 }
128
IsReservedAccelerator(const ui::KeyEvent * event)129 bool IsReservedAccelerator(const ui::KeyEvent* event) {
130 for (const auto& accelerator : kReservedAccelerators) {
131 if (event->flags() == accelerator.modifiers &&
132 event->key_code() == accelerator.keycode) {
133 return true;
134 }
135 }
136 return false;
137 }
138
139 // Returns false if an accelerator is not reserved or it's not enabled.
ProcessAcceleratorIfReserved(Surface * surface,ui::KeyEvent * event)140 bool ProcessAcceleratorIfReserved(Surface* surface, ui::KeyEvent* event) {
141 return IsReservedAccelerator(event) && ProcessAccelerator(surface, event);
142 }
143
144 // Returns true if the surface needs to support IME.
145 // TODO(yhanada, https://crbug.com/847500): Remove this when we find a way
146 // to fix https://crbug.com/847500 without breaking ARC apps/Lacros browser.
IsImeSupportedSurface(Surface * surface)147 bool IsImeSupportedSurface(Surface* surface) {
148 aura::Window* window = surface->window();
149 for (; window; window = window->parent()) {
150 const auto app_type =
151 static_cast<ash::AppType>(window->GetProperty(aura::client::kAppType));
152 switch (app_type) {
153 case ash::AppType::ARC_APP:
154 case ash::AppType::LACROS:
155 return true;
156 default:
157 // Do nothing.
158 break;
159 }
160 }
161 return false;
162 }
163
164 } // namespace
165
166 ////////////////////////////////////////////////////////////////////////////////
167 // Keyboard, public:
168
Keyboard(std::unique_ptr<KeyboardDelegate> delegate,Seat * seat)169 Keyboard::Keyboard(std::unique_ptr<KeyboardDelegate> delegate, Seat* seat)
170 : delegate_(std::move(delegate)),
171 seat_(seat),
172 expiration_delay_for_pending_key_acks_(base::TimeDelta::FromMilliseconds(
173 kExpirationDelayForPendingKeyAcksMs)) {
174 AddEventHandler();
175 seat_->AddObserver(this);
176 ash::KeyboardController::Get()->AddObserver(this);
177 ash::ImeControllerImpl* ime_controller = ash::Shell::Get()->ime_controller();
178 ime_controller->AddObserver(this);
179
180 delegate_->OnKeyboardLayoutUpdated(seat_->xkb_tracker()->GetKeymap().get());
181 OnSurfaceFocused(seat_->GetFocusedSurface());
182 OnKeyRepeatSettingsChanged(
183 ash::KeyboardController::Get()->GetKeyRepeatSettings());
184 }
185
~Keyboard()186 Keyboard::~Keyboard() {
187 for (KeyboardObserver& observer : observer_list_)
188 observer.OnKeyboardDestroying(this);
189 if (focus_)
190 focus_->RemoveSurfaceObserver(this);
191
192 ash::Shell::Get()->ime_controller()->RemoveObserver(this);
193 ash::KeyboardController::Get()->RemoveObserver(this);
194 seat_->RemoveObserver(this);
195 RemoveEventHandler();
196 }
197
HasDeviceConfigurationDelegate() const198 bool Keyboard::HasDeviceConfigurationDelegate() const {
199 return !!device_configuration_delegate_;
200 }
201
SetDeviceConfigurationDelegate(KeyboardDeviceConfigurationDelegate * delegate)202 void Keyboard::SetDeviceConfigurationDelegate(
203 KeyboardDeviceConfigurationDelegate* delegate) {
204 device_configuration_delegate_ = delegate;
205 OnKeyboardEnabledChanged(IsVirtualKeyboardEnabled());
206 }
207
AddObserver(KeyboardObserver * observer)208 void Keyboard::AddObserver(KeyboardObserver* observer) {
209 observer_list_.AddObserver(observer);
210 }
211
HasObserver(KeyboardObserver * observer) const212 bool Keyboard::HasObserver(KeyboardObserver* observer) const {
213 return observer_list_.HasObserver(observer);
214 }
215
RemoveObserver(KeyboardObserver * observer)216 void Keyboard::RemoveObserver(KeyboardObserver* observer) {
217 observer_list_.RemoveObserver(observer);
218 }
219
SetNeedKeyboardKeyAcks(bool need_acks)220 void Keyboard::SetNeedKeyboardKeyAcks(bool need_acks) {
221 RemoveEventHandler();
222 are_keyboard_key_acks_needed_ = need_acks;
223 AddEventHandler();
224 }
225
AreKeyboardKeyAcksNeeded() const226 bool Keyboard::AreKeyboardKeyAcksNeeded() const {
227 // Keyboard class doesn't need key acks while the spoken feedback is enabled.
228 // While the spoken feedback is enabled, a key event is sent to both of a
229 // wayland client and Chrome to give a chance to work to Chrome OS's
230 // shortcuts.
231 return are_keyboard_key_acks_needed_;
232 }
233
AckKeyboardKey(uint32_t serial,bool handled)234 void Keyboard::AckKeyboardKey(uint32_t serial, bool handled) {
235 auto it = pending_key_acks_.find(serial);
236 if (it == pending_key_acks_.end())
237 return;
238
239 if (!handled && focus_)
240 ProcessAccelerator(focus_, &it->second.first);
241 pending_key_acks_.erase(serial);
242 }
243
244 ////////////////////////////////////////////////////////////////////////////////
245 // ui::EventHandler overrides:
246
OnKeyEvent(ui::KeyEvent * event)247 void Keyboard::OnKeyEvent(ui::KeyEvent* event) {
248 if (!focus_)
249 return;
250
251 // If the event target is not an exo::Surface, let another handler process the
252 // event. This check may not be necessary once https://crbug.com/624168 is
253 // resolved.
254 if (!GetShellMainSurface(static_cast<aura::Window*>(event->target())) &&
255 !Surface::AsSurface(static_cast<aura::Window*>(event->target()))) {
256 return;
257 }
258
259 // Ignore synthetic key repeat events.
260 if (event->is_repeat()) {
261 // Clients should not see key repeat events and instead handle them on the
262 // client side.
263 // Mark the key repeat events as handled to avoid them from invoking
264 // accelerators.
265 event->SetHandled();
266 return;
267 }
268
269 TRACE_EXO_INPUT_EVENT(event);
270
271 // Process reserved accelerators before sending it to client.
272 if (ProcessAcceleratorIfReserved(focus_, event)) {
273 // Discard a key press event if it's a reserved accelerator and it's
274 // enabled.
275 event->SetHandled();
276 }
277
278 // When IME ate a key event, we use the event only for tracking key states and
279 // ignore for further processing. Otherwise it is handled in two places (IME
280 // and client) and causes undesired behavior.
281 // If the window should receive a key event before IME, Exo should send any
282 // key events to a client. The client will send back the events to IME if
283 // needed.
284 const bool consumed_by_ime =
285 !focus_->window()->GetProperty(aura::client::kSkipImeProcessing) &&
286 ConsumedByIme(focus_, event);
287
288 // Always update modifiers.
289 // XkbTracker must be updated in the Seat, before calling this method.
290 // Ensured by the observer registration order.
291 delegate_->OnKeyboardModifiers(seat_->xkb_tracker()->GetModifiers());
292
293 // Currently, physical keycode is tracked in Seat, assuming that the
294 // Keyboard::OnKeyEvent is called between Seat::WillProcessEvent and
295 // Seat::DidProcessEvent. However, if IME is enabled, it is no longer true,
296 // because IME work in async approach, and on its dispatching, call stack
297 // is split so actually Keyboard::OnKeyEvent is called after
298 // Seat::DidProcessEvent.
299 // TODO(yhanada): This is a quick fix for https://crbug.com/859071. Remove
300 // ARC-/Lacros-specific code path once we can find a way to manage
301 // press/release events pair for synthetic events.
302 ui::DomCode physical_code =
303 seat_->physical_code_for_currently_processing_event();
304 if (physical_code == ui::DomCode::NONE && focused_on_ime_supported_surface_) {
305 // This key event is a synthetic event.
306 // Consider DomCode field of the event as a physical code
307 // for synthetic events when focus surface belongs to an ARC application.
308 physical_code = event->code();
309 }
310
311 switch (event->type()) {
312 case ui::ET_KEY_PRESSED: {
313 auto it = pressed_keys_.find(physical_code);
314 if (it == pressed_keys_.end() && !consumed_by_ime && !event->handled() &&
315 physical_code != ui::DomCode::NONE) {
316 // Process key press event if not already handled and not already
317 // pressed.
318 uint32_t serial =
319 delegate_->OnKeyboardKey(event->time_stamp(), event->code(), true);
320 if (AreKeyboardKeyAcksNeeded()) {
321 pending_key_acks_.insert(
322 {serial,
323 {*event, base::TimeTicks::Now() +
324 expiration_delay_for_pending_key_acks_}});
325 event->SetHandled();
326 }
327 // Keep track of both the physical code and potentially re-written
328 // code that this event generated.
329 pressed_keys_.insert({physical_code, event->code()});
330 } else if (it != pressed_keys_.end() && !event->handled()) {
331 // Non-repeate key events for already pressed key can be sent in some
332 // cases (e.g. Holding 'A' key then holding 'B' key then releasing 'A'
333 // key sends a non-repeat 'B' key press event).
334 // When it happens, we don't want to send the press event to a client
335 // and also want to avoid it from invoking any accelerator.
336 if (AreKeyboardKeyAcksNeeded())
337 event->SetHandled();
338 }
339 } break;
340 case ui::ET_KEY_RELEASED: {
341 // Process key release event if currently pressed.
342 auto it = pressed_keys_.find(physical_code);
343 if (it != pressed_keys_.end()) {
344 // We use the code that was generate when the physical key was
345 // pressed rather than the current event code. This allows events
346 // to be re-written before dispatch, while still allowing the
347 // client to track the state of the physical keyboard.
348 uint32_t serial =
349 delegate_->OnKeyboardKey(event->time_stamp(), it->second, false);
350 if (AreKeyboardKeyAcksNeeded()) {
351 pending_key_acks_.insert(
352 {serial,
353 {*event, base::TimeTicks::Now() +
354 expiration_delay_for_pending_key_acks_}});
355 event->SetHandled();
356 }
357 pressed_keys_.erase(it);
358 }
359 } break;
360 default:
361 NOTREACHED();
362 break;
363 }
364
365 if (pending_key_acks_.empty())
366 return;
367 if (process_expired_pending_key_acks_pending_)
368 return;
369
370 ScheduleProcessExpiredPendingKeyAcks(expiration_delay_for_pending_key_acks_);
371 }
372
373 ////////////////////////////////////////////////////////////////////////////////
374 // SurfaceObserver overrides:
375
OnSurfaceDestroying(Surface * surface)376 void Keyboard::OnSurfaceDestroying(Surface* surface) {
377 DCHECK(surface == focus_);
378 SetFocus(nullptr);
379 }
380
381 ////////////////////////////////////////////////////////////////////////////////
382 // SeatObserver overrides:
383
OnSurfaceFocusing(Surface * gaining_focus)384 void Keyboard::OnSurfaceFocusing(Surface* gaining_focus) {}
385
OnSurfaceFocused(Surface * gained_focus)386 void Keyboard::OnSurfaceFocused(Surface* gained_focus) {
387 Surface* gained_focus_surface =
388 gained_focus && delegate_->CanAcceptKeyboardEventsForSurface(gained_focus)
389 ? gained_focus
390 : nullptr;
391 if (gained_focus_surface != focus_)
392 SetFocus(gained_focus_surface);
393 }
394
395 ////////////////////////////////////////////////////////////////////////////////
396 // ash::KeyboardControllerObserver overrides:
397
OnKeyboardEnabledChanged(bool enabled)398 void Keyboard::OnKeyboardEnabledChanged(bool enabled) {
399 if (device_configuration_delegate_) {
400 // Ignore kAndroidDisabled which affects |enabled| and just test for a11y
401 // and touch enabled keyboards. TODO(yhanada): Fix this using an Android
402 // specific KeyboardUI implementation. https://crbug.com/897655.
403 bool is_physical = !IsVirtualKeyboardEnabled();
404 device_configuration_delegate_->OnKeyboardTypeChanged(is_physical);
405 }
406 }
407
OnKeyRepeatSettingsChanged(const ash::KeyRepeatSettings & settings)408 void Keyboard::OnKeyRepeatSettingsChanged(
409 const ash::KeyRepeatSettings& settings) {
410 delegate_->OnKeyRepeatSettingsChanged(settings.enabled, settings.delay,
411 settings.interval);
412 }
413
414 ////////////////////////////////////////////////////////////////////////////////
415 // ash::ImeControllerImpl::Observer overrides:
416
OnCapsLockChanged(bool enabled)417 void Keyboard::OnCapsLockChanged(bool enabled) {}
418
OnKeyboardLayoutNameChanged(const std::string & layout_name)419 void Keyboard::OnKeyboardLayoutNameChanged(const std::string& layout_name) {
420 // XkbTracker must be updated in the Seat, before calling this method.
421 // Ensured by the observer registration order.
422 delegate_->OnKeyboardLayoutUpdated(seat_->xkb_tracker()->GetKeymap().get());
423 }
424
425 ////////////////////////////////////////////////////////////////////////////////
426 // Keyboard, private:
427
SetFocus(Surface * surface)428 void Keyboard::SetFocus(Surface* surface) {
429 if (focus_) {
430 delegate_->OnKeyboardLeave(focus_);
431 focus_->RemoveSurfaceObserver(this);
432 focus_ = nullptr;
433 pending_key_acks_.clear();
434 }
435 if (surface) {
436 pressed_keys_ = seat_->pressed_keys();
437 delegate_->OnKeyboardModifiers(seat_->xkb_tracker()->GetModifiers());
438 delegate_->OnKeyboardEnter(surface, pressed_keys_);
439 focus_ = surface;
440 focus_->AddSurfaceObserver(this);
441 focused_on_ime_supported_surface_ = IsImeSupportedSurface(surface);
442 }
443 }
444
ProcessExpiredPendingKeyAcks()445 void Keyboard::ProcessExpiredPendingKeyAcks() {
446 DCHECK(process_expired_pending_key_acks_pending_);
447 process_expired_pending_key_acks_pending_ = false;
448
449 // Check pending acks and process them as if it is handled if
450 // expiration time passed.
451 base::TimeTicks current_time = base::TimeTicks::Now();
452
453 while (!pending_key_acks_.empty()) {
454 auto it = pending_key_acks_.begin();
455 const ui::KeyEvent event = it->second.first;
456
457 if (it->second.second > current_time)
458 break;
459
460 // Expiration time has passed, assume the event was handled.
461 pending_key_acks_.erase(it);
462 }
463
464 if (pending_key_acks_.empty())
465 return;
466
467 base::TimeDelta delay_until_next_process_expired_pending_key_acks =
468 pending_key_acks_.begin()->second.second - current_time;
469 ScheduleProcessExpiredPendingKeyAcks(
470 delay_until_next_process_expired_pending_key_acks);
471 }
472
ScheduleProcessExpiredPendingKeyAcks(base::TimeDelta delay)473 void Keyboard::ScheduleProcessExpiredPendingKeyAcks(base::TimeDelta delay) {
474 DCHECK(!process_expired_pending_key_acks_pending_);
475 process_expired_pending_key_acks_pending_ = true;
476 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
477 FROM_HERE,
478 base::BindOnce(&Keyboard::ProcessExpiredPendingKeyAcks,
479 weak_ptr_factory_.GetWeakPtr()),
480 delay);
481 }
482
AddEventHandler()483 void Keyboard::AddEventHandler() {
484 auto* helper = WMHelper::GetInstance();
485 if (are_keyboard_key_acks_needed_)
486 helper->AddPreTargetHandler(this);
487 else
488 helper->AddPostTargetHandler(this);
489 }
490
RemoveEventHandler()491 void Keyboard::RemoveEventHandler() {
492 auto* helper = WMHelper::GetInstance();
493 if (are_keyboard_key_acks_needed_)
494 helper->RemovePreTargetHandler(this);
495 else
496 helper->RemovePostTargetHandler(this);
497 }
498
499 } // namespace exo
500