1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
6 #define CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
7
8 #include <map>
9 #include <memory>
10 #include <string>
11 #include <vector>
12
13 #include "ash/public/cpp/assistant/controller/assistant_screen_context_controller.h"
14 #include "base/memory/scoped_refptr.h"
15 #include "base/observer_list.h"
16 #include "base/optional.h"
17 #include "base/scoped_observer.h"
18 #include "base/synchronization/lock.h"
19 #include "base/threading/thread.h"
20 #include "chromeos/assistant/internal/action/cros_action_module.h"
21 #include "chromeos/assistant/internal/cros_display_connection.h"
22 #include "chromeos/assistant/internal/internal_util.h"
23 #include "chromeos/services/assistant/assistant_manager_service.h"
24 #include "chromeos/services/assistant/assistant_settings_impl.h"
25 #include "chromeos/services/assistant/chromium_api_delegate.h"
26 #include "chromeos/services/assistant/public/cpp/assistant_notification.h"
27 #include "chromeos/services/assistant/public/cpp/assistant_service.h"
28 #include "chromeos/services/assistant/public/cpp/device_actions.h"
29 #include "chromeos/services/assistant/public/shared/utils.h"
30 #include "libassistant/shared/internal_api/assistant_manager_delegate.h"
31 #include "libassistant/shared/public/conversation_state_listener.h"
32 #include "libassistant/shared/public/device_state_listener.h"
33 #include "libassistant/shared/public/media_manager.h"
34 #include "mojo/public/cpp/bindings/receiver.h"
35 #include "mojo/public/cpp/bindings/remote.h"
36 #include "services/device/public/mojom/battery_monitor.mojom.h"
37 #include "services/media_session/public/mojom/media_controller.mojom.h"
38 #include "services/media_session/public/mojom/media_session.mojom.h"
39 #include "ui/accessibility/ax_assistant_structure.h"
40 #include "ui/accessibility/mojom/ax_assistant_structure.mojom.h"
41
42 namespace ash {
43 class AssistantAlarmTimerController;
44 class AssistantNotificationController;
45 class AssistantStateBase;
46 } // namespace ash
47
48 namespace assistant_client {
49 class AssistantManager;
50 class AssistantManagerInternal;
51 } // namespace assistant_client
52
53 namespace network {
54 class PendingSharedURLLoaderFactory;
55 } // namespace network
56
57 namespace chromeos {
58 namespace assistant {
59
60 class AssistantMediaSession;
61 class CrosPlatformApi;
62 class ServiceContext;
63 class AssistantManagerServiceDelegate;
64 class AssistantDeviceSettingsDelegate;
65
66 // Enumeration of Assistant query response type, also recorded in histograms.
67 // These values are persisted to logs. Entries should not be renumbered and
68 // numeric values should never be reused. Only append to this enum is allowed
69 // if the possible type grows.
70 enum class AssistantQueryResponseType {
71 // Query without response.
72 kUnspecified = 0,
73 // Query results in device actions (e.g. turn on bluetooth/WiFi).
74 kDeviceAction = 1,
75 // Query results in answer cards with contents rendered inside the
76 // Assistant UI.
77 kInlineElement = 2,
78 // Query results in searching on Google, indicating that Assistant
79 // doesn't know what to do.
80 kSearchFallback = 3,
81 // Query results in specific actions (e.g. opening a web app such as YouTube
82 // or Facebook, some deeplink actions such as taking a screenshot or opening
83 // chrome settings page), indicating that Assistant knows what to do.
84 kTargetedAction = 4,
85 // Special enumerator value used by histogram macros.
86 kMaxValue = kTargetedAction
87 };
88
89 // Implementation of AssistantManagerService based on LibAssistant.
90 // This is the main class that interacts with LibAssistant.
91 // Since LibAssistant is a standalone library, all callbacks come from it
92 // running on threads not owned by Chrome. Thus we need to post the callbacks
93 // onto the main thread.
94 // NOTE: this class may start/stop LibAssistant multiple times throughout its
95 // lifetime. This may occur, for example, if the user manually toggles Assistant
96 // enabled/disabled in settings or switches to a non-primary profile.
COMPONENT_EXPORT(ASSISTANT_SERVICE)97 class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
98 : public AssistantManagerService,
99 public ::chromeos::assistant::action::AssistantActionObserver,
100 public AssistantEventObserver,
101 public assistant_client::ConversationStateListener,
102 public assistant_client::AssistantManagerDelegate,
103 public assistant_client::DeviceStateListener,
104 public assistant_client::MediaManager::Listener,
105 public media_session::mojom::MediaControllerObserver,
106 public AppListEventSubscriber {
107 public:
108 // |service| owns this class and must outlive this class.
109 AssistantManagerServiceImpl(
110 ServiceContext* context,
111 std::unique_ptr<AssistantManagerServiceDelegate> delegate,
112 std::unique_ptr<network::PendingSharedURLLoaderFactory>
113 pending_url_loader_factory,
114 base::Optional<std::string> s3_server_uri_override,
115 base::Optional<std::string> device_id_override);
116
117 ~AssistantManagerServiceImpl() override;
118
119 // assistant::AssistantManagerService overrides:
120 void Start(const base::Optional<UserInfo>& user,
121 bool enable_hotword) override;
122 void Stop() override;
123 State GetState() const override;
124 void SetUser(const base::Optional<UserInfo>& user) override;
125 void EnableAmbientMode(bool enabled) override;
126 void EnableListening(bool enable) override;
127 void EnableHotword(bool enable) override;
128 void SetArcPlayStoreEnabled(bool enable) override;
129 void SetAssistantContextEnabled(bool enable) override;
130 AssistantSettings* GetAssistantSettings() override;
131 void AddCommunicationErrorObserver(
132 CommunicationErrorObserver* observer) override;
133 void RemoveCommunicationErrorObserver(
134 const CommunicationErrorObserver* observer) override;
135 void AddAndFireStateObserver(StateObserver* observer) override;
136 void RemoveStateObserver(const StateObserver* observer) override;
137 void SyncDeviceAppsStatus() override;
138 void UpdateInternalMediaPlayerStatus(
139 media_session::mojom::MediaSessionAction action) override;
140
141 // Assistant overrides:
142 void StartEditReminderInteraction(const std::string& client_id) override;
143 void StartScreenContextInteraction(
144 ax::mojom::AssistantStructurePtr assistant_structure,
145 const std::vector<uint8_t>& assistant_screenshot) override;
146 void StartTextInteraction(const std::string& query,
147 AssistantQuerySource source,
148 bool allow_tts) override;
149 void StartVoiceInteraction() override;
150 void StopActiveInteraction(bool cancel_conversation) override;
151 void AddAssistantInteractionSubscriber(
152 AssistantInteractionSubscriber* subscriber) override;
153 void RemoveAssistantInteractionSubscriber(
154 AssistantInteractionSubscriber* subscriber) override;
155 void RetrieveNotification(const AssistantNotification& notification,
156 int action_index) override;
157 void DismissNotification(const AssistantNotification& notification) override;
158 void OnAccessibilityStatusChanged(bool spoken_feedback_enabled) override;
159 void SendAssistantFeedback(const AssistantFeedback& feedback) override;
160 void NotifyEntryIntoAssistantUi(AssistantEntryPoint entry_point) override;
161 void AddTimeToTimer(const std::string& id, base::TimeDelta duration) override;
162 void PauseTimer(const std::string& id) override;
163 void RemoveAlarmOrTimer(const std::string& id) override;
164 void ResumeTimer(const std::string& id) override;
165
166 // AssistantActionObserver overrides:
167 void OnScheduleWait(int id, int time_ms) override;
168 void OnShowContextualQueryFallback() override;
169 void OnShowHtml(const std::string& html,
170 const std::string& fallback) override;
171 void OnShowSuggestions(
172 const std::vector<action::Suggestion>& suggestions) override;
173 void OnShowText(const std::string& text) override;
174 void OnOpenUrl(const std::string& url, bool in_background) override;
175 void OnShowNotification(const action::Notification& notification) override;
176 void OnOpenAndroidApp(const AndroidAppInfo& app_info,
177 const InteractionInfo& interaction) override;
178 void OnVerifyAndroidApp(const std::vector<AndroidAppInfo>& apps_info,
179 const InteractionInfo& interaction) override;
180 void OnModifyDeviceSetting(
181 const ::assistant::api::client_op::ModifySettingArgs& args) override;
182 void OnGetDeviceSettings(
183 int interaction_id,
184 const ::assistant::api::client_op::GetDeviceSettingsArgs& args) override;
185
186 // AssistantEventObserver overrides:
187 void OnSpeechLevelUpdated(float speech_level) override;
188
189 // assistant_client::ConversationStateListener overrides:
190 void OnConversationTurnFinished(
191 assistant_client::ConversationStateListener::Resolution resolution)
192 override;
193 void OnRecognitionStateChanged(
194 assistant_client::ConversationStateListener::RecognitionState state,
195 const assistant_client::ConversationStateListener::RecognitionResult&
196 recognition_result) override;
197 void OnRespondingStarted(bool is_error_response) override;
198
199 // AssistantManagerDelegate overrides:
200 void OnConversationTurnStartedInternal(
201 const assistant_client::ConversationTurnMetadata& metadata) override;
202 void OnNotificationRemoved(const std::string& grouping_key) override;
203 void OnCommunicationError(int error_code) override;
204 // Last search source will be cleared after it is retrieved.
205 std::string GetLastSearchSource() override;
206
207 // assistant_client::DeviceStateListener overrides:
208 void OnStartFinished() override;
209
210 // AppListEventSubscriber overrides:
211 void OnAndroidAppListRefreshed(
212 const std::vector<AndroidAppInfo>& apps_info) override;
213
214 assistant_client::AssistantManager* assistant_manager() {
215 return assistant_manager_.get();
216 }
217 assistant_client::AssistantManagerInternal* assistant_manager_internal() {
218 return assistant_manager_internal_;
219 }
220 CrosPlatformApi* platform_api() { return platform_api_.get(); }
221
222 // assistant_client::MediaManager::Listener overrides:
223 void OnPlaybackStateChange(
224 const assistant_client::MediaStatus& status) override;
225
226 // media_session::mojom::MediaControllerObserver overrides:
227 void MediaSessionInfoChanged(
228 media_session::mojom::MediaSessionInfoPtr info) override;
229 void MediaSessionMetadataChanged(
230 const base::Optional<media_session::MediaMetadata>& metadata) override;
231 void MediaSessionActionsChanged(
232 const std::vector<media_session::mojom::MediaSessionAction>& action)
233 override {}
234 void MediaSessionChanged(
235 const base::Optional<base::UnguessableToken>& request_id) override;
236 void MediaSessionPositionChanged(
237 const base::Optional<media_session::MediaPosition>& position) override {}
238
239 // The start runs in the background. This will wait until the background
240 // thread is finished.
241 void WaitUntilStartIsFinishedForTesting();
242
243 // Get the action module for testing.
244 action::CrosActionModule* action_module_for_testing() {
245 return action_module_.get();
246 }
247
248 private:
249 void StartAssistantInternal(const base::Optional<UserInfo>& user,
250 const std::string& locale);
251 void PostInitAssistant();
252
253 // Update device id, type and locale
254 void UpdateDeviceSettings();
255
256 // Sync speaker id enrollment status.
257 void SyncSpeakerIdEnrollmentStatus();
258
259 void HandleLaunchMediaIntentResponse(bool app_opened);
260
261 void OnAlarmTimerStateChanged();
262 void OnModifySettingsAction(const std::string& modify_setting_args_proto);
263 void OnOpenMediaAndroidIntent(const std::string& play_media_args_proto,
264 AndroidAppInfo* app_info);
265 void OnPlayMedia(const std::string& play_media_args_proto);
266 void OnMediaControlAction(const std::string& action_name,
267 const std::string& media_action_args_proto);
268
269 void OnDeviceAppsEnabled(bool enabled);
270
271 void RegisterFallbackMediaHandler();
272 void AddMediaControllerObserver();
273 void RemoveMediaControllerObserver();
274 void RegisterAlarmsTimersListener();
275
276 void FillServerExperimentIds(std::vector<std::string>* server_experiment_ids);
277
278 // Record the response type for each query. Note that query on device
279 // actions (e.g. turn on Bluetooth, turn on WiFi) will cause duplicate
280 // record because it interacts with server twice on on the same query.
281 // The first round interaction checks if a setting is supported with no
282 // responses sent back and ends normally (will be recorded as kUnspecified),
283 // and settings modification proto along with any text/voice responses would
284 // be sent back in the second round (recorded as kDeviceAction).
285 void RecordQueryResponseTypeUMA();
286
287 void UpdateMediaState();
288 void ResetMediaState();
289
290 std::string NewPendingInteraction(AssistantInteractionType interaction_type,
291 AssistantQuerySource source,
292 const std::string& query);
293
294 std::string ConsumeLastTriggerSource();
295
296 void SendVoicelessInteraction(const std::string& interaction,
297 const std::string& description,
298 bool is_user_initiated);
299
300 ash::AssistantAlarmTimerController* assistant_alarm_timer_controller();
301 ash::AssistantNotificationController* assistant_notification_controller();
302 ash::AssistantScreenContextController* assistant_screen_context_controller();
303 ash::AssistantStateBase* assistant_state();
304 DeviceActions* device_actions();
305 scoped_refptr<base::SequencedTaskRunner> main_task_runner();
306
307 void SetStateAndInformObservers(State new_state);
308
309 State state_ = State::STOPPED;
310 std::unique_ptr<AssistantMediaSession> media_session_;
311 std::unique_ptr<CrosPlatformApi> platform_api_;
312 std::unique_ptr<action::CrosActionModule> action_module_;
313 ChromiumApiDelegate chromium_api_delegate_;
314 // NOTE: |display_connection_| is used by |assistant_manager_| and must be
315 // declared before so it will be destructed after.
316 std::unique_ptr<CrosDisplayConnection> display_connection_;
317 // Similar to |new_asssistant_manager_|, created on |background_thread_| then
318 // posted to main thread to finish initialization then move to
319 // |display_connection_|.
320 std::unique_ptr<CrosDisplayConnection> new_display_connection_;
321 std::unique_ptr<assistant_client::AssistantManager> assistant_manager_;
322 std::unique_ptr<AssistantSettingsImpl> assistant_settings_;
323 // |new_assistant_manager_| is created on |background_thread_| then posted to
324 // main thread to finish initialization then move to |assistant_manager_|.
325 std::unique_ptr<assistant_client::AssistantManager> new_assistant_manager_;
326 // Same ownership as |new_assistant_manager_|.
327 assistant_client::AssistantManagerInternal* new_assistant_manager_internal_ =
328 nullptr;
329 base::Lock new_assistant_manager_lock_;
330 // same ownership as |assistant_manager_|.
331 assistant_client::AssistantManagerInternal* assistant_manager_internal_ =
332 nullptr;
333 base::ObserverList<AssistantInteractionSubscriber> interaction_subscribers_;
334 mojo::Remote<media_session::mojom::MediaController> media_controller_;
335
336 // Owned by the parent |Service| which will destroy |this| before |context_|.
337 ServiceContext* const context_;
338
339 std::unique_ptr<AssistantManagerServiceDelegate> delegate_;
340 std::unique_ptr<AssistantDeviceSettingsDelegate> settings_delegate_;
341
342 bool spoken_feedback_enabled_ = false;
343
344 std::string last_trigger_source_;
345 base::Lock last_trigger_source_lock_;
346 base::TimeTicks started_time_;
347
348 base::Thread background_thread_;
349
350 int next_interaction_id_ = 1;
351 std::map<std::string, std::unique_ptr<AssistantInteractionMetadata>>
352 pending_interactions_;
353
354 bool receive_modify_settings_proto_response_ = false;
355 bool receive_inline_response_ = false;
356 std::string receive_url_response_;
357
358 mojo::Receiver<media_session::mojom::MediaControllerObserver>
359 media_controller_observer_receiver_{this};
360
361 // Info associated to the active media session.
362 media_session::mojom::MediaSessionInfoPtr media_session_info_ptr_;
363 // The metadata for the active media session. It can be null to be reset, e.g.
364 // the media that was being played has been stopped.
365 base::Optional<media_session::MediaMetadata> media_metadata_ = base::nullopt;
366
367 base::UnguessableToken media_session_audio_focus_id_ =
368 base::UnguessableToken::Null();
369
370 // Configuration passed to libassistant.
371 std::string libassistant_config_;
372
373 ScopedObserver<DeviceActions,
374 AppListEventSubscriber,
375 &DeviceActions::AddAppListEventSubscriber,
376 &DeviceActions::RemoveAppListEventSubscriber>
377 scoped_app_list_event_subscriber_{this};
378 base::ObserverList<CommunicationErrorObserver> error_observers_;
379 base::ObserverList<StateObserver> state_observers_;
380
381 base::WeakPtrFactory<AssistantManagerServiceImpl> weak_factory_;
382
383 DISALLOW_COPY_AND_ASSIGN(AssistantManagerServiceImpl);
384 };
385
386 } // namespace assistant
387 } // namespace chromeos
388
389 #endif // CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
390