1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
6 #define CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
7 
8 #include <map>
9 #include <memory>
10 #include <string>
11 #include <vector>
12 
13 #include "ash/public/cpp/assistant/controller/assistant_screen_context_controller.h"
14 #include "base/memory/scoped_refptr.h"
15 #include "base/observer_list.h"
16 #include "base/optional.h"
17 #include "base/scoped_observer.h"
18 #include "base/synchronization/lock.h"
19 #include "base/threading/thread.h"
20 #include "chromeos/assistant/internal/action/cros_action_module.h"
21 #include "chromeos/assistant/internal/cros_display_connection.h"
22 #include "chromeos/assistant/internal/internal_util.h"
23 #include "chromeos/services/assistant/assistant_manager_service.h"
24 #include "chromeos/services/assistant/assistant_settings_impl.h"
25 #include "chromeos/services/assistant/chromium_api_delegate.h"
26 #include "chromeos/services/assistant/public/cpp/assistant_notification.h"
27 #include "chromeos/services/assistant/public/cpp/assistant_service.h"
28 #include "chromeos/services/assistant/public/cpp/device_actions.h"
29 #include "chromeos/services/assistant/public/shared/utils.h"
30 #include "libassistant/shared/internal_api/assistant_manager_delegate.h"
31 #include "libassistant/shared/public/conversation_state_listener.h"
32 #include "libassistant/shared/public/device_state_listener.h"
33 #include "libassistant/shared/public/media_manager.h"
34 #include "mojo/public/cpp/bindings/receiver.h"
35 #include "mojo/public/cpp/bindings/remote.h"
36 #include "services/device/public/mojom/battery_monitor.mojom.h"
37 #include "services/media_session/public/mojom/media_controller.mojom.h"
38 #include "services/media_session/public/mojom/media_session.mojom.h"
39 #include "ui/accessibility/ax_assistant_structure.h"
40 #include "ui/accessibility/mojom/ax_assistant_structure.mojom.h"
41 
42 namespace ash {
43 class AssistantAlarmTimerController;
44 class AssistantNotificationController;
45 class AssistantStateBase;
46 }  // namespace ash
47 
48 namespace assistant_client {
49 class AssistantManager;
50 class AssistantManagerInternal;
51 }  // namespace assistant_client
52 
53 namespace network {
54 class PendingSharedURLLoaderFactory;
55 }  // namespace network
56 
57 namespace chromeos {
58 namespace assistant {
59 
60 class AssistantMediaSession;
61 class CrosPlatformApi;
62 class ServiceContext;
63 class AssistantManagerServiceDelegate;
64 class AssistantDeviceSettingsDelegate;
65 
66 // Enumeration of Assistant query response type, also recorded in histograms.
67 // These values are persisted to logs. Entries should not be renumbered and
68 // numeric values should never be reused. Only append to this enum is allowed
69 // if the possible type grows.
70 enum class AssistantQueryResponseType {
71   // Query without response.
72   kUnspecified = 0,
73   // Query results in device actions (e.g. turn on bluetooth/WiFi).
74   kDeviceAction = 1,
75   // Query results in answer cards with contents rendered inside the
76   // Assistant UI.
77   kInlineElement = 2,
78   // Query results in searching on Google, indicating that Assistant
79   // doesn't know what to do.
80   kSearchFallback = 3,
81   // Query results in specific actions (e.g. opening a web app such as YouTube
82   // or Facebook, some deeplink actions such as taking a screenshot or opening
83   // chrome settings page), indicating that Assistant knows what to do.
84   kTargetedAction = 4,
85   // Special enumerator value used by histogram macros.
86   kMaxValue = kTargetedAction
87 };
88 
89 // Implementation of AssistantManagerService based on LibAssistant.
90 // This is the main class that interacts with LibAssistant.
91 // Since LibAssistant is a standalone library, all callbacks come from it
92 // running on threads not owned by Chrome. Thus we need to post the callbacks
93 // onto the main thread.
94 // NOTE: this class may start/stop LibAssistant multiple times throughout its
95 // lifetime. This may occur, for example, if the user manually toggles Assistant
96 // enabled/disabled in settings or switches to a non-primary profile.
COMPONENT_EXPORT(ASSISTANT_SERVICE)97 class COMPONENT_EXPORT(ASSISTANT_SERVICE) AssistantManagerServiceImpl
98     : public AssistantManagerService,
99       public ::chromeos::assistant::action::AssistantActionObserver,
100       public AssistantEventObserver,
101       public assistant_client::ConversationStateListener,
102       public assistant_client::AssistantManagerDelegate,
103       public assistant_client::DeviceStateListener,
104       public assistant_client::MediaManager::Listener,
105       public media_session::mojom::MediaControllerObserver,
106       public AppListEventSubscriber {
107  public:
108   // |service| owns this class and must outlive this class.
109   AssistantManagerServiceImpl(
110       ServiceContext* context,
111       std::unique_ptr<AssistantManagerServiceDelegate> delegate,
112       std::unique_ptr<network::PendingSharedURLLoaderFactory>
113           pending_url_loader_factory,
114       base::Optional<std::string> s3_server_uri_override,
115       base::Optional<std::string> device_id_override);
116 
117   ~AssistantManagerServiceImpl() override;
118 
119   // assistant::AssistantManagerService overrides:
120   void Start(const base::Optional<UserInfo>& user,
121              bool enable_hotword) override;
122   void Stop() override;
123   State GetState() const override;
124   void SetUser(const base::Optional<UserInfo>& user) override;
125   void EnableAmbientMode(bool enabled) override;
126   void EnableListening(bool enable) override;
127   void EnableHotword(bool enable) override;
128   void SetArcPlayStoreEnabled(bool enable) override;
129   void SetAssistantContextEnabled(bool enable) override;
130   AssistantSettings* GetAssistantSettings() override;
131   void AddCommunicationErrorObserver(
132       CommunicationErrorObserver* observer) override;
133   void RemoveCommunicationErrorObserver(
134       const CommunicationErrorObserver* observer) override;
135   void AddAndFireStateObserver(StateObserver* observer) override;
136   void RemoveStateObserver(const StateObserver* observer) override;
137   void SyncDeviceAppsStatus() override;
138   void UpdateInternalMediaPlayerStatus(
139       media_session::mojom::MediaSessionAction action) override;
140 
141   // Assistant overrides:
142   void StartEditReminderInteraction(const std::string& client_id) override;
143   void StartScreenContextInteraction(
144       ax::mojom::AssistantStructurePtr assistant_structure,
145       const std::vector<uint8_t>& assistant_screenshot) override;
146   void StartTextInteraction(const std::string& query,
147                             AssistantQuerySource source,
148                             bool allow_tts) override;
149   void StartVoiceInteraction() override;
150   void StopActiveInteraction(bool cancel_conversation) override;
151   void AddAssistantInteractionSubscriber(
152       AssistantInteractionSubscriber* subscriber) override;
153   void RemoveAssistantInteractionSubscriber(
154       AssistantInteractionSubscriber* subscriber) override;
155   void RetrieveNotification(const AssistantNotification& notification,
156                             int action_index) override;
157   void DismissNotification(const AssistantNotification& notification) override;
158   void OnAccessibilityStatusChanged(bool spoken_feedback_enabled) override;
159   void SendAssistantFeedback(const AssistantFeedback& feedback) override;
160   void NotifyEntryIntoAssistantUi(AssistantEntryPoint entry_point) override;
161   void AddTimeToTimer(const std::string& id, base::TimeDelta duration) override;
162   void PauseTimer(const std::string& id) override;
163   void RemoveAlarmOrTimer(const std::string& id) override;
164   void ResumeTimer(const std::string& id) override;
165 
166   // AssistantActionObserver overrides:
167   void OnScheduleWait(int id, int time_ms) override;
168   void OnShowContextualQueryFallback() override;
169   void OnShowHtml(const std::string& html,
170                   const std::string& fallback) override;
171   void OnShowSuggestions(
172       const std::vector<action::Suggestion>& suggestions) override;
173   void OnShowText(const std::string& text) override;
174   void OnOpenUrl(const std::string& url, bool in_background) override;
175   void OnShowNotification(const action::Notification& notification) override;
176   void OnOpenAndroidApp(const AndroidAppInfo& app_info,
177                         const InteractionInfo& interaction) override;
178   void OnVerifyAndroidApp(const std::vector<AndroidAppInfo>& apps_info,
179                           const InteractionInfo& interaction) override;
180   void OnModifyDeviceSetting(
181       const ::assistant::api::client_op::ModifySettingArgs& args) override;
182   void OnGetDeviceSettings(
183       int interaction_id,
184       const ::assistant::api::client_op::GetDeviceSettingsArgs& args) override;
185 
186   // AssistantEventObserver overrides:
187   void OnSpeechLevelUpdated(float speech_level) override;
188 
189   // assistant_client::ConversationStateListener overrides:
190   void OnConversationTurnFinished(
191       assistant_client::ConversationStateListener::Resolution resolution)
192       override;
193   void OnRecognitionStateChanged(
194       assistant_client::ConversationStateListener::RecognitionState state,
195       const assistant_client::ConversationStateListener::RecognitionResult&
196           recognition_result) override;
197   void OnRespondingStarted(bool is_error_response) override;
198 
199   // AssistantManagerDelegate overrides:
200   void OnConversationTurnStartedInternal(
201       const assistant_client::ConversationTurnMetadata& metadata) override;
202   void OnNotificationRemoved(const std::string& grouping_key) override;
203   void OnCommunicationError(int error_code) override;
204   // Last search source will be cleared after it is retrieved.
205   std::string GetLastSearchSource() override;
206 
207   // assistant_client::DeviceStateListener overrides:
208   void OnStartFinished() override;
209 
210   // AppListEventSubscriber overrides:
211   void OnAndroidAppListRefreshed(
212       const std::vector<AndroidAppInfo>& apps_info) override;
213 
214   assistant_client::AssistantManager* assistant_manager() {
215     return assistant_manager_.get();
216   }
217   assistant_client::AssistantManagerInternal* assistant_manager_internal() {
218     return assistant_manager_internal_;
219   }
220   CrosPlatformApi* platform_api() { return platform_api_.get(); }
221 
222   // assistant_client::MediaManager::Listener overrides:
223   void OnPlaybackStateChange(
224       const assistant_client::MediaStatus& status) override;
225 
226   // media_session::mojom::MediaControllerObserver overrides:
227   void MediaSessionInfoChanged(
228       media_session::mojom::MediaSessionInfoPtr info) override;
229   void MediaSessionMetadataChanged(
230       const base::Optional<media_session::MediaMetadata>& metadata) override;
231   void MediaSessionActionsChanged(
232       const std::vector<media_session::mojom::MediaSessionAction>& action)
233       override {}
234   void MediaSessionChanged(
235       const base::Optional<base::UnguessableToken>& request_id) override;
236   void MediaSessionPositionChanged(
237       const base::Optional<media_session::MediaPosition>& position) override {}
238 
239   // The start runs in the background. This will wait until the background
240   // thread is finished.
241   void WaitUntilStartIsFinishedForTesting();
242 
243   // Get the action module for testing.
244   action::CrosActionModule* action_module_for_testing() {
245     return action_module_.get();
246   }
247 
248  private:
249   void StartAssistantInternal(const base::Optional<UserInfo>& user,
250                               const std::string& locale);
251   void PostInitAssistant();
252 
253   // Update device id, type and locale
254   void UpdateDeviceSettings();
255 
256   // Sync speaker id enrollment status.
257   void SyncSpeakerIdEnrollmentStatus();
258 
259   void HandleLaunchMediaIntentResponse(bool app_opened);
260 
261   void OnAlarmTimerStateChanged();
262   void OnModifySettingsAction(const std::string& modify_setting_args_proto);
263   void OnOpenMediaAndroidIntent(const std::string& play_media_args_proto,
264                                 AndroidAppInfo* app_info);
265   void OnPlayMedia(const std::string& play_media_args_proto);
266   void OnMediaControlAction(const std::string& action_name,
267                             const std::string& media_action_args_proto);
268 
269   void OnDeviceAppsEnabled(bool enabled);
270 
271   void RegisterFallbackMediaHandler();
272   void AddMediaControllerObserver();
273   void RemoveMediaControllerObserver();
274   void RegisterAlarmsTimersListener();
275 
276   void FillServerExperimentIds(std::vector<std::string>* server_experiment_ids);
277 
278   // Record the response type for each query. Note that query on device
279   // actions (e.g. turn on Bluetooth, turn on WiFi) will cause duplicate
280   // record because it interacts with server twice on on the same query.
281   // The first round interaction checks if a setting is supported with no
282   // responses sent back and ends normally (will be recorded as kUnspecified),
283   // and settings modification proto along with any text/voice responses would
284   // be sent back in the second round (recorded as kDeviceAction).
285   void RecordQueryResponseTypeUMA();
286 
287   void UpdateMediaState();
288   void ResetMediaState();
289 
290   std::string NewPendingInteraction(AssistantInteractionType interaction_type,
291                                     AssistantQuerySource source,
292                                     const std::string& query);
293 
294   std::string ConsumeLastTriggerSource();
295 
296   void SendVoicelessInteraction(const std::string& interaction,
297                                 const std::string& description,
298                                 bool is_user_initiated);
299 
300   ash::AssistantAlarmTimerController* assistant_alarm_timer_controller();
301   ash::AssistantNotificationController* assistant_notification_controller();
302   ash::AssistantScreenContextController* assistant_screen_context_controller();
303   ash::AssistantStateBase* assistant_state();
304   DeviceActions* device_actions();
305   scoped_refptr<base::SequencedTaskRunner> main_task_runner();
306 
307   void SetStateAndInformObservers(State new_state);
308 
309   State state_ = State::STOPPED;
310   std::unique_ptr<AssistantMediaSession> media_session_;
311   std::unique_ptr<CrosPlatformApi> platform_api_;
312   std::unique_ptr<action::CrosActionModule> action_module_;
313   ChromiumApiDelegate chromium_api_delegate_;
314   // NOTE: |display_connection_| is used by |assistant_manager_| and must be
315   // declared before so it will be destructed after.
316   std::unique_ptr<CrosDisplayConnection> display_connection_;
317   // Similar to |new_asssistant_manager_|, created on |background_thread_| then
318   // posted to main thread to finish initialization then move to
319   // |display_connection_|.
320   std::unique_ptr<CrosDisplayConnection> new_display_connection_;
321   std::unique_ptr<assistant_client::AssistantManager> assistant_manager_;
322   std::unique_ptr<AssistantSettingsImpl> assistant_settings_;
323   // |new_assistant_manager_| is created on |background_thread_| then posted to
324   // main thread to finish initialization then move to |assistant_manager_|.
325   std::unique_ptr<assistant_client::AssistantManager> new_assistant_manager_;
326   // Same ownership as |new_assistant_manager_|.
327   assistant_client::AssistantManagerInternal* new_assistant_manager_internal_ =
328       nullptr;
329   base::Lock new_assistant_manager_lock_;
330   // same ownership as |assistant_manager_|.
331   assistant_client::AssistantManagerInternal* assistant_manager_internal_ =
332       nullptr;
333   base::ObserverList<AssistantInteractionSubscriber> interaction_subscribers_;
334   mojo::Remote<media_session::mojom::MediaController> media_controller_;
335 
336   // Owned by the parent |Service| which will destroy |this| before |context_|.
337   ServiceContext* const context_;
338 
339   std::unique_ptr<AssistantManagerServiceDelegate> delegate_;
340   std::unique_ptr<AssistantDeviceSettingsDelegate> settings_delegate_;
341 
342   bool spoken_feedback_enabled_ = false;
343 
344   std::string last_trigger_source_;
345   base::Lock last_trigger_source_lock_;
346   base::TimeTicks started_time_;
347 
348   base::Thread background_thread_;
349 
350   int next_interaction_id_ = 1;
351   std::map<std::string, std::unique_ptr<AssistantInteractionMetadata>>
352       pending_interactions_;
353 
354   bool receive_modify_settings_proto_response_ = false;
355   bool receive_inline_response_ = false;
356   std::string receive_url_response_;
357 
358   mojo::Receiver<media_session::mojom::MediaControllerObserver>
359       media_controller_observer_receiver_{this};
360 
361   // Info associated to the active media session.
362   media_session::mojom::MediaSessionInfoPtr media_session_info_ptr_;
363   // The metadata for the active media session. It can be null to be reset, e.g.
364   // the media that was being played has been stopped.
365   base::Optional<media_session::MediaMetadata> media_metadata_ = base::nullopt;
366 
367   base::UnguessableToken media_session_audio_focus_id_ =
368       base::UnguessableToken::Null();
369 
370   // Configuration passed to libassistant.
371   std::string libassistant_config_;
372 
373   ScopedObserver<DeviceActions,
374                  AppListEventSubscriber,
375                  &DeviceActions::AddAppListEventSubscriber,
376                  &DeviceActions::RemoveAppListEventSubscriber>
377       scoped_app_list_event_subscriber_{this};
378   base::ObserverList<CommunicationErrorObserver> error_observers_;
379   base::ObserverList<StateObserver> state_observers_;
380 
381   base::WeakPtrFactory<AssistantManagerServiceImpl> weak_factory_;
382 
383   DISALLOW_COPY_AND_ASSIGN(AssistantManagerServiceImpl);
384 };
385 
386 }  // namespace assistant
387 }  // namespace chromeos
388 
389 #endif  // CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
390