chromeos/services/assistant/assistant_manager_service_impl.h

// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_
#define CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_

#include <memory>
#include <string>
#include <vector>

#include "ash/public/cpp/assistant/default_voice_interaction_observer.h"
#include "ash/public/interfaces/ash_message_center_controller.mojom.h"
#include "ash/public/interfaces/assistant_controller.mojom.h"
#include "ash/public/interfaces/voice_interaction_controller.mojom.h"
#include "base/threading/thread.h"
#include "chromeos/assistant/internal/action/cros_action_module.h"
#include "chromeos/assistant/internal/cros_display_connection.h"
#include "chromeos/assistant/internal/internal_util.h"
#include "chromeos/services/assistant/assistant_manager_service.h"
#include "chromeos/services/assistant/assistant_settings_manager_impl.h"
#include "chromeos/services/assistant/chromium_api_delegate.h"
#include "chromeos/services/assistant/platform_api_impl.h"
#include "chromeos/services/assistant/public/mojom/assistant.mojom.h"
#include "libassistant/shared/internal_api/assistant_manager_delegate.h"
#include "libassistant/shared/public/conversation_state_listener.h"
#include "libassistant/shared/public/device_state_listener.h"
#include "services/device/public/mojom/battery_monitor.mojom.h"
#include "ui/accessibility/ax_assistant_structure.h"
#include "ui/accessibility/mojom/ax_assistant_structure.mojom.h"

namespace assistant_client {
class AssistantManager;
class AssistantManagerInternal;
}  // namespace assistant_client

namespace service_manager {
class Connector;
}  // namespace service_manager

namespace chromeos {
namespace assistant {

class Service;
class AssistantMediaSession;

// Enumeration of Assistant query response type, also recorded in histograms.
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused. Only append to this enum is allowed
// if the possible type grows.
enum class AssistantQueryResponseType {
  // Query without response.
  kUnspecified = 0,
  // Query results in device actions (e.g. turn on bluetooth/WiFi).
  kDeviceAction = 1,
  // Query results in answer cards with contents rendered inside the
  // Assistant UI.
  kInlineElement = 2,
  // Query results in searching on Google, indicating that Assistant
  // doesn't know what to do.
  kSearchFallback = 3,
  // Query results in specific actions (e.g. opening a web app such as YouTube
  // or Facebook, some deeplink actions such as taking a screenshot or opening
  // chrome settings page), indicating that Assistant knows what to do.
  kTargetedAction = 4,
  // Special enumerator value used by histogram macros.
  kMaxValue = kTargetedAction
};

// Implementation of AssistantManagerService based on LibAssistant.
// This is the main class that ineracts with LibAssistant.
// Since LibAssistant is a standalone library, all callbacks come from it
// running on threads not owned by Chrome. Thus we need to post the callbacks
// onto the main thread.
class AssistantManagerServiceImpl
    : public AssistantManagerService,
      public ::chromeos::assistant::action::AssistantActionObserver,
      public AssistantEventObserver,
      public assistant_client::ConversationStateListener,
      public assistant_client::AssistantManagerDelegate,
      public assistant_client::DeviceStateListener {
 public:
  // |service| owns this class and must outlive this class.
  AssistantManagerServiceImpl(
      service_manager::Connector* connector,
      device::mojom::BatteryMonitorPtr battery_monitor,
      Service* service,
      network::NetworkConnectionTracker* network_connection_tracker);

  ~AssistantManagerServiceImpl() override;

  // assistant::AssistantManagerService overrides
  void Start(const std::string& access_token,
             bool enable_hotword,
             base::OnceClosure callback) override;
  void Stop() override;
  State GetState() const override;
  void SetAccessToken(const std::string& access_token) override;
  void EnableListening(bool enable) override;
  void EnableHotword(bool enable) override;
  AssistantSettingsManager* GetAssistantSettingsManager() override;

  // mojom::Assistant overrides:
  void StartCachedScreenContextInteraction() override;
  void StartMetalayerInteraction(const gfx::Rect& region) override;
  void StartTextInteraction(const std::string& query, bool allow_tts) override;
  void StartVoiceInteraction() override;
  void StartWarmerWelcomeInteraction(int num_warmer_welcome_triggered,
                                     bool allow_tts) override;
  void StopActiveInteraction(bool cancel_conversation) override;
  void AddAssistantInteractionSubscriber(
      mojom::AssistantInteractionSubscriberPtr subscriber) override;
  void RetrieveNotification(mojom::AssistantNotificationPtr notification,
                            int action_index) override;
  void DismissNotification(
      mojom::AssistantNotificationPtr notification) override;
  void CacheScreenContext(CacheScreenContextCallback callback) override;
  void ClearScreenContextCache() override;
  void OnAccessibilityStatusChanged(bool spoken_feedback_enabled) override;
  void SendAssistantFeedback(
      mojom::AssistantFeedbackPtr assistant_feedback) override;

  // AssistantActionObserver overrides:
  void OnShowContextualQueryFallback() override;
  void OnShowHtml(const std::string& html,
                  const std::string& fallback) override;
  void OnShowSuggestions(
      const std::vector<action::Suggestion>& suggestions) override;
  void OnShowText(const std::string& text) override;
  void OnOpenUrl(const std::string& url) override;
  void OnShowNotification(const action::Notification& notification) override;
  void OnOpenAndroidApp(const action::AndroidAppInfo& app_info,
                        const action::InteractionInfo& interaction) override;
  void OnVerifyAndroidApp(const std::vector<action::AndroidAppInfo>& apps_info,
                          const action::InteractionInfo& interaction) override;

  // AssistantEventObserver overrides:
  void OnSpeechLevelUpdated(float speech_level) override;

  // assistant_client::ConversationStateListener overrides:
  void OnConversationTurnStarted(bool is_mic_open) override;
  void OnConversationTurnFinished(
      assistant_client::ConversationStateListener::Resolution resolution)
      override;
  void OnRecognitionStateChanged(
      assistant_client::ConversationStateListener::RecognitionState state,
      const assistant_client::ConversationStateListener::RecognitionResult&
          recognition_result) override;
  void OnRespondingStarted(bool is_error_response) override;

  // AssistantManagerDelegate overrides
  assistant_client::ActionModule::Result HandleModifySettingClientOp(
      const std::string& modify_setting_args_proto) override;
  bool IsSettingSupported(const std::string& setting_id) override;
  bool SupportsModifySettings() override;
  void OnNotificationRemoved(const std::string& grouping_key) override;
  void OnCommunicationError(int error_code) override;
  // Last search source will be cleared after it is retrieved.
  std::string GetLastSearchSource() override;

  // assistant_client::DeviceStateListener overrides:
  void OnStartFinished() override;
  void OnTimerSoundingStarted() override;
  void OnTimerSoundingFinished() override;

  void UpdateInternalOptions(
      assistant_client::AssistantManagerInternal* assistant_manager_internal);

  assistant_client::AssistantManager* assistant_manager() {
    return assistant_manager_.get();
  }
  assistant_client::AssistantManagerInternal* assistant_manager_internal() {
    return assistant_manager_internal_;
  }

 private:
  std::unique_ptr<assistant_client::AssistantManager> StartAssistantInternal(
      const std::string& access_token);
  void PostInitAssistant(
      base::OnceClosure post_init_callback,
      std::unique_ptr<assistant_client::AssistantManager>* assistant_manager);

  // Update device id, type and locale
  void UpdateDeviceSettings();

  // Sync speaker id enrollment status.
  void SyncSpeakerIdEnrollmentStatus();

  void HandleOpenAndroidAppResponse(const action::InteractionInfo& interaction,
                                    bool app_opened);
  void HandleVerifyAndroidAppResponse(
      const action::InteractionInfo& interaction,
      std::vector<mojom::AndroidAppInfoPtr> apps_info);

  void OnConversationTurnStartedOnMainThread(bool is_mic_open);
  void OnConversationTurnFinishedOnMainThread(
      assistant_client::ConversationStateListener::Resolution resolution);
  void OnShowHtmlOnMainThread(const std::string& html,
                              const std::string& fallback);
  void OnShowSuggestionsOnMainThread(
      const std::vector<mojom::AssistantSuggestionPtr>& suggestions);
  void OnShowTextOnMainThread(const std::string& text);
  void OnOpenUrlOnMainThread(const std::string& url);
  void OnShowNotificationOnMainThread(
      const mojom::AssistantNotificationPtr& notification);
  void OnNotificationRemovedOnMainThread(const std::string& grouping_id);
  void OnCommunicationErrorOnMainThread(int error_code);
  void OnRecognitionStateChangedOnMainThread(
      assistant_client::ConversationStateListener::RecognitionState state,
      const assistant_client::ConversationStateListener::RecognitionResult&
          recognition_result);
  void OnRespondingStartedOnMainThread(bool is_error_response);
  void OnSpeechLevelUpdatedOnMainThread(const float speech_level);
  void OnModifySettingsAction(const std::string& modify_setting_args_proto);

  void RegisterFallbackMediaHandler();

  void CacheAssistantStructure(
      base::OnceClosure on_done,
      ax::mojom::AssistantExtraPtr assistant_extra,
      std::unique_ptr<ui::AssistantTree> assistant_tree);

  void CacheAssistantScreenshot(
      base::OnceClosure on_done,
      const std::vector<uint8_t>& assistant_screenshot);

  void SendScreenContextRequest(
      ax::mojom::AssistantExtra* assistant_extra,
      ui::AssistantTree* assistant_tree,
      const std::vector<uint8_t>& assistant_screenshot);

  void FillServerExperimentIds(std::vector<std::string>* server_experiment_ids);

  // Record the response type for each query. Note that query on device
  // actions (e.g. turn on Bluetooth, turn on WiFi) will cause duplicate
  // record because it interacts with server twice on on the same query.
  // The first round interaction checks IsSettingSupported with no responses
  // sent back and ends normally (will be recorded as kUnspecified), and
  // settings modification proto along with any text/voice responses would
  // be sent back in the second round (recorded as kDeviceAction).
  void RecordQueryResponseTypeUMA();

  State state_ = State::STOPPED;
  std::unique_ptr<AssistantMediaSession> media_session_;
  std::unique_ptr<PlatformApiImpl> platform_api_;
  std::unique_ptr<action::CrosActionModule> action_module_;
  ChromiumApiDelegate chromium_api_delegate_;
  // NOTE: |display_connection_| is used by |assistant_manager_| and must be
  // declared before so it will be destructed after.
  std::unique_ptr<CrosDisplayConnection> display_connection_;
  std::unique_ptr<assistant_client::AssistantManager> assistant_manager_;
  std::unique_ptr<AssistantSettingsManagerImpl> assistant_settings_manager_;
  // same ownership as assistant_manager_.
  assistant_client::AssistantManagerInternal* assistant_manager_internal_ =
      nullptr;
  mojo::InterfacePtrSet<mojom::AssistantInteractionSubscriber>
      interaction_subscribers_;
  ash::mojom::AshMessageCenterControllerPtr ash_message_center_controller_;

  Service* service_;  // unowned.

  bool spoken_feedback_enabled_ = false;

  ax::mojom::AssistantExtraPtr assistant_extra_;
  std::unique_ptr<ui::AssistantTree> assistant_tree_;
  std::vector<uint8_t> assistant_screenshot_;
  std::string last_search_source_;
  base::Lock last_search_source_lock_;
  base::TimeTicks started_time_;

  base::Thread background_thread_;

  bool receive_modify_settings_proto_response_ = false;
  bool receive_inline_response_ = false;
  std::string receive_url_response_;

  bool is_first_client_discourse_context_query_ = true;

  base::WeakPtrFactory<AssistantManagerServiceImpl> weak_factory_;

  DISALLOW_COPY_AND_ASSIGN(AssistantManagerServiceImpl);
};

}  // namespace assistant
}  // namespace chromeos

#endif  // CHROMEOS_SERVICES_ASSISTANT_ASSISTANT_MANAGER_SERVICE_IMPL_H_