Updates SelectToSpeakEventHandler to change state when extension state changes.
This is part of a change to allow Select-to-Speak to work in tablet/touch only
mode.
This change allows the extension to tell the EventHandler that capturing should
being, allowing Select-to-Speak to start capturing touch events or mouse events
without the 'search' key having been clicked.
This change does not cause any behavior changes at this time, because there is
no current way to enter the SELECTING state without holding down the search key
first.
In a follow-up change, this pathway will be activated when a user will click
a button in the ash system shelf. That button is not yet hooked up. To see a
complete demo, see https://chromium-review.googlesource.com/c/chromium/src/+/1031550.
Bug: 753018
Change-Id: I3be60fa338ce32e2a99ce7e21d9d7cab503ac4e0
Reviewed-on: https://chromium-review.googlesource.com/1036534
Commit-Queue: Katie Dektar <[email protected]>
Reviewed-by: David Tseng <[email protected]>
Cr-Commit-Position: refs/heads/master@{#555540}diff --git a/chrome/browser/chromeos/accessibility/accessibility_manager.cc b/chrome/browser/chromeos/accessibility/accessibility_manager.cc
index 2db2d916..8c3a7afc9 100644
--- a/chrome/browser/chromeos/accessibility/accessibility_manager.cc
+++ b/chrome/browser/chromeos/accessibility/accessibility_manager.cc
@@ -716,10 +716,13 @@
void AccessibilityManager::OnSelectToSpeakStateChanged(
ash::mojom::SelectToSpeakState state) {
- // TODO(katie): Forward the state change event to
- // select_to_speak_event_handler_. The extension may have requested that the
- // handler enter SELECTING state. Prepare to start capturing events from
- // stylus, mouse or touch. http://crbug.com/753018.
+ // Forward the state change event to select_to_speak_event_handler_.
+ // The extension may have requested that the handler enter SELECTING state.
+ // Prepare to start capturing events from stylus, mouse or touch.
+ if (select_to_speak_event_handler_) {
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(
+ state == ash::mojom::SelectToSpeakState::kSelectToSpeakStateSelecting);
+ }
accessibility_controller_->SetSelectToSpeakState(state);
}
diff --git a/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.cc b/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.cc
index e14f449..17b01f7 100644
--- a/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.cc
+++ b/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.cc
@@ -62,6 +62,18 @@
aura::Env::GetInstanceDontCreate()->RemovePreTargetHandler(this);
}
+void SelectToSpeakEventHandler::SetSelectToSpeakStateSelecting(
+ bool is_selecting) {
+ if (is_selecting && state_ == INACTIVE) {
+ // The extension has requested that it enter SELECTING state, and we
+ // aren't already in a SELECTING state. Prepare to start capturing events
+ // from stylus, mouse or touch.
+ // If we are already in any state besides INACTIVE then there is no
+ // work that needs to be done.
+ state_ = SELECTION_REQUESTED;
+ }
+}
+
void SelectToSpeakEventHandler::CaptureForwardedEventsForTesting(
SelectToSpeakEventDelegateForTesting* delegate) {
event_delegate_for_testing_ = delegate;
@@ -152,9 +164,11 @@
if (state_ == INACTIVE)
return;
- if ((state_ == SEARCH_DOWN || state_ == MOUSE_RELEASED) &&
- event->type() == ui::ET_MOUSE_PRESSED) {
- state_ = CAPTURING_MOUSE;
+ if (event->type() == ui::ET_MOUSE_PRESSED) {
+ if (state_ == SEARCH_DOWN || state_ == MOUSE_RELEASED)
+ state_ = CAPTURING_MOUSE;
+ else if (state_ == SELECTION_REQUESTED)
+ state_ = CAPTURING_MOUSE_ONLY;
}
if (state_ == WAIT_FOR_MOUSE_RELEASE &&
@@ -163,39 +177,96 @@
return;
}
- if (state_ != CAPTURING_MOUSE)
+ // Only forward the event to the extension if we are capturing mouse
+ // events.
+ if (state_ != CAPTURING_MOUSE && state_ != CAPTURING_MOUSE_ONLY)
return;
- if (event->type() == ui::ET_MOUSE_RELEASED)
- state_ = MOUSE_RELEASED;
+ if (event->type() == ui::ET_MOUSE_RELEASED) {
+ if (state_ == CAPTURING_MOUSE)
+ state_ = MOUSE_RELEASED;
+ else if (state_ == CAPTURING_MOUSE_ONLY)
+ state_ = INACTIVE;
+ }
ui::MouseEvent mutable_event(*event);
-
- // If we're in the capturing mouse state, forward the mouse event to
- // select-to-speak.
- if (event_delegate_for_testing_) {
- event_delegate_for_testing_->OnForwardEventToSelectToSpeakExtension(
- mutable_event);
- } else {
- extensions::ExtensionHost* host = chromeos::GetAccessibilityExtensionHost(
- extension_misc::kSelectToSpeakExtensionId);
- if (!host)
- return;
-
- content::RenderViewHost* rvh = host->render_view_host();
- if (!rvh)
- return;
-
- const blink::WebMouseEvent web_event = ui::MakeWebMouseEvent(
- mutable_event, base::Bind(&GetScreenLocationFromEvent));
- rvh->GetWidget()->ForwardMouseEvent(web_event);
- }
+ ForwardMouseEventToExtension(&mutable_event);
if (event->type() == ui::ET_MOUSE_PRESSED ||
event->type() == ui::ET_MOUSE_RELEASED)
CancelEvent(event);
}
+void SelectToSpeakEventHandler::OnTouchEvent(ui::TouchEvent* event) {
+ if (!IsSelectToSpeakEnabled())
+ return;
+
+ DCHECK(event);
+ // Only capture touch events if selection was requested or we are capturing
+ // touch events already.
+ if (state_ != SELECTION_REQUESTED && state_ != CAPTURING_TOUCH_ONLY)
+ return;
+
+ // On a touch-down event, if selection was requested, we begin capturing
+ // touch events.
+ if (event->type() == ui::ET_TOUCH_PRESSED && state_ == SELECTION_REQUESTED)
+ state_ = CAPTURING_TOUCH_ONLY;
+
+ // On a touch-up event, we go back to inactive state, but still forward the
+ // event to the extension.
+ if (event->type() == ui::ET_TOUCH_RELEASED && state_ == CAPTURING_TOUCH_ONLY)
+ state_ = INACTIVE;
+
+ // Create a mouse event to send to the extension, describing the touch.
+ // This is done because there is no RenderWidgetHost::ForwardTouchEvent,
+ // and we already have mouse event plumbing in place for Select-to-Speak.
+ ui::EventType type;
+ switch (event->type()) {
+ case ui::ET_TOUCH_PRESSED:
+ type = ui::ET_MOUSE_PRESSED;
+ break;
+ case ui::ET_TOUCH_RELEASED:
+ case ui::ET_TOUCH_CANCELLED:
+ type = ui::ET_MOUSE_RELEASED;
+ break;
+ case ui::ET_TOUCH_MOVED:
+ type = ui::ET_MOUSE_DRAGGED;
+ break;
+ default:
+ return;
+ }
+ int flags = ui::EF_LEFT_MOUSE_BUTTON;
+ ui::MouseEvent mutable_event(type, event->location(), event->root_location(),
+ event->time_stamp(), flags, flags);
+
+ ForwardMouseEventToExtension(&mutable_event);
+
+ if (event->type() != ui::ET_TOUCH_MOVED) {
+ // Don't cancel move events in case focus needs to change.
+ CancelEvent(event);
+ }
+}
+
+void SelectToSpeakEventHandler::ForwardMouseEventToExtension(
+ ui::MouseEvent* event) {
+ if (event_delegate_for_testing_) {
+ event_delegate_for_testing_->OnForwardEventToSelectToSpeakExtension(*event);
+ return;
+ }
+ extensions::ExtensionHost* host = chromeos::GetAccessibilityExtensionHost(
+ extension_misc::kSelectToSpeakExtensionId);
+ if (!host)
+ return;
+
+ content::RenderViewHost* rvh = host->render_view_host();
+ if (!rvh)
+ return;
+
+ const blink::WebMouseEvent web_event =
+ ui::MakeWebMouseEvent(*event, base::Bind(&GetScreenLocationFromEvent));
+ rvh->GetWidget()->ForwardMouseEvent(web_event);
+}
+
void SelectToSpeakEventHandler::CancelEvent(ui::Event* event) {
DCHECK(event);
if (event->cancelable()) {
diff --git a/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.h b/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.h
index 50dd54a..5034acb 100644
--- a/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.h
+++ b/chrome/browser/chromeos/accessibility/select_to_speak_event_handler.h
@@ -25,6 +25,12 @@
SelectToSpeakEventHandler();
~SelectToSpeakEventHandler() override;
+ // Called when the Select-to-Speak extension changes state. |is_selecting| is
+ // true if the extension wants to begin capturing mouse or touch events, in
+ // which case this handler needs to start forwarding those events if it was
+ // in an inactive state.
+ void SetSelectToSpeakStateSelecting(bool is_selecting);
+
void CaptureForwardedEventsForTesting(
SelectToSpeakEventDelegateForTesting* delegate);
@@ -32,6 +38,7 @@
// EventHandler
void OnKeyEvent(ui::KeyEvent* event) override;
void OnMouseEvent(ui::MouseEvent* event) override;
+ void OnTouchEvent(ui::TouchEvent* event) override;
// Returns true if Select to Speak is enabled.
bool IsSelectToSpeakEnabled();
@@ -41,8 +48,12 @@
// Converts an event in pixels to the same event in DIPs.
void ConvertMouseEventToDIPs(ui::MouseEvent* mouse_event);
+ // Forwards a mouse event to the Select-to-Speak extension.
+ void ForwardMouseEventToExtension(ui::MouseEvent* event);
+
enum State {
- // The search key is not down. No other keys or mouse events are captured.
+ // The search key is not down, no selection has been requested.
+ // No other keys or mouse events are captured.
INACTIVE,
// The Search key is down but the mouse button and 'S' key are not.
@@ -79,7 +90,21 @@
// The Search key was released while the selection key was still down. Stay
// in this mode until the speak selection key is released too.
- WAIT_FOR_SPEAK_SELECTION_KEY_RELEASE
+ WAIT_FOR_SPEAK_SELECTION_KEY_RELEASE,
+
+ // The user has clicked a button in the Chrome UI indicating they want to
+ // start capturing mouse or touch events. The next mouse or touch events
+ // should be captured.
+ SELECTION_REQUESTED,
+
+ // Mouse events are being captured, but do not need to wait for any key
+ // events when the mouse is released, unlike WAIT_FOR_MOUSE_RELEASE.
+ // Only mouse events will be captured until the mouse released event is
+ // received.
+ CAPTURING_MOUSE_ONLY,
+
+ // Touch events are being captured, similar to CAPTURING_MOUSE_ONLY.
+ CAPTURING_TOUCH_ONLY
};
State state_ = INACTIVE;
diff --git a/chrome/browser/chromeos/accessibility/select_to_speak_event_handler_unittest.cc b/chrome/browser/chromeos/accessibility/select_to_speak_event_handler_unittest.cc
index 9bed21d7..9373f197 100644
--- a/chrome/browser/chromeos/accessibility/select_to_speak_event_handler_unittest.cc
+++ b/chrome/browser/chromeos/accessibility/select_to_speak_event_handler_unittest.cc
@@ -34,10 +34,12 @@
void Reset() {
last_key_event_.reset(nullptr);
last_mouse_event_.reset(nullptr);
+ last_touch_event_.reset(nullptr);
}
ui::KeyEvent* last_key_event() { return last_key_event_.get(); }
ui::MouseEvent* last_mouse_event() { return last_mouse_event_.get(); }
+ ui::TouchEvent* last_touch_event() { return last_touch_event_.get(); }
private:
void OnMouseEvent(ui::MouseEvent* event) override {
@@ -46,9 +48,13 @@
void OnKeyEvent(ui::KeyEvent* event) override {
last_key_event_.reset(new ui::KeyEvent(*event));
}
+ void OnTouchEvent(ui::TouchEvent* event) override {
+ last_touch_event_.reset(new ui::TouchEvent(*event));
+ }
std::unique_ptr<ui::KeyEvent> last_key_event_;
std::unique_ptr<ui::MouseEvent> last_mouse_event_;
+ std::unique_ptr<ui::TouchEvent> last_touch_event_;
DISALLOW_COPY_AND_ASSIGN(EventCapturer);
};
@@ -113,8 +119,6 @@
EventCapturer event_capturer_;
TestingProfile profile_;
std::unique_ptr<SelectToSpeakMouseEventDelegate> mouse_event_delegate_;
-
- private:
std::unique_ptr<SelectToSpeakEventHandler> select_to_speak_event_handler_;
DISALLOW_COPY_AND_ASSIGN(SelectToSpeakEventHandlerTest);
@@ -408,4 +412,142 @@
EXPECT_FALSE(event_capturer_.last_key_event());
}
+TEST_F(SelectToSpeakEventHandlerTest, DoesntStartSelectionModeIfNotInactive) {
+ generator_->PressKey(ui::VKEY_LWIN, ui::EF_COMMAND_DOWN);
+
+ // This shouldn't cause any changes since the state is not inactive.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(true);
+
+ // Mouse event still captured.
+ gfx::Point click_location = gfx::Point(100, 12);
+ generator_->set_current_location(click_location);
+ generator_->PressLeftButton();
+ EXPECT_FALSE(event_capturer_.last_mouse_event());
+
+ // This shouldn't cause any changes since the state is not inactive.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(true);
+
+ generator_->ReleaseLeftButton();
+
+ // Releasing the search key is still captured per the end of the search+click
+ // mode.
+ event_capturer_.Reset();
+ generator_->ReleaseKey(ui::VKEY_LWIN, ui::EF_COMMAND_DOWN);
+ EXPECT_FALSE(event_capturer_.last_key_event());
+}
+
+TEST_F(SelectToSpeakEventHandlerTest, SelectionRequestedWorksWithMouse) {
+ gfx::Point click_location = gfx::Point(100, 12);
+ generator_->set_current_location(click_location);
+
+ // Mouse events are let through normally before entering selecting state.
+ // Another mouse event is let through normally.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(false);
+ generator_->PressLeftButton();
+ EXPECT_TRUE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+ generator_->ReleaseLeftButton();
+ EXPECT_TRUE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+
+ // Start selection mode.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(true);
+
+ generator_->PressLeftButton();
+ EXPECT_FALSE(event_capturer_.last_mouse_event());
+ EXPECT_TRUE(mouse_event_delegate_->CapturedMouseEvent(ui::ET_MOUSE_PRESSED));
+ event_capturer_.Reset();
+
+ gfx::Point drag_location = gfx::Point(120, 32);
+ generator_->DragMouseTo(drag_location);
+ EXPECT_EQ(drag_location, mouse_event_delegate_->last_mouse_event_location());
+ EXPECT_TRUE(mouse_event_delegate_->CapturedMouseEvent(ui::ET_MOUSE_DRAGGED));
+ EXPECT_TRUE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+
+ // Mouse up is the last event captured in the sequence
+ generator_->ReleaseLeftButton();
+ EXPECT_FALSE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+
+ // Another mouse event is let through normally.
+ generator_->PressLeftButton();
+ EXPECT_TRUE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+}
+
+TEST_F(SelectToSpeakEventHandlerTest, SelectionRequestedWorksWithTouch) {
+ gfx::Point touch_location = gfx::Point(100, 12);
+ generator_->set_current_location(touch_location);
+
+ // Mouse events are let through normally before entering selecting state.
+ // Another mouse event is let through normally.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(false);
+ generator_->PressTouch();
+ EXPECT_TRUE(event_capturer_.last_touch_event());
+ event_capturer_.Reset();
+ generator_->ReleaseTouch();
+ EXPECT_TRUE(event_capturer_.last_touch_event());
+ event_capturer_.Reset();
+
+ // Start selection mode.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(true);
+
+ generator_->PressTouch();
+ EXPECT_FALSE(event_capturer_.last_touch_event());
+ // Touch events are converted to mouse events for the extension.
+ EXPECT_TRUE(mouse_event_delegate_->CapturedMouseEvent(ui::ET_MOUSE_PRESSED));
+ event_capturer_.Reset();
+
+ gfx::Point drag_location = gfx::Point(120, 32);
+ generator_->MoveTouch(drag_location);
+ EXPECT_EQ(drag_location, mouse_event_delegate_->last_mouse_event_location());
+ EXPECT_TRUE(mouse_event_delegate_->CapturedMouseEvent(ui::ET_MOUSE_DRAGGED));
+ EXPECT_TRUE(event_capturer_.last_touch_event());
+ event_capturer_.Reset();
+
+ // Touch up is the last event captured in the sequence
+ generator_->ReleaseTouch();
+ EXPECT_FALSE(event_capturer_.last_touch_event());
+ event_capturer_.Reset();
+
+ // Another mouse event is let through normally.
+ generator_->PressTouch();
+ EXPECT_TRUE(event_capturer_.last_touch_event());
+ event_capturer_.Reset();
+}
+
+TEST_F(SelectToSpeakEventHandlerTest, SelectionRequestedIgnoresOtherInput) {
+ // Start selection mode.
+ select_to_speak_event_handler_->SetSelectToSpeakStateSelecting(true);
+
+ // Search key events are not impacted.
+ generator_->PressKey(ui::VKEY_LWIN, ui::EF_COMMAND_DOWN);
+ EXPECT_TRUE(event_capturer_.last_key_event());
+ event_capturer_.Reset();
+ generator_->ReleaseKey(ui::VKEY_LWIN, ui::EF_COMMAND_DOWN);
+ EXPECT_TRUE(event_capturer_.last_key_event());
+ event_capturer_.Reset();
+
+ // Start a touch selection, it should get captured and forwarded.
+ generator_->PressTouch();
+ EXPECT_FALSE(event_capturer_.last_touch_event());
+ EXPECT_TRUE(mouse_event_delegate_->CapturedMouseEvent(ui::ET_MOUSE_PRESSED));
+ event_capturer_.Reset();
+
+ // Mouse event happening during the touch selection are not impacted;
+ // we are locked into a touch selection mode.
+ generator_->PressLeftButton();
+ EXPECT_TRUE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+ generator_->ReleaseLeftButton();
+ EXPECT_TRUE(event_capturer_.last_mouse_event());
+ event_capturer_.Reset();
+
+ // Complete the touch selection.
+ generator_->ReleaseTouch();
+ EXPECT_FALSE(event_capturer_.last_touch_event());
+ event_capturer_.Reset();
+}
+
} // namespace chromeos
