1 // Copyright 2018 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_WEB_CONTROLLER_H_ 6 #define COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_WEB_CONTROLLER_H_ 7 8 #include <memory> 9 #include <string> 10 #include <vector> 11 12 #include "base/callback_forward.h" 13 #include "base/macros.h" 14 #include "base/memory/weak_ptr.h" 15 #include "components/autofill_assistant/browser/actions/click_action.h" 16 #include "components/autofill_assistant/browser/batch_element_checker.h" 17 #include "components/autofill_assistant/browser/client_status.h" 18 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_dom.h" 19 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_input.h" 20 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_network.h" 21 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_runtime.h" 22 #include "components/autofill_assistant/browser/devtools/devtools_client.h" 23 #include "components/autofill_assistant/browser/rectf.h" 24 #include "components/autofill_assistant/browser/selector.h" 25 #include "components/autofill_assistant/browser/top_padding.h" 26 #include "components/autofill_assistant/browser/web/element_finder.h" 27 #include "components/autofill_assistant/browser/web/element_position_getter.h" 28 #include "components/autofill_assistant/browser/web/element_rect_getter.h" 29 #include "components/autofill_assistant/browser/web/web_controller_worker.h" 30 #include "third_party/icu/source/common/unicode/umachine.h" 31 #include "url/gurl.h" 32 33 namespace autofill { 34 class AutofillProfile; 35 class CreditCard; 36 struct FormData; 37 struct FormFieldData; 38 } // namespace autofill 39 40 namespace content { 41 class WebContents; 42 class RenderFrameHost; 43 } // namespace content 44 45 namespace autofill_assistant { 46 struct ClientSettings; 47 48 // Controller to interact with the web pages. 49 // 50 // WARNING: Accessing or modifying page elements must be run in sequence: wait 51 // until the result of the first operation has been given to the callback before 52 // starting a new operation. 53 // 54 // TODO(crbug.com/806868): Figure out the reason for this limitation and fix it. 55 // Also, consider structuring the WebController to make it easier to run 56 // multiple operations, whether in sequence or in parallel. 57 class WebController { 58 public: 59 // Create web controller for a given |web_contents|. |settings| must be valid 60 // for the lifetime of the controller. 61 static std::unique_ptr<WebController> CreateForWebContents( 62 content::WebContents* web_contents, 63 const ClientSettings* settings); 64 65 // |web_contents| and |settings| must outlive this web controller. 66 WebController(content::WebContents* web_contents, 67 std::unique_ptr<DevtoolsClient> devtools_client, 68 const ClientSettings* settings); 69 virtual ~WebController(); 70 71 // Load |url| in the current tab. Returns immediately, before the new page has 72 // been loaded. 73 virtual void LoadURL(const GURL& url); 74 75 // Perform a mouse left button click or a touch tap on the element given by 76 // |selector| and return the result through callback. 77 virtual void ClickOrTapElement( 78 const Selector& selector, 79 ClickAction::ClickType click_type, 80 base::OnceCallback<void(const ClientStatus&)> callback); 81 82 // Fill the address form given by |selector| with the given address 83 // |profile|. 84 virtual void FillAddressForm( 85 const autofill::AutofillProfile* profile, 86 const Selector& selector, 87 base::OnceCallback<void(const ClientStatus&)> callback); 88 89 // Fill the card form given by |selector| with the given |card| and its 90 // |cvc|. 91 virtual void FillCardForm( 92 std::unique_ptr<autofill::CreditCard> card, 93 const base::string16& cvc, 94 const Selector& selector, 95 base::OnceCallback<void(const ClientStatus&)> callback); 96 97 // Return |FormData| and |FormFieldData| for the element identified with 98 // |selector|. The result is returned asynchronously through |callback|. 99 virtual void RetrieveElementFormAndFieldData( 100 const Selector& selector, 101 base::OnceCallback<void(const ClientStatus&, 102 const autofill::FormData& form_data, 103 const autofill::FormFieldData& field_data)> 104 callback); 105 106 // Select the option given by |selector| and the value of the option to be 107 // picked. 108 virtual void SelectOption( 109 const Selector& selector, 110 const std::string& value, 111 DropdownSelectStrategy select_strategy, 112 base::OnceCallback<void(const ClientStatus&)> callback); 113 114 // Highlight an element given by |selector|. 115 virtual void HighlightElement( 116 const Selector& selector, 117 base::OnceCallback<void(const ClientStatus&)> callback); 118 119 // Focus on element given by |selector|. |top_padding| specifies the padding 120 // between focused element and the top. 121 virtual void FocusElement( 122 const Selector& selector, 123 const TopPadding& top_padding, 124 base::OnceCallback<void(const ClientStatus&)> callback); 125 126 // Get the value of |selector| and return the result through |callback|. The 127 // returned value might be false, if the element cannot be found, true and the 128 // empty string in case of error or empty value. 129 // 130 // Normally done through BatchElementChecker. 131 virtual void GetFieldValue( 132 const Selector& selector, 133 base::OnceCallback<void(const ClientStatus&, const std::string&)> 134 callback); 135 136 // Set the |value| of field |selector| and return the result through 137 // |callback|. The strategy used to fill the value is defined by 138 // |fill_strategy|, see the proto for further explanation. 139 virtual void SetFieldValue( 140 const Selector& selector, 141 const std::string& value, 142 KeyboardValueFillStrategy fill_strategy, 143 int key_press_delay_in_millisecond, 144 base::OnceCallback<void(const ClientStatus&)> callback); 145 146 // Set the |value| of the |attribute| of the element given by |selector|. 147 virtual void SetAttribute( 148 const Selector& selector, 149 const std::vector<std::string>& attribute, 150 const std::string& value, 151 base::OnceCallback<void(const ClientStatus&)> callback); 152 153 // Sets the keyboard focus to |selector| and inputs |codepoints|, one 154 // character at a time. Key presses will have a delay of |delay_in_milli| 155 // between them. 156 // Returns the result through |callback|. 157 virtual void SendKeyboardInput( 158 const Selector& selector, 159 const std::vector<UChar32>& codepoints, 160 int delay_in_milli, 161 base::OnceCallback<void(const ClientStatus&)> callback); 162 163 // Return the outerHTML of |selector|. 164 virtual void GetOuterHtml( 165 const Selector& selector, 166 base::OnceCallback<void(const ClientStatus&, const std::string&)> 167 callback); 168 169 // Return the tag of |selector|. 170 virtual void GetElementTag( 171 const Selector& selector, 172 base::OnceCallback<void(const ClientStatus&, const std::string&)> 173 callback); 174 175 // Gets the visual viewport coordinates and size. 176 // 177 // The rectangle is expressed in absolute CSS coordinates. 178 virtual void GetVisualViewport( 179 base::OnceCallback<void(bool, const RectF&)> callback); 180 181 // Gets the position of the element identified by the selector. 182 // 183 // If unsuccessful, the callback gets (false, 0, 0, 0, 0). 184 // 185 // If successful, the callback gets (true, left, top, right, bottom), with 186 // coordinates expressed in absolute CSS coordinates. 187 virtual void GetElementPosition( 188 const Selector& selector, 189 base::OnceCallback<void(bool, const RectF&)> callback); 190 191 // Checks whether an element matches the given selector. 192 // 193 // If strict, there must be exactly one matching element for the check to 194 // pass. Otherwise, there must be at least one. 195 // 196 // To check multiple elements, use a BatchElementChecker. 197 virtual void ElementCheck( 198 const Selector& selector, 199 bool strict, 200 base::OnceCallback<void(const ClientStatus&)> callback); 201 202 // Calls the callback once the main document window has been resized. 203 virtual void WaitForWindowHeightChange( 204 base::OnceCallback<void(const ClientStatus&)> callback); 205 206 // Gets the value of document.readyState for |optional_frame| or, if it is 207 // empty, in the main document. 208 virtual void GetDocumentReadyState( 209 const Selector& optional_frame, 210 base::OnceCallback<void(const ClientStatus&, 211 DocumentReadyState end_state)> callback); 212 213 // Waits for the value of Document.readyState to satisfy |min_ready_state| in 214 // |optional_frame| or, if it is empty, in the main document. 215 virtual void WaitForDocumentReadyState( 216 const Selector& optional_frame, 217 DocumentReadyState min_ready_state, 218 base::OnceCallback<void(const ClientStatus&, 219 DocumentReadyState end_state)> callback); 220 221 private: 222 friend class WebControllerBrowserTest; 223 224 struct FillFormInputData { 225 FillFormInputData(); 226 ~FillFormInputData(); 227 228 // Data for filling address form. 229 std::unique_ptr<autofill::AutofillProfile> profile; 230 231 // Data for filling card form. 232 std::unique_ptr<autofill::CreditCard> card; 233 base::string16 cvc; 234 }; 235 236 void OnFindElementForClickOrTap( 237 base::OnceCallback<void(const ClientStatus&)> callback, 238 ClickAction::ClickType click_type, 239 const ClientStatus& status, 240 std::unique_ptr<ElementFinder::Result> result); 241 void OnWaitDocumentToBecomeInteractiveForClickOrTap( 242 base::OnceCallback<void(const ClientStatus&)> callback, 243 ClickAction::ClickType click_type, 244 std::unique_ptr<ElementFinder::Result> target_element, 245 bool result); 246 void OnFindElementForTap( 247 base::OnceCallback<void(const ClientStatus&)> callback, 248 const ClientStatus& status, 249 std::unique_ptr<ElementFinder::Result> result); 250 void ClickOrTapElement( 251 std::unique_ptr<ElementFinder::Result> target_element, 252 ClickAction::ClickType click_type, 253 base::OnceCallback<void(const ClientStatus&)> callback); 254 void OnClickJS(base::OnceCallback<void(const ClientStatus&)> callback, 255 const DevtoolsClient::ReplyStatus& reply_status, 256 std::unique_ptr<runtime::CallFunctionOnResult> result); 257 void OnScrollIntoView(std::unique_ptr<ElementFinder::Result> target_element, 258 base::OnceCallback<void(const ClientStatus&)> callback, 259 ClickAction::ClickType click_type, 260 const DevtoolsClient::ReplyStatus& reply_status, 261 std::unique_ptr<runtime::CallFunctionOnResult> result); 262 void TapOrClickOnCoordinates( 263 ElementPositionGetter* getter_to_release, 264 base::OnceCallback<void(const ClientStatus&)> callback, 265 const std::string& node_frame_id, 266 ClickAction::ClickType click_type, 267 bool has_coordinates, 268 int x, 269 int y); 270 void OnDispatchPressMouseEvent( 271 base::OnceCallback<void(const ClientStatus&)> callback, 272 const std::string& node_frame_id, 273 int x, 274 int y, 275 const DevtoolsClient::ReplyStatus& reply_status, 276 std::unique_ptr<input::DispatchMouseEventResult> result); 277 void OnDispatchReleaseMouseEvent( 278 base::OnceCallback<void(const ClientStatus&)> callback, 279 const DevtoolsClient::ReplyStatus& reply_status, 280 std::unique_ptr<input::DispatchMouseEventResult> result); 281 void OnDispatchTouchEventStart( 282 base::OnceCallback<void(const ClientStatus&)> callback, 283 const std::string& node_frame_id, 284 const DevtoolsClient::ReplyStatus& reply_status, 285 std::unique_ptr<input::DispatchTouchEventResult> result); 286 void OnDispatchTouchEventEnd( 287 base::OnceCallback<void(const ClientStatus&)> callback, 288 const DevtoolsClient::ReplyStatus& reply_status, 289 std::unique_ptr<input::DispatchTouchEventResult> result); 290 void OnFindElementForCheck( 291 base::OnceCallback<void(const ClientStatus&)> callback, 292 const ClientStatus& status, 293 std::unique_ptr<ElementFinder::Result> result); 294 void OnWaitForWindowHeightChange( 295 base::OnceCallback<void(const ClientStatus&)> callback, 296 const DevtoolsClient::ReplyStatus& reply_status, 297 std::unique_ptr<runtime::EvaluateResult> result); 298 299 // Find the element given by |selector|. If multiple elements match 300 // |selector| and if |strict_mode| is false, return the first one that is 301 // found. Otherwise if |strict-mode| is true, do not return any. 302 void FindElement(const Selector& selector, 303 bool strict_mode, 304 ElementFinder::Callback callback); 305 void OnFindElementResult(ElementFinder* finder_to_release, 306 ElementFinder::Callback callback, 307 const ClientStatus& status, 308 std::unique_ptr<ElementFinder::Result> result); 309 void OnFindElementForFillingForm( 310 std::unique_ptr<FillFormInputData> data_to_autofill, 311 const Selector& selector, 312 base::OnceCallback<void(const ClientStatus&)> callback, 313 const ClientStatus& status, 314 std::unique_ptr<ElementFinder::Result> element_result); 315 void OnGetFormAndFieldDataForFillingForm( 316 std::unique_ptr<FillFormInputData> data_to_autofill, 317 base::OnceCallback<void(const ClientStatus&)> callback, 318 content::RenderFrameHost* container_frame_host, 319 const autofill::FormData& form_data, 320 const autofill::FormFieldData& form_field); 321 void OnFindElementToRetrieveFormAndFieldData( 322 const Selector& selector, 323 base::OnceCallback<void(const ClientStatus&, 324 const autofill::FormData& form_data, 325 const autofill::FormFieldData& form_field)> 326 callback, 327 const ClientStatus& status, 328 std::unique_ptr<ElementFinder::Result> element_result); 329 void OnGetFormAndFieldDataForRetrieving( 330 base::OnceCallback<void(const ClientStatus&, 331 const autofill::FormData& form_data, 332 const autofill::FormFieldData& form_field)> 333 callback, 334 const autofill::FormData& form_data, 335 const autofill::FormFieldData& form_field); 336 void OnFindElementForFocusElement( 337 const TopPadding& top_padding, 338 base::OnceCallback<void(const ClientStatus&)> callback, 339 const ClientStatus& status, 340 std::unique_ptr<ElementFinder::Result> element_result); 341 void OnWaitDocumentToBecomeInteractiveForFocusElement( 342 const TopPadding& top_padding, 343 base::OnceCallback<void(const ClientStatus&)> callback, 344 std::unique_ptr<ElementFinder::Result> target_element, 345 bool result); 346 void OnFocusElement(base::OnceCallback<void(const ClientStatus&)> callback, 347 const DevtoolsClient::ReplyStatus& reply_status, 348 std::unique_ptr<runtime::CallFunctionOnResult> result); 349 void OnFindElementForSelectOption( 350 const std::string& value, 351 DropdownSelectStrategy select_strategy, 352 base::OnceCallback<void(const ClientStatus&)> callback, 353 const ClientStatus& status, 354 std::unique_ptr<ElementFinder::Result> element_result); 355 void OnSelectOption(base::OnceCallback<void(const ClientStatus&)> callback, 356 const DevtoolsClient::ReplyStatus& reply_status, 357 std::unique_ptr<runtime::CallFunctionOnResult> result); 358 void OnFindElementForHighlightElement( 359 base::OnceCallback<void(const ClientStatus&)> callback, 360 const ClientStatus& status, 361 std::unique_ptr<ElementFinder::Result> element_result); 362 void OnHighlightElement( 363 base::OnceCallback<void(const ClientStatus&)> callback, 364 const DevtoolsClient::ReplyStatus& reply_status, 365 std::unique_ptr<runtime::CallFunctionOnResult> result); 366 void OnFindElementForGetFieldValue( 367 base::OnceCallback<void(const ClientStatus&, const std::string&)> 368 callback, 369 const ClientStatus& status, 370 std::unique_ptr<ElementFinder::Result> element_result); 371 void OnGetValueAttribute( 372 base::OnceCallback<void(const ClientStatus&, const std::string&)> 373 callback, 374 const DevtoolsClient::ReplyStatus& reply_status, 375 std::unique_ptr<runtime::CallFunctionOnResult> result); 376 void OnClearFieldForSendKeyboardInput( 377 const Selector& selector, 378 const std::vector<UChar32>& codepoints, 379 int key_press_delay_in_millisecond, 380 base::OnceCallback<void(const ClientStatus&)> callback, 381 const ClientStatus& clear_status); 382 void SelectFieldValueForReplace( 383 const Selector& selector, 384 base::OnceCallback<void(std::unique_ptr<ElementFinder::Result>, 385 const ClientStatus&)> callback); 386 void OnFindElementForSelectValue( 387 base::OnceCallback<void(std::unique_ptr<ElementFinder::Result>, 388 const ClientStatus&)> callback, 389 const ClientStatus& element_status, 390 std::unique_ptr<ElementFinder::Result> element_result); 391 void OnSelectFieldValueForDispatchKeys( 392 std::unique_ptr<ElementFinder::Result> element_result, 393 base::OnceCallback<void(std::unique_ptr<ElementFinder::Result>, 394 const ClientStatus&)> callback, 395 const DevtoolsClient::ReplyStatus& reply_status, 396 std::unique_ptr<runtime::CallFunctionOnResult> result); 397 void OnFieldValueSelectedForDispatchKeys( 398 const std::vector<UChar32>& codepoints, 399 int key_press_delay_in_millisecond, 400 base::OnceCallback<void(const ClientStatus&)> callback, 401 std::unique_ptr<ElementFinder::Result> element_result, 402 const ClientStatus& select_status); 403 void DispatchKeyboardTextDownEvent( 404 const std::string& node_frame_id, 405 const std::vector<UChar32>& codepoints, 406 size_t index, 407 bool delay, 408 int delay_in_milli, 409 base::OnceCallback<void(const ClientStatus&)> callback); 410 void DispatchKeyboardTextUpEvent( 411 const std::string& node_frame_id, 412 const std::vector<UChar32>& codepoints, 413 size_t index, 414 int delay_in_milli, 415 base::OnceCallback<void(const ClientStatus&)> callback); 416 void InternalSetFieldValue( 417 const Selector& selector, 418 const std::string& value, 419 base::OnceCallback<void(const ClientStatus&)> callback); 420 void OnFindElementForSetFieldValue( 421 const std::string& value, 422 base::OnceCallback<void(const ClientStatus&)> callback, 423 const ClientStatus& status, 424 std::unique_ptr<ElementFinder::Result> element_result); 425 void OnSetValueAttribute( 426 base::OnceCallback<void(const ClientStatus&)> callback, 427 const DevtoolsClient::ReplyStatus& reply_status, 428 std::unique_ptr<runtime::CallFunctionOnResult> result); 429 void OnFindElementForSetAttribute( 430 const std::vector<std::string>& attribute, 431 const std::string& value, 432 base::OnceCallback<void(const ClientStatus&)> callback, 433 const ClientStatus& status, 434 std::unique_ptr<ElementFinder::Result> element_result); 435 void OnSetAttribute(base::OnceCallback<void(const ClientStatus&)> callback, 436 const DevtoolsClient::ReplyStatus& reply_status, 437 std::unique_ptr<runtime::CallFunctionOnResult> result); 438 void OnFindElementForSendKeyboardInput( 439 const Selector& selector, 440 const std::vector<UChar32>& codepoints, 441 int delay_in_milli, 442 base::OnceCallback<void(const ClientStatus&)> callback, 443 const ClientStatus& status, 444 std::unique_ptr<ElementFinder::Result> element_result); 445 void OnClickElementForSendKeyboardInput( 446 const std::string& node_frame_id, 447 const std::vector<UChar32>& codepoints, 448 int delay_in_milli, 449 base::OnceCallback<void(const ClientStatus&)> callback, 450 const ClientStatus& click_status); 451 void OnFindElementForGetOuterHtml( 452 base::OnceCallback<void(const ClientStatus&, const std::string&)> 453 callback, 454 const ClientStatus& status, 455 std::unique_ptr<ElementFinder::Result> element_result); 456 void OnGetOuterHtml(base::OnceCallback<void(const ClientStatus&, 457 const std::string&)> callback, 458 const DevtoolsClient::ReplyStatus& reply_status, 459 std::unique_ptr<runtime::CallFunctionOnResult> result); 460 void OnFindElementForGetElementTag( 461 base::OnceCallback<void(const ClientStatus&, const std::string&)> 462 callback, 463 const ClientStatus& status, 464 std::unique_ptr<ElementFinder::Result> element_result); 465 void OnGetElementTag(base::OnceCallback<void(const ClientStatus&, 466 const std::string&)> callback, 467 const DevtoolsClient::ReplyStatus& reply_status, 468 std::unique_ptr<runtime::CallFunctionOnResult> result); 469 void OnFindElementForPosition( 470 base::OnceCallback<void(bool, const RectF&)> callback, 471 const ClientStatus& status, 472 std::unique_ptr<ElementFinder::Result> result); 473 void OnGetVisualViewport( 474 base::OnceCallback<void(bool, const RectF&)> callback, 475 const DevtoolsClient::ReplyStatus& reply_status, 476 std::unique_ptr<runtime::EvaluateResult> result); 477 void OnGetElementRectResult( 478 ElementRectGetter* getter_to_release, 479 base::OnceCallback<void(bool, const RectF&)> callback, 480 bool has_rect, 481 const RectF& element_rect); 482 483 // Creates a new instance of DispatchKeyEventParams for the specified type and 484 // unicode codepoint. 485 using DispatchKeyEventParamsPtr = 486 std::unique_ptr<autofill_assistant::input::DispatchKeyEventParams>; 487 static DispatchKeyEventParamsPtr CreateKeyEventParamsForCharacter( 488 autofill_assistant::input::DispatchKeyEventType type, 489 const UChar32 codepoint); 490 491 // Waits for the document.readyState to be 'interactive' or 'complete'. 492 void WaitForDocumentToBecomeInteractive( 493 int remaining_rounds, 494 const std::string& object_id, 495 const std::string& node_frame_id, 496 base::OnceCallback<void(bool)> callback); 497 void OnWaitForDocumentToBecomeInteractive( 498 int remaining_rounds, 499 const std::string& object_id, 500 const std::string& node_frame_id, 501 base::OnceCallback<void(bool)> callback, 502 const DevtoolsClient::ReplyStatus& reply_status, 503 std::unique_ptr<runtime::CallFunctionOnResult> result); 504 void OnFindElementForWaitForDocumentReadyState( 505 DocumentReadyState min_ready_state, 506 base::OnceCallback<void(const ClientStatus&, DocumentReadyState)> 507 callback, 508 const ClientStatus& status, 509 std::unique_ptr<ElementFinder::Result> element); 510 511 // Weak pointer is fine here since it must outlive this web controller, which 512 // is guaranteed by the owner of this object. 513 content::WebContents* web_contents_; 514 std::unique_ptr<DevtoolsClient> devtools_client_; 515 const ClientSettings* const settings_; 516 517 // Currently running workers. 518 std::vector<std::unique_ptr<WebControllerWorker>> pending_workers_; 519 520 base::WeakPtrFactory<WebController> weak_ptr_factory_{this}; 521 DISALLOW_COPY_AND_ASSIGN(WebController); 522 }; 523 } // namespace autofill_assistant 524 #endif // COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_WEB_CONTROLLER_H_ 525