1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_WEB_CONTROLLER_H_
6 #define COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_WEB_CONTROLLER_H_
7 
8 #include <memory>
9 #include <string>
10 #include <vector>
11 
12 #include "base/callback_forward.h"
13 #include "base/macros.h"
14 #include "base/memory/weak_ptr.h"
15 #include "components/autofill_assistant/browser/actions/click_action.h"
16 #include "components/autofill_assistant/browser/batch_element_checker.h"
17 #include "components/autofill_assistant/browser/client_status.h"
18 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_dom.h"
19 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_input.h"
20 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_network.h"
21 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_runtime.h"
22 #include "components/autofill_assistant/browser/devtools/devtools_client.h"
23 #include "components/autofill_assistant/browser/rectf.h"
24 #include "components/autofill_assistant/browser/selector.h"
25 #include "components/autofill_assistant/browser/top_padding.h"
26 #include "components/autofill_assistant/browser/web/element_finder.h"
27 #include "components/autofill_assistant/browser/web/element_position_getter.h"
28 #include "components/autofill_assistant/browser/web/element_rect_getter.h"
29 #include "components/autofill_assistant/browser/web/web_controller_worker.h"
30 #include "third_party/icu/source/common/unicode/umachine.h"
31 #include "url/gurl.h"
32 
33 namespace autofill {
34 class AutofillProfile;
35 class CreditCard;
36 struct FormData;
37 struct FormFieldData;
38 }  // namespace autofill
39 
40 namespace content {
41 class WebContents;
42 class RenderFrameHost;
43 }  // namespace content
44 
45 namespace autofill_assistant {
46 struct ClientSettings;
47 
48 // Controller to interact with the web pages.
49 //
50 // WARNING: Accessing or modifying page elements must be run in sequence: wait
51 // until the result of the first operation has been given to the callback before
52 // starting a new operation.
53 //
54 // TODO(crbug.com/806868): Figure out the reason for this limitation and fix it.
55 // Also, consider structuring the WebController to make it easier to run
56 // multiple operations, whether in sequence or in parallel.
57 class WebController {
58  public:
59   // Create web controller for a given |web_contents|. |settings| must be valid
60   // for the lifetime of the controller.
61   static std::unique_ptr<WebController> CreateForWebContents(
62       content::WebContents* web_contents,
63       const ClientSettings* settings);
64 
65   // |web_contents| and |settings| must outlive this web controller.
66   WebController(content::WebContents* web_contents,
67                 std::unique_ptr<DevtoolsClient> devtools_client,
68                 const ClientSettings* settings);
69   virtual ~WebController();
70 
71   // Load |url| in the current tab. Returns immediately, before the new page has
72   // been loaded.
73   virtual void LoadURL(const GURL& url);
74 
75   // Perform a mouse left button click or a touch tap on the element given by
76   // |selector| and return the result through callback.
77   virtual void ClickOrTapElement(
78       const Selector& selector,
79       ClickAction::ClickType click_type,
80       base::OnceCallback<void(const ClientStatus&)> callback);
81 
82   // Fill the address form given by |selector| with the given address
83   // |profile|.
84   virtual void FillAddressForm(
85       const autofill::AutofillProfile* profile,
86       const Selector& selector,
87       base::OnceCallback<void(const ClientStatus&)> callback);
88 
89   // Fill the card form given by |selector| with the given |card| and its
90   // |cvc|.
91   virtual void FillCardForm(
92       std::unique_ptr<autofill::CreditCard> card,
93       const base::string16& cvc,
94       const Selector& selector,
95       base::OnceCallback<void(const ClientStatus&)> callback);
96 
97   // Return |FormData| and |FormFieldData| for the element identified with
98   // |selector|. The result is returned asynchronously through |callback|.
99   virtual void RetrieveElementFormAndFieldData(
100       const Selector& selector,
101       base::OnceCallback<void(const ClientStatus&,
102                               const autofill::FormData& form_data,
103                               const autofill::FormFieldData& field_data)>
104           callback);
105 
106   // Select the option given by |selector| and the value of the option to be
107   // picked.
108   virtual void SelectOption(
109       const Selector& selector,
110       const std::string& value,
111       DropdownSelectStrategy select_strategy,
112       base::OnceCallback<void(const ClientStatus&)> callback);
113 
114   // Highlight an element given by |selector|.
115   virtual void HighlightElement(
116       const Selector& selector,
117       base::OnceCallback<void(const ClientStatus&)> callback);
118 
119   // Focus on element given by |selector|. |top_padding| specifies the padding
120   // between focused element and the top.
121   virtual void FocusElement(
122       const Selector& selector,
123       const TopPadding& top_padding,
124       base::OnceCallback<void(const ClientStatus&)> callback);
125 
126   // Get the value of |selector| and return the result through |callback|. The
127   // returned value might be false, if the element cannot be found, true and the
128   // empty string in case of error or empty value.
129   //
130   // Normally done through BatchElementChecker.
131   virtual void GetFieldValue(
132       const Selector& selector,
133       base::OnceCallback<void(const ClientStatus&, const std::string&)>
134           callback);
135 
136   // Set the |value| of field |selector| and return the result through
137   // |callback|. The strategy used to fill the value is defined by
138   // |fill_strategy|, see the proto for further explanation.
139   virtual void SetFieldValue(
140       const Selector& selector,
141       const std::string& value,
142       KeyboardValueFillStrategy fill_strategy,
143       int key_press_delay_in_millisecond,
144       base::OnceCallback<void(const ClientStatus&)> callback);
145 
146   // Set the |value| of the |attribute| of the element given by |selector|.
147   virtual void SetAttribute(
148       const Selector& selector,
149       const std::vector<std::string>& attribute,
150       const std::string& value,
151       base::OnceCallback<void(const ClientStatus&)> callback);
152 
153   // Sets the keyboard focus to |selector| and inputs |codepoints|, one
154   // character at a time. Key presses will have a delay of |delay_in_milli|
155   // between them.
156   // Returns the result through |callback|.
157   virtual void SendKeyboardInput(
158       const Selector& selector,
159       const std::vector<UChar32>& codepoints,
160       int delay_in_milli,
161       base::OnceCallback<void(const ClientStatus&)> callback);
162 
163   // Return the outerHTML of |selector|.
164   virtual void GetOuterHtml(
165       const Selector& selector,
166       base::OnceCallback<void(const ClientStatus&, const std::string&)>
167           callback);
168 
169   // Return the tag of |selector|.
170   virtual void GetElementTag(
171       const Selector& selector,
172       base::OnceCallback<void(const ClientStatus&, const std::string&)>
173           callback);
174 
175   // Gets the visual viewport coordinates and size.
176   //
177   // The rectangle is expressed in absolute CSS coordinates.
178   virtual void GetVisualViewport(
179       base::OnceCallback<void(bool, const RectF&)> callback);
180 
181   // Gets the position of the element identified by the selector.
182   //
183   // If unsuccessful, the callback gets (false, 0, 0, 0, 0).
184   //
185   // If successful, the callback gets (true, left, top, right, bottom), with
186   // coordinates expressed in absolute CSS coordinates.
187   virtual void GetElementPosition(
188       const Selector& selector,
189       base::OnceCallback<void(bool, const RectF&)> callback);
190 
191   // Checks whether an element matches the given selector.
192   //
193   // If strict, there must be exactly one matching element for the check to
194   // pass. Otherwise, there must be at least one.
195   //
196   // To check multiple elements, use a BatchElementChecker.
197   virtual void ElementCheck(
198       const Selector& selector,
199       bool strict,
200       base::OnceCallback<void(const ClientStatus&)> callback);
201 
202   // Calls the callback once the main document window has been resized.
203   virtual void WaitForWindowHeightChange(
204       base::OnceCallback<void(const ClientStatus&)> callback);
205 
206   // Gets the value of document.readyState for |optional_frame| or, if it is
207   // empty, in the main document.
208   virtual void GetDocumentReadyState(
209       const Selector& optional_frame,
210       base::OnceCallback<void(const ClientStatus&,
211                               DocumentReadyState end_state)> callback);
212 
213   // Waits for the value of Document.readyState to satisfy |min_ready_state| in
214   // |optional_frame| or, if it is empty, in the main document.
215   virtual void WaitForDocumentReadyState(
216       const Selector& optional_frame,
217       DocumentReadyState min_ready_state,
218       base::OnceCallback<void(const ClientStatus&,
219                               DocumentReadyState end_state)> callback);
220 
221  private:
222   friend class WebControllerBrowserTest;
223 
224   struct FillFormInputData {
225     FillFormInputData();
226     ~FillFormInputData();
227 
228     // Data for filling address form.
229     std::unique_ptr<autofill::AutofillProfile> profile;
230 
231     // Data for filling card form.
232     std::unique_ptr<autofill::CreditCard> card;
233     base::string16 cvc;
234   };
235 
236   void OnFindElementForClickOrTap(
237       base::OnceCallback<void(const ClientStatus&)> callback,
238       ClickAction::ClickType click_type,
239       const ClientStatus& status,
240       std::unique_ptr<ElementFinder::Result> result);
241   void OnWaitDocumentToBecomeInteractiveForClickOrTap(
242       base::OnceCallback<void(const ClientStatus&)> callback,
243       ClickAction::ClickType click_type,
244       std::unique_ptr<ElementFinder::Result> target_element,
245       bool result);
246   void OnFindElementForTap(
247       base::OnceCallback<void(const ClientStatus&)> callback,
248       const ClientStatus& status,
249       std::unique_ptr<ElementFinder::Result> result);
250   void ClickOrTapElement(
251       std::unique_ptr<ElementFinder::Result> target_element,
252       ClickAction::ClickType click_type,
253       base::OnceCallback<void(const ClientStatus&)> callback);
254   void OnClickJS(base::OnceCallback<void(const ClientStatus&)> callback,
255                  const DevtoolsClient::ReplyStatus& reply_status,
256                  std::unique_ptr<runtime::CallFunctionOnResult> result);
257   void OnScrollIntoView(std::unique_ptr<ElementFinder::Result> target_element,
258                         base::OnceCallback<void(const ClientStatus&)> callback,
259                         ClickAction::ClickType click_type,
260                         const DevtoolsClient::ReplyStatus& reply_status,
261                         std::unique_ptr<runtime::CallFunctionOnResult> result);
262   void TapOrClickOnCoordinates(
263       ElementPositionGetter* getter_to_release,
264       base::OnceCallback<void(const ClientStatus&)> callback,
265       const std::string& node_frame_id,
266       ClickAction::ClickType click_type,
267       bool has_coordinates,
268       int x,
269       int y);
270   void OnDispatchPressMouseEvent(
271       base::OnceCallback<void(const ClientStatus&)> callback,
272       const std::string& node_frame_id,
273       int x,
274       int y,
275       const DevtoolsClient::ReplyStatus& reply_status,
276       std::unique_ptr<input::DispatchMouseEventResult> result);
277   void OnDispatchReleaseMouseEvent(
278       base::OnceCallback<void(const ClientStatus&)> callback,
279       const DevtoolsClient::ReplyStatus& reply_status,
280       std::unique_ptr<input::DispatchMouseEventResult> result);
281   void OnDispatchTouchEventStart(
282       base::OnceCallback<void(const ClientStatus&)> callback,
283       const std::string& node_frame_id,
284       const DevtoolsClient::ReplyStatus& reply_status,
285       std::unique_ptr<input::DispatchTouchEventResult> result);
286   void OnDispatchTouchEventEnd(
287       base::OnceCallback<void(const ClientStatus&)> callback,
288       const DevtoolsClient::ReplyStatus& reply_status,
289       std::unique_ptr<input::DispatchTouchEventResult> result);
290   void OnFindElementForCheck(
291       base::OnceCallback<void(const ClientStatus&)> callback,
292       const ClientStatus& status,
293       std::unique_ptr<ElementFinder::Result> result);
294   void OnWaitForWindowHeightChange(
295       base::OnceCallback<void(const ClientStatus&)> callback,
296       const DevtoolsClient::ReplyStatus& reply_status,
297       std::unique_ptr<runtime::EvaluateResult> result);
298 
299   // Find the element given by |selector|. If multiple elements match
300   // |selector| and if |strict_mode| is false, return the first one that is
301   // found. Otherwise if |strict-mode| is true, do not return any.
302   void FindElement(const Selector& selector,
303                    bool strict_mode,
304                    ElementFinder::Callback callback);
305   void OnFindElementResult(ElementFinder* finder_to_release,
306                            ElementFinder::Callback callback,
307                            const ClientStatus& status,
308                            std::unique_ptr<ElementFinder::Result> result);
309   void OnFindElementForFillingForm(
310       std::unique_ptr<FillFormInputData> data_to_autofill,
311       const Selector& selector,
312       base::OnceCallback<void(const ClientStatus&)> callback,
313       const ClientStatus& status,
314       std::unique_ptr<ElementFinder::Result> element_result);
315   void OnGetFormAndFieldDataForFillingForm(
316       std::unique_ptr<FillFormInputData> data_to_autofill,
317       base::OnceCallback<void(const ClientStatus&)> callback,
318       content::RenderFrameHost* container_frame_host,
319       const autofill::FormData& form_data,
320       const autofill::FormFieldData& form_field);
321   void OnFindElementToRetrieveFormAndFieldData(
322       const Selector& selector,
323       base::OnceCallback<void(const ClientStatus&,
324                               const autofill::FormData& form_data,
325                               const autofill::FormFieldData& form_field)>
326           callback,
327       const ClientStatus& status,
328       std::unique_ptr<ElementFinder::Result> element_result);
329   void OnGetFormAndFieldDataForRetrieving(
330       base::OnceCallback<void(const ClientStatus&,
331                               const autofill::FormData& form_data,
332                               const autofill::FormFieldData& form_field)>
333           callback,
334       const autofill::FormData& form_data,
335       const autofill::FormFieldData& form_field);
336   void OnFindElementForFocusElement(
337       const TopPadding& top_padding,
338       base::OnceCallback<void(const ClientStatus&)> callback,
339       const ClientStatus& status,
340       std::unique_ptr<ElementFinder::Result> element_result);
341   void OnWaitDocumentToBecomeInteractiveForFocusElement(
342       const TopPadding& top_padding,
343       base::OnceCallback<void(const ClientStatus&)> callback,
344       std::unique_ptr<ElementFinder::Result> target_element,
345       bool result);
346   void OnFocusElement(base::OnceCallback<void(const ClientStatus&)> callback,
347                       const DevtoolsClient::ReplyStatus& reply_status,
348                       std::unique_ptr<runtime::CallFunctionOnResult> result);
349   void OnFindElementForSelectOption(
350       const std::string& value,
351       DropdownSelectStrategy select_strategy,
352       base::OnceCallback<void(const ClientStatus&)> callback,
353       const ClientStatus& status,
354       std::unique_ptr<ElementFinder::Result> element_result);
355   void OnSelectOption(base::OnceCallback<void(const ClientStatus&)> callback,
356                       const DevtoolsClient::ReplyStatus& reply_status,
357                       std::unique_ptr<runtime::CallFunctionOnResult> result);
358   void OnFindElementForHighlightElement(
359       base::OnceCallback<void(const ClientStatus&)> callback,
360       const ClientStatus& status,
361       std::unique_ptr<ElementFinder::Result> element_result);
362   void OnHighlightElement(
363       base::OnceCallback<void(const ClientStatus&)> callback,
364       const DevtoolsClient::ReplyStatus& reply_status,
365       std::unique_ptr<runtime::CallFunctionOnResult> result);
366   void OnFindElementForGetFieldValue(
367       base::OnceCallback<void(const ClientStatus&, const std::string&)>
368           callback,
369       const ClientStatus& status,
370       std::unique_ptr<ElementFinder::Result> element_result);
371   void OnGetValueAttribute(
372       base::OnceCallback<void(const ClientStatus&, const std::string&)>
373           callback,
374       const DevtoolsClient::ReplyStatus& reply_status,
375       std::unique_ptr<runtime::CallFunctionOnResult> result);
376   void OnClearFieldForSendKeyboardInput(
377       const Selector& selector,
378       const std::vector<UChar32>& codepoints,
379       int key_press_delay_in_millisecond,
380       base::OnceCallback<void(const ClientStatus&)> callback,
381       const ClientStatus& clear_status);
382   void SelectFieldValueForReplace(
383       const Selector& selector,
384       base::OnceCallback<void(std::unique_ptr<ElementFinder::Result>,
385                               const ClientStatus&)> callback);
386   void OnFindElementForSelectValue(
387       base::OnceCallback<void(std::unique_ptr<ElementFinder::Result>,
388                               const ClientStatus&)> callback,
389       const ClientStatus& element_status,
390       std::unique_ptr<ElementFinder::Result> element_result);
391   void OnSelectFieldValueForDispatchKeys(
392       std::unique_ptr<ElementFinder::Result> element_result,
393       base::OnceCallback<void(std::unique_ptr<ElementFinder::Result>,
394                               const ClientStatus&)> callback,
395       const DevtoolsClient::ReplyStatus& reply_status,
396       std::unique_ptr<runtime::CallFunctionOnResult> result);
397   void OnFieldValueSelectedForDispatchKeys(
398       const std::vector<UChar32>& codepoints,
399       int key_press_delay_in_millisecond,
400       base::OnceCallback<void(const ClientStatus&)> callback,
401       std::unique_ptr<ElementFinder::Result> element_result,
402       const ClientStatus& select_status);
403   void DispatchKeyboardTextDownEvent(
404       const std::string& node_frame_id,
405       const std::vector<UChar32>& codepoints,
406       size_t index,
407       bool delay,
408       int delay_in_milli,
409       base::OnceCallback<void(const ClientStatus&)> callback);
410   void DispatchKeyboardTextUpEvent(
411       const std::string& node_frame_id,
412       const std::vector<UChar32>& codepoints,
413       size_t index,
414       int delay_in_milli,
415       base::OnceCallback<void(const ClientStatus&)> callback);
416   void InternalSetFieldValue(
417       const Selector& selector,
418       const std::string& value,
419       base::OnceCallback<void(const ClientStatus&)> callback);
420   void OnFindElementForSetFieldValue(
421       const std::string& value,
422       base::OnceCallback<void(const ClientStatus&)> callback,
423       const ClientStatus& status,
424       std::unique_ptr<ElementFinder::Result> element_result);
425   void OnSetValueAttribute(
426       base::OnceCallback<void(const ClientStatus&)> callback,
427       const DevtoolsClient::ReplyStatus& reply_status,
428       std::unique_ptr<runtime::CallFunctionOnResult> result);
429   void OnFindElementForSetAttribute(
430       const std::vector<std::string>& attribute,
431       const std::string& value,
432       base::OnceCallback<void(const ClientStatus&)> callback,
433       const ClientStatus& status,
434       std::unique_ptr<ElementFinder::Result> element_result);
435   void OnSetAttribute(base::OnceCallback<void(const ClientStatus&)> callback,
436                       const DevtoolsClient::ReplyStatus& reply_status,
437                       std::unique_ptr<runtime::CallFunctionOnResult> result);
438   void OnFindElementForSendKeyboardInput(
439       const Selector& selector,
440       const std::vector<UChar32>& codepoints,
441       int delay_in_milli,
442       base::OnceCallback<void(const ClientStatus&)> callback,
443       const ClientStatus& status,
444       std::unique_ptr<ElementFinder::Result> element_result);
445   void OnClickElementForSendKeyboardInput(
446       const std::string& node_frame_id,
447       const std::vector<UChar32>& codepoints,
448       int delay_in_milli,
449       base::OnceCallback<void(const ClientStatus&)> callback,
450       const ClientStatus& click_status);
451   void OnFindElementForGetOuterHtml(
452       base::OnceCallback<void(const ClientStatus&, const std::string&)>
453           callback,
454       const ClientStatus& status,
455       std::unique_ptr<ElementFinder::Result> element_result);
456   void OnGetOuterHtml(base::OnceCallback<void(const ClientStatus&,
457                                               const std::string&)> callback,
458                       const DevtoolsClient::ReplyStatus& reply_status,
459                       std::unique_ptr<runtime::CallFunctionOnResult> result);
460   void OnFindElementForGetElementTag(
461       base::OnceCallback<void(const ClientStatus&, const std::string&)>
462           callback,
463       const ClientStatus& status,
464       std::unique_ptr<ElementFinder::Result> element_result);
465   void OnGetElementTag(base::OnceCallback<void(const ClientStatus&,
466                                                const std::string&)> callback,
467                        const DevtoolsClient::ReplyStatus& reply_status,
468                        std::unique_ptr<runtime::CallFunctionOnResult> result);
469   void OnFindElementForPosition(
470       base::OnceCallback<void(bool, const RectF&)> callback,
471       const ClientStatus& status,
472       std::unique_ptr<ElementFinder::Result> result);
473   void OnGetVisualViewport(
474       base::OnceCallback<void(bool, const RectF&)> callback,
475       const DevtoolsClient::ReplyStatus& reply_status,
476       std::unique_ptr<runtime::EvaluateResult> result);
477   void OnGetElementRectResult(
478       ElementRectGetter* getter_to_release,
479       base::OnceCallback<void(bool, const RectF&)> callback,
480       bool has_rect,
481       const RectF& element_rect);
482 
483   // Creates a new instance of DispatchKeyEventParams for the specified type and
484   // unicode codepoint.
485   using DispatchKeyEventParamsPtr =
486       std::unique_ptr<autofill_assistant::input::DispatchKeyEventParams>;
487   static DispatchKeyEventParamsPtr CreateKeyEventParamsForCharacter(
488       autofill_assistant::input::DispatchKeyEventType type,
489       const UChar32 codepoint);
490 
491   // Waits for the document.readyState to be 'interactive' or 'complete'.
492   void WaitForDocumentToBecomeInteractive(
493       int remaining_rounds,
494       const std::string& object_id,
495       const std::string& node_frame_id,
496       base::OnceCallback<void(bool)> callback);
497   void OnWaitForDocumentToBecomeInteractive(
498       int remaining_rounds,
499       const std::string& object_id,
500       const std::string& node_frame_id,
501       base::OnceCallback<void(bool)> callback,
502       const DevtoolsClient::ReplyStatus& reply_status,
503       std::unique_ptr<runtime::CallFunctionOnResult> result);
504   void OnFindElementForWaitForDocumentReadyState(
505       DocumentReadyState min_ready_state,
506       base::OnceCallback<void(const ClientStatus&, DocumentReadyState)>
507           callback,
508       const ClientStatus& status,
509       std::unique_ptr<ElementFinder::Result> element);
510 
511   // Weak pointer is fine here since it must outlive this web controller, which
512   // is guaranteed by the owner of this object.
513   content::WebContents* web_contents_;
514   std::unique_ptr<DevtoolsClient> devtools_client_;
515   const ClientSettings* const settings_;
516 
517   // Currently running workers.
518   std::vector<std::unique_ptr<WebControllerWorker>> pending_workers_;
519 
520   base::WeakPtrFactory<WebController> weak_ptr_factory_{this};
521   DISALLOW_COPY_AND_ASSIGN(WebController);
522 };
523 }  // namespace autofill_assistant
524 #endif  // COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_WEB_CONTROLLER_H_
525