add angualar date picker support (#1955)

Co-authored-by: lawyzheng <lawyzheng1106@gmail.com>

skyvern/webeye/actions/handler.py

@@ -1101,22 +1101,9 @@ async def handle_select_option_action(
         await incremental_scraped.start_listen_dom_increment()
         await skyvern_element.scroll_into_view()
 
-        try:
+        await skyvern_element.click(page=page, dom=dom, timeout=timeout)
-            await skyvern_element.get_locator().click(timeout=timeout)
-        except Exception:
-            LOG.info(
-                "fail to open dropdown by clicking, try to press ArrowDown to open",
-                exc_info=True,
-                element_id=skyvern_element.get_id(),
-                task_id=task.task_id,
-                step_id=step.step_id,
-            )
-            await skyvern_element.scroll_into_view()
-            await skyvern_element.press_key("ArrowDown")
-
         # wait 5s for options to load
         await asyncio.sleep(5)
-        is_open = True
 
         incremental_element = await incremental_scraped.get_incremental_element_tree(
             clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
@@ -1140,6 +1127,7 @@ async def handle_select_option_action(
         if len(incremental_element) == 0:
             raise NoIncrementalElementFoundForCustomSelection(element_id=skyvern_element.get_id())
 
+        is_open = True
         # TODO: support sequetially select from dropdown by value, just support single select now
         result = await sequentially_select_from_dropdown(
             action=action,
@@ -2222,7 +2210,7 @@ async def select_from_dropdown(
             return single_select_result
 
         await selected_element.scroll_into_view()
-        await selected_element.get_locator().click(timeout=timeout)
+        await selected_element.click(page=page, timeout=timeout)
         single_select_result.action_result = ActionSuccess()
         return single_select_result
     except (MissingElement, MissingElementDict, MissingElementInCSSMap, MultipleElementsFound):

skyvern/webeye/scraper/domUtils.js

@@ -920,6 +920,19 @@ function hasNgAttribute(element) {
   return false;
 }
 
+function isAngularMaterial(element) {
+  if (!element.attributes[Symbol.iterator]) {
+    return false;
+  }
+
+  for (let attr of element.attributes) {
+    if (attr.name.startsWith("mat")) {
+      return true;
+    }
+  }
+  return false;
+}
+
 const isAngularDropdown = (element) => {
   if (!hasNgAttribute(element)) {
     return false;
@@ -936,6 +949,20 @@ const isAngularDropdown = (element) => {
   return false;
 };
 
+const isAngularMaterialDatePicker = (element) => {
+  if (!isAngularMaterial(element)) {
+    return false;
+  }
+
+  const tagName = element.tagName.toLowerCase();
+  if (tagName !== "input") return false;
+
+  return (
+    (element.closest("mat-datepicker") ||
+      element.closest("mat-formio-date")) !== null
+  );
+};
+
 function getPseudoContent(element, pseudo) {
   const pseudoStyle = getElementComputedStyle(element, pseudo);
   if (!pseudoStyle) {
@@ -1325,6 +1352,7 @@ async function buildElementObject(
       isDivComboboxDropdown(element) ||
       isDropdownButton(element) ||
       isAngularDropdown(element) ||
+      isAngularMaterialDatePicker(element) ||
       isSelect2Dropdown(element) ||
       isSelect2MultiChoice(element),
     isCheckable: isCheckableDiv(element),

skyvern/webeye/utils/dom.py

@@ -13,6 +13,7 @@ from skyvern.config import settings
 from skyvern.constants import SKYVERN_ID_ATTR
 from skyvern.exceptions import (
     ElementIsNotLabel,
+    InteractWithDisabledElement,
     MissingElement,
     MissingElementDict,
     MissingElementInCSSMap,
@@ -580,6 +581,45 @@ class SkyvernElement:
 
         return dest_x, dest_y
 
+    async def click(
+        self,
+        page: Page,
+        dom: DomUtil | None = None,
+        incremental_page: IncrementalScrapePage | None = None,
+        timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+    ) -> None:
+        if await self.is_disabled(dynamic=True):
+            raise InteractWithDisabledElement(element_id=self.get_id())
+
+        try:
+            await self.get_locator().click(timeout=timeout)
+            return
+        except Exception:
+            LOG.info("Failed to click by playwright", exc_info=True, element_id=self.get_id())
+
+        if dom is not None:
+            # try to click on the blocking element
+            try:
+                await self.scroll_into_view(timeout=timeout)
+                blocking_element, _ = await self.find_blocking_element(dom=dom, incremental_page=incremental_page)
+                if blocking_element:
+                    LOG.debug("Find the blocking element", element_id=blocking_element.get_id())
+                    await blocking_element.get_locator().click(timeout=timeout)
+                    return
+            except Exception:
+                LOG.info("Failed to click on the blocking element", exc_info=True, element_id=self.get_id())
+
+        try:
+            await self.scroll_into_view(timeout=timeout)
+            await self.coordinate_click(page=page, timeout=timeout)
+            return
+        except Exception:
+            LOG.info("Failed to click by coordinate", exc_info=True, element_id=self.get_id())
+
+        await self.scroll_into_view(timeout=timeout)
+        await self.click_in_javascript()
+        return
+
     async def click_in_javascript(self) -> None:
         skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
         await skyvern_frame.click_element_in_javascript(await self.get_element_handler())