vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix observer - get url block back; get mark workflow run completion back (#1726)

Browse Source
This commit is contained in:
Shuchang Zheng
2025-02-06 00:50:32 +08:00
committed by GitHub
parent b4bdcfd53c
commit ffdb823401
1 changed files with 200 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
skyvern/forge/sdk/services/observer_service.py
View File

@@ -34,6 +34,7 @@ from skyvern.forge.sdk.workflow.models.block import (
ForLoopBlock,
NavigationBlock,
TaskBlock,
UrlBlock,
)
from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, ContextParameter
from skyvern.forge.sdk.workflow.models.workflow import (
@@ -51,6 +52,7 @@ from skyvern.forge.sdk.workflow.models.yaml import (
ForLoopBlockYAML,
NavigationBlockYAML,
TaskBlockYAML,
UrlBlockYAML,
WorkflowCreateYAMLRequest,
WorkflowDefinitionYAML,
)
@@ -118,7 +120,9 @@ async def initialize_observer_task(
metadata_prompt = prompt_engine.load_prompt("observer_generate_metadata", user_goal=user_prompt, user_url=user_url)
metadata_response = await app.LLM_API_HANDLER(
prompt=metadata_prompt, observer_thought=observer_thought, prompt_name="observer-generate-metadata"
prompt=metadata_prompt,
observer_thought=observer_thought,
prompt_name="observer-generate-metadata",
)
# validate
LOG.info(f"Initialized observer initial response: {metadata_response}")
@@ -350,6 +354,23 @@ async def run_observer_task_helper(
max_iterations = int_max_iterations_override or DEFAULT_MAX_ITERATIONS
for i in range(max_iterations):
LOG.info(f"Observer iteration i={i}", workflow_run_id=workflow_run_id, url=url)
task_type = ""
plan = ""
block: BlockTypeVar | None = None
task_history_record: dict[str, Any] = {}
context = skyvern_context.ensure_context()
if i == 0:
# The first iteration is always a GOTO_URL task
task_type = "goto_url"
plan = f"Go to this website: {url}"
task_history_record = {"type": task_type, "task": plan}
block, block_yaml_list, parameter_yaml_list = await _generate_goto_url_task(
workflow_id=workflow_id,
url=url,
)
task_history_record = {"type": task_type, "task": plan}
else:
try:
browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
workflow_run=workflow_run,
@@ -365,13 +386,14 @@ async def run_observer_task_helper(
element_tree_in_prompt: str = scraped_page.build_element_tree(ElementTreeFormat.HTML)
page = await browser_state.get_working_page()
except Exception:
LOG.exception("Failed to get browser state or scrape website in observer iteration", iteration=i, url=url)
LOG.exception(
"Failed to get browser state or scrape website in observer iteration", iteration=i, url=url
)
continue
current_url = str(
await SkyvernFrame.evaluate(frame=page, expression="() => document.location.href") if page else url
)
context = skyvern_context.ensure_context()
observer_prompt = prompt_engine.load_prompt(
"observer",
current_url=current_url,
@@ -393,7 +415,7 @@ async def run_observer_task_helper(
prompt=observer_prompt,
screenshots=scraped_page.screenshots,
observer_thought=observer_thought,
prompt_name="observer-generate-plan",
prompt_name="observer",
)
LOG.info(
"Observer response",
@@ -406,8 +428,8 @@ async def run_observer_task_helper(
user_goal_achieved = observer_response.get("user_goal_achieved", False)
observation = observer_response.get("page_info", "")
thoughts: str = observer_response.get("thoughts", "")
plan: str = observer_response.get("plan", "")
task_type: str = observer_response.get("task_type", "")
plan = observer_response.get("plan", "")
task_type = observer_response.get("task_type", "")
# Create and save observer thought
await app.DATABASE.update_observer_thought(
observer_thought_id=observer_thought.observer_thought_id,
@@ -445,8 +467,6 @@ async def run_observer_task_helper(
)
break
block: BlockTypeVar | None = None
task_history_record: dict[str, Any] = {}
if task_type == "extract":
block, block_yaml_list, parameter_yaml_list = await _generate_extraction_task(
observer_cruise=observer_task,
@@ -565,6 +585,11 @@ async def run_observer_task_helper(
if block_result.success is True:
completion_screenshots = []
try:
browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
workflow_run=workflow_run,
url=url,
browser_session_id=browser_session_id,
)
scraped_page = await scrape_website(
browser_state,
url,
@@ -595,7 +620,7 @@ async def run_observer_task_helper(
prompt=observer_completion_prompt,
screenshots=completion_screenshots,
observer_thought=observer_thought,
prompt_name="observer-check-completion",
prompt_name="observer_check_completion",
)
LOG.info(
"Observer completion check response",
@@ -900,7 +925,7 @@ async def _generate_loop_task(
task_in_loop_metadata_prompt,
screenshots=scraped_page.screenshots,
observer_thought=observer_thought_task_in_loop,
prompt_name="observer-generate-task-block",
prompt_name="observer_generate_task_block",
)
LOG.info("Task in loop metadata response", task_in_loop_metadata_response=task_in_loop_metadata_response)
navigation_goal = task_in_loop_metadata_response.get("navigation_goal")
@@ -996,7 +1021,7 @@ async def _generate_extraction_task(
generate_extraction_task_response = await app.LLM_API_HANDLER(
generate_extraction_task_prompt,
observer_cruise=observer_cruise,
prompt_name="observer-generate-extraction-task",
prompt_name="observer_generate_extraction_task",
)
LOG.info("Data extraction response", data_extraction_response=generate_extraction_task_response)
@@ -1067,6 +1092,34 @@ async def _generate_navigation_task(
)
async def _generate_goto_url_task(
workflow_id: str,
url: str,
) -> tuple[UrlBlock, list[BLOCK_YAML_TYPES], list[PARAMETER_YAML_TYPES]]:
LOG.info("Generating goto url task", url=url)
# create OutputParameter for the data_extraction block
label = f"goto_url_{_generate_random_string()}"
url_block_yaml = UrlBlockYAML(
label=label,
url=url,
)
output_parameter = await app.WORKFLOW_SERVICE.create_output_parameter_for_block(
workflow_id=workflow_id,
block_yaml=url_block_yaml,
)
# create UrlBlock
return (
UrlBlock(
label=label,
url=url,
output_parameter=output_parameter,
),
[url_block_yaml],
[],
)
def _generate_random_string(length: int = 5) -> str:
# Use the current timestamp as the seed
random.seed(os.urandom(16))
@@ -1127,13 +1180,14 @@ async def mark_observer_task_as_completed(
output: dict[str, Any] | None = None,
) -> ObserverTask:
observer_task = await app.DATABASE.update_observer_cruise(
observer_cruise_id=observer_cruise_id,
observer_cruise_id,
organization_id=organization_id,
status=ObserverTaskStatus.completed,
workflow_run_id=workflow_run_id,
summary=summary,
output=output,
)
if workflow_run_id:
await app.WORKFLOW_SERVICE.mark_workflow_run_as_completed(workflow_run_id)
# Track observer cruise duration when completed
duration_seconds = (datetime.now(UTC) - observer_task.created_at.replace(tzinfo=UTC)).total_seconds()
@@ -1245,7 +1299,7 @@ async def _summarize_observer_task(
prompt=observer_summary_prompt,
screenshots=screenshots,
observer_thought=observer_thought,
prompt_name="observer-generate-summary",
prompt_name="observer_summary",
)
LOG.info("Observer summary response", observer_summary_resp=observer_summary_resp)