Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -45,7 +45,7 @@ def navigate(screenshot, task: str, platform: str, history):
|
|
45 |
"""
|
46 |
|
47 |
# βββββββββββββββββββββ normalise history input ββββββββββββββββββββββββββ
|
48 |
-
messages=
|
49 |
|
50 |
if isinstance(history, str):
|
51 |
try:
|
@@ -55,16 +55,13 @@ def navigate(screenshot, task: str, platform: str, history):
|
|
55 |
else:
|
56 |
messages = history
|
57 |
|
58 |
-
|
59 |
-
|
60 |
-
"You
|
61 |
-
"
|
62 |
-
"the task.\n\n## Output Format\n```\nThought: ...\nAction: ...\n```\n\n"
|
63 |
-
"## Action Space\nclick(start_box='...') / type(...)\n\n"
|
64 |
-
f"### Task\n{task}"
|
65 |
)
|
66 |
-
|
67 |
-
|
68 |
|
69 |
# βββββββββββββββββββββββββββ model forward βββββββββββββββββββββββββββββ
|
70 |
text = processor.apply_chat_template(
|
|
|
45 |
"""
|
46 |
|
47 |
# βββββββββββββββββββββ normalise history input ββββββββββββββββββββββββββ
|
48 |
+
messages=[]
|
49 |
|
50 |
if isinstance(history, str):
|
51 |
try:
|
|
|
55 |
else:
|
56 |
messages = history
|
57 |
|
58 |
+
prompt_header = (
|
59 |
+
"You are a GUI agent. You are given a task and your action history, with screenshots."
|
60 |
+
"You need to perform the next action to complete the task. \n\n## Output Format\n```\nThought: ...\nAction: ...\n```\n\n## Action Space\n\nclick(start_box='\u003c|box_start|\u003e(x1, y1)\u003c|box_end|\u003e')\nleft_double(start_box='\u003c|box_start|\u003e(x1, y1)\u003c|box_end|\u003e')\nright_single(start_box='\u003c|box_start|\u003e(x1, y1)\u003c|box_end|\u003e')\ndrag(start_box='\u003c|box_start|\u003e(x1, y1)\u003c|box_end|\u003e', end_box='\u003c|box_start|\u003e(x3, y3)\u003c|box_end|\u003e')\nhotkey(key='')\ntype(content='') #If you want to submit your input, use \"\\n\" at the end of `content`.\nscroll(start_box='\u003c|box_start|\u003e(x1, y1)\u003c|box_end|\u003e', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use English in `Thought` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.\n\n"
|
61 |
+
f"## User Instruction\n{task}"
|
|
|
|
|
|
|
62 |
)
|
63 |
+
current = [{"role":"user","content":[{"type":"text","text":prompt_header},{"type": "image_url", "image_url":screenshot}]}]
|
64 |
+
messages.append(current)
|
65 |
|
66 |
# βββββββββββββββββββββββββββ model forward βββββββββββββββββββββββββββββ
|
67 |
text = processor.apply_chat_template(
|