Tongyi UI Agent

Preparations

• Obtain and configure an API key

Request URL

https://dashscope.aliyuncs.com/api/v2/apps/gui-owl/gui_agent_server

Request headers

Request parameters

[
              {
                "image": "http://" # Replace with a valid URL
              },
              {
                "instruction": "Open Weibo"
              },
              {
                "session_id": ""
              },
              {
                "device_type": "mobile"
              },
              {
                "pipeline_type": "agent"
              },
              {
                "model_name": "pre-gui_owl_7b"
              },
              {
                "thought_language": "chinese"
              },
              {
                "param_list": [
                  {
                    "add_info": ""
                  }
                ]
              }
            ]

Request example

curl --location --request POST 'https://dashscope.aliyuncs.com/api/v2/apps/gui-owl/gui_agent_server' \
--header 'Authorization: Bearer sk-' \
--header 'Content-Type: application/json' \
--data-raw '{
  "app_id": "gui-owl",
  "input": [
    {
      "role": "user",
      "content": [
        {
          "type": "data",
          "data": {
            "messages": [
              {
                "image": "http://" # Replace with a valid URL
              },
              {
                "instruction": "Open Weibo"
              },
              {
                "session_id": ""
              },
              {
                "device_type": "mobile"
              },
              {
                "pipeline_type": "agent"
              },
              {
                "model_name": "pre-gui_owl_7b"
              },
              {
                "thought_language": "chinese"
              },
              {
                "param_list": [
                  {
                    "add_info": ""
                  }
                ]
              }
            ]
          }
        }
      ]
    }
  ]
}'

Response example

import time
import uuid
import requests
import json
import os

session_id = ""

# Define the request URL
url = "https://dashscope.aliyuncs.com/api/v2/apps/gui-owl/gui_agent_server"
# Prepare the request headers
headers = {
    "Authorization": f"Bearer {os.getenv('DASHSCOPE_API_KEY')}",  # Replace with a valid API key
    "Content-Type": "application/json"
}

content = [
    {
        "type": "data",
        "data": {
            "messages": [
                {
                    "image": "http://" # Replace with a valid URL
                },
                {
                    "instruction": "Open the browser"
                },
                {
                    "session_id": session_id
                },
                {
                    "device_type": "mobile"
                },
                {
                    "pipeline_type": "agent"
                },
                {
                    "model_name": "pre-gui_owl_7b"
                },
                {
                    "thought_language": "chinese"
                },
                {
                    "param_list": [
                        {
                            "add_info": ""
                        }
                    ]
                }
            ]
        }
    }
]

try:
    payload = {
        "app_id": "gui-owl",
        "input": [
            {
                "role": "user",
                "content": content
            }
        ]
    }

    start_time = time.time()
    # Send a POST request
    response = requests.post(url, headers=headers, data=json.dumps(payload))
    print(f"Response Body: {response.text}")
    # Check the response status
    response.raise_for_status()
    response_data = response.json()
    session_id = response_data['session_id']

    print(f"Time elapsed:\n", time.time() - start_time)


except requests.exceptions.RequestException as e:
    print(f"An error occurred during the request: {e}")

Response parameters

Full response example

{
    "created_at": 1758548962,
    "id": "401a9031-3fa5-4655-a34d-090523a6dd1d",
    "object": "response",
    "output": [
        {
            "code": "200",
            "content": [
                {
                    "data": {
                        "Explanation": "Swipe up from the bottom of the screen to open the app drawer.",
                        "Operation": "Swipe (512, 708, 512, 353)",
                        "Thought": "To achieve the user's request of opening the Google app, I need to first access the app drawer. Since the Google app is not visible on the home screen, swiping up or accessing the app drawer is necessary. This will allow me to locate and tap on the Google app icon."
                    },
                    "delta": false,
                    "object": "content",
                    "type": "data"
                }
            ],
            "id": "401a9031-3fa5-4655-a34d-090523a6dd1d",
            "message": "Success",
            "object": "message",
            "role": "assistant",
            "status": "Success",
            "type": "message"
        }
    ],
    "session_id": "7c86289e-127a-4edf-8055-5727489aef49",
    "status": "created",
    "usage": {
        "input_tokens": 0,
        "output_tokens": 0
    }
}

Request parameters

Request example

curl --location --request POST 'https://dashscope.aliyuncs.com/api/v2/apps/gui-owl/gui_agent_server' \
--header 'Authorization: Bearer sk-' \
--header 'Content-Type: application/json' \
--data-raw '{
  "app_id": "gui-owl",
  "input": [
    {
      "role": "user",
      "content": [
        {
          "type": "data",
          "data": {
            "messages": [
              {
                "image": "http://" # Replace with a valid URL
              },
              {
                "instruction": "Open the calculator"
              },
              {
                "session_id": ""
              },
              {
                "device_type": "pc"
              },
              {
                "pipeline_type": "agent"
              },
              {
                "model_name": "pre-gui_owl_7b"
              },
              {
                "thought_language": "chinese"
              },
              {
                "param_list": [
                  {
                    "add_info": ""
                  },
                  {
                    "enable_reflector": true
                  },
                  {
                    "enable_notetaker": true
                  },
                  {
                    "worker_model": "pre-gui_owl_7b"
                  },
                  {
                    "manager_model": "pre-gui_owl_7b"
                  },
                  {
                    "reflector_model": "pre-gui_owl_7b"
                  },
                  {
                    "notetaker_model": "pre-gui_owl_7b"
                  }
                ]
              }
            ]
          }
        }
      ]
    }
  ]
}'

Response example

import time
import uuid
import requests
import json
import os

session_id = ""

# Define the request URL
url = "https://dashscope.aliyuncs.com/api/v2/apps/gui-owl/gui_agent_server"
# Prepare the request headers
headers = {
    "Authorization": f"Bearer {os.getenv('DASHSCOPE_API_KEY')}",  # Replace with a valid API key
    "Content-Type": "application/json"
}

content = [
    {
        "type": "data",
        "data": {
            "messages": [
                {
                    "image": "http://" # Replace with a valid URL
                },
                {
                    "instruction": "Open the browser"
                },
                {
                    "session_id": session_id
                },
                {
                    "device_type": "pc"
                },
                {
                    "pipeline_type": "agent"
                },
                {
                    "model_name": "pre-gui_owl_7b"
                },
                {
                    "thought_language": "chinese"
                },
                {
                    "param_list": [
                        {
                            "add_info": ""
                        },
                        {
                            "enable_reflector": True
                        },
                        {
                            "enable_notetaker": True
                        },
                        {
                            "worker_model": "pre-gui_owl_7b"
                        },
                        {
                            "manager_model": "pre-gui_owl_7b"
                        },
                        {
                            "reflector_model": "pre-gui_owl_7b"
                        },
                        {
                            "notetaker_model": "pre-gui_owl_7b"
                        }
                    ]
                }
            ]
        }
    }
]


try:
      # Randomly generate a request_id
      payload = {
          "app_id": "gui-owl",
          "input": [
              {
                  "role": "user",
                  "content": content
              }
          ]
      }

      start_time = time.time()
      # Send a POST request
      response = requests.post(url, headers=headers, data=json.dumps(payload))
      print(f"Response Body: {response.text}")
      # Check the response status
      response.raise_for_status()
      response_data = response.json()
      session_id = response_data['session_id']

      print(f"Time elapsed:\n", time.time() - start_time)


except requests.exceptions.RequestException as e:
    print(f"An error occurred during the request: {e}")

Response parameters

Full response example

{
    "created_at": 1758182731,
    "object": "response",
    "output": [
        {
            "code": "200",
            "content": [
                {
                    "data": {
                        "action_parameter": {
                            "count": 1,
                            "position": [
                                69,
                                2124
                            ]
                        },
                        "action_type": "click",
                        "explanation": "Click the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application.",
                        "global_state": {
                            "manager": "### Thought ###\nThe user wants to open the calculator application. The screenshot shows the Windows desktop with various icons, but the calculator icon is not immediately visible. To achieve this, I need to locate and open the calculator. Since it's not on the desktop, I'll assume it might be in the Start menu or another location like the All Apps menu. Therefore, the plan involves accessing the Start menu and finding the calculator there.\n\n### Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'}\n2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Current Subgoal ###\nOpen the Start Menu",
                            "worker": "### Thought ###\nTo open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n### Action ###\n{\"action\": \"click\", \"coordinate\": [35, 1074]}\n\n### Description ###\nClick the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application."
                        },
                        "meta_data": {
                            "executor": {
                                "messages": [
                                    {
                                        "content": [
                                            {
                                                "text": "You are a helpful assistant.",
                                                "type": "text"
                                            }
                                        ],
                                        "role": "system"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "You are an agent who can operate a computer on behalf of a user. Your goal is to decide the next action to perform based on the current state of the phone and the user's request.\n\n### User Request ###\nOpen the calculator\nYou can follow these tips:\n\nOn Windows, you absolutely have to double-click to open any desktop icon or folder in File Explorer—one single click will NOT open them. Always double-click.\n\n### Overall Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'} 2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Progress Status ###\nNo progress yet.\n\n### Current Subgoal ###\nOpen the Start Menu\n\n### Collected Task-Related Notes ###\nNo important notes recorded.\n\n---\nCarefully examine all the information provided above and decide on the next action to perform. If you notice an unsolved error in the previous action, think as a human user and attempt to rectify them. You must choose your action from one of the atomic actions.\n\n#### Atomic Actions ####\nThe atomic action functions are listed in the format of `action(arguments): description` as follows:\n- open_app(app_name): Open the App with app_name such as Chrome and Wechat. Usage Example: {\"action\": \"open_app\", \"app_name\": \"chrome\"}\n- click(coordinate): Click the point on the screen with specified (x, y) coordinates. Usage Example: {\"action\": \"click\", \"coordinate\": [x, y]}\n- double_click(coordinate): Double click on the position (x, y) on the screen. Usage Example: {\"action\": \"double_click\", \"coordinate\": [x, y]}\n- right_click(coordinate): Right-click using the mouse on the position (x, y) on the screen. Usage Example: {\"action\": \"double_click\", \"coordinate\": [x, y]}\n- type(coordinate, text, clear, enter): Type text into the position (x, y) on the screen. Use escape characters \\', \\\", and \\n in the `text` part to ensure we can parse the content in normal python string format. If you want to clear the existing content, set the `clear` parameter to 1; otherwise, set it to 0. If you want to press `enter` after input, set the `enter` parameter to 1; otherwise, set it to 0. Usage Example: {\"action\": \"type\", \"coordinate\": [x, y], \"text\": \"the text you want to type\", \"clear\": 1, \"enter\": 1}\n- hotkey(keys): Press a hotkey combination. The `keys` parameter is a list of keys represented as a string, such as \"['ctrl', 'c']\". Usage Example: {\"action\": \"hotkey\", \"keys\": \"['ctrl', 'c']\"}\n- scroll(coordinate, value): Scroll at the position (x, y) on the screen. The `value` parameter can be positive (scroll up) or negative (scroll down), which is usually set to 5 or -5. Usage Example: {\"action\": \"scroll\", \"coordinate\": [x, y], \"value\": 5}\n- wait(time): Wait for a specified amount of time, such as 3s. Usage Example: {\"action\": \"wait\", \"time\": 3}\n- call_user(): Call the user when the task is unsolvable, or when you need the user's help, such as log in or close the pop up. Usage Example: {\"action\": \"call_user\"}\n- drag(coordinate, coordinate2): drag from the position with coordinate to the position with coordinate2. Usage Example: {\"action\": \"drag\", \"coordinate\": [x1, y1], \"coordinate2\": [x2, y2]}\n- set_cell_values(cell_values, file_name, sheet_name): set individual cell values (a Dict) in the sheet with sheet_name and in the spreadsheet with file_name. Usage Example: {\"action\": \"set_cell_value\", \"cell_values\": {\"A2\": \"hello\"}, \"file_name\": 'Untitled 1', \"sheet_name\": 'Sheet1'}\n- done(): Output this action when you think this instruction is done. Usage Example: {\"action\": \"done\"}\n\n### Latest Action History ###\nNo actions have been taken yet.\n\n### Tips\n\n* On Windows, you absolutely have to double-click to open any desktop icon or folder in File Explorer—one single click will NOT open them. Always double-click.\n* To log on or encounter a pop-up window, use the call_user tool to ask the user to take over. Do not enter account credentials or perform unauthorized operations.\n* Use app-specific tools first, rather than generic cv tools such as cv_click or cv_key.\n\n\n---\nProvide your output in the following format, which contains three parts:\n### Thought ###\nProvide a detailed explanation of your rationale for the chosen action.  Please output in Chinese.\n\n### Action ###\nChoose only one action or shortcut from the options provided. IMPORTANT: Do NOT return invalid actions like null or stop. Do NOT repeat previously failed actions multiple times.\nYou must provide your decision using a valid JSON format specifying the `action` and the arguments of the action.\n\n### Description ###\nA brief description of the chosen action and the expected outcome.  Please output in Chinese."
                                            },
                                            {
                                                "image": "C:\\0421d463-47ac-4307-8bed-a22df6568cad.png"
                                            }
                                        ],
                                        "role": "user"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "### Thought ###\nTo open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n### Action ###\n{\"action\": \"click\", \"coordinate\": [35, 1074]}\n\n### Description ###\nClick the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application."
                                            }
                                        ],
                                        "role": "assistant"
                                    }
                                ],
                                "name": "operator",
                                "response": "### Thought ###\nTo open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n### Action ###\n{\"action\": \"click\", \"coordinate\": [35, 1074]}\n\n### Description ###\nClick the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application."
                            },
                            "manager": {
                                "messages": [
                                    {
                                        "content": [
                                            {
                                                "text": "You are a helpful assistant.",
                                                "type": "text"
                                            }
                                        ],
                                        "role": "system"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "You are an agent who can operate a windows computer on behalf of a user. Your goal is to track progress and devise high-level plans to achieve the user's requests.\n\n### User Request ###\nOpen the calculator\nYou can follow these tips:\n\nOn Windows, you absolutely have to double-click to open any desktop icon or folder in File Explorer—one single click will NOT open them. Always double-click.\n\n---\nMake a high-level plan to achieve the user's request. If the request is complex, break it down into subgoals. The screenshot displays the starting state of the computer.\nPlease respond in Chinese.\n\n##### Important Notes #####\n1. Before generating your plan, carefully observe and understand the current state of the computer.\n2. Your plan should contain only necessary steps; however, please include information useful for executing the subgoal in the 'info' field.\n3. Do not include verification steps in your plan. Steps that confirm or validate other subtasks should not be included.\n4. Do not include optional steps in your plan.\n\nProvide your output in the following format which contains three parts:\n\n### Thought ###\nA detailed explanation of your rationale for the plan and subgoals.\n\n### Plan ###\n1. {'name': 'brief description of the first subgoal.', 'info': 'detailed information about executing the first subgoal.'}\n2. {'name': 'brief description of the second subgoal.', 'info': 'detailed information about executing the second subgoal.'}\n...\n\n### Current Subgoal ###\nThe first subgoal's name you should work on.\n\n"
                                            },
                                            {
                                                "image": "C:\\Usersimages\\b5bf3eff-ae32-4dd0-85ee-7ee31c359142.png"
                                            }
                                        ],
                                        "role": "user"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "### Thought ###\nThe user wants to open the calculator application. The screenshot shows the Windows desktop with various icons, but the calculator icon is not immediately visible. To achieve this, I need to locate and open the calculator. Since it's not on the desktop, I'll assume it might be in the Start menu or another location like the All Apps menu. Therefore, the plan involves accessing the Start menu and finding the calculator there.\n\n### Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'}\n2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Current Subgoal ###\nOpen the Start Menu"
                                            }
                                        ],
                                        "role": "assistant"
                                    }
                                ],
                                "name": "manager",
                                "response": "### Thought ###\nThe user wants to open the calculator application. The screenshot shows the Windows desktop with various icons, but the calculator icon is not immediately visible. To achieve this, I need to locate and open the calculator. Since it's not on the desktop, I'll assume it might be in the Start menu or another location like the All Apps menu. Therefore, the plan involves accessing the Start menu and finding the calculator there.\n\n### Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'}\n2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Current Subgoal ###\nOpen the Start Menu"
                            }
                        },
                        "thought": "To open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n"
                    },
                    "delta": false,
                    "object": "content",
                    "type": "data"
                }
            ],
            "id": "msg_e9dd72ea-19fc-4a22-93ee-d97588645f88",
            "message": "Success",
            "object": "message",
            "role": "assistant",
            "status": "Success",
            "type": "message"
        }
    ],
    "session_id": "bee1915d-6f4d-4bfc-b657-ecb9d0ed8dad",
    "status": "created"
}{
    "created_at": 1758182731,
    "object": "response",
    "output": [
        {
            "code": "200",
            "content": [
                {
                    "data": {
                        "action_parameter": {
                            "count": 1,
                            "position": [
                                69,
                                2124
                            ]
                        },
                        "action_type": "click",
                        "explanation": "Click the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application.",
                        "global_state": {
                            "manager": "### Thought ###\nThe user wants to open the calculator application. The screenshot shows the Windows desktop with various icons, but the calculator icon is not immediately visible. To achieve this, I need to locate and open the calculator. Since it's not on the desktop, I'll assume it might be in the Start menu or another location like the All Apps menu. Therefore, the plan involves accessing the Start menu and finding the calculator there.\n\n### Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'}\n2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Current Subgoal ###\nOpen the Start Menu",
                            "worker": "### Thought ###\nTo open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n### Action ###\n{\"action\": \"click\", \"coordinate\": [35, 1074]}\n\n### Description ###\nClick the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application."
                        },
                        "meta_data": {
                            "executor": {
                                "messages": [
                                    {
                                        "content": [
                                            {
                                                "text": "You are a helpful assistant.",
                                                "type": "text"
                                            }
                                        ],
                                        "role": "system"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "You are an agent who can operate a computer on behalf of a user. Your goal is to decide the next action to perform based on the current state of the phone and the user's request.\n\n### User Request ###\nOpen the calculator\nYou can follow these tips:\n\nOn Windows, you absolutely have to double-click to open any desktop icon or folder in File Explorer—one single click will NOT open them. Always double-click.\n\n### Overall Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'} 2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Progress Status ###\nNo progress yet.\n\n### Current Subgoal ###\nOpen the Start Menu\n\n### Collected Task-Related Notes ###\nNo important notes recorded.\n\n---\nCarefully examine all the information provided above and decide on the next action to perform. If you notice an unsolved error in the previous action, think as a human user and attempt to rectify them. You must choose your action from one of the atomic actions.\n\n#### Atomic Actions ####\nThe atomic action functions are listed in the format of `action(arguments): description` as follows:\n- open_app(app_name): Open the App with app_name such as Chrome and Wechat. Usage Example: {\"action\": \"open_app\", \"app_name\": \"chrome\"}\n- click(coordinate): Click the point on the screen with specified (x, y) coordinates. Usage Example: {\"action\": \"click\", \"coordinate\": [x, y]}\n- double_click(coordinate): Double click on the position (x, y) on the screen. Usage Example: {\"action\": \"double_click\", \"coordinate\": [x, y]}\n- right_click(coordinate): Right-click using the mouse on the position (x, y) on the screen. Usage Example: {\"action\": \"double_click\", \"coordinate\": [x, y]}\n- type(coordinate, text, clear, enter): Type text into the position (x, y) on the screen. Use escape characters \\', \\\", and \\n in the `text` part to ensure we can parse the content in normal python string format. If you want to clear the existing content, set the `clear` parameter to 1; otherwise, set it to 0. If you want to press `enter` after input, set the `enter` parameter to 1; otherwise, set it to 0. Usage Example: {\"action\": \"type\", \"coordinate\": [x, y], \"text\": \"the text you want to type\", \"clear\": 1, \"enter\": 1}\n- hotkey(keys): Press a hotkey combination. The `keys` parameter is a list of keys represented as a string, such as \"['ctrl', 'c']\". Usage Example: {\"action\": \"hotkey\", \"keys\": \"['ctrl', 'c']\"}\n- scroll(coordinate, value): Scroll at the position (x, y) on the screen. The `value` parameter can be positive (scroll up) or negative (scroll down), which is usually set to 5 or -5. Usage Example: {\"action\": \"scroll\", \"coordinate\": [x, y], \"value\": 5}\n- wait(time): Wait for a specified amount of time, such as 3s. Usage Example: {\"action\": \"wait\", \"time\": 3}\n- call_user(): Call the user when the task is unsolvable, or when you need the user's help, such as log in or close the pop up. Usage Example: {\"action\": \"call_user\"}\n- drag(coordinate, coordinate2): drag from the position with coordinate to the position with coordinate2. Usage Example: {\"action\": \"drag\", \"coordinate\": [x1, y1], \"coordinate2\": [x2, y2]}\n- set_cell_values(cell_values, file_name, sheet_name): set individual cell values (a Dict) in the sheet with sheet_name and in the spreadsheet with file_name. Usage Example: {\"action\": \"set_cell_value\", \"cell_values\": {\"A2\": \"hello\"}, \"file_name\": 'Untitled 1', \"sheet_name\": 'Sheet1'}\n- done(): Output this action when you think this instruction is done. Usage Example: {\"action\": \"done\"}\n\n### Latest Action History ###\nNo actions have been taken yet.\n\n### Tips\n\n* On Windows, you absolutely have to double-click to open any desktop icon or folder in File Explorer—one single click will NOT open them. Always double-click.\n* To log on or encounter a pop-up window, use the call_user tool to ask the user to take over. Do not enter account credentials or perform unauthorized operations.\n* Use app-specific tools first, rather than generic cv tools such as cv_click or cv_key.\n\n\n---\nProvide your output in the following format, which contains three parts:\n### Thought ###\nProvide a detailed explanation of your rationale for the chosen action.  Please output in Chinese.\n\n### Action ###\nChoose only one action or shortcut from the options provided. IMPORTANT: Do NOT return invalid actions like null or stop. Do NOT repeat previously failed actions multiple times.\nYou must provide your decision using a valid JSON format specifying the `action` and the arguments of the action.\n\n### Description ###\nA brief description of the chosen action and the expected outcome.  Please output in Chinese."
                                            },
                                            {
                                                "image": "C:\\0421d463-47ac-4307-8bed-a22df6568cad.png"
                                            }
                                        ],
                                        "role": "user"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "### Thought ###\nTo open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n### Action ###\n{\"action\": \"click\", \"coordinate\": [35, 1074]}\n\n### Description ###\nClick the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application."
                                            }
                                        ],
                                        "role": "assistant"
                                    }
                                ],
                                "name": "operator",
                                "response": "### Thought ###\nTo open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n### Action ###\n{\"action\": \"click\", \"coordinate\": [35, 1074]}\n\n### Description ###\nClick the Start button in the lower-left corner of the screen. This opens the Windows Start menu, which lets you search for and open the Calculator application."
                            },
                            "manager": {
                                "messages": [
                                    {
                                        "content": [
                                            {
                                                "text": "You are a helpful assistant.",
                                                "type": "text"
                                            }
                                        ],
                                        "role": "system"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "You are an agent who can operate a windows computer on behalf of a user. Your goal is to track progress and devise high-level plans to achieve the user's requests.\n\n### User Request ###\nOpen the calculator\nYou can follow these tips:\n\nOn Windows, you absolutely have to double-click to open any desktop icon or folder in File Explorer—one single click will NOT open them. Always double-click.\n\n---\nMake a high-level plan to achieve the user's request. If the request is complex, break it down into subgoals. The screenshot displays the starting state of the computer.\nPlease respond in Chinese.\n\n##### Important Notes #####\n1. Before generating your plan, carefully observe and understand the current state of the computer.\n2. Your plan should contain only necessary steps; however, please include information useful for executing the subgoal in the 'info' field.\n3. Do not include verification steps in your plan. Steps that confirm or validate other subtasks should not be included.\n4. Do not include optional steps in your plan.\n\nProvide your output in the following format which contains three parts:\n\n### Thought ###\nA detailed explanation of your rationale for the plan and subgoals.\n\n### Plan ###\n1. {'name': 'brief description of the first subgoal.', 'info': 'detailed information about executing the first subgoal.'}\n2. {'name': 'brief description of the second subgoal.', 'info': 'detailed information about executing the second subgoal.'}\n...\n\n### Current Subgoal ###\nThe first subgoal's name you should work on.\n\n"
                                            },
                                            {
                                                "image": "C:\\Usersimages\\b5bf3eff-ae32-4dd0-85ee-7ee31c359142.png"
                                            }
                                        ],
                                        "role": "user"
                                    },
                                    {
                                        "content": [
                                            {
                                                "text": "### Thought ###\nThe user wants to open the calculator application. The screenshot shows the Windows desktop with various icons, but the calculator icon is not immediately visible. To achieve this, I need to locate and open the calculator. Since it's not on the desktop, I'll assume it might be in the Start menu or another location like the All Apps menu. Therefore, the plan involves accessing the Start menu and finding the calculator there.\n\n### Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'}\n2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Current Subgoal ###\nOpen the Start Menu"
                                            }
                                        ],
                                        "role": "assistant"
                                    }
                                ],
                                "name": "manager",
                                "response": "### Thought ###\nThe user wants to open the calculator application. The screenshot shows the Windows desktop with various icons, but the calculator icon is not immediately visible. To achieve this, I need to locate and open the calculator. Since it's not on the desktop, I'll assume it might be in the Start menu or another location like the All Apps menu. Therefore, the plan involves accessing the Start menu and finding the calculator there.\n\n### Plan ###\n1. {'name': 'Open the Start Menu', 'info': 'Click on the Start button located at the bottom left corner of the screen to open the Start menu.'}\n2. {'name': 'Locate Calculator', 'info': 'In the Start menu, scroll through the list of applications or use the search bar at the top to find \"Calculator\".'}\n\n### Current Subgoal ###\nOpen the Start Menu"
                            }
                        },
                        "thought": "To open the calculator, I must first open the Start menu. According to the Windows operating system standard, clicking the Start button in the lower-left corner opens the Start menu. This button is usually in the lower-left corner of the screen and has a circular icon with the Windows logo.\n\n"
                    },
                    "delta": false,
                    "object": "content",
                    "type": "data"
                }
            ],
            "id": "msg_e9dd72ea-19fc-4a22-93ee-d97588645f88",
            "message": "Success",
            "object": "message",
            "role": "assistant",
            "status": "Success",
            "type": "message"
        }
    ],
    "session_id": "bee1915d-6f4d-4bfc-b657-ecb9d0ed8dad",
    "status": "created"
}