File size: 8,663 Bytes
5655a42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | """Schema for the generic `computer_use` tool (model-facing; value is byte-frozen —
the schema goes to the model every turn, so prompt-cache parity depends on it).
Model-agnostic: any tool-calling model can drive this. Vision-capable models
should prefer `capture(mode='som')` then `click(element=N)` — much more reliable
than pixel coordinates, which remain supported for models trained on them.
"""
from __future__ import annotations
from typing import Any, Dict
# One consolidated tool with an `action` discriminator keeps the schema compact
# and the per-turn token cost low. Property groups: capture (mode, app, pid,
# window_id) / targeting (element, coordinate, button, modifiers) / drag / scroll /
# set_value / type-key-wait / focus_app / delivery ladder / return shape.
_PROPERTIES: Dict[str, Any] = {
"action": {
"type": "string",
"enum": [
"capture",
"click",
"double_click",
"right_click",
"middle_click",
"drag",
"scroll",
"type",
"key",
"set_value",
"wait",
"list_apps",
"list_windows",
"focus_app",
],
"description": (
"Which action to perform. `capture` is free (no side effects). All other actions "
"require approval unless auto-approved. Use `set_value` for select/popup elements and "
"sliders — it selects the matching option directly without opening the native menu (no "
"focus steal)."
),
},
"mode": {
"type": "string",
"enum": ["som", "vision", "ax"],
"description": (
"Capture mode. `som` (default) is a screenshot with numbered overlays on every "
"interactable element plus the AX tree — best for vision models, lets you click by "
"element index. `vision` is a plain screenshot. `ax` is the accessibility tree only "
"(no image; useful for text-only models)."
),
},
"app": {
"type": "string",
"description": (
"Optional. Limit capture/action to one app (name e.g. 'Safari', or bundle ID). Omitted "
"= frontmost window. app='screen' = composited full-screen grab (image only, no "
"clickable elements); app='desktop' = the OS desktop/shell surface (wallpaper, icons, "
"taskbar) with its elements."
),
},
"pid": {
"type": "integer",
"description": (
"Optional exact process target for action='capture'. Pair with window_id when "
"discovery cannot resolve an X11 app."
),
},
"window_id": {
"type": "integer",
"description": (
"Optional exact native window target for action='capture'. Pair with pid when an "
"external cua-driver list_windows lookup has already identified the window."
),
},
"element": {
"type": "integer",
"description": (
"The 1-based SOM index returned by the last `capture(mode='som')` call. Strongly "
"preferred over raw coordinates."
),
},
"coordinate": {
"type": "array",
"items": {"type": "integer"},
"minItems": 2,
"maxItems": 2,
"description": (
"Pixel coordinates [x, y] relative to the captured window screenshot (top-left "
"origin). Only use this if no element index is available."
),
},
"button": {
"type": "string",
"enum": ["left", "right", "middle"],
"description": "Mouse button. Defaults to left.",
},
"modifiers": {
"type": "array",
"items": {
"type": "string",
"enum": [
"cmd",
"shift",
"option",
"alt",
"ctrl",
"fn",
"win",
"windows",
"super",
"meta",
],
},
"description": "Modifier keys held during the action.",
},
"from_element": {"type": "integer", "description": "Source element index (drag)."},
"to_element": {"type": "integer", "description": "Target element index (drag)."},
"from_coordinate": {
"type": "array",
"items": {"type": "integer"},
"minItems": 2,
"maxItems": 2,
"description": "Source [x,y] (drag; use when no element available).",
},
"to_coordinate": {
"type": "array",
"items": {"type": "integer"},
"minItems": 2,
"maxItems": 2,
"description": "Target [x,y] (drag; use when no element available).",
},
"direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "Scroll direction."},
"amount": {"type": "integer", "description": "Scroll wheel ticks. Default 3."},
"value": {
"type": "string",
"description": (
"For action='set_value': the value to set on the element. For AXPopUpButton / select "
"dropdowns, pass the option's display label (e.g. 'Blue'). For sliders and other "
"AXValue-settable elements, pass the numeric or string value."
),
},
"text": {"type": "string", "description": "Text to type (respects the current layout)."},
"keys": {
"type": "string",
"description": (
"Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine."
),
},
"seconds": {"type": "number", "description": "Seconds to wait. Max 30."},
"raise_window": {
"type": "boolean",
"description": (
"Only for action='focus_app'. If true, brings the window to front (DISRUPTS the user). "
"Default false — input is routed to the app without raising, matching the background "
"co-work model."
),
},
"delivery_mode": {
"type": "string",
"enum": ["background", "foreground"],
"description": (
"For input actions (click, type, key, drag, scroll). `background` (DEFAULT) delivers "
"without raising the window or stealing focus. `foreground` briefly fronts the window "
"then restores focus — a visible change needing its own approval; use it only when a "
"result's verdict tells you to escalate there. Each result's `verdict` carries the "
"next step; follow it rather than guessing."
),
},
"bring_to_front": {
"type": "boolean",
"description": (
"Optional and only valid with delivery_mode='foreground'. Explicitly invokes "
"cua-driver's standalone bring_to_front tool before the input; it is never passed as "
"an input property. This persistent focus change has a separate approval scope. "
"Default false."
),
},
"capture_after": {
"type": "boolean",
"description": (
"If true, take a follow-up capture after the action and include it in the response. "
"Saves a round-trip when you need to verify an action's effect."
),
},
}
COMPUTER_USE_SCHEMA: Dict[str, Any] = {
"name": "computer_use",
"description": (
"Drive the desktop via cua-driver — screenshots, mouse, keyboard, scroll, drag — on macOS, "
"Windows, and Linux. Input is background-FIRST, not background-only: the default delivery "
"routes to the target window without stealing the user's cursor or focus (works even on "
"hidden/minimized windows), and when a result's `verdict` says to escalate you climb — "
"pixel coordinates, or delivery_mode='foreground' (briefly fronts the window; separate "
"approval). Each result carries a `verdict` with the next step; follow it — never repeat "
"confirmed input, and re-capture to verify an unverifiable one before retrying. Workflow: "
"action='capture' (mode='som' gives numbered element overlays), then click by `element` "
"index; re-capture after state-changing actions (or pass capture_after=true). Image "
"captures include a shareable `screenshot_path`; deliver it via the platform's MEDIA "
"syntax when the user asks to see it — not for captures used only for control."
),
"parameters": {"type": "object", "properties": _PROPERTIES, "required": ["action"]},
}
def get_computer_use_schema() -> Dict[str, Any]:
"""Return the generic OpenAI function-calling schema."""
return COMPUTER_USE_SCHEMA
|