| """Schema for the generic `computer_use` tool (model-facing; value is byte-frozen — |
| the schema goes to the model every turn, so prompt-cache parity depends on it). |
| |
| Model-agnostic: any tool-calling model can drive this. Vision-capable models |
| should prefer `capture(mode='som')` then `click(element=N)` — much more reliable |
| than pixel coordinates, which remain supported for models trained on them. |
| """ |
|
|
| from __future__ import annotations |
|
|
| from typing import Any, Dict |
|
|
| |
| |
| |
| |
| _PROPERTIES: Dict[str, Any] = { |
| "action": { |
| "type": "string", |
| "enum": [ |
| "capture", |
| "click", |
| "double_click", |
| "right_click", |
| "middle_click", |
| "drag", |
| "scroll", |
| "type", |
| "key", |
| "set_value", |
| "wait", |
| "list_apps", |
| "list_windows", |
| "focus_app", |
| ], |
| "description": ( |
| "Which action to perform. `capture` is free (no side effects). All other actions " |
| "require approval unless auto-approved. Use `set_value` for select/popup elements and " |
| "sliders — it selects the matching option directly without opening the native menu (no " |
| "focus steal)." |
| ), |
| }, |
| "mode": { |
| "type": "string", |
| "enum": ["som", "vision", "ax"], |
| "description": ( |
| "Capture mode. `som` (default) is a screenshot with numbered overlays on every " |
| "interactable element plus the AX tree — best for vision models, lets you click by " |
| "element index. `vision` is a plain screenshot. `ax` is the accessibility tree only " |
| "(no image; useful for text-only models)." |
| ), |
| }, |
| "app": { |
| "type": "string", |
| "description": ( |
| "Optional. Limit capture/action to one app (name e.g. 'Safari', or bundle ID). Omitted " |
| "= frontmost window. app='screen' = composited full-screen grab (image only, no " |
| "clickable elements); app='desktop' = the OS desktop/shell surface (wallpaper, icons, " |
| "taskbar) with its elements." |
| ), |
| }, |
| "pid": { |
| "type": "integer", |
| "description": ( |
| "Optional exact process target for action='capture'. Pair with window_id when " |
| "discovery cannot resolve an X11 app." |
| ), |
| }, |
| "window_id": { |
| "type": "integer", |
| "description": ( |
| "Optional exact native window target for action='capture'. Pair with pid when an " |
| "external cua-driver list_windows lookup has already identified the window." |
| ), |
| }, |
| "element": { |
| "type": "integer", |
| "description": ( |
| "The 1-based SOM index returned by the last `capture(mode='som')` call. Strongly " |
| "preferred over raw coordinates." |
| ), |
| }, |
| "coordinate": { |
| "type": "array", |
| "items": {"type": "integer"}, |
| "minItems": 2, |
| "maxItems": 2, |
| "description": ( |
| "Pixel coordinates [x, y] relative to the captured window screenshot (top-left " |
| "origin). Only use this if no element index is available." |
| ), |
| }, |
| "button": { |
| "type": "string", |
| "enum": ["left", "right", "middle"], |
| "description": "Mouse button. Defaults to left.", |
| }, |
| "modifiers": { |
| "type": "array", |
| "items": { |
| "type": "string", |
| "enum": [ |
| "cmd", |
| "shift", |
| "option", |
| "alt", |
| "ctrl", |
| "fn", |
| "win", |
| "windows", |
| "super", |
| "meta", |
| ], |
| }, |
| "description": "Modifier keys held during the action.", |
| }, |
| "from_element": {"type": "integer", "description": "Source element index (drag)."}, |
| "to_element": {"type": "integer", "description": "Target element index (drag)."}, |
| "from_coordinate": { |
| "type": "array", |
| "items": {"type": "integer"}, |
| "minItems": 2, |
| "maxItems": 2, |
| "description": "Source [x,y] (drag; use when no element available).", |
| }, |
| "to_coordinate": { |
| "type": "array", |
| "items": {"type": "integer"}, |
| "minItems": 2, |
| "maxItems": 2, |
| "description": "Target [x,y] (drag; use when no element available).", |
| }, |
| "direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "Scroll direction."}, |
| "amount": {"type": "integer", "description": "Scroll wheel ticks. Default 3."}, |
| "value": { |
| "type": "string", |
| "description": ( |
| "For action='set_value': the value to set on the element. For AXPopUpButton / select " |
| "dropdowns, pass the option's display label (e.g. 'Blue'). For sliders and other " |
| "AXValue-settable elements, pass the numeric or string value." |
| ), |
| }, |
| "text": {"type": "string", "description": "Text to type (respects the current layout)."}, |
| "keys": { |
| "type": "string", |
| "description": ( |
| "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine." |
| ), |
| }, |
| "seconds": {"type": "number", "description": "Seconds to wait. Max 30."}, |
| "raise_window": { |
| "type": "boolean", |
| "description": ( |
| "Only for action='focus_app'. If true, brings the window to front (DISRUPTS the user). " |
| "Default false — input is routed to the app without raising, matching the background " |
| "co-work model." |
| ), |
| }, |
| "delivery_mode": { |
| "type": "string", |
| "enum": ["background", "foreground"], |
| "description": ( |
| "For input actions (click, type, key, drag, scroll). `background` (DEFAULT) delivers " |
| "without raising the window or stealing focus. `foreground` briefly fronts the window " |
| "then restores focus — a visible change needing its own approval; use it only when a " |
| "result's verdict tells you to escalate there. Each result's `verdict` carries the " |
| "next step; follow it rather than guessing." |
| ), |
| }, |
| "bring_to_front": { |
| "type": "boolean", |
| "description": ( |
| "Optional and only valid with delivery_mode='foreground'. Explicitly invokes " |
| "cua-driver's standalone bring_to_front tool before the input; it is never passed as " |
| "an input property. This persistent focus change has a separate approval scope. " |
| "Default false." |
| ), |
| }, |
| "capture_after": { |
| "type": "boolean", |
| "description": ( |
| "If true, take a follow-up capture after the action and include it in the response. " |
| "Saves a round-trip when you need to verify an action's effect." |
| ), |
| }, |
| } |
|
|
| COMPUTER_USE_SCHEMA: Dict[str, Any] = { |
| "name": "computer_use", |
| "description": ( |
| "Drive the desktop via cua-driver — screenshots, mouse, keyboard, scroll, drag — on macOS, " |
| "Windows, and Linux. Input is background-FIRST, not background-only: the default delivery " |
| "routes to the target window without stealing the user's cursor or focus (works even on " |
| "hidden/minimized windows), and when a result's `verdict` says to escalate you climb — " |
| "pixel coordinates, or delivery_mode='foreground' (briefly fronts the window; separate " |
| "approval). Each result carries a `verdict` with the next step; follow it — never repeat " |
| "confirmed input, and re-capture to verify an unverifiable one before retrying. Workflow: " |
| "action='capture' (mode='som' gives numbered element overlays), then click by `element` " |
| "index; re-capture after state-changing actions (or pass capture_after=true). Image " |
| "captures include a shareable `screenshot_path`; deliver it via the platform's MEDIA " |
| "syntax when the user asks to see it — not for captures used only for control." |
| ), |
| "parameters": {"type": "object", "properties": _PROPERTIES, "required": ["action"]}, |
| } |
|
|
| def get_computer_use_schema() -> Dict[str, Any]: |
| """Return the generic OpenAI function-calling schema.""" |
| return COMPUTER_USE_SCHEMA |
|
|
