Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/linux_take_photo.png filter=lfs diff=lfs merge=lfs -text
+examples/vscode_split_editor.png filter=lfs diff=lfs merge=lfs -text
+examples/windows_mute.png filter=lfs diff=lfs merge=lfs -text
+examples/word_redo.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,45 @@
 ---
-title: Groundgui
-emoji: 🌖
-colorFrom: red
-colorTo: purple
+title: GroundGUI GUI Grounding
+emoji: 🖱️
+colorFrom: pink
+colorTo: blue
 sdk: gradio
 sdk_version: 6.30.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+short_description: Ground click coordinates on any GUI screenshot.
+python_version: "3.12"
+startup_duration_timeout: 1h
+license: apache-2.0
+models:
+  - PrentisAI/GroundGUI-8B
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# GroundGUI · GUI Grounding
+
+This demo runs [PrentisAI/GroundGUI-8B](https://huggingface.co/PrentisAI/GroundGUI-8B),
+an 8B GUI-grounding vision-language model fine-tuned from Qwen3-VL-8B-Instruct via
+high-quality data curation and distillation-to-reinforcement learning.
+
+Give it a GUI screenshot plus a short instruction, and it emits the click action
+as a tool call with coordinates in a **normalized 0–1000 space** —
+`{"name": "computer_use", "arguments": {"action": "left_click", "coordinate": [x, y]}}`.
+The demo maps the coordinates back onto your original image and marks the
+predicted click point with a crosshair.
+
+The system prompt is the exact frozen per-platform grounding prompt the model was
+trained with (`scalecua_toolcall`): pick **desktop**, **web**, or **mobile** to
+serve the matching tool schema (`computer_use` / `browser_use` / `mobile_use`).
+Inference is greedy (`do_sample=False`), matching the paper's evaluation.
+
+- Paper: [GroundGUI: GUI Grounding by High-Quality Data and Distillation-to-Reinforcement Learning](https://huggingface.co/papers/2610.19100)
+- Code & data: [github.com/PrentisAI/GroundGUI](https://github.com/PrentisAI/GroundGUI)
+- Model: [PrentisAI/GroundGUI-8B](https://huggingface.co/PrentisAI/GroundGUI-8B)
+
+**Benchmark results** (from the paper): 95.2 ScreenSpot-v2 · 63.9 ScreenSpot-Pro ·
+86.8 MMBench-GUI L2 · 38.3 UI-Vision · 65.4 OSWorld-G.
+
+## Example screenshots
+
+The bundled example screenshots come from
+[likaixin/ScreenSpot-Pro](https://huggingface.co/datasets/likaixin/ScreenSpot-Pro)
+(MIT license), a professional high-resolution GUI grounding benchmark.

app.py

@@ -0,0 +1,299 @@
+import spaces  # MUST come before torch / transformers (ZeroGPU)
+
+import json
+import os
+import re
+import time
+
+import gradio as gr
+import torch
+from PIL import Image, ImageDraw
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+# GroundGUI-8B: a Qwen3-VL-8B derivative trained for GUI grounding with the
+# ScaleCUA-recipe per-platform tool-call system prompts (frozen verbatim in
+# prompts.json, extracted from ScaleCUAToolCallPrompt._GROUNDING_SYSTEM in
+# github.com/PrentisAI/GroundGUI eval/prompts.py).
+MODEL = "PrentisAI/GroundGUI-8B"
+
+# Eval-side image budget (run_eval.sh: MAX_IMAGE_PIXELS=16777216). Larger
+# screenshots are downscaled with smart_resize (factor = patch 16 * merge 2 =
+# 32, the Qwen3-VL grid) before encoding; the predicted point is mapped back
+# onto the ORIGINAL image afterwards, so the overlay is unaffected.
+FACTOR = 32
+MAX_PIXELS = 16_777_216
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+with open(os.path.join(HERE, "prompts.json"), encoding="utf-8") as f:
+    GROUNDING_SYSTEM = json.load(f)
+
+TOOL_CALL_OPEN = chr(60) + "tool_call" + chr(62)
+TOOL_CALL_CLOSE = chr(60) + "/" + "tool_call" + chr(62)
+
+
+def smart_resize(h, w, factor=FACTOR, min_pixels=56 * 56, max_pixels=MAX_PIXELS):
+    """Qwen3-VL smart_resize: snap sides to the factor grid under a pixel budget.
+
+    Mirrors qwen_vl_utils.smart_resize as used by the GroundGUI eval
+    (IMAGE_FACTOR=32, MAX_IMAGE_PIXELS=16777216).
+    """
+    h_bar = round(h / factor) * factor
+    w_bar = round(w / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = (h * w / max_pixels) ** 0.5
+        h_bar = round(h / beta / factor) * factor
+        w_bar = round(w / beta / factor) * factor
+    if h_bar * w_bar < min_pixels:
+        beta2 = (min_pixels / (h * w)) ** 0.5
+        h_bar = round(h * beta2 / factor) * factor
+        w_bar = round(w * beta2 / factor) * factor
+    return max(h_bar, factor), max(w_bar, factor)
+
+
+def fit(image):
+    """Downscale a screenshot to the eval's vision budget (factor-32 grid)."""
+    w, h = image.size
+    nh, nw = smart_resize(h, w)
+    if (nh, nw) != (h, w):
+        image = image.resize((nw, nh), Image.LANCZOS)
+    return image
+
+
+def extract_point(response):
+    """Parse the model's tool_call block into a [x, y] point in 0-1000 space.
+
+    Follows the official eval parse (Qwen3VLPrompt.extract_coordinates):
+    the JSON object inside the tool-call tags, terminate/abstain -> [-1, -1],
+    falling back to the GUI-Owl "(x, y)" then "[x, y]" regexes (last match).
+
+    Returns (point, refusal, tool_name, action_name) where point is None if
+    nothing parsed.
+    """
+    if ('"action": "terminate"' in response or '"status": "failure"' in response
+            or '"action": "abstain"' in response):
+        return None, True, "", ""
+    blobs = re.findall(
+        re.escape(TOOL_CALL_OPEN) + r"\s*(\{.*?\})\s*" + re.escape(TOOL_CALL_CLOSE),
+        response, re.DOTALL,
+    )
+    for blob in reversed(blobs):
+        try:
+            obj = json.loads(blob)
+        except json.JSONDecodeError:
+            continue
+        name = obj.get("name") or ""
+        args = obj.get("arguments") or {}
+        if args.get("action") in ("terminate", "abstain"):
+            return None, True, name, args.get("action")
+        coord = args.get("coordinate")
+        if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+            try:
+                return [float(coord[0]), float(coord[1])], False, name, args.get("action")
+            except (TypeError, ValueError):
+                continue
+    matches = re.findall(r"\((\d+),\s*(\d+)\)", response)
+    if not matches:
+        matches = re.findall(r"\[(\d+),\s*(\d+)\]", response)
+    if not matches:
+        return None, False, "", ""
+    x, y = matches[-1]
+    return [int(x), int(y)], False, "", ""
+
+
+def norm_to_pixels(point, width, height):
+    """Map a 0-1000 normalized point onto image pixels (GUIOwlPrompt.calculate_metrics)."""
+    return round(point[0] / 1000.0 * width), round(point[1] / 1000.0 * height)
+
+
+def draw_marker(image, point):
+    """Draw a crosshair + circle at the predicted click point.
+
+    A white underlay keeps the marker visible on both light and dark UIs.
+    """
+    out = image.copy()
+    d = ImageDraw.Draw(out)
+    x, y = point
+    r = max(10, min(out.width, out.height) // 60)
+    # white underlay, then orange marker on top
+    d.ellipse([x - r, y - r, x + r, y + r], outline=(255, 255, 255), width=6)
+    d.ellipse([x - r, y - r, x + r, y + r], outline=(255, 64, 0), width=3)
+    for (x0, y0, x1, y1) in [
+        (x - r * 1.8, y, x + r * 1.8, y),  # horizontal
+        (x, y - r * 1.8, x, y + r * 1.8),  # vertical
+    ]:
+        d.line([x0, y0, x1, y1], fill=(255, 255, 255), width=9)
+        d.line([x0, y0, x1, y1], fill=(255, 64, 0), width=4)
+    return out
+
+
+processor = AutoProcessor.from_pretrained(MODEL)
+model = (
+    AutoModelForImageTextToText.from_pretrained(
+        MODEL, dtype=torch.bfloat16, attn_implementation="sdpa"
+    )
+    .eval()
+    .to("cuda")
+)
+
+
+@spaces.GPU(duration=120)
+def predict(instruction: str, screenshot, platform: str = "desktop", max_new_tokens: int = 256):
+    """Locate a UI element in a screenshot.
+
+    Args:
+        instruction: referring expression naming the element to find,
+            e.g. "the redo button".
+        screenshot: a GUI screenshot (desktop, web, or mobile).
+        platform: one of "desktop", "web", "mobile" -- selects the
+            per-platform grounding tool schema the model was trained with
+            (computer_use / browser_use / mobile_use).
+        max_new_tokens: generation length cap for the emitted tool call.
+
+    Returns:
+        Tuple of (annotated screenshot with the predicted click point,
+        raw model output, predicted action, coordinates on the original
+        image, inference seconds).
+    """
+    if screenshot is None:
+        raise gr.Error("Please provide a GUI screenshot first.")
+    if not instruction or not instruction.strip():
+        raise gr.Error("Please enter an instruction, e.g. 'the redo button'.")
+    platform = platform if platform in GROUNDING_SYSTEM else "desktop"
+
+    if screenshot.mode != "RGB":
+        screenshot = screenshot.convert("RGB")
+    orig_w, orig_h = screenshot.size
+
+    # Eval-order prompt: system = frozen per-platform grounding prompt,
+    # user = image first, then the bare instruction (scalecua_toolcall).
+    messages = [
+        {"role": "system", "content": GROUNDING_SYSTEM[platform]},
+        {
+            "role": "user",
+            "content": [
+                {"type": "image"},
+                {"type": "text", "text": instruction.strip()},
+            ],
+        },
+    ]
+    prompt = processor.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+    inputs = processor(
+        text=[prompt], images=[fit(screenshot)], return_tensors="pt"
+    ).to(model.device)
+
+    t0 = time.perf_counter()
+    with torch.inference_mode():
+        output = model.generate(
+            **inputs,
+            max_new_tokens=int(max_new_tokens),
+            do_sample=False,
+            use_cache=True,
+        )
+    elapsed = time.perf_counter() - t0
+
+    response = processor.decode(
+        output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True
+    ).strip()
+
+    # Unwrap the chat-template's tool-call rendering (skip_special_tokens=True
+    # keeps the non-special tool-call tags, which the parser strips itself).
+    body = response
+    if TOOL_CALL_OPEN in body:
+        body = body.split(TOOL_CALL_OPEN, 1)[1]
+        if TOOL_CALL_CLOSE in body:
+            body = body.split(TOOL_CALL_CLOSE, 1)[0]
+        body = body.strip()
+
+    point, refusal, tool_name, action_name = extract_point(body)
+    if refusal or point is None:
+        note = "Model abstained: element not found on screen." if refusal else "No coordinates parsed from the model output."
+        return screenshot.copy(), body, "abstain" if refusal else "", note, f"{elapsed:.1f}s"
+
+    px = norm_to_pixels(point, orig_w, orig_h)
+    clipped = (max(0, min(orig_w - 1, px[0])), max(0, min(orig_h - 1, px[1])))
+    annotated = draw_marker(screenshot, clipped)
+    coords_text = f"click point ({clipped[0]}, {clipped[1]}) px on {orig_w}x{orig_h} image; model space [{point[0]:g}, {point[1]:g}] / 1000"
+    action_text = (tool_name or "?") + "." + (action_name or "?") + f" -> ({clipped[0]}, {clipped[1]})"
+    return annotated, body, action_text, coords_text, f"{elapsed:.1f}s"
+
+
+CSS = """
+#col-container { max-width: 1400px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+with gr.Blocks() as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(
+            """
+            # :mag: GroundGUI · GUI Grounding
+
+            **GroundGUI-8B** ([PrentisAI/GroundGUI-8B](https://huggingface.co/PrentisAI/GroundGUI-8B))
+            is an 8B GUI-grounding vision-language model fine-tuned from Qwen3-VL-8B-Instruct with
+            high-quality grounding data and distillation-to-reinforcement learning.
+            Give it a GUI screenshot and a short instruction, and it emits the click
+            action with coordinates in a **normalized 0–1000 space**, which are mapped onto
+            your original image and marked with a crosshair.
+
+            Paper: [GroundGUI: GUI Grounding by High-Quality Data and Distillation-to-Reinforcement Learning](https://huggingface.co/papers/2610.19100) ·
+            Code: [github.com/PrentisAI/GroundGUI](https://github.com/PrentisAI/GroundGUI)
+
+            Benchmarks: **95.2** ScreenSpot-v2 · **63.9** ScreenSpot-Pro · **86.8** MMBench-GUI L2 ·
+            **38.3** UI-Vision · **65.4** OSWorld-G.
+            """
+        )
+        with gr.Row():
+            with gr.Column(scale=1):
+                screenshot_in = gr.Image(
+                    type="pil", label="GUI screenshot", height=420,
+                    sources=["upload", "clipboard"],
+                )
+                instruction = gr.Textbox(
+                    label="Instruction",
+                    placeholder="e.g. the redo button",
+                    lines=2,
+                )
+                platform = gr.Radio(
+                    ["desktop", "web", "mobile"], value="desktop",
+                    label="Platform (tool schema used in the system prompt)",
+                )
+                run_btn = gr.Button("Ground it", variant="primary")
+                with gr.Accordion("Advanced", open=False):
+                    max_new_tokens = gr.Slider(
+                        64, 512, value=256, step=32, label="Max new tokens",
+                    )
+            with gr.Column(scale=1):
+                screenshot_out = gr.Image(
+                    type="pil", label="Annotated screenshot (predicted click point)",
+                    height=420,
+                )
+                action_out = gr.Textbox(label="Predicted action", lines=1)
+                coords_out = gr.Textbox(label="Coordinates", lines=2)
+                raw_out = gr.Textbox(label="Raw model output", lines=6)
+                time_out = gr.Textbox(label="Inference time", lines=1)
+
+        gr.Examples(
+            examples=[
+                ["redo", "examples/word_redo.png", "desktop"],
+                ["Split editor.", "examples/vscode_split_editor.png", "desktop"],
+                ["mute", "examples/windows_mute.png", "desktop"],
+                ["take a photo", "examples/linux_take_photo.png", "desktop"],
+            ],
+            inputs=[instruction, screenshot_in, platform],
+            fn=predict,
+            outputs=[screenshot_out, raw_out, action_out, coords_out, time_out],
+            cache_examples=True,
+            cache_mode="lazy",
+        )
+
+    run_btn.click(
+        predict,
+        inputs=[instruction, screenshot_in, platform, max_new_tokens],
+        outputs=[screenshot_out, raw_out, action_out, coords_out, time_out],
+        api_name="predict",
+    )
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)

prompts.json

@@ -0,0 +1,5 @@
+{
+ "desktop": "You are an autonomous GUI agent capable of operating on desktops, mobile devices, and web browsers. Your primary function is to analyze a screen capture, locate the target UI element, and issue the single appropriate action that fulfills the given instruction.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"type\": \"function\", \"function\": {\"name\": \"computer_use\", \"description\": \"Use a mouse and keyboard to interact with a computer, and take screenshots.\\n* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\\n* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\\n* The screen's resolution is 1000x1000.\\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.\", \"parameters\": {\"properties\": {\"action\": {\"description\": \"The action to perform. The available actions are:\\n* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.\\n* `left_click`: Click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `left_click_drag`: Press the left mouse button at the start point with coordinate (x, y), drag to the end point with coordinate2 (x2, y2), and release.\\n* `right_click`: Click the right mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `middle_click`: Click the middle mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `double_click`: Double-click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `triple_click`: Triple-click the left mouse button at a specified (x, y) pixel coordinate on the screen.\", \"enum\": [\"mouse_move\", \"left_click\", \"left_click_drag\", \"right_click\", \"middle_click\", \"double_click\", \"triple_click\"], \"type\": \"string\"}, \"coordinate\": {\"description\": \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=mouse_move`, `action=left_click`, `action=left_click_drag`, `action=right_click`, `action=middle_click`, `action=double_click`, and `action=triple_click`.\", \"type\": \"array\"}, \"coordinate2\": {\"description\": \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to drag the cursor to. Required only by `action=left_click_drag`.\", \"type\": \"array\"}}, \"required\": [\"action\"], \"type\": \"object\"}}}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>\n\n## Input Specification\n- A screenshot of the current screen + an instruction describing the action to perform.\n\n## Output Format\n- Emit exactly one <tool_call></tool_call> block containing a single JSON object with the function name and its arguments.\n- Coordinates are integers in a 1000x1000 space (x measured from the left edge, y from the top edge).\n\n## Note\n- Choose exactly one function whose name and arguments are defined in the tool schema above.\n- Do not output anything other than the <tool_call></tool_call> block.",
+ "mobile": "You are an autonomous GUI agent capable of operating on desktops, mobile devices, and web browsers. Your primary function is to analyze a screen capture, locate the target UI element, and issue the single appropriate action that fulfills the given instruction.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"type\": \"function\", \"function\": {\"name\": \"mobile_use\", \"description\": \"Use a touchscreen to interact with a mobile device, and take screenshots.\\n* This is an interface to a mobile device with touchscreen. You can perform actions like clicking, typing, swiping, etc.\\n* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions.\\n* The screen's resolution is 1000x1000.\\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.\", \"parameters\": {\"properties\": {\"action\": {\"description\": \"The action to perform. The available actions are:\\n* `click`: Click the point on the screen with coordinate (x, y).\\n* `long_press`: Press the point on the screen with coordinate (x, y) for specified seconds.\", \"enum\": [\"click\", \"long_press\"], \"type\": \"string\"}, \"coordinate\": {\"description\": \"(x, y): the point on the screen the action acts on — the element to touch, the region to scroll, or the object to start dragging. x is measured from the left edge and y from the top edge, both on a 0-1000 scale, so y gets larger toward the bottom of the screen. Required only by `action=click` and `action=long_press`.\", \"type\": \"array\"}, \"time\": {\"description\": \"The seconds to wait. Required only by `action=long_press`.\", \"type\": \"number\"}}, \"required\": [\"action\"], \"type\": \"object\"}}}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>\n\n## Input Specification\n- A screenshot of the current screen + an instruction describing the action to perform.\n\n## Output Format\n- Emit exactly one <tool_call></tool_call> block containing a single JSON object with the function name and its arguments.\n- Coordinates are integers in a 1000x1000 space (x measured from the left edge, y from the top edge).\n\n## Note\n- Choose exactly one function whose name and arguments are defined in the tool schema above.\n- Do not output anything other than the <tool_call></tool_call> block.",
+ "web": "You are an autonomous GUI agent capable of operating on desktops, mobile devices, and web browsers. Your primary function is to analyze a screen capture, locate the target UI element, and issue the single appropriate action that fulfills the given instruction.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"type\": \"function\", \"function\": {\"name\": \"browser_use\", \"description\": \"Use a mouse and keyboard to interact with a web browser, and take screenshots.\\n* This is an interface to a web browser. You interact with the rendered web page by clicking, typing, scrolling, selecting options and navigating between pages.\\n* Some pages may take time to load or process actions, so you may need to wait and take successive screenshots to see the results of your actions.\\n* The screen's resolution is 1000x1000.\\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.\", \"parameters\": {\"properties\": {\"action\": {\"description\": \"The action to perform. The available actions are:\\n* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.\\n* `left_click`: Click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `left_click_drag`: Press the left mouse button at the start point with coordinate (x, y), drag to the end point with coordinate2 (x2, y2), and release.\\n* `right_click`: Click the right mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `middle_click`: Click the middle mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `double_click`: Double-click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `triple_click`: Triple-click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `select`: Select an option in a dropdown or list element at the specified (x, y) coordinate; the option to choose is given by text.\", \"enum\": [\"mouse_move\", \"left_click\", \"left_click_drag\", \"right_click\", \"middle_click\", \"double_click\", \"triple_click\", \"select\"], \"type\": \"string\"}, \"coordinate\": {\"description\": \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=mouse_move`, `action=left_click`, `action=left_click_drag`, `action=right_click`, `action=middle_click`, `action=double_click`, `action=triple_click`, and `action=select`.\", \"type\": \"array\"}, \"coordinate2\": {\"description\": \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to drag the cursor to. Required only by `action=left_click_drag`.\", \"type\": \"array\"}, \"text\": {\"description\": \"Required only by `action=select`.\", \"type\": \"string\"}}, \"required\": [\"action\"], \"type\": \"object\"}}}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>\n\n## Input Specification\n- A screenshot of the current screen + an instruction describing the action to perform.\n\n## Output Format\n- Emit exactly one <tool_call></tool_call> block containing a single JSON object with the function name and its arguments.\n- Coordinates are integers in a 1000x1000 space (x measured from the left edge, y from the top edge).\n\n## Note\n- Choose exactly one function whose name and arguments are defined in the tool schema above.\n- Do not output anything other than the <tool_call></tool_call> block."
+}

requirements.txt

@@ -0,0 +1,4 @@
+transformers==5.19.0
+accelerate
+torchvision==0.26.0
+pillow