GUI-Decisions Space

intro.md

@@ -8,7 +8,7 @@ The heatmap is that distribution.
 Gemma 4 31B with a LoRA ([GUI-Decisions-31B-LoRA](https://huggingface.co/infinitylogesh/GUI-Decisions-31B-LoRA),
 merged: [GUI-Decisions-31B](https://huggingface.co/infinitylogesh/GUI-Decisions-31B)).
 Blog post: [Stop Decoding Coordinates](https://logeshumapathi.com/blog/2026/10/06/slot-reading.html) ·
-code: [slot-reading](https://github.com/infinitylogesh/slot-reading).
+code: [gui-decisions](https://github.com/infinitylogesh/gui-decisions).
 
 *Timings here are this Space's shared GPU running bf16 in plain transformers. Served on vLLM (NVFP4, one
 RTX PRO 6000) a step takes ~146 ms against ~470–550 ms for the same model writing its native JSON.*

slotread/client.py

@@ -61,12 +61,32 @@ def generate(model, url, text, max_tokens=64, think=False, system=None, **extra)
     return d["choices"][0]["message"].get("content") or "", ms, d.get("usage", {})
 
 
+CONFIG_NAME = "slotread_config.json"
+
+
+def load_config(path):
+    """A model's slot config: a run folder (config.json, as slotread.train writes it), a folder with
+    slotread_config.json (the released models), or a Hugging Face repo id holding slotread_config.json."""
+    if os.path.isdir(path):
+        for name in (CONFIG_NAME, "config.json"):
+            p = os.path.join(path, name)
+            if os.path.exists(p):
+                c = json.load(open(p))
+                if "tasks" in c:
+                    return c
+        raise FileNotFoundError(f"no {CONFIG_NAME} in {path}")
+    from huggingface_hub import hf_hub_download
+    return json.load(open(hf_hub_download(path, CONFIG_NAME)))
+
+
 class SlotModel:
-    """A served slot adapter: `name` as vLLM serves it, `path` the run folder holding config.json."""
+    """A served slot model: `name` as vLLM serves it (the adapter's --lora-modules name, or the merged
+    model's --served-model-name), `path` its slot config (run folder, released model folder or HF repo id).
+    `text_model`: the served base model that writes text steps (what to type, the answer)."""
 
-    def __init__(self, name, path):
-        self.name, self.path = name, path
-        c = json.load(open(os.path.join(path, "config.json")))
+    def __init__(self, name, path, text_model=None):
+        self.name, self.path, self.text_model = name, path, text_model
+        c = load_config(path)
         self.cfg = c
         self.arm = Arm(c["arm"], c["coarse"], c["bins"], c["tokens"], c["fine"])
         self.tasks = {"agent": AgentTask(), "box": BoxTask(), "seg": SegTask(c["rays"])}
@@ -178,6 +198,45 @@ class SlotModel:
         return out
 
 
+    def image_url(self, image):
+        """A PIL image as the data URL sent to the server: JPEG at the model's input size by default (the
+        fast path; IMG_FMT / IMG_MAX_PX), capped as in training when the config says so (GUI-Owl: 1 MP).
+        A string is taken as a URL already."""
+        if isinstance(image, str):
+            return image
+        from .images import cap_image, data_url
+        cap = self.cfg.get("image_cap")
+        return data_url(cap_image(image.convert("RGB"), cap), max_px=cap) if cap else data_url(image)
+
+    def decide(self, image, instruction, previous="None"):
+        """A computer-use step: action and point; the key, scroll or swipe end when the action needs them.
+        Reads action, x and y first, then the action's other slots (lazy)."""
+        from .data.aguvis import KEYS, SDIR
+        t = time.perf_counter()
+        lp = self.read("agent", self.image_url(image), agent_user(instruction, previous), lazy=True)
+        d = self.tasks["agent"].decode(lp, self.arm)
+        if "key" in lp:
+            d["key"] = KEYS[int(lp["key"].argmax())]
+        if "sdir" in lp:
+            d.update(sdir=SDIR[int(lp["sdir"].argmax())], samt=int(lp["samt"].argmax()))
+        return dict(d, ms=round((time.perf_counter() - t) * 1000, 1))
+
+    def box(self, image, description):
+        """An element's or object's box [x1, y1, x2, y2] in [0, 1]."""
+        t = time.perf_counter()
+        lp = self.read("box", self.image_url(image), self.tasks["box"].user({"instruction": description}))
+        return dict(box=self.tasks["box"].decode(lp, self.arm), ms=round((time.perf_counter() - t) * 1000, 1))
+
+    def write_text(self, image, instruction, action, previous="None", max_tokens=24):
+        """The text a type / answer / open step needs, written by the served base model (text_model)."""
+        from .agent import TEXT_ASK
+        if not self.text_model:
+            raise RuntimeError("text steps need the base model: serve it and pass text_model=<its served name>")
+        text, _, _ = generate(self.text_model, self.image_url(image),
+                              agent_user(instruction, previous) + "\n\n" + TEXT_ASK[action], max_tokens=max_tokens)
+        return text.strip().strip('"').split("\n")[0][:120]
+
+
 _PROC = {}
 
 

slotread/local.py

@@ -11,30 +11,16 @@ The slot template and label ids come from slotread_config.json in the model (or
 config.json that slotread.train writes, renamed so it does not collide with the model's own config.json.
 """
 
-import json, os, time
+import json, time
 
 import torch
 
 from . import labels as L
-from .client import agent_user
+from .client import agent_user, load_config
 from .images import cap_image
 from .labels import Arm
 from .tasks import AgentTask, BoxTask, SegTask
 
-CONFIG_NAME = "slotread_config.json"
-
-
-def _config(repo_or_dir):
-    if os.path.isdir(repo_or_dir):
-        for name in (CONFIG_NAME, "config.json"):
-            p = os.path.join(repo_or_dir, name)
-            if os.path.exists(p) and "tasks" in json.load(open(p)):
-                return json.load(open(p))
-        raise FileNotFoundError(f"no {CONFIG_NAME} in {repo_or_dir}")
-    from huggingface_hub import hf_hub_download
-    return json.load(open(hf_hub_download(repo_or_dir, CONFIG_NAME)))
-
-
 class LocalSlotModel:
     def __init__(self, model, processor, cfg):
         self.model, self.proc, self.cfg = model, processor, cfg
@@ -54,7 +40,7 @@ class LocalSlotModel:
             kw = {"torch_device": "cpu"} if device_map == "cpu" else {}
             model = PeftModel.from_pretrained(model, adapter, **kw)
         model.eval()
-        cfg = json.load(open(config)) if config else _config(adapter or model_id)
+        cfg = json.load(open(config)) if config else load_config(adapter or model_id)
         return cls(model, AutoProcessor.from_pretrained(processor or model_id), cfg)
 
     @torch.no_grad()