computaco-seekui / handler.py
jacob-valdez's picture
ComputaCo's inference handler for SeekUI, with attribution
68538cf verified
Raw History Blame Contribute Delete
4.29 kB
"""SeekUI behind one call: a screen and the text of a target in, a predicted visual-search scanpath out.
This is the Hugging Face Inference Endpoints custom handler (`EndpointHandler`, the file at the root of the model's
repository). serve.py runs the same class on a machine of ours, so a live frame and a historical frame are predicted by
the same code wherever it runs. The prompt, the greedy decoding and the 512-token limit are those of the authors' release:
Guo, Jiang, Leiva and Oulasvirta, "SeekUI", CHI 2026, https://doi.org/10.1145/3772318.3791178.
The fixations are a model's guess of how people search a screen. They are not an observation of anyone.
"""
import base64
import io
import pathlib
import re
import time
import torch
from PIL import Image
from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration
# The authors' weights, at the revision measured. A directory that holds a checkpoint (serve.py's) is used instead.
MODEL = "sushizixin1/SeekUI"
REVISION = "f776d40e4beaf7f7fb4f00abcbcfa717e7ae5039"
MAX_NEW_TOKENS = 512
MAX_TARGET = 200
PROMPT = (
"Given the image with width {width} and height {height}, "
"what is the scanpath for the visual search task on this GUI? "
'The text on the target element is "{target}".\n'
"Output the thinking process in <think> </think> and final answer in <answer> </answer> tags. "
"The output answer format should be as follows:\n"
"<think> ... </think> <answer>The scanpath is [x1, y1] [x2, y2] ...</answer>\n"
"Please strictly follow the format.\n"
)
def parse(content, width, height):
"""The model's text as (explanation, fixations). A point outside the image is dropped and counted, never moved."""
thought = re.search(r"<think>(.*?)</think>", content, re.S)
answer = re.search(r"<answer>(.*?)</answer>", content, re.S)
text = answer.group(1) if answer else content
points = [(int(x), int(y)) for x, y in re.findall(r"\[(\d+)\s*[\s,]\s*(\d+)\]", text)]
inside = [{"x": x, "y": y} for x, y in points if x < width and y < height]
return (thought.group(1).strip() if thought else ""), inside, len(points) - len(inside)
class EndpointHandler:
def __init__(self, path=""):
self.device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if self.device == "cuda" else torch.float32
local = (pathlib.Path(path) / "model.safetensors.index.json").exists()
source = {"pretrained_model_name_or_path": path if local else MODEL, "revision": None if local else REVISION}
self.processor = AutoProcessor.from_pretrained(**source)
self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(**source, dtype=dtype).to(self.device).eval()
def __call__(self, data):
given = data.get("inputs", data)
target = str(given.get("target", "")).strip()
if not target or len(target) > MAX_TARGET:
return {"error": f"target is the text of the element searched for, 1 to {MAX_TARGET} characters"}
image = Image.open(io.BytesIO(base64.b64decode(given["image"]))).convert("RGB")
width, height = image.size
started = time.perf_counter()
messages = [{"role": "user", "content": [
{"type": "text", "text": PROMPT.format(width=width, height=height, target=target)},
{"type": "image", "image": image},
]}]
text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = self.processor(text=[text], images=[image], padding=True, return_tensors="pt").to(self.device)
with torch.inference_mode():
out = self.model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False)
fresh = out[0][inputs.input_ids.shape[1]:]
content = self.processor.decode(fresh, skip_special_tokens=True, clean_up_tokenization_spaces=False)
explanation, fixations, outside = parse(content, width, height)
return {
"width": width, "height": height, "target": target, "fixations": fixations, "outside": outside,
"explanation": explanation, "seconds": round(time.perf_counter() - started, 3),
"tokens_in": int(inputs.input_ids.shape[1]), "tokens_out": int(fresh.shape[0]),
}