File size: 4,293 Bytes
68538cf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
"""SeekUI behind one call: a screen and the text of a target in, a predicted visual-search scanpath out.

This is the Hugging Face Inference Endpoints custom handler (`EndpointHandler`, the file at the root of the model's
repository). serve.py runs the same class on a machine of ours, so a live frame and a historical frame are predicted by
the same code wherever it runs. The prompt, the greedy decoding and the 512-token limit are those of the authors' release:
Guo, Jiang, Leiva and Oulasvirta, "SeekUI", CHI 2026, https://doi.org/10.1145/3772318.3791178.

The fixations are a model's guess of how people search a screen. They are not an observation of anyone.
"""

import base64
import io
import pathlib
import re
import time

import torch
from PIL import Image
from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration

# The authors' weights, at the revision measured. A directory that holds a checkpoint (serve.py's) is used instead.
MODEL = "sushizixin1/SeekUI"
REVISION = "f776d40e4beaf7f7fb4f00abcbcfa717e7ae5039"
MAX_NEW_TOKENS = 512
MAX_TARGET = 200

PROMPT = (
    "Given the image with width {width} and height {height}, "
    "what is the scanpath for the visual search task on this GUI? "
    'The text on the target element is "{target}".\n'
    "Output the thinking process in <think> </think> and final answer in <answer> </answer> tags. "
    "The output answer format should be as follows:\n"
    "<think> ... </think> <answer>The scanpath is [x1, y1] [x2, y2] ...</answer>\n"
    "Please strictly follow the format.\n"
)


def parse(content, width, height):
    """The model's text as (explanation, fixations). A point outside the image is dropped and counted, never moved."""
    thought = re.search(r"<think>(.*?)</think>", content, re.S)
    answer = re.search(r"<answer>(.*?)</answer>", content, re.S)
    text = answer.group(1) if answer else content
    points = [(int(x), int(y)) for x, y in re.findall(r"\[(\d+)\s*[\s,]\s*(\d+)\]", text)]
    inside = [{"x": x, "y": y} for x, y in points if x < width and y < height]
    return (thought.group(1).strip() if thought else ""), inside, len(points) - len(inside)


class EndpointHandler:
    def __init__(self, path=""):
        self.device = "cuda" if torch.cuda.is_available() else "cpu"
        dtype = torch.bfloat16 if self.device == "cuda" else torch.float32
        local = (pathlib.Path(path) / "model.safetensors.index.json").exists()
        source = {"pretrained_model_name_or_path": path if local else MODEL, "revision": None if local else REVISION}
        self.processor = AutoProcessor.from_pretrained(**source)
        self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(**source, dtype=dtype).to(self.device).eval()

    def __call__(self, data):
        given = data.get("inputs", data)
        target = str(given.get("target", "")).strip()
        if not target or len(target) > MAX_TARGET:
            return {"error": f"target is the text of the element searched for, 1 to {MAX_TARGET} characters"}
        image = Image.open(io.BytesIO(base64.b64decode(given["image"]))).convert("RGB")
        width, height = image.size
        started = time.perf_counter()
        messages = [{"role": "user", "content": [
            {"type": "text", "text": PROMPT.format(width=width, height=height, target=target)},
            {"type": "image", "image": image},
        ]}]
        text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        inputs = self.processor(text=[text], images=[image], padding=True, return_tensors="pt").to(self.device)
        with torch.inference_mode():
            out = self.model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False)
        fresh = out[0][inputs.input_ids.shape[1]:]
        content = self.processor.decode(fresh, skip_special_tokens=True, clean_up_tokenization_spaces=False)
        explanation, fixations, outside = parse(content, width, height)
        return {
            "width": width, "height": height, "target": target, "fixations": fixations, "outside": outside,
            "explanation": explanation, "seconds": round(time.perf_counter() - started, 3),
            "tokens_in": int(inputs.input_ids.shape[1]), "tokens_out": int(fresh.shape[0]),
        }