Instructions to use jacob-valdez/computaco-seekui with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use jacob-valdez/computaco-seekui with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("jacob-valdez/computaco-seekui", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download handler.py from jacob-valdez/computaco-seekui: direct link, hf CLI and curl.
- Browser
- Download file 4.29 kB
-
https://huggingface.co/jacob-valdez/computaco-seekui/resolve/main/handler.py
- Command line
-
hf download hf://jacob-valdez/computaco-seekui/handler.py
-
curl -L -o handler.py https://huggingface.co/jacob-valdez/computaco-seekui/resolve/main/handler.py
4.29 kB
| """SeekUI behind one call: a screen and the text of a target in, a predicted visual-search scanpath out. | |
| This is the Hugging Face Inference Endpoints custom handler (`EndpointHandler`, the file at the root of the model's | |
| repository). serve.py runs the same class on a machine of ours, so a live frame and a historical frame are predicted by | |
| the same code wherever it runs. The prompt, the greedy decoding and the 512-token limit are those of the authors' release: | |
| Guo, Jiang, Leiva and Oulasvirta, "SeekUI", CHI 2026, https://doi.org/10.1145/3772318.3791178. | |
| The fixations are a model's guess of how people search a screen. They are not an observation of anyone. | |
| """ | |
| import base64 | |
| import io | |
| import pathlib | |
| import re | |
| import time | |
| import torch | |
| from PIL import Image | |
| from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration | |
| # The authors' weights, at the revision measured. A directory that holds a checkpoint (serve.py's) is used instead. | |
| MODEL = "sushizixin1/SeekUI" | |
| REVISION = "f776d40e4beaf7f7fb4f00abcbcfa717e7ae5039" | |
| MAX_NEW_TOKENS = 512 | |
| MAX_TARGET = 200 | |
| PROMPT = ( | |
| "Given the image with width {width} and height {height}, " | |
| "what is the scanpath for the visual search task on this GUI? " | |
| 'The text on the target element is "{target}".\n' | |
| "Output the thinking process in <think> </think> and final answer in <answer> </answer> tags. " | |
| "The output answer format should be as follows:\n" | |
| "<think> ... </think> <answer>The scanpath is [x1, y1] [x2, y2] ...</answer>\n" | |
| "Please strictly follow the format.\n" | |
| ) | |
| def parse(content, width, height): | |
| """The model's text as (explanation, fixations). A point outside the image is dropped and counted, never moved.""" | |
| thought = re.search(r"<think>(.*?)</think>", content, re.S) | |
| answer = re.search(r"<answer>(.*?)</answer>", content, re.S) | |
| text = answer.group(1) if answer else content | |
| points = [(int(x), int(y)) for x, y in re.findall(r"\[(\d+)\s*[\s,]\s*(\d+)\]", text)] | |
| inside = [{"x": x, "y": y} for x, y in points if x < width and y < height] | |
| return (thought.group(1).strip() if thought else ""), inside, len(points) - len(inside) | |
| class EndpointHandler: | |
| def __init__(self, path=""): | |
| self.device = "cuda" if torch.cuda.is_available() else "cpu" | |
| dtype = torch.bfloat16 if self.device == "cuda" else torch.float32 | |
| local = (pathlib.Path(path) / "model.safetensors.index.json").exists() | |
| source = {"pretrained_model_name_or_path": path if local else MODEL, "revision": None if local else REVISION} | |
| self.processor = AutoProcessor.from_pretrained(**source) | |
| self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(**source, dtype=dtype).to(self.device).eval() | |
| def __call__(self, data): | |
| given = data.get("inputs", data) | |
| target = str(given.get("target", "")).strip() | |
| if not target or len(target) > MAX_TARGET: | |
| return {"error": f"target is the text of the element searched for, 1 to {MAX_TARGET} characters"} | |
| image = Image.open(io.BytesIO(base64.b64decode(given["image"]))).convert("RGB") | |
| width, height = image.size | |
| started = time.perf_counter() | |
| messages = [{"role": "user", "content": [ | |
| {"type": "text", "text": PROMPT.format(width=width, height=height, target=target)}, | |
| {"type": "image", "image": image}, | |
| ]}] | |
| text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) | |
| inputs = self.processor(text=[text], images=[image], padding=True, return_tensors="pt").to(self.device) | |
| with torch.inference_mode(): | |
| out = self.model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False) | |
| fresh = out[0][inputs.input_ids.shape[1]:] | |
| content = self.processor.decode(fresh, skip_special_tokens=True, clean_up_tokenization_spaces=False) | |
| explanation, fixations, outside = parse(content, width, height) | |
| return { | |
| "width": width, "height": height, "target": target, "fixations": fixations, "outside": outside, | |
| "explanation": explanation, "seconds": round(time.perf_counter() - started, 3), | |
| "tokens_in": int(inputs.input_ids.shape[1]), "tokens_out": int(fresh.shape[0]), | |
| } | |