Spaces:
Sleeping
Sleeping
Download scripts/predict_single.py from oussamab2n/GazeLIFT: direct link, hf CLI and curl.
- Browser
- Download file 7.64 kB
-
https://huggingface.co/spaces/oussamab2n/GazeLIFT/resolve/main/scripts/predict_single.py
- Command line
-
hf download hf://spaces/oussamab2n/GazeLIFT/scripts/predict_single.py
-
curl -L -o predict_single.py https://huggingface.co/spaces/oussamab2n/GazeLIFT/resolve/main/scripts/predict_single.py
7.64 kB
| """Run a verified trained GazeLIFT adapter on one image and fixation CSV.""" | |
| from __future__ import annotations | |
| import csv | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| import logging | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import numpy as np | |
| from PIL import Image | |
| import torch | |
| import torch.nn.functional as F | |
| # CPU preprocessing uses the Space's allocated cores. | |
| torch.set_num_threads(max(1, int(os.getenv("GAZELIFT_CPU_THREADS", "2")))) | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT / "model_code")) | |
| from heatmap import make_image_heatmap | |
| from model import TransposeCNNGazePyramidDecoder | |
| PRESETS = {"colonoscopy": "kvasir", "mri": "prostate"} | |
| def _manifest(): | |
| return json.loads((ROOT / "weights" / "manifest.json").read_text(encoding="utf-8")) | |
| def _checkpoint(dataset: str) -> tuple[Path, dict]: | |
| record = _manifest()[dataset] | |
| path = Path(os.getenv(record["environment_variable"], ROOT / "weights" / record["filename"])) | |
| if not path.is_file(): | |
| repository = os.getenv("GAZELIFT_WEIGHTS_REPO") | |
| if not repository: | |
| raise FileNotFoundError("The verified trained adapter is unavailable. Add it to weights/ or set GAZELIFT_WEIGHTS_REPO.") | |
| from huggingface_hub import hf_hub_download | |
| path = Path(hf_hub_download(repository, record["filename"], token=os.getenv("HF_TOKEN"))) | |
| expected = record["sha256"] | |
| if expected.startswith("REPLACE_"): | |
| raise ValueError("Set the verified safetensors SHA-256 in weights/manifest.json before inference.") | |
| if hashlib.sha256(path.read_bytes()).hexdigest() != expected: | |
| raise ValueError("Checkpoint hash does not match the selected GazeLIFT adapter.") | |
| return path, record | |
| def _adapter(dataset: str, device: str): | |
| path, record = _checkpoint(dataset) | |
| if path.suffix == ".safetensors": | |
| from safetensors.torch import load_file | |
| state = load_file(str(path), device="cpu") | |
| else: | |
| state = torch.load(path, map_location="cpu", weights_only=True)["model"] | |
| model = TransposeCNNGazePyramidDecoder() | |
| model.load_state_dict(state, strict=True) | |
| return model.eval().requires_grad_(False).to(device), record | |
| def _encoder(device: str): | |
| import timm | |
| checkpoint = os.getenv("GAZELIFT_DINO_CHECKPOINT") | |
| model = timm.create_model("vit_large_patch16_dinov3.lvd1689m", pretrained=not bool(checkpoint), num_classes=0, img_size=512, dynamic_img_size=True) | |
| if checkpoint: | |
| timm.models.load_checkpoint(model, checkpoint, strict=True) | |
| return model.eval().requires_grad_(False).to(device) | |
| def preload_for_zerogpu(): | |
| """Place available models on emulated CUDA during ZeroGPU startup. | |
| The real GPU is allocated only by the UI's decorated segmentation callback. | |
| Missing adapters leave the UI available and are rejected again at inference. | |
| """ | |
| if os.getenv("SPACES_ZERO_GPU", "").lower() not in ("true", "1", "t"): | |
| return | |
| loaded = False | |
| for dataset in ("kvasir", "prostate"): | |
| try: | |
| _adapter(dataset, "cuda") | |
| loaded = True | |
| except Exception as error: | |
| logging.getLogger(__name__).warning( | |
| "Could not preload the %s adapter (%s). Inference requires its verified trained weights.", | |
| dataset, type(error).__name__, | |
| ) | |
| if loaded: | |
| _encoder("cuda") | |
| def _load_points(path: str | Path, size: tuple[int, int]) -> np.ndarray: | |
| with open(path, encoding="utf-8-sig", newline="") as stream: | |
| rows = list(csv.DictReader(stream)) | |
| if not rows: | |
| raise ValueError("The fixation CSV is empty.") | |
| key = {item.lower(): item for item in rows[0]} | |
| x, y = key.get("x", key.get("current_fix_x")), key.get("y", key.get("current_fix_y")) | |
| duration = key.get("duration", key.get("current_fix_duration")) | |
| if not x or not y: | |
| raise ValueError("Fixation CSV needs x/y or CURRENT_FIX_X/CURRENT_FIX_Y columns.") | |
| values = np.asarray([[float(row[x]), float(row[y]), float(row[duration]) if duration else 1] for row in rows], np.float32) | |
| if not np.isfinite(values).all() or (values[:, 2] < 0).any(): | |
| raise ValueError("Fixations must be finite and have nonnegative durations.") | |
| w, h = size | |
| # The UI always writes native-pixel coordinates here. Never infer units | |
| # from their magnitude: a real click near (0, 1) is still in pixels. | |
| values[:, 0] = np.clip(values[:, 0], 0, w - 1) | |
| values[:, 1] = np.clip(values[:, 1], 0, h - 1) | |
| return values | |
| def _overlay_heatmap(image: Image.Image, heatmap: np.ndarray) -> Image.Image: | |
| heat = Image.fromarray(np.uint8(np.clip(heatmap, 0, 1) * 255)).resize(image.size, Image.Resampling.BILINEAR) | |
| hue = np.zeros((*np.asarray(heat).shape, 3), dtype=np.uint8) | |
| hue[..., 0], hue[..., 1] = np.asarray(heat), np.asarray(heat) // 2 | |
| return Image.blend(image.convert("RGB"), Image.fromarray(hue), 0.45) | |
| def predict(image_path, fixation_csv, preset="colonoscopy", threshold=0.5, device=None, return_all=False): | |
| if preset not in PRESETS: | |
| raise ValueError(f"Unknown configuration: {preset}") | |
| image = image_path.convert("RGB") if isinstance(image_path, Image.Image) else Image.open(image_path).convert("RGB") | |
| values = _load_points(fixation_csv, image.size) | |
| dataset = PRESETS[preset] | |
| zero_gpu = os.getenv("SPACES_ZERO_GPU", "").lower() in ("true", "1", "t") | |
| if zero_gpu and device is not None and str(device) != "cuda": | |
| raise ValueError("The ZeroGPU deployment requires device='cuda'.") | |
| runtime = "cuda" if zero_gpu else str(device or ("cuda" if torch.cuda.is_available() else "cpu")) | |
| adapter, record = _adapter(dataset, runtime) | |
| xy = values[:, :2] / np.array([max(image.width - 1, 1), max(image.height - 1, 1)], np.float32) | |
| duration = values[:, 2] | |
| duration = (duration - duration.min()) / (duration.max() - duration.min() + 1e-8) if duration.max() > duration.min() else np.ones_like(duration) | |
| heatmap = make_image_heatmap(xy, duration, image_size=(512, 512), sigma_px=record["sigma_px"])[None].to(runtime) | |
| reduced = image.resize((512, 512), Image.Resampling.BILINEAR) | |
| tensor = torch.from_numpy(np.asarray(reduced).copy()).permute(2, 0, 1).float() / 255 | |
| tensor = (tensor - torch.tensor([.485, .456, .406])[:, None, None]) / torch.tensor([.229, .224, .225])[:, None, None] | |
| encoder = _encoder(runtime) | |
| captured, hooks = {}, [] | |
| for index in [5, 11, 17, 23]: | |
| hooks.append(encoder.blocks[index].register_forward_hook(lambda module, args, output, index=index: captured.update({index: output[:, encoder.num_prefix_tokens:]}))) | |
| try: | |
| encoder(tensor[None].to(runtime)) | |
| finally: | |
| for hook in hooks: | |
| hook.remove() | |
| features = [captured[index].transpose(1, 2).reshape(1, 1024, 32, 32).half().float() for index in [5, 11, 17, 23]] | |
| logits = adapter(tensor[None].half().float().to(runtime), features, heatmap) | |
| probability = F.interpolate(logits.sigmoid(), size=(image.height, image.width), mode="bilinear", align_corners=False)[0, 0].cpu().numpy() | |
| mask = probability > threshold | |
| overlay = np.asarray(image).copy() | |
| overlay[mask] = (0.55 * overlay[mask] + 0.45 * np.array([0, 205, 235])).astype(np.uint8) | |
| result = {"mask": Image.fromarray((255 * mask).astype(np.uint8)), "mask_overlay": Image.fromarray(overlay), "gaze_overlay": _overlay_heatmap(image, heatmap[0, 0].cpu().numpy()), "preds": probability, "gaze_heatmap": heatmap[0, 0].cpu().numpy()} | |
| return result if return_all else result["mask"] | |