"""GazeLIFT interactive segmentation Space."""
from __future__ import annotations
import csv
import tempfile
from pathlib import Path
# Import before torch/model code so ZeroGPU can install CUDA emulation.
import spaces
import gradio as gr
import numpy as np
from PIL import Image, ImageDraw
from scripts.predict_single import predict, preload_for_zerogpu
preload_for_zerogpu()
CONFIGURATIONS = {
"Colonoscopy / polyp — Kvasir-SEG": "colonoscopy",
"Prostate MRI — NCI-ISBI": "mri",
}
def read_image(path):
if not path:
return None, "", [], None, [], None, None, None, "Upload an image to begin."
try:
if Path(path).suffix.lower() == ".dcm":
import pydicom
array = pydicom.dcmread(path).pixel_array.astype(np.float32)
if array.ndim != 2:
raise ValueError("Upload one grayscale DICOM slice.")
array = (array - array.min()) / max(float(array.max() - array.min()), 1e-8)
image = Image.fromarray((array * 255).astype(np.uint8)).convert("RGB")
else:
image = Image.open(path).convert("RGB")
except Exception as error:
raise gr.Error(f"Could not read image: {error}") from error
return image, Path(path).name, [], image, [], None, None, None, "Click the image to add fixation points, or import a CSV."
def draw_points(image, points):
if image is None:
return None
view = image.copy()
draw = ImageDraw.Draw(view)
radius = max(5, min(image.size) // 80)
for index, (x, y, duration) in enumerate(points):
r = radius * (0.6 + 0.8 * float(duration))
draw.ellipse((x-r, y-r, x+r, y+r), fill="#f9d423", outline="#725b00", width=2)
draw.text((x+r+2, y-r), str(index+1), fill="#f9d423")
return view
def add_point(image, points, duration, event: gr.SelectData):
if image is None:
raise gr.Error("Upload an image first.")
items = [list(point) for point in (points or [])]
items.append([float(event.index[0]), float(event.index[1]), float(duration)])
return items, draw_points(image, items), items
def apply_table(image, rows):
if image is None:
raise gr.Error("Upload an image first.")
values = np.asarray(rows, dtype=float)
if values.ndim != 2 or values.shape[1] != 3 or len(values) == 0 or not np.isfinite(values).all():
raise gr.Error("Enter at least one finite x, y, duration row.")
if (values[:, 2] < 0).any() or (values[:, 0] < 0).any() or (values[:, 1] < 0).any() or (values[:, 0] >= image.width).any() or (values[:, 1] >= image.height).any():
raise gr.Error("Fixations must be inside the image and durations must be nonnegative.")
points = values.tolist()
return points, draw_points(image, points)
def read_fixation_table(path):
import pandas as pd
if Path(path).suffix.lower() in (".xlsx", ".xls"):
return pd.read_excel(path)
return pd.read_csv(path, sep=None, engine="python")
def fixation_columns(path):
if not path:
return [gr.update(choices=[], value=None) for _ in range(4)]
try:
columns = [str(c) for c in read_fixation_table(path).columns]
keys = {c.lower(): c for c in columns}
def guess(*names):
return next((keys[name] for name in names if name in keys), None)
image = guess("image", "image_name", "filename", "id")
x, y = guess("x", "current_fix_x", "gaze_x"), guess("y", "current_fix_y", "gaze_y")
duration = guess("duration", "current_fix_duration")
return [gr.update(choices=["None"]+columns, value=image or "None"),
gr.update(choices=columns, value=x), gr.update(choices=columns, value=y),
gr.update(choices=["None"]+columns, value=duration or "None")]
except Exception as error:
raise gr.Error(f"Could not read fixation file: {error}") from error
def import_csv(path, image, image_name, coordinate_units, image_column, x, y, duration):
if image is None or not path:
raise gr.Error("Upload an image and fixation CSV first.")
try:
table = read_fixation_table(path)
duration = None if duration == "None" else duration
image_column = None if image_column == "None" else image_column
if not x or not y:
raise ValueError("Select the X and Y coordinate columns.")
if image_column:
selected = table[table[image_column].astype(str).map(lambda value: Path(value).stem == Path(image_name).stem)]
if selected.empty:
raise ValueError("No fixation rows match the uploaded image filename.")
else:
selected = table
values = np.column_stack([selected[x].astype(float), selected[y].astype(float), selected[duration].astype(float) if duration else np.ones(len(selected))])
if len(values) == 0:
raise ValueError("The fixation file has no matching rows.")
if coordinate_units == "Normalized [0, 1]":
if ((values[:, :2] < 0) | (values[:, :2] > 1)).any():
raise ValueError("Normalized coordinates must be in [0, 1].")
values[:, :2] *= np.array([max(image.width-1, 1), max(image.height-1, 1)])
if not np.isfinite(values).all() or (values[:, 2] < 0).any():
raise ValueError("CSV contains invalid fixation values.")
values[:, 0] = np.clip(values[:, 0], 0, image.width-1)
values[:, 1] = np.clip(values[:, 1], 0, image.height-1)
points = values.tolist()
return points, draw_points(image, points), points
except Exception as error:
raise gr.Error(str(error)) from error
@spaces.GPU(duration=120)
def segment(image, points, configuration, threshold):
if image is None:
raise gr.Error("Upload an image first.")
if not points:
raise gr.Error("Add at least one fixation point.")
with tempfile.TemporaryDirectory(prefix="gazelift-") as folder:
csv_path = Path(folder) / "fixations.csv"
with csv_path.open("w", newline="") as stream:
writer = csv.writer(stream)
writer.writerow(["x", "y", "duration"])
writer.writerows(points)
try:
output = predict(image, csv_path, preset=CONFIGURATIONS[configuration], threshold=float(threshold), return_all=True)
except Exception as error:
raise gr.Error(f"Segmentation did not run: {error}") from error
return output["gaze_overlay"], output["mask_overlay"], output["mask"], "Segmentation complete."
with gr.Blocks(title="GazeLIFT — Gaze-Guided Medical Image Segmentation", analytics_enabled=False,
theme=gr.themes.Soft(primary_hue="blue", secondary_hue="teal"),
css=(Path(__file__).resolve().parent / "theme.css").read_text()) as demo:
gr.HTML('''
Segmentation demo
GazeLIFT
Upload an image and provide gaze fixations to segment a polyp or the prostate.
Kvasir-SEG · polypsNCI-ISBI · prostate MRI
''')
image_state, name_state, points_state = gr.State(None), gr.State(""), gr.State([])
with gr.Row():
with gr.Column(scale=5, elem_classes=["workflow-column"]):
gr.Markdown("### 1 · Image and fixations\nUpload an image, then click the target to add fixation points.", elem_classes=["step-heading"])
image_file = gr.File(label="Image or single grayscale DICOM slice", file_types=["image", ".dcm"], type="filepath")
image_panel = gr.Image(label="Click to place fixations", type="pil", height=380, interactive=False)
with gr.Row():
duration = gr.Slider(0.1, 1.0, value=1.0, step=0.1, label="Next fixation duration weight")
clear = gr.Button("Clear fixations")
with gr.Column(scale=4, elem_classes=["workflow-column"]):
gr.Markdown("### 2 · Model and settings\nSelect the model matching your image.", elem_classes=["step-heading"])
configuration = gr.Radio(list(CONFIGURATIONS), value=list(CONFIGURATIONS)[0], label="Trained configuration")
with gr.Accordion("Import fixation records", open=False):
fixation_file = gr.File(label="Fixation file (CSV / XLSX / XLS)", file_types=[".csv", ".xlsx", ".xls", ".tsv", ".txt"], type="filepath")
with gr.Row():
id_column = gr.Dropdown(label="Image / ID column", choices=[])
duration_column = gr.Dropdown(label="Duration column (optional)", choices=[])
with gr.Row():
x_column = gr.Dropdown(label="X column", choices=[])
y_column = gr.Dropdown(label="Y column", choices=[])
units = gr.Radio(["Normalized [0, 1]", "Native pixels"], value="Normalized [0, 1]", label="CSV coordinate units")
import_button = gr.Button("Import fixations")
with gr.Accordion("Review and edit fixation points", open=False):
table = gr.Dataframe(headers=["x", "y", "duration"], datatype=["number", "number", "number"], type="array", value=[], col_count=(3, "fixed"), label="Fixations in native pixels", interactive=True)
apply = gr.Button("Apply table edits")
with gr.Accordion("Mask settings", open=False):
threshold = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Mask threshold")
run = gr.Button("Run segmentation", variant="primary", elem_id="segment-button")
status = gr.Markdown("Upload an image and add at least one fixation.", elem_id="demo-status")
gr.Markdown("Runs on ZeroGPU. Requests may queue.")
gr.Markdown("### 3 · Prediction\nCyan marks the predicted region. Each image can be downloaded from its tab.", elem_classes=["result-heading"])
with gr.Tabs():
with gr.Tab("Segmentation overlay"):
mask_out = gr.Image(label="Segmentation overlay", type="pil", height=440)
with gr.Tab("Binary mask"):
binary_out = gr.Image(label="Binary mask", type="pil", height=440)
with gr.Tab("Gaze prior"):
gaze_out = gr.Image(label="Duration-weighted gaze prior", type="pil", height=440)
with gr.Accordion("Model architecture", open=False):
gr.Image(value=str(Path(__file__).resolve().parent / "assets" / "architecture.jpg"),
label="Frozen multi-depth DINOv3, gaze conditioning, lifting wavelets and trained decoder",
interactive=False, elem_classes=["architecture"])
gr.HTML('''''')
image_file.change(read_image, image_file, [image_state, name_state, points_state, image_panel, table, gaze_out, mask_out, binary_out, status])
image_panel.select(add_point, [image_state, points_state, duration], [points_state, image_panel, table])
clear.click(lambda image: ([], draw_points(image, []), []), image_state, [points_state, image_panel, table])
fixation_file.change(fixation_columns, fixation_file, [id_column, x_column, y_column, duration_column])
import_button.click(import_csv, [fixation_file, image_state, name_state, units, id_column, x_column, y_column, duration_column], [points_state, image_panel, table])
apply.click(apply_table, [image_state, table], [points_state, image_panel])
run.click(segment, [image_state, points_state, configuration, threshold], [gaze_out, mask_out, binary_out, status], concurrency_limit=1)
if __name__ == "__main__":
demo.queue(default_concurrency_limit=1).launch(ssr_mode=False)