"""GazeLIFT interactive segmentation Space.""" from __future__ import annotations import csv import tempfile from pathlib import Path # Import before torch/model code so ZeroGPU can install CUDA emulation. import spaces import gradio as gr import numpy as np from PIL import Image, ImageDraw from scripts.predict_single import predict, preload_for_zerogpu preload_for_zerogpu() CONFIGURATIONS = { "Colonoscopy / polyp — Kvasir-SEG": "colonoscopy", "Prostate MRI — NCI-ISBI": "mri", } def read_image(path): if not path: return None, "", [], None, [], None, None, None, "Upload an image to begin." try: if Path(path).suffix.lower() == ".dcm": import pydicom array = pydicom.dcmread(path).pixel_array.astype(np.float32) if array.ndim != 2: raise ValueError("Upload one grayscale DICOM slice.") array = (array - array.min()) / max(float(array.max() - array.min()), 1e-8) image = Image.fromarray((array * 255).astype(np.uint8)).convert("RGB") else: image = Image.open(path).convert("RGB") except Exception as error: raise gr.Error(f"Could not read image: {error}") from error return image, Path(path).name, [], image, [], None, None, None, "Click the image to add fixation points, or import a CSV." def draw_points(image, points): if image is None: return None view = image.copy() draw = ImageDraw.Draw(view) radius = max(5, min(image.size) // 80) for index, (x, y, duration) in enumerate(points): r = radius * (0.6 + 0.8 * float(duration)) draw.ellipse((x-r, y-r, x+r, y+r), fill="#f9d423", outline="#725b00", width=2) draw.text((x+r+2, y-r), str(index+1), fill="#f9d423") return view def add_point(image, points, duration, event: gr.SelectData): if image is None: raise gr.Error("Upload an image first.") items = [list(point) for point in (points or [])] items.append([float(event.index[0]), float(event.index[1]), float(duration)]) return items, draw_points(image, items), items def apply_table(image, rows): if image is None: raise gr.Error("Upload an image first.") values = np.asarray(rows, dtype=float) if values.ndim != 2 or values.shape[1] != 3 or len(values) == 0 or not np.isfinite(values).all(): raise gr.Error("Enter at least one finite x, y, duration row.") if (values[:, 2] < 0).any() or (values[:, 0] < 0).any() or (values[:, 1] < 0).any() or (values[:, 0] >= image.width).any() or (values[:, 1] >= image.height).any(): raise gr.Error("Fixations must be inside the image and durations must be nonnegative.") points = values.tolist() return points, draw_points(image, points) def read_fixation_table(path): import pandas as pd if Path(path).suffix.lower() in (".xlsx", ".xls"): return pd.read_excel(path) return pd.read_csv(path, sep=None, engine="python") def fixation_columns(path): if not path: return [gr.update(choices=[], value=None) for _ in range(4)] try: columns = [str(c) for c in read_fixation_table(path).columns] keys = {c.lower(): c for c in columns} def guess(*names): return next((keys[name] for name in names if name in keys), None) image = guess("image", "image_name", "filename", "id") x, y = guess("x", "current_fix_x", "gaze_x"), guess("y", "current_fix_y", "gaze_y") duration = guess("duration", "current_fix_duration") return [gr.update(choices=["None"]+columns, value=image or "None"), gr.update(choices=columns, value=x), gr.update(choices=columns, value=y), gr.update(choices=["None"]+columns, value=duration or "None")] except Exception as error: raise gr.Error(f"Could not read fixation file: {error}") from error def import_csv(path, image, image_name, coordinate_units, image_column, x, y, duration): if image is None or not path: raise gr.Error("Upload an image and fixation CSV first.") try: table = read_fixation_table(path) duration = None if duration == "None" else duration image_column = None if image_column == "None" else image_column if not x or not y: raise ValueError("Select the X and Y coordinate columns.") if image_column: selected = table[table[image_column].astype(str).map(lambda value: Path(value).stem == Path(image_name).stem)] if selected.empty: raise ValueError("No fixation rows match the uploaded image filename.") else: selected = table values = np.column_stack([selected[x].astype(float), selected[y].astype(float), selected[duration].astype(float) if duration else np.ones(len(selected))]) if len(values) == 0: raise ValueError("The fixation file has no matching rows.") if coordinate_units == "Normalized [0, 1]": if ((values[:, :2] < 0) | (values[:, :2] > 1)).any(): raise ValueError("Normalized coordinates must be in [0, 1].") values[:, :2] *= np.array([max(image.width-1, 1), max(image.height-1, 1)]) if not np.isfinite(values).all() or (values[:, 2] < 0).any(): raise ValueError("CSV contains invalid fixation values.") values[:, 0] = np.clip(values[:, 0], 0, image.width-1) values[:, 1] = np.clip(values[:, 1], 0, image.height-1) points = values.tolist() return points, draw_points(image, points), points except Exception as error: raise gr.Error(str(error)) from error @spaces.GPU(duration=120) def segment(image, points, configuration, threshold): if image is None: raise gr.Error("Upload an image first.") if not points: raise gr.Error("Add at least one fixation point.") with tempfile.TemporaryDirectory(prefix="gazelift-") as folder: csv_path = Path(folder) / "fixations.csv" with csv_path.open("w", newline="") as stream: writer = csv.writer(stream) writer.writerow(["x", "y", "duration"]) writer.writerows(points) try: output = predict(image, csv_path, preset=CONFIGURATIONS[configuration], threshold=float(threshold), return_all=True) except Exception as error: raise gr.Error(f"Segmentation did not run: {error}") from error return output["gaze_overlay"], output["mask_overlay"], output["mask"], "Segmentation complete." with gr.Blocks(title="GazeLIFT — Gaze-Guided Medical Image Segmentation", analytics_enabled=False, theme=gr.themes.Soft(primary_hue="blue", secondary_hue="teal"), css=(Path(__file__).resolve().parent / "theme.css").read_text()) as demo: gr.HTML('''
Segmentation demo

GazeLIFT

Upload an image and provide gaze fixations to segment a polyp or the prostate.

Kvasir-SEG · polypsNCI-ISBI · prostate MRI
''') image_state, name_state, points_state = gr.State(None), gr.State(""), gr.State([]) with gr.Row(): with gr.Column(scale=5, elem_classes=["workflow-column"]): gr.Markdown("### 1 · Image and fixations\nUpload an image, then click the target to add fixation points.", elem_classes=["step-heading"]) image_file = gr.File(label="Image or single grayscale DICOM slice", file_types=["image", ".dcm"], type="filepath") image_panel = gr.Image(label="Click to place fixations", type="pil", height=380, interactive=False) with gr.Row(): duration = gr.Slider(0.1, 1.0, value=1.0, step=0.1, label="Next fixation duration weight") clear = gr.Button("Clear fixations") with gr.Column(scale=4, elem_classes=["workflow-column"]): gr.Markdown("### 2 · Model and settings\nSelect the model matching your image.", elem_classes=["step-heading"]) configuration = gr.Radio(list(CONFIGURATIONS), value=list(CONFIGURATIONS)[0], label="Trained configuration") with gr.Accordion("Import fixation records", open=False): fixation_file = gr.File(label="Fixation file (CSV / XLSX / XLS)", file_types=[".csv", ".xlsx", ".xls", ".tsv", ".txt"], type="filepath") with gr.Row(): id_column = gr.Dropdown(label="Image / ID column", choices=[]) duration_column = gr.Dropdown(label="Duration column (optional)", choices=[]) with gr.Row(): x_column = gr.Dropdown(label="X column", choices=[]) y_column = gr.Dropdown(label="Y column", choices=[]) units = gr.Radio(["Normalized [0, 1]", "Native pixels"], value="Normalized [0, 1]", label="CSV coordinate units") import_button = gr.Button("Import fixations") with gr.Accordion("Review and edit fixation points", open=False): table = gr.Dataframe(headers=["x", "y", "duration"], datatype=["number", "number", "number"], type="array", value=[], col_count=(3, "fixed"), label="Fixations in native pixels", interactive=True) apply = gr.Button("Apply table edits") with gr.Accordion("Mask settings", open=False): threshold = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Mask threshold") run = gr.Button("Run segmentation", variant="primary", elem_id="segment-button") status = gr.Markdown("Upload an image and add at least one fixation.", elem_id="demo-status") gr.Markdown("Runs on ZeroGPU. Requests may queue.") gr.Markdown("### 3 · Prediction\nCyan marks the predicted region. Each image can be downloaded from its tab.", elem_classes=["result-heading"]) with gr.Tabs(): with gr.Tab("Segmentation overlay"): mask_out = gr.Image(label="Segmentation overlay", type="pil", height=440) with gr.Tab("Binary mask"): binary_out = gr.Image(label="Binary mask", type="pil", height=440) with gr.Tab("Gaze prior"): gaze_out = gr.Image(label="Duration-weighted gaze prior", type="pil", height=440) with gr.Accordion("Model architecture", open=False): gr.Image(value=str(Path(__file__).resolve().parent / "assets" / "architecture.jpg"), label="Frozen multi-depth DINOv3, gaze conditioning, lifting wavelets and trained decoder", interactive=False, elem_classes=["architecture"]) gr.HTML('''''') image_file.change(read_image, image_file, [image_state, name_state, points_state, image_panel, table, gaze_out, mask_out, binary_out, status]) image_panel.select(add_point, [image_state, points_state, duration], [points_state, image_panel, table]) clear.click(lambda image: ([], draw_points(image, []), []), image_state, [points_state, image_panel, table]) fixation_file.change(fixation_columns, fixation_file, [id_column, x_column, y_column, duration_column]) import_button.click(import_csv, [fixation_file, image_state, name_state, units, id_column, x_column, y_column, duration_column], [points_state, image_panel, table]) apply.click(apply_table, [image_state, table], [points_state, image_panel]) run.click(segment, [image_state, points_state, configuration, threshold], [gaze_out, mask_out, binary_out, status], concurrency_limit=1) if __name__ == "__main__": demo.queue(default_concurrency_limit=1).launch(ssr_mode=False)