import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 # must precede any torch/CUDA import on ZeroGPU import time import tempfile from typing import Dict, List, Tuple import cv2 import gradio as gr import numpy as np import torch import torch.nn.functional as F from PIL import Image from omegaconf import OmegaConf from hydra.utils import instantiate from huggingface_hub import hf_hub_download from training.dataset.transforms import ( ComposeAPI, RandomResizeAPI, ToTensorAPI, NormalizeAPI, ) from training.utils.data_utils import collate_fn, Frame, Object, VideoDatapoint from training.model.sam2 import SAM2Train # --------------------------------------------------------------------------- # Model: load eagerly at module scope (ZeroGPU streams it into VRAM on the # first @spaces.GPU call). VQ-SAM = SAM 2.1 Hiera B+ + two-stage memory + DAM, # trained for visual query segmentation at 768x768 with 8-frame clips. # --------------------------------------------------------------------------- CONFIG_PATH = os.path.join( "sam2", "configs", "sam2.1_training", "sam2.1_hiera_b+_DAM_AdaptiveGate_Jan28.yaml" ) RESOLUTION = 768 # cfg.scratch.resolution CLIP_FRAMES = 8 # cfg.scratch.num_frames (1 query + 7 video frames) MAX_FRAMES = 49 # demo cap: keeps ZeroGPU duration bounded OBJECT_ID = 1 # single target object per visual query SCORE_THRESH = 0.5 # reference default (--score_thresh) print("Loading VQ-SAM config...") cfg = OmegaConf.load(CONFIG_PATH) print("Instantiating model...") model: SAM2Train = instantiate(cfg.trainer.model, _recursive_=True) print("Downloading VQ-SAM checkpoint (bf0820/VQ-SAM)...") ckpt_path = hf_hub_download(repo_id="bf0820/VQ-SAM", filename="vqsam_hiera_bplus.pt") checkpoint = torch.load(ckpt_path, map_location="cpu", weights_only=False) state_dict = checkpoint.get("model", checkpoint) missing, unexpected = model.load_state_dict(state_dict, strict=False) print(f"Loaded checkpoint (missing={len(missing)}, unexpected={len(unexpected)})") model = model.to("cuda") model.eval() EVAL_TRANSFORMS = ComposeAPI( transforms=[ RandomResizeAPI(sizes=RESOLUTION, square=True, consistent_transform=True), ToTensorAPI(), NormalizeAPI(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ] ) DEVICE = torch.device("cuda") # --------------------------------------------------------------------------- # Video I/O helpers # --------------------------------------------------------------------------- def load_video_frames(video_path: str, max_frames: int) -> Tuple[List[np.ndarray], float]: """Decode up to `max_frames` frames of a video with OpenCV (CPU).""" cap = cv2.VideoCapture(video_path) if not cap.isOpened(): raise ValueError("Could not open the uploaded video file.") fps = cap.get(cv2.CAP_PROP_FPS) or 25.0 frames: List[np.ndarray] = [] while len(frames) < max_frames: ok, frame = cap.read() if not ok: break frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) cap.release() if not frames: raise ValueError("No frames could be decoded from the video.") return frames, float(fps) def extract_query_mask(editor_value) -> np.ndarray: """Extract a binary mask from a gr.ImageEditor value dict. With ``type="numpy"`` and the default ``image_mode="RGBA"``, the editor's ``layers`` arrive as RGBA arrays where the alpha channel IS the paint stroke. We union the alpha of all layers, and fall back to a composite-vs-background difference if no layer alpha is present. """ if editor_value is None: raise gr.Error( "Please upload a reference image and brush over the target object." ) background = editor_value.get("background") layers = editor_value.get("layers") or [] mask = None # 1) Layer alpha — the direct, robust signal. alpha_union = None for layer in layers: lay = np.array(layer) if lay.ndim == 2: a = (lay > 10).astype(np.uint8) * 255 elif lay.shape[-1] == 4: a = (lay[..., 3] > 10).astype(np.uint8) * 255 else: a = (lay.sum(axis=-1) > 30).astype(np.uint8) * 255 if alpha_union is None: alpha_union = a elif a.shape == alpha_union.shape: alpha_union = np.maximum(alpha_union, a) if alpha_union is not None and alpha_union.sum() > 0: mask = alpha_union # 2) Fallback: composite minus background difference. if mask is None and background is not None and layers: bg = np.array(background) comp = np.array(editor_value.get("composite")) if comp.shape == bg.shape and bg.ndim == 3: diff = np.abs(comp.astype(np.int16) - bg.astype(np.int16)) diff_mask = (diff.sum(axis=-1) > 30).astype(np.uint8) * 255 if diff_mask.sum() > 0: mask = diff_mask if mask is None or mask.sum() == 0: raise gr.Error( "No mask detected: brush over the target object in the reference image." ) return mask # --------------------------------------------------------------------------- # Inference: 1:1 with VQS tools/test_video_segmentation.py # --------------------------------------------------------------------------- @torch.inference_mode() def _run_segments( frames_rgb: List[np.ndarray], query_image: Image.Image, query_mask: np.ndarray, score_thresh: float, ) -> Dict[int, np.ndarray]: """Run VQ-SAM over all segments of the video. Returns {global_frame_idx: binary mask (H, W) uint8 with 0/255 values} for frames with a non-empty prediction. """ h, w = frames_rgb[0].shape[:2] pil_frames = [Image.fromarray(f) for f in frames_rgb] query_pil = query_image.convert("RGB") # --- chunking (TestChunkSampler.sample_chunks) --- # Each chunk: [query frame, 7 video frames]; tail padded by repeating # the last frame index (duplicate predictions are dropped, first wins). # Unlike the reference (VQS-4K eval layout, where frame 00000 IS the # query), the demo's query is EXTERNAL — so chunks cover video frames # 0..N-1 and the query occupies local slot 0 with a sentinel index -1 # whose own (echoed) prediction is simply not stored. QUERY_SENTINEL = -1 num_video_frames = len(pil_frames) chunks: List[List[int]] = [] start = 0 while start < num_video_frames: end = min(start + (CLIP_FRAMES - 1), num_video_frames) idxs = list(range(start, end)) last = idxs[-1] while len(idxs) < (CLIP_FRAMES - 1): idxs.append(last) chunks.append(idxs) start = end query_mask_tensor = torch.from_numpy(query_mask).to(torch.uint8) all_predictions: Dict[int, np.ndarray] = {} orig_size_hw = (h, w) for chunk_video_idx in chunks: # --- build_datapoint_from_chunk --- # Local slot 0 = the external query (sentinel frame_index -1); # local slots 1..7 = video frames. chunk_indices = [QUERY_SENTINEL] + chunk_video_idx images: List[Frame] = [] for local_idx, frame_idx in enumerate(chunk_indices): if frame_idx == QUERY_SENTINEL: data = query_pil segment = query_mask_tensor else: data = pil_frames[frame_idx] segment = torch.zeros(h, w, dtype=torch.uint8) frame_obj = Frame(data=data, objects=[]) frame_obj.objects.append( Object( object_id=OBJECT_ID, frame_index=frame_idx, segment=segment, ) ) images.append(frame_obj) datapoint = VideoDatapoint(frames=images, video_id=0, size=orig_size_hw) datapoint = EVAL_TRANSFORMS(datapoint, epoch=0) batched = collate_fn([datapoint], dict_key="all") batched = batched.to(DEVICE) # --- forward: reference wraps the entire inference in bf16 autocast # (tools/test_video_segmentation.py runs the whole body under it) --- with torch.autocast(device_type="cuda", dtype=torch.bfloat16): backbone_out = model.forward_image(batched.flat_img_batch) model.prepare_prompt_inputs(backbone_out, batched, start_frame_idx=0) backbone_out["init_cond_frames"] = [0] backbone_out["frames_to_add_correction_pt"] = [] backbone_out["use_pt_input"] = False if 0 in backbone_out["gt_masks_per_frame"]: query_masks = backbone_out["gt_masks_per_frame"][0] obj_to_frame_idx = batched.obj_to_frame_idx[0] video_indices = obj_to_frame_idx[:, 1].to(torch.long) batch_mask_list = [] for vid_idx in range(batched.img_batch.shape[1]): vid_mask = query_masks[video_indices == vid_idx] if vid_mask.numel() == 0: zeros = torch.zeros( (1, 1, query_masks.shape[-2], query_masks.shape[-1]), dtype=query_masks.dtype, device=query_masks.device, ).float() batch_mask_list.append(zeros) else: batch_mask_list.append( vid_mask.any(dim=0, keepdim=True).float() ) if batch_mask_list: combined_masks = torch.cat(batch_mask_list, dim=0) backbone_out["mask_inputs_per_frame"] = {0: combined_masks} # two-stage memory refinement (enable_two_stage_memory=True) if getattr(model, "enable_two_stage_memory", False): stage1_outputs = model.forward_tracking( backbone_out, batched, return_dict=True ) outputs = model._run_second_stage( backbone_out, batched, stage1_outputs ) else: outputs = model.forward_tracking( backbone_out, batched, return_dict=False ) # --- output extraction --- for frame_pos, output in enumerate(outputs): if frame_pos == 0: # slot 0 is the external query itself — its prediction is # just the input mask echoed back; never store it. continue if "pred_masks_high_res" in output: pred_masks = output["pred_masks_high_res"] else: pred_masks = output.get("pred_masks") if pred_masks is None: continue if pred_masks.dim() == 3: pred_masks = pred_masks.unsqueeze(0) frame_idx = chunk_indices[frame_pos] per_frame_pred = pred_masks[0] if OBJECT_ID - 1 >= per_frame_pred.shape[0]: continue mask_logits = per_frame_pred[OBJECT_ID - 1] mask_probs = torch.sigmoid(mask_logits.float()) mask_binary = (mask_probs > score_thresh).float() if mask_binary.shape != orig_size_hw: mask_binary = F.interpolate( mask_binary.unsqueeze(0).unsqueeze(0), size=orig_size_hw, mode="nearest", ).squeeze(0).squeeze(0) if frame_idx not in all_predictions: mask_np = (mask_binary.cpu().numpy() * 255).astype(np.uint8) if mask_np.sum() > 0: # sparse: absent object -> no mask all_predictions[frame_idx] = mask_np return all_predictions def _render_overlay( frames_rgb: List[np.ndarray], predictions: Dict[int, np.ndarray], fps: float, ) -> str: """Render the video with the predicted masklets overlaid in color.""" h, w = frames_rgb[0].shape[:2] out_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) out_file.close() writer = cv2.VideoWriter( out_file.name, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h) ) if not writer.isOpened(): raise RuntimeError("Could not open the output video writer.") for i, frame in enumerate(frames_rgb): overlay = frame.copy() mask = predictions.get(i) if mask is not None: color = np.array([255, 170, 0], dtype=np.uint8) # orange masklet m = mask > 0 tint = np.zeros_like(frame) tint[..., 0], tint[..., 1], tint[..., 2] = color overlay[m] = cv2.addWeighted( frame[m].reshape(-1, 3).astype(np.uint8), 0.4, tint[m].reshape(-1, 3), 0.6, 0, ).reshape(-1, 3) contours, _ = cv2.findContours( mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE ) cv2.drawContours(overlay, contours, -1, (255, 255, 255), 2) writer.write(cv2.cvtColor(overlay, cv2.COLOR_RGB2BGR)) writer.release() return out_file.name # --------------------------------------------------------------------------- # Gradio handler # --------------------------------------------------------------------------- @spaces.GPU(duration=45) # measured: 3.9s cold / 2.7s steady for 49 frames def segment_video( video, query, score_thresh: float = SCORE_THRESH, max_frames: int = MAX_FRAMES, progress=gr.Progress(), ): """Segment the target object across a video, given a visual query. Args: video: input video (first ~49 frames are used). query: reference image with the target object brushed over (gr.ImageEditor). score_thresh: probability threshold for output masks. max_frames: maximum number of video frames to process. progress: Gradio progress bar. Returns: (overlay_video, status_text): the video with predicted masklets overlaid, and a short status string. """ if video is None: raise gr.Error("Please upload a video.") if query is None: raise gr.Error("Please provide a visual query image with the target brushed.") t_start = time.perf_counter() frames_rgb, fps = load_video_frames(video, int(max_frames)) query_mask = extract_query_mask(query) bg = np.array(query["background"]) if bg.ndim == 2: bg = np.stack([bg] * 3, axis=-1) if bg.shape[-1] == 4: # background is flattened onto white by preprocess already; drop alpha bg = bg[..., :3] query_image = Image.fromarray(bg.astype(np.uint8)).convert("RGB") progress((0.1, 1.0), desc=f"Tracking {len(frames_rgb)} frames...") predictions = _run_segments( frames_rgb, query_image, query_mask, float(score_thresh) ) out_path = _render_overlay(frames_rgb, predictions, fps) n_masks = len(predictions) elapsed = time.perf_counter() - t_start status = ( f"Segmented object in {n_masks}/{len(frames_rgb)} frames " f"(absent frames are expected — VQS masks are sparse). " f"Took {elapsed:.1f}s." ) return out_path, status # --------------------------------------------------------------------------- # UI # --------------------------------------------------------------------------- CUSTOM_CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks() as demo: gr.HTML( """

VQ-SAM: Visual Query Segmentation in the Wild

Give an external reference image with a brushed target object, and a short video — VQ-SAM segments the target everywhere it appears (masklets, spatio-temporal, sparse). Weights: bf0820/VQ-SAM · Paper: Towards Visual Query Segmentation in the Wild

""" ) with gr.Column(elem_id="col-container"): with gr.Row(): with gr.Column(): video_in = gr.Video( label="Video (first ~49 frames are used)", sources=["upload"], ) query_in = gr.ImageEditor( label="Visual query — upload a reference image, then brush over the target object", type="numpy", brush=gr.Brush(colors=["#FFAA00"], color_mode="fixed"), layers=False, ) run_btn = gr.Button("Segment video", variant="primary") with gr.Column(): video_out = gr.Video(label="Segmented masklets (orange overlay)") status_out = gr.Textbox(label="Status", interactive=False) with gr.Accordion("Advanced options", open=False): score_thresh_in = gr.Slider( 0.1, 0.9, value=SCORE_THRESH, step=0.05, label="Score threshold", info="Probability threshold for output masks (reference default 0.5)", ) max_frames_in = gr.Slider( 8, MAX_FRAMES, value=MAX_FRAMES, step=1, label="Max video frames", info=f"Higher values take longer; capped at {MAX_FRAMES} in this demo", ) gr.Examples( examples=[ ["assets/poodle_running.mp4", {"background": "assets/query_poodle.jpg", "layers": ["assets/query_poodle_overlay.png"], "composite": "assets/query_poodle.jpg"}], ["assets/jellyfish_glide.mp4", {"background": "assets/query_jellyfish.jpg", "layers": ["assets/query_jellyfish_overlay.png"], "composite": "assets/query_jellyfish.jpg"}], ], inputs=[video_in, query_in], fn=segment_video, outputs=[video_out, status_out], cache_examples=True, cache_mode="lazy", label="Examples (video + visual query pairs; press an example, then Segment video)", ) run_btn.click( fn=segment_video, inputs=[video_in, query_in, score_thresh_in, max_frames_in], outputs=[video_out, status_out], api_name="segment_video", ) if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CUSTOM_CSS, mcp_server=True)