Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/vq-sam: direct link, hf CLI and curl.
- Browser
- Download file 18.8 kB
-
https://huggingface.co/spaces/hugging-apps/vq-sam/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/vq-sam/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/vq-sam/resolve/main/app.py
18.8 kB
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 # must precede any torch/CUDA import on ZeroGPU | |
| import time | |
| import tempfile | |
| from typing import Dict, List, Tuple | |
| import cv2 | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| from omegaconf import OmegaConf | |
| from hydra.utils import instantiate | |
| from huggingface_hub import hf_hub_download | |
| from training.dataset.transforms import ( | |
| ComposeAPI, | |
| RandomResizeAPI, | |
| ToTensorAPI, | |
| NormalizeAPI, | |
| ) | |
| from training.utils.data_utils import collate_fn, Frame, Object, VideoDatapoint | |
| from training.model.sam2 import SAM2Train | |
| # --------------------------------------------------------------------------- | |
| # Model: load eagerly at module scope (ZeroGPU streams it into VRAM on the | |
| # first @spaces.GPU call). VQ-SAM = SAM 2.1 Hiera B+ + two-stage memory + DAM, | |
| # trained for visual query segmentation at 768x768 with 8-frame clips. | |
| # --------------------------------------------------------------------------- | |
| CONFIG_PATH = os.path.join( | |
| "sam2", "configs", "sam2.1_training", "sam2.1_hiera_b+_DAM_AdaptiveGate_Jan28.yaml" | |
| ) | |
| RESOLUTION = 768 # cfg.scratch.resolution | |
| CLIP_FRAMES = 8 # cfg.scratch.num_frames (1 query + 7 video frames) | |
| MAX_FRAMES = 49 # demo cap: keeps ZeroGPU duration bounded | |
| OBJECT_ID = 1 # single target object per visual query | |
| SCORE_THRESH = 0.5 # reference default (--score_thresh) | |
| print("Loading VQ-SAM config...") | |
| cfg = OmegaConf.load(CONFIG_PATH) | |
| print("Instantiating model...") | |
| model: SAM2Train = instantiate(cfg.trainer.model, _recursive_=True) | |
| print("Downloading VQ-SAM checkpoint (bf0820/VQ-SAM)...") | |
| ckpt_path = hf_hub_download(repo_id="bf0820/VQ-SAM", filename="vqsam_hiera_bplus.pt") | |
| checkpoint = torch.load(ckpt_path, map_location="cpu", weights_only=False) | |
| state_dict = checkpoint.get("model", checkpoint) | |
| missing, unexpected = model.load_state_dict(state_dict, strict=False) | |
| print(f"Loaded checkpoint (missing={len(missing)}, unexpected={len(unexpected)})") | |
| model = model.to("cuda") | |
| model.eval() | |
| EVAL_TRANSFORMS = ComposeAPI( | |
| transforms=[ | |
| RandomResizeAPI(sizes=RESOLUTION, square=True, consistent_transform=True), | |
| ToTensorAPI(), | |
| NormalizeAPI(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), | |
| ] | |
| ) | |
| DEVICE = torch.device("cuda") | |
| # --------------------------------------------------------------------------- | |
| # Video I/O helpers | |
| # --------------------------------------------------------------------------- | |
| def load_video_frames(video_path: str, max_frames: int) -> Tuple[List[np.ndarray], float]: | |
| """Decode up to `max_frames` frames of a video with OpenCV (CPU).""" | |
| cap = cv2.VideoCapture(video_path) | |
| if not cap.isOpened(): | |
| raise ValueError("Could not open the uploaded video file.") | |
| fps = cap.get(cv2.CAP_PROP_FPS) or 25.0 | |
| frames: List[np.ndarray] = [] | |
| while len(frames) < max_frames: | |
| ok, frame = cap.read() | |
| if not ok: | |
| break | |
| frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) | |
| cap.release() | |
| if not frames: | |
| raise ValueError("No frames could be decoded from the video.") | |
| return frames, float(fps) | |
| def extract_query_mask(editor_value) -> np.ndarray: | |
| """Extract a binary mask from a gr.ImageEditor value dict. | |
| With ``type="numpy"`` and the default ``image_mode="RGBA"``, the editor's | |
| ``layers`` arrive as RGBA arrays where the alpha channel IS the paint | |
| stroke. We union the alpha of all layers, and fall back to a | |
| composite-vs-background difference if no layer alpha is present. | |
| """ | |
| if editor_value is None: | |
| raise gr.Error( | |
| "Please upload a reference image and brush over the target object." | |
| ) | |
| background = editor_value.get("background") | |
| layers = editor_value.get("layers") or [] | |
| mask = None | |
| # 1) Layer alpha — the direct, robust signal. | |
| alpha_union = None | |
| for layer in layers: | |
| lay = np.array(layer) | |
| if lay.ndim == 2: | |
| a = (lay > 10).astype(np.uint8) * 255 | |
| elif lay.shape[-1] == 4: | |
| a = (lay[..., 3] > 10).astype(np.uint8) * 255 | |
| else: | |
| a = (lay.sum(axis=-1) > 30).astype(np.uint8) * 255 | |
| if alpha_union is None: | |
| alpha_union = a | |
| elif a.shape == alpha_union.shape: | |
| alpha_union = np.maximum(alpha_union, a) | |
| if alpha_union is not None and alpha_union.sum() > 0: | |
| mask = alpha_union | |
| # 2) Fallback: composite minus background difference. | |
| if mask is None and background is not None and layers: | |
| bg = np.array(background) | |
| comp = np.array(editor_value.get("composite")) | |
| if comp.shape == bg.shape and bg.ndim == 3: | |
| diff = np.abs(comp.astype(np.int16) - bg.astype(np.int16)) | |
| diff_mask = (diff.sum(axis=-1) > 30).astype(np.uint8) * 255 | |
| if diff_mask.sum() > 0: | |
| mask = diff_mask | |
| if mask is None or mask.sum() == 0: | |
| raise gr.Error( | |
| "No mask detected: brush over the target object in the reference image." | |
| ) | |
| return mask | |
| # --------------------------------------------------------------------------- | |
| # Inference: 1:1 with VQS tools/test_video_segmentation.py | |
| # --------------------------------------------------------------------------- | |
| def _run_segments( | |
| frames_rgb: List[np.ndarray], | |
| query_image: Image.Image, | |
| query_mask: np.ndarray, | |
| score_thresh: float, | |
| ) -> Dict[int, np.ndarray]: | |
| """Run VQ-SAM over all segments of the video. | |
| Returns {global_frame_idx: binary mask (H, W) uint8 with 0/255 values} | |
| for frames with a non-empty prediction. | |
| """ | |
| h, w = frames_rgb[0].shape[:2] | |
| pil_frames = [Image.fromarray(f) for f in frames_rgb] | |
| query_pil = query_image.convert("RGB") | |
| # --- chunking (TestChunkSampler.sample_chunks) --- | |
| # Each chunk: [query frame, 7 video frames]; tail padded by repeating | |
| # the last frame index (duplicate predictions are dropped, first wins). | |
| # Unlike the reference (VQS-4K eval layout, where frame 00000 IS the | |
| # query), the demo's query is EXTERNAL — so chunks cover video frames | |
| # 0..N-1 and the query occupies local slot 0 with a sentinel index -1 | |
| # whose own (echoed) prediction is simply not stored. | |
| QUERY_SENTINEL = -1 | |
| num_video_frames = len(pil_frames) | |
| chunks: List[List[int]] = [] | |
| start = 0 | |
| while start < num_video_frames: | |
| end = min(start + (CLIP_FRAMES - 1), num_video_frames) | |
| idxs = list(range(start, end)) | |
| last = idxs[-1] | |
| while len(idxs) < (CLIP_FRAMES - 1): | |
| idxs.append(last) | |
| chunks.append(idxs) | |
| start = end | |
| query_mask_tensor = torch.from_numpy(query_mask).to(torch.uint8) | |
| all_predictions: Dict[int, np.ndarray] = {} | |
| orig_size_hw = (h, w) | |
| for chunk_video_idx in chunks: | |
| # --- build_datapoint_from_chunk --- | |
| # Local slot 0 = the external query (sentinel frame_index -1); | |
| # local slots 1..7 = video frames. | |
| chunk_indices = [QUERY_SENTINEL] + chunk_video_idx | |
| images: List[Frame] = [] | |
| for local_idx, frame_idx in enumerate(chunk_indices): | |
| if frame_idx == QUERY_SENTINEL: | |
| data = query_pil | |
| segment = query_mask_tensor | |
| else: | |
| data = pil_frames[frame_idx] | |
| segment = torch.zeros(h, w, dtype=torch.uint8) | |
| frame_obj = Frame(data=data, objects=[]) | |
| frame_obj.objects.append( | |
| Object( | |
| object_id=OBJECT_ID, | |
| frame_index=frame_idx, | |
| segment=segment, | |
| ) | |
| ) | |
| images.append(frame_obj) | |
| datapoint = VideoDatapoint(frames=images, video_id=0, size=orig_size_hw) | |
| datapoint = EVAL_TRANSFORMS(datapoint, epoch=0) | |
| batched = collate_fn([datapoint], dict_key="all") | |
| batched = batched.to(DEVICE) | |
| # --- forward: reference wraps the entire inference in bf16 autocast | |
| # (tools/test_video_segmentation.py runs the whole body under it) --- | |
| with torch.autocast(device_type="cuda", dtype=torch.bfloat16): | |
| backbone_out = model.forward_image(batched.flat_img_batch) | |
| model.prepare_prompt_inputs(backbone_out, batched, start_frame_idx=0) | |
| backbone_out["init_cond_frames"] = [0] | |
| backbone_out["frames_to_add_correction_pt"] = [] | |
| backbone_out["use_pt_input"] = False | |
| if 0 in backbone_out["gt_masks_per_frame"]: | |
| query_masks = backbone_out["gt_masks_per_frame"][0] | |
| obj_to_frame_idx = batched.obj_to_frame_idx[0] | |
| video_indices = obj_to_frame_idx[:, 1].to(torch.long) | |
| batch_mask_list = [] | |
| for vid_idx in range(batched.img_batch.shape[1]): | |
| vid_mask = query_masks[video_indices == vid_idx] | |
| if vid_mask.numel() == 0: | |
| zeros = torch.zeros( | |
| (1, 1, query_masks.shape[-2], query_masks.shape[-1]), | |
| dtype=query_masks.dtype, | |
| device=query_masks.device, | |
| ).float() | |
| batch_mask_list.append(zeros) | |
| else: | |
| batch_mask_list.append( | |
| vid_mask.any(dim=0, keepdim=True).float() | |
| ) | |
| if batch_mask_list: | |
| combined_masks = torch.cat(batch_mask_list, dim=0) | |
| backbone_out["mask_inputs_per_frame"] = {0: combined_masks} | |
| # two-stage memory refinement (enable_two_stage_memory=True) | |
| if getattr(model, "enable_two_stage_memory", False): | |
| stage1_outputs = model.forward_tracking( | |
| backbone_out, batched, return_dict=True | |
| ) | |
| outputs = model._run_second_stage( | |
| backbone_out, batched, stage1_outputs | |
| ) | |
| else: | |
| outputs = model.forward_tracking( | |
| backbone_out, batched, return_dict=False | |
| ) | |
| # --- output extraction --- | |
| for frame_pos, output in enumerate(outputs): | |
| if frame_pos == 0: | |
| # slot 0 is the external query itself — its prediction is | |
| # just the input mask echoed back; never store it. | |
| continue | |
| if "pred_masks_high_res" in output: | |
| pred_masks = output["pred_masks_high_res"] | |
| else: | |
| pred_masks = output.get("pred_masks") | |
| if pred_masks is None: | |
| continue | |
| if pred_masks.dim() == 3: | |
| pred_masks = pred_masks.unsqueeze(0) | |
| frame_idx = chunk_indices[frame_pos] | |
| per_frame_pred = pred_masks[0] | |
| if OBJECT_ID - 1 >= per_frame_pred.shape[0]: | |
| continue | |
| mask_logits = per_frame_pred[OBJECT_ID - 1] | |
| mask_probs = torch.sigmoid(mask_logits.float()) | |
| mask_binary = (mask_probs > score_thresh).float() | |
| if mask_binary.shape != orig_size_hw: | |
| mask_binary = F.interpolate( | |
| mask_binary.unsqueeze(0).unsqueeze(0), | |
| size=orig_size_hw, | |
| mode="nearest", | |
| ).squeeze(0).squeeze(0) | |
| if frame_idx not in all_predictions: | |
| mask_np = (mask_binary.cpu().numpy() * 255).astype(np.uint8) | |
| if mask_np.sum() > 0: # sparse: absent object -> no mask | |
| all_predictions[frame_idx] = mask_np | |
| return all_predictions | |
| def _render_overlay( | |
| frames_rgb: List[np.ndarray], | |
| predictions: Dict[int, np.ndarray], | |
| fps: float, | |
| ) -> str: | |
| """Render the video with the predicted masklets overlaid in color.""" | |
| h, w = frames_rgb[0].shape[:2] | |
| out_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) | |
| out_file.close() | |
| writer = cv2.VideoWriter( | |
| out_file.name, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h) | |
| ) | |
| if not writer.isOpened(): | |
| raise RuntimeError("Could not open the output video writer.") | |
| for i, frame in enumerate(frames_rgb): | |
| overlay = frame.copy() | |
| mask = predictions.get(i) | |
| if mask is not None: | |
| color = np.array([255, 170, 0], dtype=np.uint8) # orange masklet | |
| m = mask > 0 | |
| tint = np.zeros_like(frame) | |
| tint[..., 0], tint[..., 1], tint[..., 2] = color | |
| overlay[m] = cv2.addWeighted( | |
| frame[m].reshape(-1, 3).astype(np.uint8), | |
| 0.4, | |
| tint[m].reshape(-1, 3), | |
| 0.6, | |
| 0, | |
| ).reshape(-1, 3) | |
| contours, _ = cv2.findContours( | |
| mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE | |
| ) | |
| cv2.drawContours(overlay, contours, -1, (255, 255, 255), 2) | |
| writer.write(cv2.cvtColor(overlay, cv2.COLOR_RGB2BGR)) | |
| writer.release() | |
| return out_file.name | |
| # --------------------------------------------------------------------------- | |
| # Gradio handler | |
| # --------------------------------------------------------------------------- | |
| # measured: 3.9s cold / 2.7s steady for 49 frames | |
| def segment_video( | |
| video, | |
| query, | |
| score_thresh: float = SCORE_THRESH, | |
| max_frames: int = MAX_FRAMES, | |
| progress=gr.Progress(), | |
| ): | |
| """Segment the target object across a video, given a visual query. | |
| Args: | |
| video: input video (first ~49 frames are used). | |
| query: reference image with the target object brushed over | |
| (gr.ImageEditor). | |
| score_thresh: probability threshold for output masks. | |
| max_frames: maximum number of video frames to process. | |
| progress: Gradio progress bar. | |
| Returns: | |
| (overlay_video, status_text): the video with predicted masklets | |
| overlaid, and a short status string. | |
| """ | |
| if video is None: | |
| raise gr.Error("Please upload a video.") | |
| if query is None: | |
| raise gr.Error("Please provide a visual query image with the target brushed.") | |
| t_start = time.perf_counter() | |
| frames_rgb, fps = load_video_frames(video, int(max_frames)) | |
| query_mask = extract_query_mask(query) | |
| bg = np.array(query["background"]) | |
| if bg.ndim == 2: | |
| bg = np.stack([bg] * 3, axis=-1) | |
| if bg.shape[-1] == 4: | |
| # background is flattened onto white by preprocess already; drop alpha | |
| bg = bg[..., :3] | |
| query_image = Image.fromarray(bg.astype(np.uint8)).convert("RGB") | |
| progress((0.1, 1.0), desc=f"Tracking {len(frames_rgb)} frames...") | |
| predictions = _run_segments( | |
| frames_rgb, query_image, query_mask, float(score_thresh) | |
| ) | |
| out_path = _render_overlay(frames_rgb, predictions, fps) | |
| n_masks = len(predictions) | |
| elapsed = time.perf_counter() - t_start | |
| status = ( | |
| f"Segmented object in {n_masks}/{len(frames_rgb)} frames " | |
| f"(absent frames are expected — VQS masks are sparse). " | |
| f"Took {elapsed:.1f}s." | |
| ) | |
| return out_path, status | |
| # --------------------------------------------------------------------------- | |
| # UI | |
| # --------------------------------------------------------------------------- | |
| CUSTOM_CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks() as demo: | |
| gr.HTML( | |
| """ | |
| <div style="text-align: center; margin-bottom: 8px;"> | |
| <h1 style="margin-bottom: 4px;">VQ-SAM: Visual Query Segmentation in the Wild</h1> | |
| <p style="color: var(--body-text-color); font-size: 15px; margin: 0;"> | |
| Give an <b>external reference image</b> with a brushed target object, and a | |
| short video — VQ-SAM segments the target everywhere it appears | |
| (masklets, spatio-temporal, sparse). Weights: | |
| <a href="https://huggingface.co/bf0820/VQ-SAM">bf0820/VQ-SAM</a> · | |
| Paper: <a href="https://huggingface.co/papers/2603.08898">Towards Visual Query Segmentation in the Wild</a> | |
| </p> | |
| </div> | |
| """ | |
| ) | |
| with gr.Column(elem_id="col-container"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| video_in = gr.Video( | |
| label="Video (first ~49 frames are used)", | |
| sources=["upload"], | |
| ) | |
| query_in = gr.ImageEditor( | |
| label="Visual query — upload a reference image, then brush over the target object", | |
| type="numpy", | |
| brush=gr.Brush(colors=["#FFAA00"], color_mode="fixed"), | |
| layers=False, | |
| ) | |
| run_btn = gr.Button("Segment video", variant="primary") | |
| with gr.Column(): | |
| video_out = gr.Video(label="Segmented masklets (orange overlay)") | |
| status_out = gr.Textbox(label="Status", interactive=False) | |
| with gr.Accordion("Advanced options", open=False): | |
| score_thresh_in = gr.Slider( | |
| 0.1, 0.9, value=SCORE_THRESH, step=0.05, | |
| label="Score threshold", | |
| info="Probability threshold for output masks (reference default 0.5)", | |
| ) | |
| max_frames_in = gr.Slider( | |
| 8, MAX_FRAMES, value=MAX_FRAMES, step=1, | |
| label="Max video frames", | |
| info=f"Higher values take longer; capped at {MAX_FRAMES} in this demo", | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| ["assets/poodle_running.mp4", | |
| {"background": "assets/query_poodle.jpg", | |
| "layers": ["assets/query_poodle_overlay.png"], | |
| "composite": "assets/query_poodle.jpg"}], | |
| ["assets/jellyfish_glide.mp4", | |
| {"background": "assets/query_jellyfish.jpg", | |
| "layers": ["assets/query_jellyfish_overlay.png"], | |
| "composite": "assets/query_jellyfish.jpg"}], | |
| ], | |
| inputs=[video_in, query_in], | |
| fn=segment_video, | |
| outputs=[video_out, status_out], | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Examples (video + visual query pairs; press an example, then Segment video)", | |
| ) | |
| run_btn.click( | |
| fn=segment_video, | |
| inputs=[video_in, query_in, score_thresh_in, max_frames_in], | |
| outputs=[video_out, status_out], | |
| api_name="segment_video", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CUSTOM_CSS, mcp_server=True) | |