vq-sam / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
65b1d69 verified
Raw History Blame Contribute Delete
18.8 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 # must precede any torch/CUDA import on ZeroGPU
import time
import tempfile
from typing import Dict, List, Tuple
import cv2
import gradio as gr
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
from omegaconf import OmegaConf
from hydra.utils import instantiate
from huggingface_hub import hf_hub_download
from training.dataset.transforms import (
ComposeAPI,
RandomResizeAPI,
ToTensorAPI,
NormalizeAPI,
)
from training.utils.data_utils import collate_fn, Frame, Object, VideoDatapoint
from training.model.sam2 import SAM2Train
# ---------------------------------------------------------------------------
# Model: load eagerly at module scope (ZeroGPU streams it into VRAM on the
# first @spaces.GPU call). VQ-SAM = SAM 2.1 Hiera B+ + two-stage memory + DAM,
# trained for visual query segmentation at 768x768 with 8-frame clips.
# ---------------------------------------------------------------------------
CONFIG_PATH = os.path.join(
"sam2", "configs", "sam2.1_training", "sam2.1_hiera_b+_DAM_AdaptiveGate_Jan28.yaml"
)
RESOLUTION = 768 # cfg.scratch.resolution
CLIP_FRAMES = 8 # cfg.scratch.num_frames (1 query + 7 video frames)
MAX_FRAMES = 49 # demo cap: keeps ZeroGPU duration bounded
OBJECT_ID = 1 # single target object per visual query
SCORE_THRESH = 0.5 # reference default (--score_thresh)
print("Loading VQ-SAM config...")
cfg = OmegaConf.load(CONFIG_PATH)
print("Instantiating model...")
model: SAM2Train = instantiate(cfg.trainer.model, _recursive_=True)
print("Downloading VQ-SAM checkpoint (bf0820/VQ-SAM)...")
ckpt_path = hf_hub_download(repo_id="bf0820/VQ-SAM", filename="vqsam_hiera_bplus.pt")
checkpoint = torch.load(ckpt_path, map_location="cpu", weights_only=False)
state_dict = checkpoint.get("model", checkpoint)
missing, unexpected = model.load_state_dict(state_dict, strict=False)
print(f"Loaded checkpoint (missing={len(missing)}, unexpected={len(unexpected)})")
model = model.to("cuda")
model.eval()
EVAL_TRANSFORMS = ComposeAPI(
transforms=[
RandomResizeAPI(sizes=RESOLUTION, square=True, consistent_transform=True),
ToTensorAPI(),
NormalizeAPI(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
]
)
DEVICE = torch.device("cuda")
# ---------------------------------------------------------------------------
# Video I/O helpers
# ---------------------------------------------------------------------------
def load_video_frames(video_path: str, max_frames: int) -> Tuple[List[np.ndarray], float]:
"""Decode up to `max_frames` frames of a video with OpenCV (CPU)."""
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
raise ValueError("Could not open the uploaded video file.")
fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
frames: List[np.ndarray] = []
while len(frames) < max_frames:
ok, frame = cap.read()
if not ok:
break
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
cap.release()
if not frames:
raise ValueError("No frames could be decoded from the video.")
return frames, float(fps)
def extract_query_mask(editor_value) -> np.ndarray:
"""Extract a binary mask from a gr.ImageEditor value dict.
With ``type="numpy"`` and the default ``image_mode="RGBA"``, the editor's
``layers`` arrive as RGBA arrays where the alpha channel IS the paint
stroke. We union the alpha of all layers, and fall back to a
composite-vs-background difference if no layer alpha is present.
"""
if editor_value is None:
raise gr.Error(
"Please upload a reference image and brush over the target object."
)
background = editor_value.get("background")
layers = editor_value.get("layers") or []
mask = None
# 1) Layer alpha — the direct, robust signal.
alpha_union = None
for layer in layers:
lay = np.array(layer)
if lay.ndim == 2:
a = (lay > 10).astype(np.uint8) * 255
elif lay.shape[-1] == 4:
a = (lay[..., 3] > 10).astype(np.uint8) * 255
else:
a = (lay.sum(axis=-1) > 30).astype(np.uint8) * 255
if alpha_union is None:
alpha_union = a
elif a.shape == alpha_union.shape:
alpha_union = np.maximum(alpha_union, a)
if alpha_union is not None and alpha_union.sum() > 0:
mask = alpha_union
# 2) Fallback: composite minus background difference.
if mask is None and background is not None and layers:
bg = np.array(background)
comp = np.array(editor_value.get("composite"))
if comp.shape == bg.shape and bg.ndim == 3:
diff = np.abs(comp.astype(np.int16) - bg.astype(np.int16))
diff_mask = (diff.sum(axis=-1) > 30).astype(np.uint8) * 255
if diff_mask.sum() > 0:
mask = diff_mask
if mask is None or mask.sum() == 0:
raise gr.Error(
"No mask detected: brush over the target object in the reference image."
)
return mask
# ---------------------------------------------------------------------------
# Inference: 1:1 with VQS tools/test_video_segmentation.py
# ---------------------------------------------------------------------------
@torch.inference_mode()
def _run_segments(
frames_rgb: List[np.ndarray],
query_image: Image.Image,
query_mask: np.ndarray,
score_thresh: float,
) -> Dict[int, np.ndarray]:
"""Run VQ-SAM over all segments of the video.
Returns {global_frame_idx: binary mask (H, W) uint8 with 0/255 values}
for frames with a non-empty prediction.
"""
h, w = frames_rgb[0].shape[:2]
pil_frames = [Image.fromarray(f) for f in frames_rgb]
query_pil = query_image.convert("RGB")
# --- chunking (TestChunkSampler.sample_chunks) ---
# Each chunk: [query frame, 7 video frames]; tail padded by repeating
# the last frame index (duplicate predictions are dropped, first wins).
# Unlike the reference (VQS-4K eval layout, where frame 00000 IS the
# query), the demo's query is EXTERNAL — so chunks cover video frames
# 0..N-1 and the query occupies local slot 0 with a sentinel index -1
# whose own (echoed) prediction is simply not stored.
QUERY_SENTINEL = -1
num_video_frames = len(pil_frames)
chunks: List[List[int]] = []
start = 0
while start < num_video_frames:
end = min(start + (CLIP_FRAMES - 1), num_video_frames)
idxs = list(range(start, end))
last = idxs[-1]
while len(idxs) < (CLIP_FRAMES - 1):
idxs.append(last)
chunks.append(idxs)
start = end
query_mask_tensor = torch.from_numpy(query_mask).to(torch.uint8)
all_predictions: Dict[int, np.ndarray] = {}
orig_size_hw = (h, w)
for chunk_video_idx in chunks:
# --- build_datapoint_from_chunk ---
# Local slot 0 = the external query (sentinel frame_index -1);
# local slots 1..7 = video frames.
chunk_indices = [QUERY_SENTINEL] + chunk_video_idx
images: List[Frame] = []
for local_idx, frame_idx in enumerate(chunk_indices):
if frame_idx == QUERY_SENTINEL:
data = query_pil
segment = query_mask_tensor
else:
data = pil_frames[frame_idx]
segment = torch.zeros(h, w, dtype=torch.uint8)
frame_obj = Frame(data=data, objects=[])
frame_obj.objects.append(
Object(
object_id=OBJECT_ID,
frame_index=frame_idx,
segment=segment,
)
)
images.append(frame_obj)
datapoint = VideoDatapoint(frames=images, video_id=0, size=orig_size_hw)
datapoint = EVAL_TRANSFORMS(datapoint, epoch=0)
batched = collate_fn([datapoint], dict_key="all")
batched = batched.to(DEVICE)
# --- forward: reference wraps the entire inference in bf16 autocast
# (tools/test_video_segmentation.py runs the whole body under it) ---
with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
backbone_out = model.forward_image(batched.flat_img_batch)
model.prepare_prompt_inputs(backbone_out, batched, start_frame_idx=0)
backbone_out["init_cond_frames"] = [0]
backbone_out["frames_to_add_correction_pt"] = []
backbone_out["use_pt_input"] = False
if 0 in backbone_out["gt_masks_per_frame"]:
query_masks = backbone_out["gt_masks_per_frame"][0]
obj_to_frame_idx = batched.obj_to_frame_idx[0]
video_indices = obj_to_frame_idx[:, 1].to(torch.long)
batch_mask_list = []
for vid_idx in range(batched.img_batch.shape[1]):
vid_mask = query_masks[video_indices == vid_idx]
if vid_mask.numel() == 0:
zeros = torch.zeros(
(1, 1, query_masks.shape[-2], query_masks.shape[-1]),
dtype=query_masks.dtype,
device=query_masks.device,
).float()
batch_mask_list.append(zeros)
else:
batch_mask_list.append(
vid_mask.any(dim=0, keepdim=True).float()
)
if batch_mask_list:
combined_masks = torch.cat(batch_mask_list, dim=0)
backbone_out["mask_inputs_per_frame"] = {0: combined_masks}
# two-stage memory refinement (enable_two_stage_memory=True)
if getattr(model, "enable_two_stage_memory", False):
stage1_outputs = model.forward_tracking(
backbone_out, batched, return_dict=True
)
outputs = model._run_second_stage(
backbone_out, batched, stage1_outputs
)
else:
outputs = model.forward_tracking(
backbone_out, batched, return_dict=False
)
# --- output extraction ---
for frame_pos, output in enumerate(outputs):
if frame_pos == 0:
# slot 0 is the external query itself — its prediction is
# just the input mask echoed back; never store it.
continue
if "pred_masks_high_res" in output:
pred_masks = output["pred_masks_high_res"]
else:
pred_masks = output.get("pred_masks")
if pred_masks is None:
continue
if pred_masks.dim() == 3:
pred_masks = pred_masks.unsqueeze(0)
frame_idx = chunk_indices[frame_pos]
per_frame_pred = pred_masks[0]
if OBJECT_ID - 1 >= per_frame_pred.shape[0]:
continue
mask_logits = per_frame_pred[OBJECT_ID - 1]
mask_probs = torch.sigmoid(mask_logits.float())
mask_binary = (mask_probs > score_thresh).float()
if mask_binary.shape != orig_size_hw:
mask_binary = F.interpolate(
mask_binary.unsqueeze(0).unsqueeze(0),
size=orig_size_hw,
mode="nearest",
).squeeze(0).squeeze(0)
if frame_idx not in all_predictions:
mask_np = (mask_binary.cpu().numpy() * 255).astype(np.uint8)
if mask_np.sum() > 0: # sparse: absent object -> no mask
all_predictions[frame_idx] = mask_np
return all_predictions
def _render_overlay(
frames_rgb: List[np.ndarray],
predictions: Dict[int, np.ndarray],
fps: float,
) -> str:
"""Render the video with the predicted masklets overlaid in color."""
h, w = frames_rgb[0].shape[:2]
out_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
out_file.close()
writer = cv2.VideoWriter(
out_file.name, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)
)
if not writer.isOpened():
raise RuntimeError("Could not open the output video writer.")
for i, frame in enumerate(frames_rgb):
overlay = frame.copy()
mask = predictions.get(i)
if mask is not None:
color = np.array([255, 170, 0], dtype=np.uint8) # orange masklet
m = mask > 0
tint = np.zeros_like(frame)
tint[..., 0], tint[..., 1], tint[..., 2] = color
overlay[m] = cv2.addWeighted(
frame[m].reshape(-1, 3).astype(np.uint8),
0.4,
tint[m].reshape(-1, 3),
0.6,
0,
).reshape(-1, 3)
contours, _ = cv2.findContours(
mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
cv2.drawContours(overlay, contours, -1, (255, 255, 255), 2)
writer.write(cv2.cvtColor(overlay, cv2.COLOR_RGB2BGR))
writer.release()
return out_file.name
# ---------------------------------------------------------------------------
# Gradio handler
# ---------------------------------------------------------------------------
@spaces.GPU(duration=45) # measured: 3.9s cold / 2.7s steady for 49 frames
def segment_video(
video,
query,
score_thresh: float = SCORE_THRESH,
max_frames: int = MAX_FRAMES,
progress=gr.Progress(),
):
"""Segment the target object across a video, given a visual query.
Args:
video: input video (first ~49 frames are used).
query: reference image with the target object brushed over
(gr.ImageEditor).
score_thresh: probability threshold for output masks.
max_frames: maximum number of video frames to process.
progress: Gradio progress bar.
Returns:
(overlay_video, status_text): the video with predicted masklets
overlaid, and a short status string.
"""
if video is None:
raise gr.Error("Please upload a video.")
if query is None:
raise gr.Error("Please provide a visual query image with the target brushed.")
t_start = time.perf_counter()
frames_rgb, fps = load_video_frames(video, int(max_frames))
query_mask = extract_query_mask(query)
bg = np.array(query["background"])
if bg.ndim == 2:
bg = np.stack([bg] * 3, axis=-1)
if bg.shape[-1] == 4:
# background is flattened onto white by preprocess already; drop alpha
bg = bg[..., :3]
query_image = Image.fromarray(bg.astype(np.uint8)).convert("RGB")
progress((0.1, 1.0), desc=f"Tracking {len(frames_rgb)} frames...")
predictions = _run_segments(
frames_rgb, query_image, query_mask, float(score_thresh)
)
out_path = _render_overlay(frames_rgb, predictions, fps)
n_masks = len(predictions)
elapsed = time.perf_counter() - t_start
status = (
f"Segmented object in {n_masks}/{len(frames_rgb)} frames "
f"(absent frames are expected — VQS masks are sparse). "
f"Took {elapsed:.1f}s."
)
return out_path, status
# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
CUSTOM_CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks() as demo:
gr.HTML(
"""
<div style="text-align: center; margin-bottom: 8px;">
<h1 style="margin-bottom: 4px;">VQ-SAM: Visual Query Segmentation in the Wild</h1>
<p style="color: var(--body-text-color); font-size: 15px; margin: 0;">
Give an <b>external reference image</b> with a brushed target object, and a
short video — VQ-SAM segments the target everywhere it appears
(masklets, spatio-temporal, sparse). Weights:
<a href="https://huggingface.co/bf0820/VQ-SAM">bf0820/VQ-SAM</a> ·
Paper: <a href="https://huggingface.co/papers/2603.08898">Towards Visual Query Segmentation in the Wild</a>
</p>
</div>
"""
)
with gr.Column(elem_id="col-container"):
with gr.Row():
with gr.Column():
video_in = gr.Video(
label="Video (first ~49 frames are used)",
sources=["upload"],
)
query_in = gr.ImageEditor(
label="Visual query — upload a reference image, then brush over the target object",
type="numpy",
brush=gr.Brush(colors=["#FFAA00"], color_mode="fixed"),
layers=False,
)
run_btn = gr.Button("Segment video", variant="primary")
with gr.Column():
video_out = gr.Video(label="Segmented masklets (orange overlay)")
status_out = gr.Textbox(label="Status", interactive=False)
with gr.Accordion("Advanced options", open=False):
score_thresh_in = gr.Slider(
0.1, 0.9, value=SCORE_THRESH, step=0.05,
label="Score threshold",
info="Probability threshold for output masks (reference default 0.5)",
)
max_frames_in = gr.Slider(
8, MAX_FRAMES, value=MAX_FRAMES, step=1,
label="Max video frames",
info=f"Higher values take longer; capped at {MAX_FRAMES} in this demo",
)
gr.Examples(
examples=[
["assets/poodle_running.mp4",
{"background": "assets/query_poodle.jpg",
"layers": ["assets/query_poodle_overlay.png"],
"composite": "assets/query_poodle.jpg"}],
["assets/jellyfish_glide.mp4",
{"background": "assets/query_jellyfish.jpg",
"layers": ["assets/query_jellyfish_overlay.png"],
"composite": "assets/query_jellyfish.jpg"}],
],
inputs=[video_in, query_in],
fn=segment_video,
outputs=[video_out, status_out],
cache_examples=True,
cache_mode="lazy",
label="Examples (video + visual query pairs; press an example, then Segment video)",
)
run_btn.click(
fn=segment_video,
inputs=[video_in, query_in, score_thresh_in, max_frames_in],
outputs=[video_out, status_out],
api_name="segment_video",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CUSTOM_CSS, mcp_server=True)