multimodalart's picture
multimodalart HF Staff
Right-size ZeroGPU duration to 60s: measured worst case (1024 tokens, 64 frames) = 38.3s; default example path = 6.9s
6c78ed6 verified
Raw History Blame Contribute Delete
18.3 kB
"""
MVRD — Multi-View Relational Distillation for Spatial Reasoning with Vision-Language Models.
Interactive demo of the MVRD student VLM: LLaVA-Video-7B-Qwen2 fine-tuned with a
LoRA adapter distilled from VGGT multi-view spatial relations (checkpoint
`vggt_sim_distill_lora_l21_lambda0.5_mlp256_GELU_nobiasall_aug2/checkpoint-1622`,
released by the authors at https://huggingface.co/kietngt00/MVRD).
Ask spatial-reasoning questions about a scene walk-through video or a set of
images: object presence and counting, relative directions (left/right,
front/behind), distances between objects, room size, route planning, etc.
The loading recipe replicates the authors' `llava/model/builder.py` +
`lmms_eval` harness (`llava_vid` model, `qwen_1_5` conv template) 1:1, adapted
for ZeroGPU (module-scope eager load, no accelerate device_map).
"""
import spaces # noqa: F401 # must be imported before torch on ZeroGPU
import os
import shutil
import tarfile
import time
import numpy as np
import torch
import gradio as gr
from PIL import Image
from transformers import AutoTokenizer
from huggingface_hub import hf_hub_download
from peft import PeftModel
from decord import VideoReader, cpu
from llava.model.language_model.llava_qwen import LlavaQwenConfig, LlavaQwenForCausalLM
from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
from llava.conversation import conv_templates, SeparatorStyle
from llava.mm_utils import (
process_images,
tokenizer_image_token,
KeywordsStoppingCriteria,
expand2square,
)
BASE_MODEL = "lmms-lab/LLaVA-Video-7B-Qwen2"
HERE = os.path.dirname(os.path.abspath(__file__))
CKPT_DIR = os.path.join(HERE, "ckpt")
# The authors release the distilled student as a LoRA adapter inside a tarball
# on their model repo (kietngt00/MVRD). We fetch just the checkpoint files we
# need at startup instead of bundling them in this Space.
CKPT_REPO = "kietngt00/MVRD"
CKPT_TARBALL = "mvrd.tar.gz"
CKPT_PREFIX = "vggt_sim_distill_lora_l21_lambda0.5_mlp256_GELU_nobiasall_aug2/checkpoint-1622/"
CKPT_FILES = {
"adapter_config.json",
"adapter_model.bin",
"config.json",
"generation_config.json",
"non_lora_trainables.bin",
}
def fetch_mvrd_checkpoint(ckpt_dir: str) -> None:
"""Download the authors' released MVRD LoRA checkpoint and extract the
files needed for inference from their tarball into `ckpt_dir`."""
if all(os.path.exists(os.path.join(ckpt_dir, f)) for f in CKPT_FILES):
return
print(f"[mvrd] fetching {CKPT_TARBALL} from {CKPT_REPO} ...", flush=True)
t0 = time.time()
tar_path = hf_hub_download(repo_id=CKPT_REPO, filename=CKPT_TARBALL)
print(f"[mvrd] downloaded in {time.time() - t0:.1f}s, extracting ...", flush=True)
os.makedirs(ckpt_dir, exist_ok=True)
with tarfile.open(tar_path, "r:gz") as tar:
for member in tar:
name = member.name[2:] if member.name.startswith("./") else member.name
if not name.startswith(CKPT_PREFIX) or not member.isfile():
continue
base = name[len(CKPT_PREFIX):]
if base not in CKPT_FILES:
continue
src = tar.extractfile(member)
if src is None:
continue
with open(os.path.join(ckpt_dir, base), "wb") as dst:
shutil.copyfileobj(src, dst)
print(f"[mvrd] extracted {base}", flush=True)
missing = [f for f in CKPT_FILES if not os.path.exists(os.path.join(ckpt_dir, f))]
if missing:
raise RuntimeError(f"checkpoint extraction incomplete, missing: {missing}")
# --------------------------------------------------------------------------- #
# Model loading (module scope, eager) — replicates the authors' builder #
# --------------------------------------------------------------------------- #
fetch_mvrd_checkpoint(CKPT_DIR)
print("[mvrd] loading config...", flush=True)
lora_cfg = LlavaQwenConfig.from_pretrained(CKPT_DIR)
# The eval harness (lmms_eval llava_vid, overwrite=True) merges this
# overwrite_config on top of the checkpoint config; the builder adds
# additional_config (vocab/pad must match the base checkpoint's embeddings).
for key, value in {
"mm_spatial_pool_stride": 2,
"mm_spatial_pool_mode": "bilinear",
"mm_pooling_position": "before",
"mm_newline_position": "grid",
"delay_load": False,
"tie_word_embeddings": False,
"use_cache": True,
"vocab_size": 152064,
}.items():
setattr(lora_cfg, key, value)
# The authors' checkpoint config.json carries `text_config` as a plain dict
# ({'model_type': 'llama'}), which is neither a PretrainedConfig nor a known
# sub-config of LlavaQwenConfig. transformers' get_text_config(decoder=True)
# searches ("decoder", "generator", "text_config") and returns the first
# non-None hit, so this dict is handed to GenerationConfig.from_model_config,
# which calls .to_dict() on it -> AttributeError. Setting it to None makes
# get_text_config skip it and fall back to the config itself.
lora_cfg.text_config = None
print("[mvrd] loading tokenizer...", flush=True)
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, use_fast=False)
print("[mvrd] loading base model + vision tower...", flush=True)
model = LlavaQwenForCausalLM.from_pretrained(
BASE_MODEL,
low_cpu_mem_usage=True,
config=lora_cfg,
attn_implementation="sdpa",
torch_dtype=torch.float16,
)
# Builder guard: reallocate lm_head/embeddings only if the base checkpoint's
# shapes disagree with the config (no-op when vocab_size matches, 152064 here).
token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
if model.lm_head.weight.shape[0] != token_num:
model.lm_head.weight = torch.nn.Parameter(
torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype)
)
model.model.embed_tokens.weight = torch.nn.Parameter(
torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype)
)
print("[mvrd] loading non-LoRA trainables (mm_projector)...", flush=True)
non_lora_trainables = torch.load(
os.path.join(CKPT_DIR, "non_lora_trainables.bin"), map_location="cpu"
)
non_lora_trainables = {
(k[11:] if k.startswith("base_model.") else k): v
for k, v in non_lora_trainables.items()
}
if any(k.startswith("model.model.") for k in non_lora_trainables):
non_lora_trainables = {
(k[6:] if k.startswith("model.") else k): v
for k, v in non_lora_trainables.items()
}
model.load_state_dict(non_lora_trainables, strict=False)
print("[mvrd] merging LoRA adapter...", flush=True)
model = PeftModel.from_pretrained(model, CKPT_DIR)
model = model.merge_and_unload()
# The authors' Qwen2 fork registers a distillation-only `matching_head`
# (hidden -> hidden -> mlp_out_dim) used only to align student hidden states
# with VGGT teacher features during training. Its weights are absent from the
# base checkpoint, so with low_cpu_mem_usage=True those parameters stay on the
# meta device; copying into them is a no-op and `.to("cuda")` then crashes
# ("Cannot copy out of meta tensor; no data!"). The head lives on
# `Qwen2ForCausalLM` (not the LLaVA wrapper) and is only touched inside
# `if features_3d is not None:` in its forward — the spatial tower that
# supplies `features_3d` is never built at inference (the checkpoint config
# has no `spatial_tower` attribute), and `generate()` never passes it. The
# LoRA adapter (language-model layers only) does not wrap it either, so
# dropping it here is safe and keeps the real weights intact.
if hasattr(model, "matching_head"):
del model.matching_head
model.resize_token_embeddings(len(tokenizer))
vision_tower = model.get_vision_tower()
if not vision_tower.is_loaded:
vision_tower.load_model()
image_processor = vision_tower.image_processor
model = model.to(torch.float16)
model.eval()
model.to("cuda")
print("[mvrd] model ready on ZeroGPU.", flush=True)
# --------------------------------------------------------------------------- #
# Inference — replicates the authors' lmms_eval llava_vid generate_until #
# --------------------------------------------------------------------------- #
def load_video(video_path: str, max_frames_num: int, fps: int = 1, force_sample: bool = False):
"""Sample frames from a video with decord, exactly like the eval harness.
1-fps stride sampling; if more frames than `max_frames_num` (or
`force_sample`), uniformly sample `max_frames_num` frames across the video.
Returns (frames ndarray [T,H,W,C] uint8, duration in seconds).
"""
vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
total_frame_num = len(vr)
video_time = total_frame_num / vr.get_avg_fps()
fps_stride = round(vr.get_avg_fps() / fps)
frame_idx = list(range(0, len(vr), fps_stride))
if len(frame_idx) > max_frames_num or force_sample:
frame_idx = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int).tolist()
return vr.get_batch(frame_idx).asnumpy(), video_time
def pad_process_pil(pil_image: Image.Image):
"""Process one image with the 'pad' aspect ratio (expand2square + resize)."""
pil_image = expand2square(
pil_image, tuple(int(x * 255) for x in image_processor.image_mean)
)
return image_processor.preprocess(pil_image, return_tensors="pt")["pixel_values"][0]
@spaces.GPU(duration=60)
def answer(
video: str | None,
question: str,
images: list[str] | None = None,
max_new_tokens: int = 64,
temperature: float = 0.0,
max_frames_num: int = 32,
force_sample: bool = False,
progress: gr.Progress = gr.Progress(),
) -> tuple[str, str]:
"""Answer a spatial-reasoning question about a scene walk-through video or
one or more images using the MVRD student VLM.
Ask about object presence and counting, relative directions (left/right,
front/behind), distances between objects (in meters), room size, or route
planning. If both a video and images are supplied, the video takes
precedence.
Args:
video: path to a scene walk-through video (or None).
question: the spatial-reasoning question to answer.
images: paths to one or more images of the scene (used if no video).
max_new_tokens: maximum number of tokens to generate.
temperature: sampling temperature; 0 means greedy decoding.
max_frames_num: number of video frames to sample.
force_sample: always resample to exactly max_frames_num frames.
Returns:
A tuple of (answer text, status line).
"""
t0 = time.time()
question = (question or "").strip()
if not question:
raise gr.Error("Please enter a question.")
if video:
progress(0.2, desc="Sampling video frames")
video_np, video_time = load_video(
video, int(max_frames_num), fps=1, force_sample=bool(force_sample)
)
video_tensor = (
image_processor.preprocess(video_np, return_tensors="pt")["pixel_values"]
.half()
.to("cuda")
)
images_arg = [video_tensor]
image_sizes = None
modality = "video"
n_tokens = 1
media_note = f"video: {len(video_np)} frames, {video_time:.1f}s"
elif images:
progress(0.2, desc="Encoding images")
pils = [Image.open(p).convert("RGB") for p in images]
image_sizes = [im.size for im in pils]
if len(pils) > 1:
# Multi-image: each image is its own media item (pad -> one tile).
images_arg = [pad_process_pil(im).half().to("cuda") for im in pils]
else:
# Single image: anyres_max_9 processing via the reference path.
images_arg = process_images(pils, image_processor, model.config)
images_arg = images_arg.half().to("cuda")
modality = ["image"] * len(pils)
n_tokens = len(pils)
media_note = f"{len(pils)} image(s)"
else:
raise gr.Error("Upload a video or at least one image first.")
# Prepend one <image> placeholder per media item (the reference harness
# inserts these before the question; without them `llava_arch` truncates
# the visual features to zero tokens and the model never sees the media).
qs = DEFAULT_IMAGE_TOKEN * n_tokens + "\n" + question
conv = conv_templates["qwen_1_5"].copy()
conv.append_message(conv.roles[0], qs)
conv.append_message(conv.roles[1], None)
prompt = conv.get_prompt()
input_ids = tokenizer_image_token(
prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt"
).unsqueeze(0).to("cuda")
pad_token_ids = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id
attention_masks = input_ids.ne(pad_token_ids).long().to("cuda")
stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
stopping_criteria = KeywordsStoppingCriteria([stop_str], tokenizer, input_ids)
generate_kwargs = dict(
inputs=input_ids,
images=images_arg,
attention_mask=attention_masks,
modalities=modality,
use_cache=True,
stopping_criteria=[stopping_criteria],
do_sample=temperature > 0,
num_beams=1,
max_new_tokens=int(max_new_tokens),
)
if image_sizes is not None:
generate_kwargs["image_sizes"] = image_sizes
if temperature > 0:
generate_kwargs["temperature"] = temperature
progress(0.5, desc="Generating answer")
with torch.inference_mode():
output_ids = model.generate(**generate_kwargs)
output = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
elapsed = time.time() - t0
status = f"⏱ answered in **{elapsed:.1f}s** on ZeroGPU • input: {media_note}"
return output, status
# --------------------------------------------------------------------------- #
# UI #
# --------------------------------------------------------------------------- #
CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
TITLE = """
# 🧭 MVRD — Spatial Reasoning with Vision-Language Models
**Multi-View Relational Distillation** ([paper](https://huggingface.co/papers/2608.10864),
[code](https://github.com/kietngt00/MVRD)) distills 3D multi-view spatial relations from
VGGT into a 7B VLM. This Space runs the distilled **student model**
([LLaVA-Video-7B-Qwen2](https://huggingface.co/lmms-lab/LLaVA-Video-7B-Qwen2) + the authors'
released LoRA checkpoint) and answers spatial questions about a scene walk-through
video or a set of images: counting, room size, distances, relative directions.
"""
with gr.Blocks() as demo:
gr.Markdown(TITLE)
with gr.Row():
with gr.Column():
video_in = gr.Video(label="Scene walk-through video")
images_in = gr.File(
label="…or image(s) of the scene",
file_count="multiple",
file_types=["image"],
)
question_in = gr.Textbox(
label="Question",
placeholder="e.g. What is the size of this room (in square meters)?",
lines=3,
)
run_btn = gr.Button("Answer", variant="primary")
with gr.Column():
answer_out = gr.Textbox(label="Answer", lines=4, buttons=["copy"])
status_out = gr.Markdown("Upload a video or images, type a question, press **Answer**.")
with gr.Accordion("Advanced options", open=False):
with gr.Row():
max_new_tokens_in = gr.Slider(
minimum=1, maximum=1024, value=64, step=1,
label="Max new tokens",
)
temperature_in = gr.Slider(
minimum=0.0, maximum=1.0, value=0.0, step=0.05,
label="Temperature (0 = greedy)",
)
with gr.Row():
max_frames_in = gr.Slider(
minimum=1, maximum=64, value=32, step=1,
label="Video frames sampled",
)
force_sample_in = gr.Checkbox(
value=False,
label="Always resample to the target frame count",
)
gr.Examples(
examples=[
[
"examples/41069025.mp4",
"These are frames of a video.\nHow many table(s) are in this room?\nPlease answer the question using a single word or phrase.",
],
[
"examples/41069025.mp4",
"These are frames of a video.\nWhat is the size of this room (in square meters)? \nIf multiple rooms are shown, estimate the size of the combined space.\nPlease answer the question using a single word or phrase.",
],
[
"examples/41069025.mp4",
"These are frames of a video.\nMeasuring from the closest point of each object, what is the distance between the sofa and the stove (in meters)?\nPlease answer the question using a single word or phrase.",
],
[
"examples/41069025.mp4",
"These are frames of a video.\nIf I am standing by the stove and facing the sofa, is the tv to the left or to the right of the sofa?\nOptions:\nA. left\nB. right\nAnswer with the option's letter from the given choices directly.",
],
[
"examples/scene0461_00.mp4",
"These are frames of a video.\nWhat is the size of this room (in square meters)? \nIf multiple rooms are shown, estimate the size of the combined space.\nPlease answer the question using a single word or phrase.",
],
],
inputs=[video_in, question_in],
outputs=[answer_out, status_out],
fn=answer,
cache_examples=True,
cache_mode="lazy",
label="VSI-Bench examples (ARKitScenes scene 41069025, ScanNet scene0461_00)",
)
run_btn.click(
fn=answer,
inputs=[video_in, question_in, images_in, max_new_tokens_in, temperature_in, max_frames_in, force_sample_in],
outputs=[answer_out, status_out],
api_name="answer",
)
if __name__ == "__main__":
demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)