Spaces:
Running on Zero
Running on Zero
multimodalart HF Staff
Right-size ZeroGPU duration to 60s: measured worst case (1024 tokens, 64 frames) = 38.3s; default example path = 6.9s
6c78ed6 verified Download app.py from hugging-apps/mvrd-spatial-reasoning: direct link, hf CLI and curl.
- Browser
- Download file 18.3 kB
-
https://huggingface.co/spaces/hugging-apps/mvrd-spatial-reasoning/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/mvrd-spatial-reasoning/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/mvrd-spatial-reasoning/resolve/main/app.py
18.3 kB
| """ | |
| MVRD — Multi-View Relational Distillation for Spatial Reasoning with Vision-Language Models. | |
| Interactive demo of the MVRD student VLM: LLaVA-Video-7B-Qwen2 fine-tuned with a | |
| LoRA adapter distilled from VGGT multi-view spatial relations (checkpoint | |
| `vggt_sim_distill_lora_l21_lambda0.5_mlp256_GELU_nobiasall_aug2/checkpoint-1622`, | |
| released by the authors at https://huggingface.co/kietngt00/MVRD). | |
| Ask spatial-reasoning questions about a scene walk-through video or a set of | |
| images: object presence and counting, relative directions (left/right, | |
| front/behind), distances between objects, room size, route planning, etc. | |
| The loading recipe replicates the authors' `llava/model/builder.py` + | |
| `lmms_eval` harness (`llava_vid` model, `qwen_1_5` conv template) 1:1, adapted | |
| for ZeroGPU (module-scope eager load, no accelerate device_map). | |
| """ | |
| import spaces # noqa: F401 # must be imported before torch on ZeroGPU | |
| import os | |
| import shutil | |
| import tarfile | |
| import time | |
| import numpy as np | |
| import torch | |
| import gradio as gr | |
| from PIL import Image | |
| from transformers import AutoTokenizer | |
| from huggingface_hub import hf_hub_download | |
| from peft import PeftModel | |
| from decord import VideoReader, cpu | |
| from llava.model.language_model.llava_qwen import LlavaQwenConfig, LlavaQwenForCausalLM | |
| from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN | |
| from llava.conversation import conv_templates, SeparatorStyle | |
| from llava.mm_utils import ( | |
| process_images, | |
| tokenizer_image_token, | |
| KeywordsStoppingCriteria, | |
| expand2square, | |
| ) | |
| BASE_MODEL = "lmms-lab/LLaVA-Video-7B-Qwen2" | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| CKPT_DIR = os.path.join(HERE, "ckpt") | |
| # The authors release the distilled student as a LoRA adapter inside a tarball | |
| # on their model repo (kietngt00/MVRD). We fetch just the checkpoint files we | |
| # need at startup instead of bundling them in this Space. | |
| CKPT_REPO = "kietngt00/MVRD" | |
| CKPT_TARBALL = "mvrd.tar.gz" | |
| CKPT_PREFIX = "vggt_sim_distill_lora_l21_lambda0.5_mlp256_GELU_nobiasall_aug2/checkpoint-1622/" | |
| CKPT_FILES = { | |
| "adapter_config.json", | |
| "adapter_model.bin", | |
| "config.json", | |
| "generation_config.json", | |
| "non_lora_trainables.bin", | |
| } | |
| def fetch_mvrd_checkpoint(ckpt_dir: str) -> None: | |
| """Download the authors' released MVRD LoRA checkpoint and extract the | |
| files needed for inference from their tarball into `ckpt_dir`.""" | |
| if all(os.path.exists(os.path.join(ckpt_dir, f)) for f in CKPT_FILES): | |
| return | |
| print(f"[mvrd] fetching {CKPT_TARBALL} from {CKPT_REPO} ...", flush=True) | |
| t0 = time.time() | |
| tar_path = hf_hub_download(repo_id=CKPT_REPO, filename=CKPT_TARBALL) | |
| print(f"[mvrd] downloaded in {time.time() - t0:.1f}s, extracting ...", flush=True) | |
| os.makedirs(ckpt_dir, exist_ok=True) | |
| with tarfile.open(tar_path, "r:gz") as tar: | |
| for member in tar: | |
| name = member.name[2:] if member.name.startswith("./") else member.name | |
| if not name.startswith(CKPT_PREFIX) or not member.isfile(): | |
| continue | |
| base = name[len(CKPT_PREFIX):] | |
| if base not in CKPT_FILES: | |
| continue | |
| src = tar.extractfile(member) | |
| if src is None: | |
| continue | |
| with open(os.path.join(ckpt_dir, base), "wb") as dst: | |
| shutil.copyfileobj(src, dst) | |
| print(f"[mvrd] extracted {base}", flush=True) | |
| missing = [f for f in CKPT_FILES if not os.path.exists(os.path.join(ckpt_dir, f))] | |
| if missing: | |
| raise RuntimeError(f"checkpoint extraction incomplete, missing: {missing}") | |
| # --------------------------------------------------------------------------- # | |
| # Model loading (module scope, eager) — replicates the authors' builder # | |
| # --------------------------------------------------------------------------- # | |
| fetch_mvrd_checkpoint(CKPT_DIR) | |
| print("[mvrd] loading config...", flush=True) | |
| lora_cfg = LlavaQwenConfig.from_pretrained(CKPT_DIR) | |
| # The eval harness (lmms_eval llava_vid, overwrite=True) merges this | |
| # overwrite_config on top of the checkpoint config; the builder adds | |
| # additional_config (vocab/pad must match the base checkpoint's embeddings). | |
| for key, value in { | |
| "mm_spatial_pool_stride": 2, | |
| "mm_spatial_pool_mode": "bilinear", | |
| "mm_pooling_position": "before", | |
| "mm_newline_position": "grid", | |
| "delay_load": False, | |
| "tie_word_embeddings": False, | |
| "use_cache": True, | |
| "vocab_size": 152064, | |
| }.items(): | |
| setattr(lora_cfg, key, value) | |
| # The authors' checkpoint config.json carries `text_config` as a plain dict | |
| # ({'model_type': 'llama'}), which is neither a PretrainedConfig nor a known | |
| # sub-config of LlavaQwenConfig. transformers' get_text_config(decoder=True) | |
| # searches ("decoder", "generator", "text_config") and returns the first | |
| # non-None hit, so this dict is handed to GenerationConfig.from_model_config, | |
| # which calls .to_dict() on it -> AttributeError. Setting it to None makes | |
| # get_text_config skip it and fall back to the config itself. | |
| lora_cfg.text_config = None | |
| print("[mvrd] loading tokenizer...", flush=True) | |
| tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, use_fast=False) | |
| print("[mvrd] loading base model + vision tower...", flush=True) | |
| model = LlavaQwenForCausalLM.from_pretrained( | |
| BASE_MODEL, | |
| low_cpu_mem_usage=True, | |
| config=lora_cfg, | |
| attn_implementation="sdpa", | |
| torch_dtype=torch.float16, | |
| ) | |
| # Builder guard: reallocate lm_head/embeddings only if the base checkpoint's | |
| # shapes disagree with the config (no-op when vocab_size matches, 152064 here). | |
| token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features | |
| if model.lm_head.weight.shape[0] != token_num: | |
| model.lm_head.weight = torch.nn.Parameter( | |
| torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype) | |
| ) | |
| model.model.embed_tokens.weight = torch.nn.Parameter( | |
| torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype) | |
| ) | |
| print("[mvrd] loading non-LoRA trainables (mm_projector)...", flush=True) | |
| non_lora_trainables = torch.load( | |
| os.path.join(CKPT_DIR, "non_lora_trainables.bin"), map_location="cpu" | |
| ) | |
| non_lora_trainables = { | |
| (k[11:] if k.startswith("base_model.") else k): v | |
| for k, v in non_lora_trainables.items() | |
| } | |
| if any(k.startswith("model.model.") for k in non_lora_trainables): | |
| non_lora_trainables = { | |
| (k[6:] if k.startswith("model.") else k): v | |
| for k, v in non_lora_trainables.items() | |
| } | |
| model.load_state_dict(non_lora_trainables, strict=False) | |
| print("[mvrd] merging LoRA adapter...", flush=True) | |
| model = PeftModel.from_pretrained(model, CKPT_DIR) | |
| model = model.merge_and_unload() | |
| # The authors' Qwen2 fork registers a distillation-only `matching_head` | |
| # (hidden -> hidden -> mlp_out_dim) used only to align student hidden states | |
| # with VGGT teacher features during training. Its weights are absent from the | |
| # base checkpoint, so with low_cpu_mem_usage=True those parameters stay on the | |
| # meta device; copying into them is a no-op and `.to("cuda")` then crashes | |
| # ("Cannot copy out of meta tensor; no data!"). The head lives on | |
| # `Qwen2ForCausalLM` (not the LLaVA wrapper) and is only touched inside | |
| # `if features_3d is not None:` in its forward — the spatial tower that | |
| # supplies `features_3d` is never built at inference (the checkpoint config | |
| # has no `spatial_tower` attribute), and `generate()` never passes it. The | |
| # LoRA adapter (language-model layers only) does not wrap it either, so | |
| # dropping it here is safe and keeps the real weights intact. | |
| if hasattr(model, "matching_head"): | |
| del model.matching_head | |
| model.resize_token_embeddings(len(tokenizer)) | |
| vision_tower = model.get_vision_tower() | |
| if not vision_tower.is_loaded: | |
| vision_tower.load_model() | |
| image_processor = vision_tower.image_processor | |
| model = model.to(torch.float16) | |
| model.eval() | |
| model.to("cuda") | |
| print("[mvrd] model ready on ZeroGPU.", flush=True) | |
| # --------------------------------------------------------------------------- # | |
| # Inference — replicates the authors' lmms_eval llava_vid generate_until # | |
| # --------------------------------------------------------------------------- # | |
| def load_video(video_path: str, max_frames_num: int, fps: int = 1, force_sample: bool = False): | |
| """Sample frames from a video with decord, exactly like the eval harness. | |
| 1-fps stride sampling; if more frames than `max_frames_num` (or | |
| `force_sample`), uniformly sample `max_frames_num` frames across the video. | |
| Returns (frames ndarray [T,H,W,C] uint8, duration in seconds). | |
| """ | |
| vr = VideoReader(video_path, ctx=cpu(0), num_threads=1) | |
| total_frame_num = len(vr) | |
| video_time = total_frame_num / vr.get_avg_fps() | |
| fps_stride = round(vr.get_avg_fps() / fps) | |
| frame_idx = list(range(0, len(vr), fps_stride)) | |
| if len(frame_idx) > max_frames_num or force_sample: | |
| frame_idx = np.linspace(0, total_frame_num - 1, max_frames_num, dtype=int).tolist() | |
| return vr.get_batch(frame_idx).asnumpy(), video_time | |
| def pad_process_pil(pil_image: Image.Image): | |
| """Process one image with the 'pad' aspect ratio (expand2square + resize).""" | |
| pil_image = expand2square( | |
| pil_image, tuple(int(x * 255) for x in image_processor.image_mean) | |
| ) | |
| return image_processor.preprocess(pil_image, return_tensors="pt")["pixel_values"][0] | |
| def answer( | |
| video: str | None, | |
| question: str, | |
| images: list[str] | None = None, | |
| max_new_tokens: int = 64, | |
| temperature: float = 0.0, | |
| max_frames_num: int = 32, | |
| force_sample: bool = False, | |
| progress: gr.Progress = gr.Progress(), | |
| ) -> tuple[str, str]: | |
| """Answer a spatial-reasoning question about a scene walk-through video or | |
| one or more images using the MVRD student VLM. | |
| Ask about object presence and counting, relative directions (left/right, | |
| front/behind), distances between objects (in meters), room size, or route | |
| planning. If both a video and images are supplied, the video takes | |
| precedence. | |
| Args: | |
| video: path to a scene walk-through video (or None). | |
| question: the spatial-reasoning question to answer. | |
| images: paths to one or more images of the scene (used if no video). | |
| max_new_tokens: maximum number of tokens to generate. | |
| temperature: sampling temperature; 0 means greedy decoding. | |
| max_frames_num: number of video frames to sample. | |
| force_sample: always resample to exactly max_frames_num frames. | |
| Returns: | |
| A tuple of (answer text, status line). | |
| """ | |
| t0 = time.time() | |
| question = (question or "").strip() | |
| if not question: | |
| raise gr.Error("Please enter a question.") | |
| if video: | |
| progress(0.2, desc="Sampling video frames") | |
| video_np, video_time = load_video( | |
| video, int(max_frames_num), fps=1, force_sample=bool(force_sample) | |
| ) | |
| video_tensor = ( | |
| image_processor.preprocess(video_np, return_tensors="pt")["pixel_values"] | |
| .half() | |
| .to("cuda") | |
| ) | |
| images_arg = [video_tensor] | |
| image_sizes = None | |
| modality = "video" | |
| n_tokens = 1 | |
| media_note = f"video: {len(video_np)} frames, {video_time:.1f}s" | |
| elif images: | |
| progress(0.2, desc="Encoding images") | |
| pils = [Image.open(p).convert("RGB") for p in images] | |
| image_sizes = [im.size for im in pils] | |
| if len(pils) > 1: | |
| # Multi-image: each image is its own media item (pad -> one tile). | |
| images_arg = [pad_process_pil(im).half().to("cuda") for im in pils] | |
| else: | |
| # Single image: anyres_max_9 processing via the reference path. | |
| images_arg = process_images(pils, image_processor, model.config) | |
| images_arg = images_arg.half().to("cuda") | |
| modality = ["image"] * len(pils) | |
| n_tokens = len(pils) | |
| media_note = f"{len(pils)} image(s)" | |
| else: | |
| raise gr.Error("Upload a video or at least one image first.") | |
| # Prepend one <image> placeholder per media item (the reference harness | |
| # inserts these before the question; without them `llava_arch` truncates | |
| # the visual features to zero tokens and the model never sees the media). | |
| qs = DEFAULT_IMAGE_TOKEN * n_tokens + "\n" + question | |
| conv = conv_templates["qwen_1_5"].copy() | |
| conv.append_message(conv.roles[0], qs) | |
| conv.append_message(conv.roles[1], None) | |
| prompt = conv.get_prompt() | |
| input_ids = tokenizer_image_token( | |
| prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt" | |
| ).unsqueeze(0).to("cuda") | |
| pad_token_ids = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id | |
| attention_masks = input_ids.ne(pad_token_ids).long().to("cuda") | |
| stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 | |
| stopping_criteria = KeywordsStoppingCriteria([stop_str], tokenizer, input_ids) | |
| generate_kwargs = dict( | |
| inputs=input_ids, | |
| images=images_arg, | |
| attention_mask=attention_masks, | |
| modalities=modality, | |
| use_cache=True, | |
| stopping_criteria=[stopping_criteria], | |
| do_sample=temperature > 0, | |
| num_beams=1, | |
| max_new_tokens=int(max_new_tokens), | |
| ) | |
| if image_sizes is not None: | |
| generate_kwargs["image_sizes"] = image_sizes | |
| if temperature > 0: | |
| generate_kwargs["temperature"] = temperature | |
| progress(0.5, desc="Generating answer") | |
| with torch.inference_mode(): | |
| output_ids = model.generate(**generate_kwargs) | |
| output = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip() | |
| elapsed = time.time() - t0 | |
| status = f"⏱ answered in **{elapsed:.1f}s** on ZeroGPU • input: {media_note}" | |
| return output, status | |
| # --------------------------------------------------------------------------- # | |
| # UI # | |
| # --------------------------------------------------------------------------- # | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| TITLE = """ | |
| # 🧭 MVRD — Spatial Reasoning with Vision-Language Models | |
| **Multi-View Relational Distillation** ([paper](https://huggingface.co/papers/2608.10864), | |
| [code](https://github.com/kietngt00/MVRD)) distills 3D multi-view spatial relations from | |
| VGGT into a 7B VLM. This Space runs the distilled **student model** | |
| ([LLaVA-Video-7B-Qwen2](https://huggingface.co/lmms-lab/LLaVA-Video-7B-Qwen2) + the authors' | |
| released LoRA checkpoint) and answers spatial questions about a scene walk-through | |
| video or a set of images: counting, room size, distances, relative directions. | |
| """ | |
| with gr.Blocks() as demo: | |
| gr.Markdown(TITLE) | |
| with gr.Row(): | |
| with gr.Column(): | |
| video_in = gr.Video(label="Scene walk-through video") | |
| images_in = gr.File( | |
| label="…or image(s) of the scene", | |
| file_count="multiple", | |
| file_types=["image"], | |
| ) | |
| question_in = gr.Textbox( | |
| label="Question", | |
| placeholder="e.g. What is the size of this room (in square meters)?", | |
| lines=3, | |
| ) | |
| run_btn = gr.Button("Answer", variant="primary") | |
| with gr.Column(): | |
| answer_out = gr.Textbox(label="Answer", lines=4, buttons=["copy"]) | |
| status_out = gr.Markdown("Upload a video or images, type a question, press **Answer**.") | |
| with gr.Accordion("Advanced options", open=False): | |
| with gr.Row(): | |
| max_new_tokens_in = gr.Slider( | |
| minimum=1, maximum=1024, value=64, step=1, | |
| label="Max new tokens", | |
| ) | |
| temperature_in = gr.Slider( | |
| minimum=0.0, maximum=1.0, value=0.0, step=0.05, | |
| label="Temperature (0 = greedy)", | |
| ) | |
| with gr.Row(): | |
| max_frames_in = gr.Slider( | |
| minimum=1, maximum=64, value=32, step=1, | |
| label="Video frames sampled", | |
| ) | |
| force_sample_in = gr.Checkbox( | |
| value=False, | |
| label="Always resample to the target frame count", | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| [ | |
| "examples/41069025.mp4", | |
| "These are frames of a video.\nHow many table(s) are in this room?\nPlease answer the question using a single word or phrase.", | |
| ], | |
| [ | |
| "examples/41069025.mp4", | |
| "These are frames of a video.\nWhat is the size of this room (in square meters)? \nIf multiple rooms are shown, estimate the size of the combined space.\nPlease answer the question using a single word or phrase.", | |
| ], | |
| [ | |
| "examples/41069025.mp4", | |
| "These are frames of a video.\nMeasuring from the closest point of each object, what is the distance between the sofa and the stove (in meters)?\nPlease answer the question using a single word or phrase.", | |
| ], | |
| [ | |
| "examples/41069025.mp4", | |
| "These are frames of a video.\nIf I am standing by the stove and facing the sofa, is the tv to the left or to the right of the sofa?\nOptions:\nA. left\nB. right\nAnswer with the option's letter from the given choices directly.", | |
| ], | |
| [ | |
| "examples/scene0461_00.mp4", | |
| "These are frames of a video.\nWhat is the size of this room (in square meters)? \nIf multiple rooms are shown, estimate the size of the combined space.\nPlease answer the question using a single word or phrase.", | |
| ], | |
| ], | |
| inputs=[video_in, question_in], | |
| outputs=[answer_out, status_out], | |
| fn=answer, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="VSI-Bench examples (ARKitScenes scene 41069025, ScanNet scene0461_00)", | |
| ) | |
| run_btn.click( | |
| fn=answer, | |
| inputs=[video_in, question_in, images_in, max_new_tokens_in, temperature_in, max_frames_in, force_sample_in], | |
| outputs=[answer_out, status_out], | |
| api_name="answer", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS) | |