Spaces:
Running on Zero
Running on Zero
| import os | |
| # Allocator config: avoid fragmentation OOM under transient activation spikes. | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| # VGGT feature-layer selection must be set before any qwenvl import: | |
| # qwenvl/geometry/constants.py bakes GEOMETRY_PATCH_DIM at import time and | |
| # the fusion block's patch_mapping input dim must match (2048 for "23"). | |
| os.environ.setdefault("VGGT_MULTI_LAYER_INDICES", "23") | |
| import spaces # MUST come before torch / any CUDA-touching import | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| # Make the bundled qwenvl package (repo root) importable. | |
| _APP_DIR = Path(__file__).resolve().parent | |
| if str(_APP_DIR) not in sys.path: | |
| sys.path.insert(0, str(_APP_DIR)) | |
| from transformers import AutoProcessor, AutoTokenizer | |
| from qwenvl.eval.evaluate_geometry_bench import EvalBuilder | |
| from qwenvl.geometry.modeling_qwen3vl_geometry import ( | |
| Qwen3VLGeometryForConditionalGeneration, | |
| ) | |
| CHECKPOINT = "yuqun/R2R-Qwen3-VL-8B" | |
| BASE_MODEL = "Qwen/Qwen3-VL-8B-Instruct" | |
| MIN_PIXELS = 784 | |
| MAX_PIXELS = 200704 | |
| DEFAULT_NUM_FRAMES = 32 # the 8B checkpoint was trained with 32 input frames | |
| print(f"Loading {CHECKPOINT} ...", flush=True) | |
| model = Qwen3VLGeometryForConditionalGeneration.from_pretrained( | |
| CHECKPOINT, dtype=torch.bfloat16 | |
| ) | |
| if model.geometry_tower is not None: | |
| model.geometry_tower.load_model() # downloads facebook/VGGT-1B (~5 GB) | |
| model.geometry_tower.to(torch.bfloat16) | |
| model = model.eval().to("cuda") | |
| tokenizer = AutoTokenizer.from_pretrained(CHECKPOINT, use_fast=False) | |
| if tokenizer.pad_token_id is None: | |
| tokenizer.pad_token_id = tokenizer.eos_token_id | |
| processor = AutoProcessor.from_pretrained(BASE_MODEL) | |
| # EvalBuilder mirrors the authors' eval pipeline: it re-configures the Qwen | |
| # processor pixel budget (784..200704 per frame) and builds the multi-frame | |
| # prompt with <t seconds> markers + <image> tokens. | |
| builder = EvalBuilder(processor, tokenizer, DEFAULT_NUM_FRAMES, MIN_PIXELS, MAX_PIXELS) | |
| print("Model ready.", flush=True) | |
| def _load_video_frames(path, num_frames): | |
| """Sample up to num_frames evenly-spaced frames + their timestamps.""" | |
| from decord import VideoReader | |
| vr = VideoReader(path, num_threads=1) | |
| idx = np.linspace(0, len(vr) - 1, min(num_frames, len(vr)), dtype=int) | |
| frames = [Image.fromarray(f) for f in vr.get_batch(idx).asnumpy()] | |
| times = [i / vr.get_avg_fps() for i in idx] | |
| return frames, times | |
| def _load_image_frames(paths, num_frames, fps=1.0): | |
| """Load a sorted set of view images + synthetic 1 fps timestamps.""" | |
| idx = np.linspace(0, len(paths) - 1, min(num_frames, len(paths)), dtype=int) | |
| frames = [Image.open(paths[i]).convert("RGB") for i in idx] | |
| times = [i / fps for i in idx] | |
| return frames, times | |
| def answer_question( | |
| video, | |
| question, | |
| images=None, | |
| num_frames=DEFAULT_NUM_FRAMES, | |
| max_new_tokens=128, | |
| progress=gr.Progress(), | |
| ): | |
| """Answer a spatial question about a video (or a set of multi-view images). | |
| Args: | |
| video: an uploaded video file. | |
| question: the question to ask about the scene, e.g. "What is behind | |
| the woman on the wall?" | |
| images: optionally, a set of multi-view images instead of a video. | |
| num_frames: how many frames to sample from the video (default 32, | |
| the frame count the model was trained with). | |
| max_new_tokens: maximum length of the generated answer. | |
| Returns: | |
| A short text answer to the question. | |
| """ | |
| if not question or not str(question).strip(): | |
| raise gr.Error("Please enter a question.") | |
| question = str(question).strip() | |
| progress(0.1, desc="Decoding input") | |
| if video is not None: | |
| frames, times = _load_video_frames(video, int(num_frames)) | |
| elif images: | |
| paths = sorted(p if isinstance(p, str) else str(p.name) for p in images) | |
| frames, times = _load_image_frames(paths, int(num_frames)) | |
| else: | |
| raise gr.Error("Please upload a video or a set of images first.") | |
| n = len(frames) | |
| progress(0.35, desc="Building geometry-aware prompt") | |
| markers = [f"<{t:.1f} seconds>" for t in times] | |
| prompt_inputs = builder.image_prompt(frames, question, markers) | |
| # Geometry pixels for the VGGT tower, on the same Qwen patch grid. | |
| geometry_pixels = builder._geometry_pixels(frames) | |
| progress(0.55, desc="Reasoning over the scene") | |
| device = model.device | |
| kw = {k: v.to(device) for k, v in prompt_inputs.items()} | |
| t0 = time.perf_counter() | |
| out = model.generate( | |
| **kw, | |
| geometry_pixel_values=geometry_pixels.to(device=device, dtype=torch.bfloat16), | |
| # Zero placeholder embeds: the model forward regenerates the VGGT | |
| # camera/patch features from geometry_pixel_values itself (the same | |
| # path the training data takes via _zero_geometry). | |
| geometry_patch_embeds=torch.zeros(n, 1, 1, device=device), | |
| geometry_camera_embeds=torch.zeros(n, 1, 1, device=device), | |
| post_decoder_num_frames=torch.tensor([n], device=device), | |
| semantic_target_frame_idx=torch.tensor([-1], device=device), | |
| max_new_tokens=int(max_new_tokens), | |
| do_sample=False, | |
| use_cache=True, | |
| ) | |
| elapsed = time.perf_counter() - t0 | |
| gen_ids = out[0, kw["input_ids"].shape[1] :] | |
| answer = tokenizer.decode(gen_ids, skip_special_tokens=True).strip() | |
| status = f"Answered in **{elapsed:.1f} s** using {n} frames." | |
| return answer, status | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="Render to Reason") as demo: | |
| gr.HTML( | |
| """ | |
| <h1 style="text-align: center; font-size: 1.6rem; margin-bottom: 0.2rem;"> | |
| 🧭 Render to Reason | |
| </h1> | |
| <p style="text-align: center; opacity: 0.85; margin-top: 0;"> | |
| Spatial understanding for video VLMs: Qwen3-VL fused with a VGGT geometry tower | |
| that predicts novel-view semantics and camera geometry. | |
| </p> | |
| <p style="text-align: center; font-size: 0.85rem; opacity: 0.75; margin-top: 0.2rem;"> | |
| <a href="https://huggingface.co/papers/2610.05417">Paper</a> · | |
| <a href="https://github.com/yuqunw/R2R">GitHub</a> · | |
| <a href="https://huggingface.co/yuqun/R2R-Qwen3-VL-8B">Model weights</a> | |
| </p> | |
| """ | |
| ) | |
| with gr.Column(elem_id="col-container"): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| video_in = gr.Video(label="Video", sources=["upload"]) | |
| images_in = gr.File( | |
| label="…or a set of multi-view images", | |
| file_count="multiple", | |
| file_types=["image"], | |
| ) | |
| question_in = gr.Textbox( | |
| label="Question", | |
| placeholder="e.g. What is behind the woman on the wall?", | |
| lines=2, | |
| ) | |
| run_btn = gr.Button("Answer", variant="primary") | |
| with gr.Column(scale=1): | |
| answer_out = gr.Textbox(label="Answer", lines=4) | |
| status_out = gr.Markdown() | |
| with gr.Accordion("Advanced settings", open=False): | |
| num_frames_in = gr.Slider( | |
| 4, 48, value=DEFAULT_NUM_FRAMES, step=2, | |
| label="Frames sampled from the video", | |
| info="The 8B model was trained with 32 frames.", | |
| ) | |
| max_new_tokens_in = gr.Slider( | |
| 16, 512, value=128, step=16, label="Max new tokens" | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| [ | |
| "examples/podcaster.mp4", | |
| "What is behind the woman on the wall? Answer in a few words.", | |
| ], | |
| [ | |
| "examples/pouring_coffee.mp4", | |
| "On what does the grey cup sit, and what is next to it?", | |
| ], | |
| [ | |
| "examples/kitten_toy.mp4", | |
| "What is behind the kitten while it plays with the toy?", | |
| ], | |
| [ | |
| "examples/cycling_pov.mp4", | |
| "In which direction is the camera moving, and what vehicle does it pass?", | |
| ], | |
| ], | |
| inputs=[video_in, question_in], | |
| outputs=[answer_out, status_out], | |
| fn=answer_question, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| run_btn.click( | |
| answer_question, | |
| inputs=[video_in, question_in, images_in, num_frames_in, max_new_tokens_in], | |
| outputs=[answer_out, status_out], | |
| api_name="generate", | |
| ) | |
| demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS) |