import os # Allocator config: avoid fragmentation OOM under transient activation spikes. os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # VGGT feature-layer selection must be set before any qwenvl import: # qwenvl/geometry/constants.py bakes GEOMETRY_PATCH_DIM at import time and # the fusion block's patch_mapping input dim must match (2048 for "23"). os.environ.setdefault("VGGT_MULTI_LAYER_INDICES", "23") import spaces # MUST come before torch / any CUDA-touching import import re import sys import time from pathlib import Path import gradio as gr import numpy as np import torch from PIL import Image # Make the bundled qwenvl package (repo root) importable. _APP_DIR = Path(__file__).resolve().parent if str(_APP_DIR) not in sys.path: sys.path.insert(0, str(_APP_DIR)) from transformers import AutoProcessor, AutoTokenizer from qwenvl.eval.evaluate_geometry_bench import EvalBuilder from qwenvl.geometry.modeling_qwen3vl_geometry import ( Qwen3VLGeometryForConditionalGeneration, ) CHECKPOINT = "yuqun/R2R-Qwen3-VL-8B" BASE_MODEL = "Qwen/Qwen3-VL-8B-Instruct" MIN_PIXELS = 784 MAX_PIXELS = 200704 DEFAULT_NUM_FRAMES = 32 # the 8B checkpoint was trained with 32 input frames print(f"Loading {CHECKPOINT} ...", flush=True) model = Qwen3VLGeometryForConditionalGeneration.from_pretrained( CHECKPOINT, dtype=torch.bfloat16 ) if model.geometry_tower is not None: model.geometry_tower.load_model() # downloads facebook/VGGT-1B (~5 GB) model.geometry_tower.to(torch.bfloat16) model = model.eval().to("cuda") tokenizer = AutoTokenizer.from_pretrained(CHECKPOINT, use_fast=False) if tokenizer.pad_token_id is None: tokenizer.pad_token_id = tokenizer.eos_token_id processor = AutoProcessor.from_pretrained(BASE_MODEL) # EvalBuilder mirrors the authors' eval pipeline: it re-configures the Qwen # processor pixel budget (784..200704 per frame) and builds the multi-frame # prompt with markers + tokens. builder = EvalBuilder(processor, tokenizer, DEFAULT_NUM_FRAMES, MIN_PIXELS, MAX_PIXELS) print("Model ready.", flush=True) def _load_video_frames(path, num_frames): """Sample up to num_frames evenly-spaced frames + their timestamps.""" from decord import VideoReader vr = VideoReader(path, num_threads=1) idx = np.linspace(0, len(vr) - 1, min(num_frames, len(vr)), dtype=int) frames = [Image.fromarray(f) for f in vr.get_batch(idx).asnumpy()] times = [i / vr.get_avg_fps() for i in idx] return frames, times def _load_image_frames(paths, num_frames, fps=1.0): """Load a sorted set of view images + synthetic 1 fps timestamps.""" idx = np.linspace(0, len(paths) - 1, min(num_frames, len(paths)), dtype=int) frames = [Image.open(paths[i]).convert("RGB") for i in idx] times = [i / fps for i in idx] return frames, times @spaces.GPU(duration=30) def answer_question( video, question, images=None, num_frames=DEFAULT_NUM_FRAMES, max_new_tokens=128, progress=gr.Progress(), ): """Answer a spatial question about a video (or a set of multi-view images). Args: video: an uploaded video file. question: the question to ask about the scene, e.g. "What is behind the woman on the wall?" images: optionally, a set of multi-view images instead of a video. num_frames: how many frames to sample from the video (default 32, the frame count the model was trained with). max_new_tokens: maximum length of the generated answer. Returns: A short text answer to the question. """ if not question or not str(question).strip(): raise gr.Error("Please enter a question.") question = str(question).strip() progress(0.1, desc="Decoding input") if video is not None: frames, times = _load_video_frames(video, int(num_frames)) elif images: paths = sorted(p if isinstance(p, str) else str(p.name) for p in images) frames, times = _load_image_frames(paths, int(num_frames)) else: raise gr.Error("Please upload a video or a set of images first.") n = len(frames) progress(0.35, desc="Building geometry-aware prompt") markers = [f"<{t:.1f} seconds>" for t in times] prompt_inputs = builder.image_prompt(frames, question, markers) # Geometry pixels for the VGGT tower, on the same Qwen patch grid. geometry_pixels = builder._geometry_pixels(frames) progress(0.55, desc="Reasoning over the scene") device = model.device kw = {k: v.to(device) for k, v in prompt_inputs.items()} t0 = time.perf_counter() out = model.generate( **kw, geometry_pixel_values=geometry_pixels.to(device=device, dtype=torch.bfloat16), # Zero placeholder embeds: the model forward regenerates the VGGT # camera/patch features from geometry_pixel_values itself (the same # path the training data takes via _zero_geometry). geometry_patch_embeds=torch.zeros(n, 1, 1, device=device), geometry_camera_embeds=torch.zeros(n, 1, 1, device=device), post_decoder_num_frames=torch.tensor([n], device=device), semantic_target_frame_idx=torch.tensor([-1], device=device), max_new_tokens=int(max_new_tokens), do_sample=False, use_cache=True, ) elapsed = time.perf_counter() - t0 gen_ids = out[0, kw["input_ids"].shape[1] :] answer = tokenizer.decode(gen_ids, skip_special_tokens=True).strip() status = f"Answered in **{elapsed:.1f} s** using {n} frames." return answer, status CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="Render to Reason") as demo: gr.HTML( """

🧭 Render to Reason

Spatial understanding for video VLMs: Qwen3-VL fused with a VGGT geometry tower that predicts novel-view semantics and camera geometry.

Paper · GitHub · Model weights

""" ) with gr.Column(elem_id="col-container"): with gr.Row(): with gr.Column(scale=1): video_in = gr.Video(label="Video", sources=["upload"]) images_in = gr.File( label="…or a set of multi-view images", file_count="multiple", file_types=["image"], ) question_in = gr.Textbox( label="Question", placeholder="e.g. What is behind the woman on the wall?", lines=2, ) run_btn = gr.Button("Answer", variant="primary") with gr.Column(scale=1): answer_out = gr.Textbox(label="Answer", lines=4) status_out = gr.Markdown() with gr.Accordion("Advanced settings", open=False): num_frames_in = gr.Slider( 4, 48, value=DEFAULT_NUM_FRAMES, step=2, label="Frames sampled from the video", info="The 8B model was trained with 32 frames.", ) max_new_tokens_in = gr.Slider( 16, 512, value=128, step=16, label="Max new tokens" ) gr.Examples( examples=[ [ "examples/podcaster.mp4", "What is behind the woman on the wall? Answer in a few words.", ], [ "examples/pouring_coffee.mp4", "On what does the grey cup sit, and what is next to it?", ], [ "examples/kitten_toy.mp4", "What is behind the kitten while it plays with the toy?", ], [ "examples/cycling_pov.mp4", "In which direction is the camera moving, and what vehicle does it pass?", ], ], inputs=[video_in, question_in], outputs=[answer_out, status_out], fn=answer_question, cache_examples=True, cache_mode="lazy", ) run_btn.click( answer_question, inputs=[video_in, question_in, images_in, num_frames_in, max_new_tokens_in], outputs=[answer_out, status_out], api_name="generate", ) demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)