import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 -- must precede torch / CUDA-touching imports import json import re import shutil import subprocess import tempfile import gradio as gr import imageio_ffmpeg import torch from huggingface_hub import hf_hub_download from PIL import Image from transformers import AutoProcessor, Qwen3VLForConditionalGeneration # --------------------------------------------------------------------------- # # Model # --------------------------------------------------------------------------- # MODEL_ID = "voyager-so/shot-reader-8b" N_FRAMES, SIZE = 16, (448, 256) HERE = os.path.dirname(os.path.abspath(__file__)) FFMPEG = os.environ.get("FFMPEG_BINARY") or shutil.which("ffmpeg") or imageio_ffmpeg.get_ffmpeg_exe() def _load_schema() -> str: """The shot-sheet field reference the model was trained against.""" local = os.path.join(HERE, "schema.md") if os.path.exists(local): with open(local) as fh: return fh.read() return open(hf_hub_download(MODEL_ID, "schema.md")).read() SCHEMA = _load_schema() # Reference prompt, verbatim from the authors' shot_reader.py. PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s (frame i is at t = {times}). Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward). Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f} (add intermediate keys when it moves). Keep the given ids. Subjects in the brief (id: class): {subjects} {schema} Reply with only the JSON object, no prose, no code fences.""" processor = AutoProcessor.from_pretrained(MODEL_ID) model = ( Qwen3VLForConditionalGeneration.from_pretrained(MODEL_ID, dtype=torch.bfloat16) .to("cuda") .eval() ) # --------------------------------------------------------------------------- # # Video -> 16 frames, exactly as in the reference implementation # --------------------------------------------------------------------------- # def video_duration(path: str) -> float: _, seconds = imageio_ffmpeg.count_frames_and_secs(path) return float(seconds) def sample_frames(path: str, dur: float): times = [i * dur / N_FRAMES for i in range(N_FRAMES)] imgs = [] with tempfile.TemporaryDirectory() as d: for i, t in enumerate(times): f = os.path.join(d, f"{i}.jpg") subprocess.run( [FFMPEG, "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, "-frames:v", "1", "-q:v", "2", f], check=True, ) imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC)) return times, imgs def build_prompt(dur, times, subjects) -> str: subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)" return PROMPT.format( nf=N_FRAMES, dur=dur, times=", ".join(f"{t:.2f}" for t in times), subjects=subs, schema=SCHEMA, ) def parse_sheet(raw: str): """Pull the JSON object out of the model output (same regex as the reference).""" text = raw.strip() fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.S) if fence: text = fence.group(1).strip() m = re.search(r"\{.*\}", text, re.S) if not m: return None try: return json.loads(m.group(0)) except json.JSONDecodeError: return None # --------------------------------------------------------------------------- # # Inference # --------------------------------------------------------------------------- # DURATION = 60 @spaces.GPU(duration=DURATION) def read_shot(video, subjects_text="", max_new_tokens=8192, progress=gr.Progress(track_tqdm=True)): """Read one continuous video shot and write its shot sheet (camera, subjects, paths, set) as JSON. Args: video: one continuous shot (a clip with no cuts). subjects_text: the brief, one subject per line, as "ID: class (optional description)". max_new_tokens: cap on generated tokens for the JSON sheet. """ if not video: raise gr.Error("Upload or pick a video shot first.") path = video.get("video") if isinstance(video, dict) else video if not path or not os.path.exists(path): raise gr.Error("Could not read the uploaded video.") dur = video_duration(path) if dur <= 0: raise gr.Error("Could not read the video duration.") times, imgs = sample_frames(path, dur) subjects = [s.strip() for s in (subjects_text or "").splitlines() if s.strip()] text = build_prompt(dur, times, subjects) msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}] chat = processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) enc = processor(text=[chat], images=imgs, return_tensors="pt").to(model.device) with torch.no_grad(): out = model.generate(**enc, max_new_tokens=int(max_new_tokens), do_sample=False) raw = processor.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True) sheet = parse_sheet(raw) if sheet is None: return {}, raw, f"⚠️ Could not parse a JSON shot sheet from the output ({len(raw)} chars shown below)." n_sub = len(sheet.get("subjects", []) or []) n_ev = len(sheet.get("events", []) or []) n_set = len(sheet.get("set", []) or []) cam = (sheet.get("camera") or {}).get("move", "?") return ( sheet, json.dumps(sheet, indent=2), f"✅ Parsed shot sheet — camera move **{cam}**, {n_sub} subject(s), {n_ev} event(s), {n_set} set piece(s) " f"· {dur:.2f} s shot, {N_FRAMES} frames at {SIZE[0]}×{SIZE[1]}", ) # --------------------------------------------------------------------------- # # UI # --------------------------------------------------------------------------- # CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ EXAMPLES = [ [ os.path.join(HERE, "assets", "dog_chasing_ball.mp4"), "O01: dog (border collie)\nO02: ball (large blue exercise ball)", ], [ os.path.join(HERE, "assets", "cycling_forest.mp4"), "P01: cyclist (person on a road bike)", ], [ os.path.join(HERE, "assets", "city_traffic_night.mp4"), "O01: car\nO02: bus (white city bus)", ], ] with gr.Blocks(title="Shot Reader 8B") as demo: gr.Markdown( """ # 🎬 Shot Reader 8B Watch one shot — get the 3D scene. Upload a single continuous shot and Shot Reader writes its **shot sheet** as JSON: camera height, angle, lens and move; who is in frame and where they stand and move; the events; and the set around them. Positions are in metres on the ground plane, ready for a 3D tool or a coding agent to rebuild in Blender. A fine-tune of [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) ([model card](https://huggingface.co/voyager-so/shot-reader-8b) · [schema](https://huggingface.co/voyager-so/shot-reader-8b/blob/main/schema.md)). """ ) with gr.Row(): with gr.Column(scale=2): video = gr.Video( label="Shot (one continuous take — cut longer videos at the edits)", sources=["upload"], height=320, ) subjects = gr.Textbox( label="Subjects in the brief (optional)", placeholder="P01: person (woman in a black polo)\nO01: dog", lines=3, info='One per line as "ID: class (optional description)". The model keeps your ids.', ) run = gr.Button("Read the shot", variant="primary") with gr.Column(scale=3): status = gr.Markdown("") sheet = gr.JSON(label="Shot sheet") raw = gr.Code(label="Raw JSON", language="json", lines=14) with gr.Accordion("Advanced settings", open=False): max_new_tokens = gr.Slider( label="Max new tokens", minimum=512, maximum=8192, step=256, value=8192, info="Cap on the generated JSON. The sheet normally stops well before this.", ) gr.Examples( examples=EXAMPLES, inputs=[video, subjects], outputs=[sheet, raw, status], fn=read_shot, cache_examples=True, cache_mode="lazy", label="Example shots", ) gr.Markdown( """ **Tips** — one continuous shot per call (cut at the edits first); 16 frames are sampled evenly across the clip at 448×256. Events and routes are the model's weakest fields on real footage, and positions are estimates from a single camera. Apache-2.0. """ ) run.click( read_shot, inputs=[video, subjects, max_new_tokens], outputs=[sheet, raw, status], api_name="read_shot", ) if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, ssr_mode=False)