Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/shot-reader-8b-2: direct link, hf CLI and curl.
- Browser
- Download file 9.49 kB
-
https://huggingface.co/spaces/hugging-apps/shot-reader-8b-2/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/shot-reader-8b-2/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/shot-reader-8b-2/resolve/main/app.py
9.49 kB
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 -- must precede torch / CUDA-touching imports | |
| import json | |
| import re | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| import gradio as gr | |
| import imageio_ffmpeg | |
| import torch | |
| from huggingface_hub import hf_hub_download | |
| from PIL import Image | |
| from transformers import AutoProcessor, Qwen3VLForConditionalGeneration | |
| # --------------------------------------------------------------------------- # | |
| # Model | |
| # --------------------------------------------------------------------------- # | |
| MODEL_ID = "voyager-so/shot-reader-8b" | |
| N_FRAMES, SIZE = 16, (448, 256) | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| FFMPEG = os.environ.get("FFMPEG_BINARY") or shutil.which("ffmpeg") or imageio_ffmpeg.get_ffmpeg_exe() | |
| def _load_schema() -> str: | |
| """The shot-sheet field reference the model was trained against.""" | |
| local = os.path.join(HERE, "schema.md") | |
| if os.path.exists(local): | |
| with open(local) as fh: | |
| return fh.read() | |
| return open(hf_hub_download(MODEL_ID, "schema.md")).read() | |
| SCHEMA = _load_schema() | |
| # Reference prompt, verbatim from the authors' shot_reader.py. | |
| PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s | |
| (frame i is at t = {times}). | |
| Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres | |
| in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward). | |
| Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to | |
| estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f} | |
| (add intermediate keys when it moves). Keep the given ids. | |
| Subjects in the brief (id: class): {subjects} | |
| {schema} | |
| Reply with only the JSON object, no prose, no code fences.""" | |
| processor = AutoProcessor.from_pretrained(MODEL_ID) | |
| model = ( | |
| Qwen3VLForConditionalGeneration.from_pretrained(MODEL_ID, dtype=torch.bfloat16) | |
| .to("cuda") | |
| .eval() | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Video -> 16 frames, exactly as in the reference implementation | |
| # --------------------------------------------------------------------------- # | |
| def video_duration(path: str) -> float: | |
| _, seconds = imageio_ffmpeg.count_frames_and_secs(path) | |
| return float(seconds) | |
| def sample_frames(path: str, dur: float): | |
| times = [i * dur / N_FRAMES for i in range(N_FRAMES)] | |
| imgs = [] | |
| with tempfile.TemporaryDirectory() as d: | |
| for i, t in enumerate(times): | |
| f = os.path.join(d, f"{i}.jpg") | |
| subprocess.run( | |
| [FFMPEG, "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, | |
| "-frames:v", "1", "-q:v", "2", f], | |
| check=True, | |
| ) | |
| imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC)) | |
| return times, imgs | |
| def build_prompt(dur, times, subjects) -> str: | |
| subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)" | |
| return PROMPT.format( | |
| nf=N_FRAMES, | |
| dur=dur, | |
| times=", ".join(f"{t:.2f}" for t in times), | |
| subjects=subs, | |
| schema=SCHEMA, | |
| ) | |
| def parse_sheet(raw: str): | |
| """Pull the JSON object out of the model output (same regex as the reference).""" | |
| text = raw.strip() | |
| fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.S) | |
| if fence: | |
| text = fence.group(1).strip() | |
| m = re.search(r"\{.*\}", text, re.S) | |
| if not m: | |
| return None | |
| try: | |
| return json.loads(m.group(0)) | |
| except json.JSONDecodeError: | |
| return None | |
| # --------------------------------------------------------------------------- # | |
| # Inference | |
| # --------------------------------------------------------------------------- # | |
| DURATION = 60 | |
| def read_shot(video, subjects_text="", max_new_tokens=8192, | |
| progress=gr.Progress(track_tqdm=True)): | |
| """Read one continuous video shot and write its shot sheet (camera, subjects, paths, set) as JSON. | |
| Args: | |
| video: one continuous shot (a clip with no cuts). | |
| subjects_text: the brief, one subject per line, as "ID: class (optional description)". | |
| max_new_tokens: cap on generated tokens for the JSON sheet. | |
| """ | |
| if not video: | |
| raise gr.Error("Upload or pick a video shot first.") | |
| path = video.get("video") if isinstance(video, dict) else video | |
| if not path or not os.path.exists(path): | |
| raise gr.Error("Could not read the uploaded video.") | |
| dur = video_duration(path) | |
| if dur <= 0: | |
| raise gr.Error("Could not read the video duration.") | |
| times, imgs = sample_frames(path, dur) | |
| subjects = [s.strip() for s in (subjects_text or "").splitlines() if s.strip()] | |
| text = build_prompt(dur, times, subjects) | |
| msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}] | |
| chat = processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) | |
| enc = processor(text=[chat], images=imgs, return_tensors="pt").to(model.device) | |
| with torch.no_grad(): | |
| out = model.generate(**enc, max_new_tokens=int(max_new_tokens), do_sample=False) | |
| raw = processor.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True) | |
| sheet = parse_sheet(raw) | |
| if sheet is None: | |
| return {}, raw, f"⚠️ Could not parse a JSON shot sheet from the output ({len(raw)} chars shown below)." | |
| n_sub = len(sheet.get("subjects", []) or []) | |
| n_ev = len(sheet.get("events", []) or []) | |
| n_set = len(sheet.get("set", []) or []) | |
| cam = (sheet.get("camera") or {}).get("move", "?") | |
| return ( | |
| sheet, | |
| json.dumps(sheet, indent=2), | |
| f"✅ Parsed shot sheet — camera move **{cam}**, {n_sub} subject(s), {n_ev} event(s), {n_set} set piece(s) " | |
| f"· {dur:.2f} s shot, {N_FRAMES} frames at {SIZE[0]}×{SIZE[1]}", | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # UI | |
| # --------------------------------------------------------------------------- # | |
| CSS = """ | |
| #col-container { max-width: 1200px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| EXAMPLES = [ | |
| [ | |
| os.path.join(HERE, "assets", "dog_chasing_ball.mp4"), | |
| "O01: dog (border collie)\nO02: ball (large blue exercise ball)", | |
| ], | |
| [ | |
| os.path.join(HERE, "assets", "cycling_forest.mp4"), | |
| "P01: cyclist (person on a road bike)", | |
| ], | |
| [ | |
| os.path.join(HERE, "assets", "city_traffic_night.mp4"), | |
| "O01: car\nO02: bus (white city bus)", | |
| ], | |
| ] | |
| with gr.Blocks(title="Shot Reader 8B") as demo: | |
| gr.Markdown( | |
| """ | |
| # 🎬 Shot Reader 8B | |
| Watch one shot — get the 3D scene. Upload a single continuous shot and Shot Reader writes | |
| its **shot sheet** as JSON: camera height, angle, lens and move; who is in frame and where | |
| they stand and move; the events; and the set around them. Positions are in metres on the | |
| ground plane, ready for a 3D tool or a coding agent to rebuild in Blender. | |
| A fine-tune of [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) | |
| ([model card](https://huggingface.co/voyager-so/shot-reader-8b) · | |
| [schema](https://huggingface.co/voyager-so/shot-reader-8b/blob/main/schema.md)). | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| video = gr.Video( | |
| label="Shot (one continuous take — cut longer videos at the edits)", | |
| sources=["upload"], | |
| height=320, | |
| ) | |
| subjects = gr.Textbox( | |
| label="Subjects in the brief (optional)", | |
| placeholder="P01: person (woman in a black polo)\nO01: dog", | |
| lines=3, | |
| info='One per line as "ID: class (optional description)". The model keeps your ids.', | |
| ) | |
| run = gr.Button("Read the shot", variant="primary") | |
| with gr.Column(scale=3): | |
| status = gr.Markdown("") | |
| sheet = gr.JSON(label="Shot sheet") | |
| raw = gr.Code(label="Raw JSON", language="json", lines=14) | |
| with gr.Accordion("Advanced settings", open=False): | |
| max_new_tokens = gr.Slider( | |
| label="Max new tokens", minimum=512, maximum=8192, step=256, value=8192, | |
| info="Cap on the generated JSON. The sheet normally stops well before this.", | |
| ) | |
| gr.Examples( | |
| examples=EXAMPLES, | |
| inputs=[video, subjects], | |
| outputs=[sheet, raw, status], | |
| fn=read_shot, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Example shots", | |
| ) | |
| gr.Markdown( | |
| """ | |
| **Tips** — one continuous shot per call (cut at the edits first); 16 frames are sampled evenly | |
| across the clip at 448×256. Events and routes are the model's weakest fields on real footage, | |
| and positions are estimates from a single camera. Apache-2.0. | |
| """ | |
| ) | |
| run.click( | |
| read_shot, | |
| inputs=[video, subjects, max_new_tokens], | |
| outputs=[sheet, raw, status], | |
| api_name="read_shot", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, ssr_mode=False) | |