multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
42806df verified
Raw History Blame Contribute Delete
9.49 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 -- must precede torch / CUDA-touching imports
import json
import re
import shutil
import subprocess
import tempfile
import gradio as gr
import imageio_ffmpeg
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
# --------------------------------------------------------------------------- #
# Model
# --------------------------------------------------------------------------- #
MODEL_ID = "voyager-so/shot-reader-8b"
N_FRAMES, SIZE = 16, (448, 256)
HERE = os.path.dirname(os.path.abspath(__file__))
FFMPEG = os.environ.get("FFMPEG_BINARY") or shutil.which("ffmpeg") or imageio_ffmpeg.get_ffmpeg_exe()
def _load_schema() -> str:
"""The shot-sheet field reference the model was trained against."""
local = os.path.join(HERE, "schema.md")
if os.path.exists(local):
with open(local) as fh:
return fh.read()
return open(hf_hub_download(MODEL_ID, "schema.md")).read()
SCHEMA = _load_schema()
# Reference prompt, verbatim from the authors' shot_reader.py.
PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s
(frame i is at t = {times}).
Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres
in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward).
Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to
estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f}
(add intermediate keys when it moves). Keep the given ids.
Subjects in the brief (id: class): {subjects}
{schema}
Reply with only the JSON object, no prose, no code fences."""
processor = AutoProcessor.from_pretrained(MODEL_ID)
model = (
Qwen3VLForConditionalGeneration.from_pretrained(MODEL_ID, dtype=torch.bfloat16)
.to("cuda")
.eval()
)
# --------------------------------------------------------------------------- #
# Video -> 16 frames, exactly as in the reference implementation
# --------------------------------------------------------------------------- #
def video_duration(path: str) -> float:
_, seconds = imageio_ffmpeg.count_frames_and_secs(path)
return float(seconds)
def sample_frames(path: str, dur: float):
times = [i * dur / N_FRAMES for i in range(N_FRAMES)]
imgs = []
with tempfile.TemporaryDirectory() as d:
for i, t in enumerate(times):
f = os.path.join(d, f"{i}.jpg")
subprocess.run(
[FFMPEG, "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path,
"-frames:v", "1", "-q:v", "2", f],
check=True,
)
imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC))
return times, imgs
def build_prompt(dur, times, subjects) -> str:
subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)"
return PROMPT.format(
nf=N_FRAMES,
dur=dur,
times=", ".join(f"{t:.2f}" for t in times),
subjects=subs,
schema=SCHEMA,
)
def parse_sheet(raw: str):
"""Pull the JSON object out of the model output (same regex as the reference)."""
text = raw.strip()
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.S)
if fence:
text = fence.group(1).strip()
m = re.search(r"\{.*\}", text, re.S)
if not m:
return None
try:
return json.loads(m.group(0))
except json.JSONDecodeError:
return None
# --------------------------------------------------------------------------- #
# Inference
# --------------------------------------------------------------------------- #
DURATION = 60
@spaces.GPU(duration=DURATION)
def read_shot(video, subjects_text="", max_new_tokens=8192,
progress=gr.Progress(track_tqdm=True)):
"""Read one continuous video shot and write its shot sheet (camera, subjects, paths, set) as JSON.
Args:
video: one continuous shot (a clip with no cuts).
subjects_text: the brief, one subject per line, as "ID: class (optional description)".
max_new_tokens: cap on generated tokens for the JSON sheet.
"""
if not video:
raise gr.Error("Upload or pick a video shot first.")
path = video.get("video") if isinstance(video, dict) else video
if not path or not os.path.exists(path):
raise gr.Error("Could not read the uploaded video.")
dur = video_duration(path)
if dur <= 0:
raise gr.Error("Could not read the video duration.")
times, imgs = sample_frames(path, dur)
subjects = [s.strip() for s in (subjects_text or "").splitlines() if s.strip()]
text = build_prompt(dur, times, subjects)
msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}]
chat = processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)
enc = processor(text=[chat], images=imgs, return_tensors="pt").to(model.device)
with torch.no_grad():
out = model.generate(**enc, max_new_tokens=int(max_new_tokens), do_sample=False)
raw = processor.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True)
sheet = parse_sheet(raw)
if sheet is None:
return {}, raw, f"⚠️ Could not parse a JSON shot sheet from the output ({len(raw)} chars shown below)."
n_sub = len(sheet.get("subjects", []) or [])
n_ev = len(sheet.get("events", []) or [])
n_set = len(sheet.get("set", []) or [])
cam = (sheet.get("camera") or {}).get("move", "?")
return (
sheet,
json.dumps(sheet, indent=2),
f"✅ Parsed shot sheet — camera move **{cam}**, {n_sub} subject(s), {n_ev} event(s), {n_set} set piece(s) "
f"· {dur:.2f} s shot, {N_FRAMES} frames at {SIZE[0]}×{SIZE[1]}",
)
# --------------------------------------------------------------------------- #
# UI
# --------------------------------------------------------------------------- #
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
EXAMPLES = [
[
os.path.join(HERE, "assets", "dog_chasing_ball.mp4"),
"O01: dog (border collie)\nO02: ball (large blue exercise ball)",
],
[
os.path.join(HERE, "assets", "cycling_forest.mp4"),
"P01: cyclist (person on a road bike)",
],
[
os.path.join(HERE, "assets", "city_traffic_night.mp4"),
"O01: car\nO02: bus (white city bus)",
],
]
with gr.Blocks(title="Shot Reader 8B") as demo:
gr.Markdown(
"""
# 🎬 Shot Reader 8B
Watch one shot — get the 3D scene. Upload a single continuous shot and Shot Reader writes
its **shot sheet** as JSON: camera height, angle, lens and move; who is in frame and where
they stand and move; the events; and the set around them. Positions are in metres on the
ground plane, ready for a 3D tool or a coding agent to rebuild in Blender.
A fine-tune of [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)
([model card](https://huggingface.co/voyager-so/shot-reader-8b) ·
[schema](https://huggingface.co/voyager-so/shot-reader-8b/blob/main/schema.md)).
"""
)
with gr.Row():
with gr.Column(scale=2):
video = gr.Video(
label="Shot (one continuous take — cut longer videos at the edits)",
sources=["upload"],
height=320,
)
subjects = gr.Textbox(
label="Subjects in the brief (optional)",
placeholder="P01: person (woman in a black polo)\nO01: dog",
lines=3,
info='One per line as "ID: class (optional description)". The model keeps your ids.',
)
run = gr.Button("Read the shot", variant="primary")
with gr.Column(scale=3):
status = gr.Markdown("")
sheet = gr.JSON(label="Shot sheet")
raw = gr.Code(label="Raw JSON", language="json", lines=14)
with gr.Accordion("Advanced settings", open=False):
max_new_tokens = gr.Slider(
label="Max new tokens", minimum=512, maximum=8192, step=256, value=8192,
info="Cap on the generated JSON. The sheet normally stops well before this.",
)
gr.Examples(
examples=EXAMPLES,
inputs=[video, subjects],
outputs=[sheet, raw, status],
fn=read_shot,
cache_examples=True,
cache_mode="lazy",
label="Example shots",
)
gr.Markdown(
"""
**Tips** — one continuous shot per call (cut at the edits first); 16 frames are sampled evenly
across the clip at 448×256. Events and routes are the model's weakest fields on real footage,
and positions are estimates from a single camera. Apache-2.0.
"""
)
run.click(
read_shot,
inputs=[video, subjects, max_new_tokens],
outputs=[sheet, raw, status],
api_name="read_shot",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True, ssr_mode=False)