Image-Text-to-Text
Transformers
Safetensors
English
qwen3_vl
vision
video-understanding
previs
previsualization
camera
3d
blender
scene-understanding
qwen3-vl
lora
conversational
Instructions to use voyager-so/shot-reader-8b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use voyager-so/shot-reader-8b with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="voyager-so/shot-reader-8b") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("voyager-so/shot-reader-8b") model = AutoModelForMultimodalLM.from_pretrained("voyager-so/shot-reader-8b", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use voyager-so/shot-reader-8b with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "voyager-so/shot-reader-8b" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "voyager-so/shot-reader-8b", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/voyager-so/shot-reader-8b
- SGLang
How to use voyager-so/shot-reader-8b with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "voyager-so/shot-reader-8b" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "voyager-so/shot-reader-8b", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "voyager-so/shot-reader-8b" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "voyager-so/shot-reader-8b", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use voyager-so/shot-reader-8b with Docker Model Runner:
docker model run hf.co/voyager-so/shot-reader-8b
File size: 6,013 Bytes
52e9143 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | #!/usr/bin/env python3
"""Shot Reader: read one video shot and write its shot sheet (camera, subjects, paths, events, set) as JSON.
python shot_reader.py clip.mp4 --subject "P01: person (woman in a black polo)" --subject "O01: dog"
python shot_reader.py clip.mp4 --subject "P01: person" --backend vllm # fastest
python shot_reader.py clip.mp4 --subject "P01: person" --load-in-4bit # ~6 GB of VRAM
One shot = one continuous take (cut longer videos into shots first). The model sees 16 frames sampled
evenly across the shot at 448x256, plus the ids and classes of the subjects you name; it keeps your ids.
Needs: torch, transformers>=4.57, pillow, ffmpeg on PATH (+ vllm or bitsandbytes for those options).
"""
import argparse, json, os, re, subprocess, sys, tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
REPO = "voyager-so/shot-reader-8b"
N_FRAMES, SIZE = 16, (448, 256)
PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s
(frame i is at t = {times}).
Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres
in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward).
Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to
estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f}
(add intermediate keys when it moves). Keep the given ids.
Subjects in the brief (id: class): {subjects}
{schema}
Reply with only the JSON object, no prose, no code fences."""
def schema_doc(model):
p = os.path.join(HERE, "schema.md")
if not os.path.exists(p):
from huggingface_hub import hf_hub_download
p = hf_hub_download(model if "/" in model and not os.path.isdir(model) else REPO, "schema.md")
return open(p).read()
def duration(path):
out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path],
capture_output=True, text=True, check=True).stdout
return float(out.strip())
def sample_frames(path, dur):
from PIL import Image
times = [i * dur / N_FRAMES for i in range(N_FRAMES)]
imgs = []
with tempfile.TemporaryDirectory() as d:
for i, t in enumerate(times):
f = os.path.join(d, f"{i}.jpg")
subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, "-frames:v", "1", "-q:v", "2", f],
check=True)
imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC))
return times, imgs
def build_prompt(dur, times, subjects, schema):
subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)"
return PROMPT.format(nf=N_FRAMES, dur=dur, times=", ".join(f"{t:.2f}" for t in times), subjects=subs, schema=schema)
def chat(processor, text):
msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}]
return processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)
def run_transformers(model_id, prompt_text, imgs, max_new, four_bit):
import torch
from transformers import AutoModelForImageTextToText, AutoProcessor
proc = AutoProcessor.from_pretrained(model_id)
kw = {"dtype": torch.bfloat16, "device_map": "auto"}
if four_bit:
from transformers import BitsAndBytesConfig
kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.bfloat16)
model = AutoModelForImageTextToText.from_pretrained(model_id, **kw).eval()
enc = proc(text=[chat(proc, prompt_text)], images=imgs, return_tensors="pt").to(model.device)
with torch.no_grad():
out = model.generate(**enc, max_new_tokens=max_new, do_sample=False)
return proc.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True)
def run_vllm(model_id, prompt_text, imgs, max_new):
from transformers import AutoProcessor
from vllm import LLM, SamplingParams
proc = AutoProcessor.from_pretrained(model_id)
llm = LLM(model=model_id, max_model_len=12288, limit_mm_per_prompt={"image": N_FRAMES})
o = llm.generate([{"prompt": chat(proc, prompt_text), "multi_modal_data": {"image": imgs}}],
SamplingParams(temperature=0, max_tokens=max_new), use_tqdm=False)[0]
return o.outputs[0].text
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("video")
ap.add_argument("--subject", action="append", default=[], help='"ID: class (optional description)", repeatable')
ap.add_argument("--model", default=REPO)
ap.add_argument("--backend", choices=["transformers", "vllm"], default="transformers")
ap.add_argument("--load-in-4bit", action="store_true", help="transformers + bitsandbytes NF4, fits small GPUs")
ap.add_argument("--max-new-tokens", type=int, default=8192)
ap.add_argument("-o", "--out", help="write the sheet here (default: stdout)")
a = ap.parse_args()
dur = duration(a.video)
times, imgs = sample_frames(a.video, dur)
text = build_prompt(dur, times, a.subject, schema_doc(a.model))
raw = (run_vllm(a.model, text, imgs, a.max_new_tokens) if a.backend == "vllm"
else run_transformers(a.model, text, imgs, a.max_new_tokens, a.load_in_4bit))
m = re.search(r"\{.*\}", raw, re.S)
try:
sheet = json.loads(m.group(0)) if m else None
except json.JSONDecodeError:
sheet = None
if sheet is None:
print(raw, file=sys.stderr); sys.exit("could not parse a JSON sheet from the model output")
s = json.dumps(sheet, indent=2)
if a.out: open(a.out, "w").write(s + "\n")
else: print(s)
if __name__ == "__main__":
main()
|