File size: 6,013 Bytes
52e9143
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
#!/usr/bin/env python3
"""Shot Reader: read one video shot and write its shot sheet (camera, subjects, paths, events, set) as JSON.

    python shot_reader.py clip.mp4 --subject "P01: person (woman in a black polo)" --subject "O01: dog"
    python shot_reader.py clip.mp4 --subject "P01: person" --backend vllm          # fastest
    python shot_reader.py clip.mp4 --subject "P01: person" --load-in-4bit          # ~6 GB of VRAM

One shot = one continuous take (cut longer videos into shots first). The model sees 16 frames sampled
evenly across the shot at 448x256, plus the ids and classes of the subjects you name; it keeps your ids.
Needs: torch, transformers>=4.57, pillow, ffmpeg on PATH (+ vllm or bitsandbytes for those options).
"""
import argparse, json, os, re, subprocess, sys, tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = "voyager-so/shot-reader-8b"
N_FRAMES, SIZE = 16, (448, 256)

PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s
(frame i is at t = {times}).

Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres
in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward).
Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to
estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f}
(add intermediate keys when it moves). Keep the given ids.

Subjects in the brief (id: class): {subjects}

{schema}

Reply with only the JSON object, no prose, no code fences."""


def schema_doc(model):
    p = os.path.join(HERE, "schema.md")
    if not os.path.exists(p):
        from huggingface_hub import hf_hub_download
        p = hf_hub_download(model if "/" in model and not os.path.isdir(model) else REPO, "schema.md")
    return open(p).read()


def duration(path):
    out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path],
                         capture_output=True, text=True, check=True).stdout
    return float(out.strip())


def sample_frames(path, dur):
    from PIL import Image
    times = [i * dur / N_FRAMES for i in range(N_FRAMES)]
    imgs = []
    with tempfile.TemporaryDirectory() as d:
        for i, t in enumerate(times):
            f = os.path.join(d, f"{i}.jpg")
            subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, "-frames:v", "1", "-q:v", "2", f],
                           check=True)
            imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC))
    return times, imgs


def build_prompt(dur, times, subjects, schema):
    subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)"
    return PROMPT.format(nf=N_FRAMES, dur=dur, times=", ".join(f"{t:.2f}" for t in times), subjects=subs, schema=schema)


def chat(processor, text):
    msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}]
    return processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)


def run_transformers(model_id, prompt_text, imgs, max_new, four_bit):
    import torch
    from transformers import AutoModelForImageTextToText, AutoProcessor
    proc = AutoProcessor.from_pretrained(model_id)
    kw = {"dtype": torch.bfloat16, "device_map": "auto"}
    if four_bit:
        from transformers import BitsAndBytesConfig
        kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
                                                       bnb_4bit_compute_dtype=torch.bfloat16)
    model = AutoModelForImageTextToText.from_pretrained(model_id, **kw).eval()
    enc = proc(text=[chat(proc, prompt_text)], images=imgs, return_tensors="pt").to(model.device)
    with torch.no_grad():
        out = model.generate(**enc, max_new_tokens=max_new, do_sample=False)
    return proc.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True)


def run_vllm(model_id, prompt_text, imgs, max_new):
    from transformers import AutoProcessor
    from vllm import LLM, SamplingParams
    proc = AutoProcessor.from_pretrained(model_id)
    llm = LLM(model=model_id, max_model_len=12288, limit_mm_per_prompt={"image": N_FRAMES})
    o = llm.generate([{"prompt": chat(proc, prompt_text), "multi_modal_data": {"image": imgs}}],
                     SamplingParams(temperature=0, max_tokens=max_new), use_tqdm=False)[0]
    return o.outputs[0].text


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("video")
    ap.add_argument("--subject", action="append", default=[], help='"ID: class (optional description)", repeatable')
    ap.add_argument("--model", default=REPO)
    ap.add_argument("--backend", choices=["transformers", "vllm"], default="transformers")
    ap.add_argument("--load-in-4bit", action="store_true", help="transformers + bitsandbytes NF4, fits small GPUs")
    ap.add_argument("--max-new-tokens", type=int, default=8192)
    ap.add_argument("-o", "--out", help="write the sheet here (default: stdout)")
    a = ap.parse_args()

    dur = duration(a.video)
    times, imgs = sample_frames(a.video, dur)
    text = build_prompt(dur, times, a.subject, schema_doc(a.model))
    raw = (run_vllm(a.model, text, imgs, a.max_new_tokens) if a.backend == "vllm"
           else run_transformers(a.model, text, imgs, a.max_new_tokens, a.load_in_4bit))
    m = re.search(r"\{.*\}", raw, re.S)
    try:
        sheet = json.loads(m.group(0)) if m else None
    except json.JSONDecodeError:
        sheet = None
    if sheet is None:
        print(raw, file=sys.stderr); sys.exit("could not parse a JSON sheet from the model output")
    s = json.dumps(sheet, indent=2)
    if a.out: open(a.out, "w").write(s + "\n")
    else: print(s)


if __name__ == "__main__":
    main()