File size: 3,240 Bytes
33ec6e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# ─────────────────────────────────────────────
#  NINA β€” keyframe extraction for the vision-language model
# ─────────────────────────────────────────────
#  Pulls a handful of frames at chosen timestamps and returns them as base64 JPEG
#  data URLs ready to drop into a chat-completion image message. Frames are
#  downscaled (long side ≀ VLM_FRAME_MAX_PX) to keep image-token cost in check.
#  Best-effort: a frame that can't be read is skipped.

import base64
import io

import ffmpeg
from PIL import Image

from config import VLM_FRAME_MAX_PX, VLM_MAX_FRAMES


def select_timestamps(regions: list[dict], times, max_frames: int = VLM_MAX_FRAMES) -> list[float]:
    """Choose representative timestamps: the midpoints of retrieved regions, padded
    with evenly-spaced samples across the video so the VLM still gets overall context.
    """
    picks: list[float] = [round((r["start"] + r["end"]) / 2.0, 2) for r in (regions or [])]

    # Span to spread context over β€” from the frame timeline if we have it.
    hi = 0.0
    if times is not None and len(times):
        hi = float(times[-1])
    for r in (regions or []):
        hi = max(hi, float(r["end"]))

    if hi > 0:
        n = max(3, max_frames - len(picks))
        picks += [round(hi * (i + 0.5) / n, 2) for i in range(n)]

    # De-duplicate within ~1s, keep order, cap.
    out: list[float] = []
    for t in sorted(picks):
        if t >= 0 and (not out or t - out[-1] >= 1.0):
            out.append(t)
    return out[:max_frames] if out else [0.0]


def _jpeg_at(video_path: str, t: float) -> bytes:
    out, _ = (
        ffmpeg.input(video_path, ss=max(0.0, t))
        .output("pipe:", vframes=1, format="image2", vcodec="mjpeg")
        .run(capture_stdout=True, capture_stderr=True)
    )
    return out


def extract_keyframes(video_path: str, timestamps: list[float],
                      max_px: int = VLM_FRAME_MAX_PX) -> list[tuple]:
    """Return [(timestamp, base64-jpeg-data-url)] for frames that decoded (downscaled)."""
    out: list[tuple] = []
    for t in timestamps:
        try:
            raw = _jpeg_at(video_path, t)
            if not raw:
                continue
            img = Image.open(io.BytesIO(raw)).convert("RGB")
            img.thumbnail((max_px, max_px))
            buf = io.BytesIO()
            img.save(buf, format="JPEG", quality=80)
            b64 = base64.b64encode(buf.getvalue()).decode("ascii")
            out.append((t, f"data:image/jpeg;base64,{b64}"))
        except Exception:
            continue
    return out


def keyframes_for(video_path: str, regions: list[dict], times,
                  max_frames: int = VLM_MAX_FRAMES) -> list[tuple]:
    """Pick representative timestamps for these regions and return decoded keyframes
    as [(timestamp, data_url)]. Best-effort β€” returns [] on failure."""
    try:
        ts = select_timestamps(regions, times, max_frames)
        return extract_keyframes(video_path, ts)
    except Exception as e:
        print(f"[NINA] Keyframe extraction failed: {e}")
        return []