# ───────────────────────────────────────────── # NINA — keyframe extraction for the vision-language model # ───────────────────────────────────────────── # Pulls a handful of frames at chosen timestamps and returns them as base64 JPEG # data URLs ready to drop into a chat-completion image message. Frames are # downscaled (long side ≤ VLM_FRAME_MAX_PX) to keep image-token cost in check. # Best-effort: a frame that can't be read is skipped. import base64 import io import ffmpeg from PIL import Image from config import VLM_FRAME_MAX_PX, VLM_MAX_FRAMES def select_timestamps(regions: list[dict], times, max_frames: int = VLM_MAX_FRAMES) -> list[float]: """Choose representative timestamps: the midpoints of retrieved regions, padded with evenly-spaced samples across the video so the VLM still gets overall context. """ picks: list[float] = [round((r["start"] + r["end"]) / 2.0, 2) for r in (regions or [])] # Span to spread context over — from the frame timeline if we have it. hi = 0.0 if times is not None and len(times): hi = float(times[-1]) for r in (regions or []): hi = max(hi, float(r["end"])) if hi > 0: n = max(3, max_frames - len(picks)) picks += [round(hi * (i + 0.5) / n, 2) for i in range(n)] # De-duplicate within ~1s, keep order, cap. out: list[float] = [] for t in sorted(picks): if t >= 0 and (not out or t - out[-1] >= 1.0): out.append(t) return out[:max_frames] if out else [0.0] def _jpeg_at(video_path: str, t: float) -> bytes: out, _ = ( ffmpeg.input(video_path, ss=max(0.0, t)) .output("pipe:", vframes=1, format="image2", vcodec="mjpeg") .run(capture_stdout=True, capture_stderr=True) ) return out def extract_keyframes(video_path: str, timestamps: list[float], max_px: int = VLM_FRAME_MAX_PX) -> list[tuple]: """Return [(timestamp, base64-jpeg-data-url)] for frames that decoded (downscaled).""" out: list[tuple] = [] for t in timestamps: try: raw = _jpeg_at(video_path, t) if not raw: continue img = Image.open(io.BytesIO(raw)).convert("RGB") img.thumbnail((max_px, max_px)) buf = io.BytesIO() img.save(buf, format="JPEG", quality=80) b64 = base64.b64encode(buf.getvalue()).decode("ascii") out.append((t, f"data:image/jpeg;base64,{b64}")) except Exception: continue return out def keyframes_for(video_path: str, regions: list[dict], times, max_frames: int = VLM_MAX_FRAMES) -> list[tuple]: """Pick representative timestamps for these regions and return decoded keyframes as [(timestamp, data_url)]. Best-effort — returns [] on failure.""" try: ts = select_timestamps(regions, times, max_frames) return extract_keyframes(video_path, ts) except Exception as e: print(f"[NINA] Keyframe extraction failed: {e}") return []