Spaces:
Paused
Paused
File size: 3,240 Bytes
33ec6e1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | # βββββββββββββββββββββββββββββββββββββββββββββ
# NINA β keyframe extraction for the vision-language model
# βββββββββββββββββββββββββββββββββββββββββββββ
# Pulls a handful of frames at chosen timestamps and returns them as base64 JPEG
# data URLs ready to drop into a chat-completion image message. Frames are
# downscaled (long side β€ VLM_FRAME_MAX_PX) to keep image-token cost in check.
# Best-effort: a frame that can't be read is skipped.
import base64
import io
import ffmpeg
from PIL import Image
from config import VLM_FRAME_MAX_PX, VLM_MAX_FRAMES
def select_timestamps(regions: list[dict], times, max_frames: int = VLM_MAX_FRAMES) -> list[float]:
"""Choose representative timestamps: the midpoints of retrieved regions, padded
with evenly-spaced samples across the video so the VLM still gets overall context.
"""
picks: list[float] = [round((r["start"] + r["end"]) / 2.0, 2) for r in (regions or [])]
# Span to spread context over β from the frame timeline if we have it.
hi = 0.0
if times is not None and len(times):
hi = float(times[-1])
for r in (regions or []):
hi = max(hi, float(r["end"]))
if hi > 0:
n = max(3, max_frames - len(picks))
picks += [round(hi * (i + 0.5) / n, 2) for i in range(n)]
# De-duplicate within ~1s, keep order, cap.
out: list[float] = []
for t in sorted(picks):
if t >= 0 and (not out or t - out[-1] >= 1.0):
out.append(t)
return out[:max_frames] if out else [0.0]
def _jpeg_at(video_path: str, t: float) -> bytes:
out, _ = (
ffmpeg.input(video_path, ss=max(0.0, t))
.output("pipe:", vframes=1, format="image2", vcodec="mjpeg")
.run(capture_stdout=True, capture_stderr=True)
)
return out
def extract_keyframes(video_path: str, timestamps: list[float],
max_px: int = VLM_FRAME_MAX_PX) -> list[tuple]:
"""Return [(timestamp, base64-jpeg-data-url)] for frames that decoded (downscaled)."""
out: list[tuple] = []
for t in timestamps:
try:
raw = _jpeg_at(video_path, t)
if not raw:
continue
img = Image.open(io.BytesIO(raw)).convert("RGB")
img.thumbnail((max_px, max_px))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=80)
b64 = base64.b64encode(buf.getvalue()).decode("ascii")
out.append((t, f"data:image/jpeg;base64,{b64}"))
except Exception:
continue
return out
def keyframes_for(video_path: str, regions: list[dict], times,
max_frames: int = VLM_MAX_FRAMES) -> list[tuple]:
"""Pick representative timestamps for these regions and return decoded keyframes
as [(timestamp, data_url)]. Best-effort β returns [] on failure."""
try:
ts = select_timestamps(regions, times, max_frames)
return extract_keyframes(video_path, ts)
except Exception as e:
print(f"[NINA] Keyframe extraction failed: {e}")
return []
|