Spaces:
Paused
Paused
Malina
Claude Opus 4.8
NINA: VLM-centric synthesis β the model now actually sees the video
33ec6e1 Download frames.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 3.24 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/frames.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/frames.py
-
curl -L -o frames.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/frames.py
3.24 kB
| # βββββββββββββββββββββββββββββββββββββββββββββ | |
| # NINA β keyframe extraction for the vision-language model | |
| # βββββββββββββββββββββββββββββββββββββββββββββ | |
| # Pulls a handful of frames at chosen timestamps and returns them as base64 JPEG | |
| # data URLs ready to drop into a chat-completion image message. Frames are | |
| # downscaled (long side β€ VLM_FRAME_MAX_PX) to keep image-token cost in check. | |
| # Best-effort: a frame that can't be read is skipped. | |
| import base64 | |
| import io | |
| import ffmpeg | |
| from PIL import Image | |
| from config import VLM_FRAME_MAX_PX, VLM_MAX_FRAMES | |
| def select_timestamps(regions: list[dict], times, max_frames: int = VLM_MAX_FRAMES) -> list[float]: | |
| """Choose representative timestamps: the midpoints of retrieved regions, padded | |
| with evenly-spaced samples across the video so the VLM still gets overall context. | |
| """ | |
| picks: list[float] = [round((r["start"] + r["end"]) / 2.0, 2) for r in (regions or [])] | |
| # Span to spread context over β from the frame timeline if we have it. | |
| hi = 0.0 | |
| if times is not None and len(times): | |
| hi = float(times[-1]) | |
| for r in (regions or []): | |
| hi = max(hi, float(r["end"])) | |
| if hi > 0: | |
| n = max(3, max_frames - len(picks)) | |
| picks += [round(hi * (i + 0.5) / n, 2) for i in range(n)] | |
| # De-duplicate within ~1s, keep order, cap. | |
| out: list[float] = [] | |
| for t in sorted(picks): | |
| if t >= 0 and (not out or t - out[-1] >= 1.0): | |
| out.append(t) | |
| return out[:max_frames] if out else [0.0] | |
| def _jpeg_at(video_path: str, t: float) -> bytes: | |
| out, _ = ( | |
| ffmpeg.input(video_path, ss=max(0.0, t)) | |
| .output("pipe:", vframes=1, format="image2", vcodec="mjpeg") | |
| .run(capture_stdout=True, capture_stderr=True) | |
| ) | |
| return out | |
| def extract_keyframes(video_path: str, timestamps: list[float], | |
| max_px: int = VLM_FRAME_MAX_PX) -> list[tuple]: | |
| """Return [(timestamp, base64-jpeg-data-url)] for frames that decoded (downscaled).""" | |
| out: list[tuple] = [] | |
| for t in timestamps: | |
| try: | |
| raw = _jpeg_at(video_path, t) | |
| if not raw: | |
| continue | |
| img = Image.open(io.BytesIO(raw)).convert("RGB") | |
| img.thumbnail((max_px, max_px)) | |
| buf = io.BytesIO() | |
| img.save(buf, format="JPEG", quality=80) | |
| b64 = base64.b64encode(buf.getvalue()).decode("ascii") | |
| out.append((t, f"data:image/jpeg;base64,{b64}")) | |
| except Exception: | |
| continue | |
| return out | |
| def keyframes_for(video_path: str, regions: list[dict], times, | |
| max_frames: int = VLM_MAX_FRAMES) -> list[tuple]: | |
| """Pick representative timestamps for these regions and return decoded keyframes | |
| as [(timestamp, data_url)]. Best-effort β returns [] on failure.""" | |
| try: | |
| ts = select_timestamps(regions, times, max_frames) | |
| return extract_keyframes(video_path, ts) | |
| except Exception as e: | |
| print(f"[NINA] Keyframe extraction failed: {e}") | |
| return [] | |