NINA / visual.py
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7
Raw History Blame
7.17 kB
import numpy as np
import ffmpeg
from PIL import Image
from config import (
CLIP_FRAME_SIZE, CLIP_MAX_FRAMES, CLIP_MAX_FPS, CLIP_MIN_FPS,
DEBUG, VISUAL_CONCEPTS, VISUAL_ONSCREEN_TEXT_SIM, VISUAL_TAG_MIN_SIM,
VISUAL_TAGS_TOP_K,
)
from models import TORCH_DEVICE, get_clip, get_concept_embs
# On CPU (no MPS/CUDA) CLIP is slow — halve the frame sample budget.
_CLIP_MAX_FRAMES_EFF = CLIP_MAX_FRAMES // 2 if TORCH_DEVICE == "cpu" else CLIP_MAX_FRAMES
from utils import fmt_time
_SEEK_SAMPLING_ABOVE_SEC = 300 # past five minutes, decoding the whole file costs minutes
def _frame_at(video_path: str, t: float, S: int):
"""One frame by input seeking (keyframe-accurate, cheap): no full decode."""
try:
out, _ = (
ffmpeg.input(video_path, ss=max(0.0, t))
.filter("scale", S, S, force_original_aspect_ratio="increase")
.filter("crop", S, S)
.output("pipe:", format="rawvideo", pix_fmt="rgb24", vframes=1)
.run(capture_stdout=True, capture_stderr=True)
)
except ffmpeg.Error:
return None
if len(out) < S * S * 3:
return None
return np.frombuffer(out[: S * S * 3], np.uint8).reshape(S, S, 3)
def extract_frames(video_path: str, duration: float):
"""
Sample frames evenly. Short files go through one ffmpeg rawvideo pipe (no temp
files); long files are sampled by seeking, so a 45-minute recording is not decoded
end to end just to keep 48 frames of it.
Returns (frames_rgb, times): uint8 (N, S, S, 3) and float (N,).
"""
S = CLIP_FRAME_SIZE
if duration and duration > _SEEK_SAMPLING_ABOVE_SEC:
n = _CLIP_MAX_FRAMES_EFF
times = [duration * (i + 0.5) / n for i in range(n)]
frames, kept = [], []
for t in times:
f = _frame_at(video_path, t, S)
if f is not None:
frames.append(f); kept.append(t)
if frames:
return np.stack(frames), np.array(kept)
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
fps = CLIP_MAX_FPS
if duration and duration > 0:
fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, _CLIP_MAX_FRAMES_EFF / duration))
try:
out, _ = (
ffmpeg.input(video_path)
.filter("fps", fps=fps)
.filter("scale", S, S, force_original_aspect_ratio="increase")
.filter("crop", S, S)
.output("pipe:", format="rawvideo", pix_fmt="rgb24")
.run(capture_stdout=True, capture_stderr=True)
)
except ffmpeg.Error:
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
if not out:
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES]
times = np.array([(i + 0.5) / fps for i in range(len(frames))])
return frames, times
def analyze_visual(video_path: str, duration: float) -> dict:
"""
Sample + CLIP-encode frames. Returns {"emb": (N,D) float32, "time": (N,)}.
Empty arrays for audio-only input.
"""
frames, times = extract_frames(video_path, duration)
if len(frames) == 0:
print("[NINA] No video frames (audio-only?) — visual analysis skipped.")
return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))}
images = [Image.fromarray(f) for f in frames]
embs = get_clip().encode(images, convert_to_numpy=True, batch_size=32, show_progress_bar=False)
print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).")
return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)}
def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]:
"""
Per-frame cosine of prompt↔frame in CLIP space.
Returns (raw_sims, min-max-normalised_sims); empty arrays when no frames.
"""
embs = visual.get("emb")
if embs is None or len(embs) == 0:
if DEBUG:
print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.")
return np.array([]), np.array([])
q = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0]
qn = q / (np.linalg.norm(q) + 1e-9)
en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
sims = en @ qn
lo, hi = float(sims.min()), float(sims.max())
norm = (sims - lo) / (hi - lo + 1e-9)
if DEBUG:
times = visual.get("time")
order = np.argsort(sims)[::-1][:3]
tops = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order)
print(
f"[DEBUG] Visual: {len(sims)} frames; raw sim "
f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; "
f"top moments: {tops}"
)
return sims, norm
def span_visual_score(times: np.ndarray, norm: np.ndarray,
start: float, end: float) -> float:
"""Best normalised visual relevance among frames inside [start, end]."""
if len(times) == 0:
return 0.0
mask = (times >= start - 0.5) & (times <= end + 0.5)
if not mask.any():
idx = int(np.argmin(np.abs(times - (start + end) / 2)))
return float(norm[idx])
return float(norm[mask].max())
def region_visual_tags(visual: dict, start: float, end: float) -> list[str]:
"""Zero-shot CLIP tags for what is on screen during [start, end]."""
embs = visual.get("emb")
times = visual.get("time")
if embs is None or len(embs) == 0:
return []
mask = (times >= start - 0.5) & (times <= end + 0.5)
if mask.any():
sel = embs[mask]
else:
idx = int(np.argmin(np.abs(times - (start + end) / 2)))
sel = embs[idx:idx + 1]
region_emb = sel.mean(axis=0)
region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9)
sims = get_concept_embs() @ region_emb
order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K]
return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM]
def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]:
for r in regions:
r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"])
return regions
# Concepts that imply readable on-screen text — the gate for the OCR fallback.
_TEXT_CONCEPT_IDX = [
i for i, c in enumerate(VISUAL_CONCEPTS)
if any(k in c.lower() for k in ("text on screen", "source code", "webpage",
"screen recording", "slides"))
]
def has_on_screen_text(visual: dict) -> bool:
"""True if any frame looks like it contains readable text — used to decide whether
OCR of burned-in captions is worth attempting."""
embs = visual.get("emb")
if embs is None or len(embs) == 0 or not _TEXT_CONCEPT_IDX:
return False
try:
concepts = get_concept_embs()[_TEXT_CONCEPT_IDX] # (k, D), normalised
en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
sims = en @ concepts.T # (frames, k)
return float(sims.max()) >= VISUAL_ONSCREEN_TEXT_SIM
except Exception:
return False