import numpy as np import ffmpeg from PIL import Image from config import ( CLIP_FRAME_SIZE, CLIP_MAX_FRAMES, CLIP_MAX_FPS, CLIP_MIN_FPS, DEBUG, VISUAL_CONCEPTS, VISUAL_ONSCREEN_TEXT_SIM, VISUAL_TAG_MIN_SIM, VISUAL_TAGS_TOP_K, ) from models import TORCH_DEVICE, get_clip, get_concept_embs # On CPU (no MPS/CUDA) CLIP is slow — halve the frame sample budget. _CLIP_MAX_FRAMES_EFF = CLIP_MAX_FRAMES // 2 if TORCH_DEVICE == "cpu" else CLIP_MAX_FRAMES from utils import fmt_time _SEEK_SAMPLING_ABOVE_SEC = 300 # past five minutes, decoding the whole file costs minutes def _frame_at(video_path: str, t: float, S: int): """One frame by input seeking (keyframe-accurate, cheap): no full decode.""" try: out, _ = ( ffmpeg.input(video_path, ss=max(0.0, t)) .filter("scale", S, S, force_original_aspect_ratio="increase") .filter("crop", S, S) .output("pipe:", format="rawvideo", pix_fmt="rgb24", vframes=1) .run(capture_stdout=True, capture_stderr=True) ) except ffmpeg.Error: return None if len(out) < S * S * 3: return None return np.frombuffer(out[: S * S * 3], np.uint8).reshape(S, S, 3) def extract_frames(video_path: str, duration: float): """ Sample frames evenly. Short files go through one ffmpeg rawvideo pipe (no temp files); long files are sampled by seeking, so a 45-minute recording is not decoded end to end just to keep 48 frames of it. Returns (frames_rgb, times): uint8 (N, S, S, 3) and float (N,). """ S = CLIP_FRAME_SIZE if duration and duration > _SEEK_SAMPLING_ABOVE_SEC: n = _CLIP_MAX_FRAMES_EFF times = [duration * (i + 0.5) / n for i in range(n)] frames, kept = [], [] for t in times: f = _frame_at(video_path, t, S) if f is not None: frames.append(f); kept.append(t) if frames: return np.stack(frames), np.array(kept) return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,)) fps = CLIP_MAX_FPS if duration and duration > 0: fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, _CLIP_MAX_FRAMES_EFF / duration)) try: out, _ = ( ffmpeg.input(video_path) .filter("fps", fps=fps) .filter("scale", S, S, force_original_aspect_ratio="increase") .filter("crop", S, S) .output("pipe:", format="rawvideo", pix_fmt="rgb24") .run(capture_stdout=True, capture_stderr=True) ) except ffmpeg.Error: return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,)) if not out: return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,)) frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES] times = np.array([(i + 0.5) / fps for i in range(len(frames))]) return frames, times def analyze_visual(video_path: str, duration: float) -> dict: """ Sample + CLIP-encode frames. Returns {"emb": (N,D) float32, "time": (N,)}. Empty arrays for audio-only input. """ frames, times = extract_frames(video_path, duration) if len(frames) == 0: print("[NINA] No video frames (audio-only?) — visual analysis skipped.") return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))} images = [Image.fromarray(f) for f in frames] embs = get_clip().encode(images, convert_to_numpy=True, batch_size=32, show_progress_bar=False) print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).") return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)} def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]: """ Per-frame cosine of prompt↔frame in CLIP space. Returns (raw_sims, min-max-normalised_sims); empty arrays when no frames. """ embs = visual.get("emb") if embs is None or len(embs) == 0: if DEBUG: print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.") return np.array([]), np.array([]) q = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0] qn = q / (np.linalg.norm(q) + 1e-9) en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) sims = en @ qn lo, hi = float(sims.min()), float(sims.max()) norm = (sims - lo) / (hi - lo + 1e-9) if DEBUG: times = visual.get("time") order = np.argsort(sims)[::-1][:3] tops = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order) print( f"[DEBUG] Visual: {len(sims)} frames; raw sim " f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; " f"top moments: {tops}" ) return sims, norm def span_visual_score(times: np.ndarray, norm: np.ndarray, start: float, end: float) -> float: """Best normalised visual relevance among frames inside [start, end].""" if len(times) == 0: return 0.0 mask = (times >= start - 0.5) & (times <= end + 0.5) if not mask.any(): idx = int(np.argmin(np.abs(times - (start + end) / 2))) return float(norm[idx]) return float(norm[mask].max()) def region_visual_tags(visual: dict, start: float, end: float) -> list[str]: """Zero-shot CLIP tags for what is on screen during [start, end].""" embs = visual.get("emb") times = visual.get("time") if embs is None or len(embs) == 0: return [] mask = (times >= start - 0.5) & (times <= end + 0.5) if mask.any(): sel = embs[mask] else: idx = int(np.argmin(np.abs(times - (start + end) / 2))) sel = embs[idx:idx + 1] region_emb = sel.mean(axis=0) region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9) sims = get_concept_embs() @ region_emb order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K] return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM] def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]: for r in regions: r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"]) return regions # Concepts that imply readable on-screen text — the gate for the OCR fallback. _TEXT_CONCEPT_IDX = [ i for i, c in enumerate(VISUAL_CONCEPTS) if any(k in c.lower() for k in ("text on screen", "source code", "webpage", "screen recording", "slides")) ] def has_on_screen_text(visual: dict) -> bool: """True if any frame looks like it contains readable text — used to decide whether OCR of burned-in captions is worth attempting.""" embs = visual.get("emb") if embs is None or len(embs) == 0 or not _TEXT_CONCEPT_IDX: return False try: concepts = get_concept_embs()[_TEXT_CONCEPT_IDX] # (k, D), normalised en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) sims = en @ concepts.T # (frames, k) return float(sims.max()) >= VISUAL_ONSCREEN_TEXT_SIM except Exception: return False