Spaces:
Paused
Paused
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7 Download visual.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 7.17 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/visual.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/visual.py
-
curl -L -o visual.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/visual.py
7.17 kB
| import numpy as np | |
| import ffmpeg | |
| from PIL import Image | |
| from config import ( | |
| CLIP_FRAME_SIZE, CLIP_MAX_FRAMES, CLIP_MAX_FPS, CLIP_MIN_FPS, | |
| DEBUG, VISUAL_CONCEPTS, VISUAL_ONSCREEN_TEXT_SIM, VISUAL_TAG_MIN_SIM, | |
| VISUAL_TAGS_TOP_K, | |
| ) | |
| from models import TORCH_DEVICE, get_clip, get_concept_embs | |
| # On CPU (no MPS/CUDA) CLIP is slow — halve the frame sample budget. | |
| _CLIP_MAX_FRAMES_EFF = CLIP_MAX_FRAMES // 2 if TORCH_DEVICE == "cpu" else CLIP_MAX_FRAMES | |
| from utils import fmt_time | |
| _SEEK_SAMPLING_ABOVE_SEC = 300 # past five minutes, decoding the whole file costs minutes | |
| def _frame_at(video_path: str, t: float, S: int): | |
| """One frame by input seeking (keyframe-accurate, cheap): no full decode.""" | |
| try: | |
| out, _ = ( | |
| ffmpeg.input(video_path, ss=max(0.0, t)) | |
| .filter("scale", S, S, force_original_aspect_ratio="increase") | |
| .filter("crop", S, S) | |
| .output("pipe:", format="rawvideo", pix_fmt="rgb24", vframes=1) | |
| .run(capture_stdout=True, capture_stderr=True) | |
| ) | |
| except ffmpeg.Error: | |
| return None | |
| if len(out) < S * S * 3: | |
| return None | |
| return np.frombuffer(out[: S * S * 3], np.uint8).reshape(S, S, 3) | |
| def extract_frames(video_path: str, duration: float): | |
| """ | |
| Sample frames evenly. Short files go through one ffmpeg rawvideo pipe (no temp | |
| files); long files are sampled by seeking, so a 45-minute recording is not decoded | |
| end to end just to keep 48 frames of it. | |
| Returns (frames_rgb, times): uint8 (N, S, S, 3) and float (N,). | |
| """ | |
| S = CLIP_FRAME_SIZE | |
| if duration and duration > _SEEK_SAMPLING_ABOVE_SEC: | |
| n = _CLIP_MAX_FRAMES_EFF | |
| times = [duration * (i + 0.5) / n for i in range(n)] | |
| frames, kept = [], [] | |
| for t in times: | |
| f = _frame_at(video_path, t, S) | |
| if f is not None: | |
| frames.append(f); kept.append(t) | |
| if frames: | |
| return np.stack(frames), np.array(kept) | |
| return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,)) | |
| fps = CLIP_MAX_FPS | |
| if duration and duration > 0: | |
| fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, _CLIP_MAX_FRAMES_EFF / duration)) | |
| try: | |
| out, _ = ( | |
| ffmpeg.input(video_path) | |
| .filter("fps", fps=fps) | |
| .filter("scale", S, S, force_original_aspect_ratio="increase") | |
| .filter("crop", S, S) | |
| .output("pipe:", format="rawvideo", pix_fmt="rgb24") | |
| .run(capture_stdout=True, capture_stderr=True) | |
| ) | |
| except ffmpeg.Error: | |
| return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,)) | |
| if not out: | |
| return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,)) | |
| frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES] | |
| times = np.array([(i + 0.5) / fps for i in range(len(frames))]) | |
| return frames, times | |
| def analyze_visual(video_path: str, duration: float) -> dict: | |
| """ | |
| Sample + CLIP-encode frames. Returns {"emb": (N,D) float32, "time": (N,)}. | |
| Empty arrays for audio-only input. | |
| """ | |
| frames, times = extract_frames(video_path, duration) | |
| if len(frames) == 0: | |
| print("[NINA] No video frames (audio-only?) — visual analysis skipped.") | |
| return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))} | |
| images = [Image.fromarray(f) for f in frames] | |
| embs = get_clip().encode(images, convert_to_numpy=True, batch_size=32, show_progress_bar=False) | |
| print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).") | |
| return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)} | |
| def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]: | |
| """ | |
| Per-frame cosine of prompt↔frame in CLIP space. | |
| Returns (raw_sims, min-max-normalised_sims); empty arrays when no frames. | |
| """ | |
| embs = visual.get("emb") | |
| if embs is None or len(embs) == 0: | |
| if DEBUG: | |
| print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.") | |
| return np.array([]), np.array([]) | |
| q = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0] | |
| qn = q / (np.linalg.norm(q) + 1e-9) | |
| en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) | |
| sims = en @ qn | |
| lo, hi = float(sims.min()), float(sims.max()) | |
| norm = (sims - lo) / (hi - lo + 1e-9) | |
| if DEBUG: | |
| times = visual.get("time") | |
| order = np.argsort(sims)[::-1][:3] | |
| tops = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order) | |
| print( | |
| f"[DEBUG] Visual: {len(sims)} frames; raw sim " | |
| f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; " | |
| f"top moments: {tops}" | |
| ) | |
| return sims, norm | |
| def span_visual_score(times: np.ndarray, norm: np.ndarray, | |
| start: float, end: float) -> float: | |
| """Best normalised visual relevance among frames inside [start, end].""" | |
| if len(times) == 0: | |
| return 0.0 | |
| mask = (times >= start - 0.5) & (times <= end + 0.5) | |
| if not mask.any(): | |
| idx = int(np.argmin(np.abs(times - (start + end) / 2))) | |
| return float(norm[idx]) | |
| return float(norm[mask].max()) | |
| def region_visual_tags(visual: dict, start: float, end: float) -> list[str]: | |
| """Zero-shot CLIP tags for what is on screen during [start, end].""" | |
| embs = visual.get("emb") | |
| times = visual.get("time") | |
| if embs is None or len(embs) == 0: | |
| return [] | |
| mask = (times >= start - 0.5) & (times <= end + 0.5) | |
| if mask.any(): | |
| sel = embs[mask] | |
| else: | |
| idx = int(np.argmin(np.abs(times - (start + end) / 2))) | |
| sel = embs[idx:idx + 1] | |
| region_emb = sel.mean(axis=0) | |
| region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9) | |
| sims = get_concept_embs() @ region_emb | |
| order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K] | |
| return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM] | |
| def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]: | |
| for r in regions: | |
| r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"]) | |
| return regions | |
| # Concepts that imply readable on-screen text — the gate for the OCR fallback. | |
| _TEXT_CONCEPT_IDX = [ | |
| i for i, c in enumerate(VISUAL_CONCEPTS) | |
| if any(k in c.lower() for k in ("text on screen", "source code", "webpage", | |
| "screen recording", "slides")) | |
| ] | |
| def has_on_screen_text(visual: dict) -> bool: | |
| """True if any frame looks like it contains readable text — used to decide whether | |
| OCR of burned-in captions is worth attempting.""" | |
| embs = visual.get("emb") | |
| if embs is None or len(embs) == 0 or not _TEXT_CONCEPT_IDX: | |
| return False | |
| try: | |
| concepts = get_concept_embs()[_TEXT_CONCEPT_IDX] # (k, D), normalised | |
| en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) | |
| sims = en @ concepts.T # (frames, k) | |
| return float(sims.max()) >= VISUAL_ONSCREEN_TEXT_SIM | |
| except Exception: | |
| return False | |