Spaces:
Paused
Paused
File size: 7,167 Bytes
05f095d 729fbc4 05f095d 55eb376 05f095d 1d697a7 05f095d 1d697a7 05f095d 1d697a7 05f095d 55eb376 05f095d 729fbc4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | import numpy as np
import ffmpeg
from PIL import Image
from config import (
CLIP_FRAME_SIZE, CLIP_MAX_FRAMES, CLIP_MAX_FPS, CLIP_MIN_FPS,
DEBUG, VISUAL_CONCEPTS, VISUAL_ONSCREEN_TEXT_SIM, VISUAL_TAG_MIN_SIM,
VISUAL_TAGS_TOP_K,
)
from models import TORCH_DEVICE, get_clip, get_concept_embs
# On CPU (no MPS/CUDA) CLIP is slow — halve the frame sample budget.
_CLIP_MAX_FRAMES_EFF = CLIP_MAX_FRAMES // 2 if TORCH_DEVICE == "cpu" else CLIP_MAX_FRAMES
from utils import fmt_time
_SEEK_SAMPLING_ABOVE_SEC = 300 # past five minutes, decoding the whole file costs minutes
def _frame_at(video_path: str, t: float, S: int):
"""One frame by input seeking (keyframe-accurate, cheap): no full decode."""
try:
out, _ = (
ffmpeg.input(video_path, ss=max(0.0, t))
.filter("scale", S, S, force_original_aspect_ratio="increase")
.filter("crop", S, S)
.output("pipe:", format="rawvideo", pix_fmt="rgb24", vframes=1)
.run(capture_stdout=True, capture_stderr=True)
)
except ffmpeg.Error:
return None
if len(out) < S * S * 3:
return None
return np.frombuffer(out[: S * S * 3], np.uint8).reshape(S, S, 3)
def extract_frames(video_path: str, duration: float):
"""
Sample frames evenly. Short files go through one ffmpeg rawvideo pipe (no temp
files); long files are sampled by seeking, so a 45-minute recording is not decoded
end to end just to keep 48 frames of it.
Returns (frames_rgb, times): uint8 (N, S, S, 3) and float (N,).
"""
S = CLIP_FRAME_SIZE
if duration and duration > _SEEK_SAMPLING_ABOVE_SEC:
n = _CLIP_MAX_FRAMES_EFF
times = [duration * (i + 0.5) / n for i in range(n)]
frames, kept = [], []
for t in times:
f = _frame_at(video_path, t, S)
if f is not None:
frames.append(f); kept.append(t)
if frames:
return np.stack(frames), np.array(kept)
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
fps = CLIP_MAX_FPS
if duration and duration > 0:
fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, _CLIP_MAX_FRAMES_EFF / duration))
try:
out, _ = (
ffmpeg.input(video_path)
.filter("fps", fps=fps)
.filter("scale", S, S, force_original_aspect_ratio="increase")
.filter("crop", S, S)
.output("pipe:", format="rawvideo", pix_fmt="rgb24")
.run(capture_stdout=True, capture_stderr=True)
)
except ffmpeg.Error:
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
if not out:
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES]
times = np.array([(i + 0.5) / fps for i in range(len(frames))])
return frames, times
def analyze_visual(video_path: str, duration: float) -> dict:
"""
Sample + CLIP-encode frames. Returns {"emb": (N,D) float32, "time": (N,)}.
Empty arrays for audio-only input.
"""
frames, times = extract_frames(video_path, duration)
if len(frames) == 0:
print("[NINA] No video frames (audio-only?) — visual analysis skipped.")
return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))}
images = [Image.fromarray(f) for f in frames]
embs = get_clip().encode(images, convert_to_numpy=True, batch_size=32, show_progress_bar=False)
print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).")
return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)}
def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]:
"""
Per-frame cosine of prompt↔frame in CLIP space.
Returns (raw_sims, min-max-normalised_sims); empty arrays when no frames.
"""
embs = visual.get("emb")
if embs is None or len(embs) == 0:
if DEBUG:
print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.")
return np.array([]), np.array([])
q = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0]
qn = q / (np.linalg.norm(q) + 1e-9)
en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
sims = en @ qn
lo, hi = float(sims.min()), float(sims.max())
norm = (sims - lo) / (hi - lo + 1e-9)
if DEBUG:
times = visual.get("time")
order = np.argsort(sims)[::-1][:3]
tops = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order)
print(
f"[DEBUG] Visual: {len(sims)} frames; raw sim "
f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; "
f"top moments: {tops}"
)
return sims, norm
def span_visual_score(times: np.ndarray, norm: np.ndarray,
start: float, end: float) -> float:
"""Best normalised visual relevance among frames inside [start, end]."""
if len(times) == 0:
return 0.0
mask = (times >= start - 0.5) & (times <= end + 0.5)
if not mask.any():
idx = int(np.argmin(np.abs(times - (start + end) / 2)))
return float(norm[idx])
return float(norm[mask].max())
def region_visual_tags(visual: dict, start: float, end: float) -> list[str]:
"""Zero-shot CLIP tags for what is on screen during [start, end]."""
embs = visual.get("emb")
times = visual.get("time")
if embs is None or len(embs) == 0:
return []
mask = (times >= start - 0.5) & (times <= end + 0.5)
if mask.any():
sel = embs[mask]
else:
idx = int(np.argmin(np.abs(times - (start + end) / 2)))
sel = embs[idx:idx + 1]
region_emb = sel.mean(axis=0)
region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9)
sims = get_concept_embs() @ region_emb
order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K]
return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM]
def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]:
for r in regions:
r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"])
return regions
# Concepts that imply readable on-screen text — the gate for the OCR fallback.
_TEXT_CONCEPT_IDX = [
i for i, c in enumerate(VISUAL_CONCEPTS)
if any(k in c.lower() for k in ("text on screen", "source code", "webpage",
"screen recording", "slides"))
]
def has_on_screen_text(visual: dict) -> bool:
"""True if any frame looks like it contains readable text — used to decide whether
OCR of burned-in captions is worth attempting."""
embs = visual.get("emb")
if embs is None or len(embs) == 0 or not _TEXT_CONCEPT_IDX:
return False
try:
concepts = get_concept_embs()[_TEXT_CONCEPT_IDX] # (k, D), normalised
en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
sims = en @ concepts.T # (frames, k)
return float(sims.max()) >= VISUAL_ONSCREEN_TEXT_SIM
except Exception:
return False
|