File size: 7,167 Bytes
05f095d
 
 
 
 
 
729fbc4
 
05f095d
 
55eb376
 
 
05f095d
 
 
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
05f095d
 
1d697a7
 
 
05f095d
 
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
05f095d
 
55eb376
05f095d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
729fbc4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
import numpy as np
import ffmpeg
from PIL import Image

from config import (
    CLIP_FRAME_SIZE, CLIP_MAX_FRAMES, CLIP_MAX_FPS, CLIP_MIN_FPS,
    DEBUG, VISUAL_CONCEPTS, VISUAL_ONSCREEN_TEXT_SIM, VISUAL_TAG_MIN_SIM,
    VISUAL_TAGS_TOP_K,
)
from models import TORCH_DEVICE, get_clip, get_concept_embs

# On CPU (no MPS/CUDA) CLIP is slow — halve the frame sample budget.
_CLIP_MAX_FRAMES_EFF = CLIP_MAX_FRAMES // 2 if TORCH_DEVICE == "cpu" else CLIP_MAX_FRAMES
from utils import fmt_time


_SEEK_SAMPLING_ABOVE_SEC = 300   # past five minutes, decoding the whole file costs minutes


def _frame_at(video_path: str, t: float, S: int):
    """One frame by input seeking (keyframe-accurate, cheap): no full decode."""
    try:
        out, _ = (
            ffmpeg.input(video_path, ss=max(0.0, t))
            .filter("scale", S, S, force_original_aspect_ratio="increase")
            .filter("crop", S, S)
            .output("pipe:", format="rawvideo", pix_fmt="rgb24", vframes=1)
            .run(capture_stdout=True, capture_stderr=True)
        )
    except ffmpeg.Error:
        return None
    if len(out) < S * S * 3:
        return None
    return np.frombuffer(out[: S * S * 3], np.uint8).reshape(S, S, 3)


def extract_frames(video_path: str, duration: float):
    """
    Sample frames evenly. Short files go through one ffmpeg rawvideo pipe (no temp
    files); long files are sampled by seeking, so a 45-minute recording is not decoded
    end to end just to keep 48 frames of it.
    Returns (frames_rgb, times): uint8 (N, S, S, 3) and float (N,).
    """
    S = CLIP_FRAME_SIZE
    if duration and duration > _SEEK_SAMPLING_ABOVE_SEC:
        n = _CLIP_MAX_FRAMES_EFF
        times = [duration * (i + 0.5) / n for i in range(n)]
        frames, kept = [], []
        for t in times:
            f = _frame_at(video_path, t, S)
            if f is not None:
                frames.append(f); kept.append(t)
        if frames:
            return np.stack(frames), np.array(kept)
        return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))

    fps = CLIP_MAX_FPS
    if duration and duration > 0:
        fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, _CLIP_MAX_FRAMES_EFF / duration))
    try:
        out, _ = (
            ffmpeg.input(video_path)
            .filter("fps", fps=fps)
            .filter("scale", S, S, force_original_aspect_ratio="increase")
            .filter("crop", S, S)
            .output("pipe:", format="rawvideo", pix_fmt="rgb24")
            .run(capture_stdout=True, capture_stderr=True)
        )
    except ffmpeg.Error:
        return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))

    if not out:
        return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))

    frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES]
    times  = np.array([(i + 0.5) / fps for i in range(len(frames))])
    return frames, times


def analyze_visual(video_path: str, duration: float) -> dict:
    """
    Sample + CLIP-encode frames. Returns {"emb": (N,D) float32, "time": (N,)}.
    Empty arrays for audio-only input.
    """
    frames, times = extract_frames(video_path, duration)
    if len(frames) == 0:
        print("[NINA] No video frames (audio-only?) — visual analysis skipped.")
        return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))}

    images = [Image.fromarray(f) for f in frames]
    embs = get_clip().encode(images, convert_to_numpy=True, batch_size=32, show_progress_bar=False)
    print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).")
    return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)}


def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]:
    """
    Per-frame cosine of prompt↔frame in CLIP space.
    Returns (raw_sims, min-max-normalised_sims); empty arrays when no frames.
    """
    embs = visual.get("emb")
    if embs is None or len(embs) == 0:
        if DEBUG:
            print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.")
        return np.array([]), np.array([])

    q  = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0]
    qn = q / (np.linalg.norm(q) + 1e-9)
    en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
    sims = en @ qn
    lo, hi = float(sims.min()), float(sims.max())
    norm = (sims - lo) / (hi - lo + 1e-9)

    if DEBUG:
        times = visual.get("time")
        order = np.argsort(sims)[::-1][:3]
        tops  = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order)
        print(
            f"[DEBUG] Visual: {len(sims)} frames; raw sim "
            f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; "
            f"top moments: {tops}"
        )
    return sims, norm


def span_visual_score(times: np.ndarray, norm: np.ndarray,
                      start: float, end: float) -> float:
    """Best normalised visual relevance among frames inside [start, end]."""
    if len(times) == 0:
        return 0.0
    mask = (times >= start - 0.5) & (times <= end + 0.5)
    if not mask.any():
        idx = int(np.argmin(np.abs(times - (start + end) / 2)))
        return float(norm[idx])
    return float(norm[mask].max())


def region_visual_tags(visual: dict, start: float, end: float) -> list[str]:
    """Zero-shot CLIP tags for what is on screen during [start, end]."""
    embs  = visual.get("emb")
    times = visual.get("time")
    if embs is None or len(embs) == 0:
        return []

    mask = (times >= start - 0.5) & (times <= end + 0.5)
    if mask.any():
        sel = embs[mask]
    else:
        idx = int(np.argmin(np.abs(times - (start + end) / 2)))
        sel = embs[idx:idx + 1]

    region_emb = sel.mean(axis=0)
    region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9)
    sims = get_concept_embs() @ region_emb

    order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K]
    return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM]


def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]:
    for r in regions:
        r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"])
    return regions


# Concepts that imply readable on-screen text — the gate for the OCR fallback.
_TEXT_CONCEPT_IDX = [
    i for i, c in enumerate(VISUAL_CONCEPTS)
    if any(k in c.lower() for k in ("text on screen", "source code", "webpage",
                                    "screen recording", "slides"))
]


def has_on_screen_text(visual: dict) -> bool:
    """True if any frame looks like it contains readable text — used to decide whether
    OCR of burned-in captions is worth attempting."""
    embs = visual.get("emb")
    if embs is None or len(embs) == 0 or not _TEXT_CONCEPT_IDX:
        return False
    try:
        concepts = get_concept_embs()[_TEXT_CONCEPT_IDX]            # (k, D), normalised
        en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
        sims = en @ concepts.T                                      # (frames, k)
        return float(sims.max()) >= VISUAL_ONSCREEN_TEXT_SIM
    except Exception:
        return False