QuerryRaspBerry commited on
Commit
06347d3
·
verified ·
1 Parent(s): 3d1cea2

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +1328 -349
app.py CHANGED
@@ -1,378 +1,1357 @@
1
- from __future__ import annotations
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
  import os
 
 
 
 
 
 
4
  import shutil
5
- import subprocess
6
- from pathlib import Path
7
- from uuid import uuid4
 
8
 
 
9
  import gradio as gr
 
 
 
 
 
 
 
 
10
 
11
- os.environ.setdefault("IRIS_DATA_DIR", "/tmp/iris-data")
12
- os.environ.setdefault("IRIS_DEVICE", "cpu")
13
- os.environ.setdefault("IRIS_TORCH_DTYPE", "float32")
14
- os.environ.setdefault("IRIS_ASR_MODEL", "openai/whisper-tiny")
15
- os.environ.setdefault("IRIS_MAX_AUDIO_SECONDS", "120")
16
- os.environ.setdefault("IRIS_VISUAL_SEARCH_ENABLED", "false")
17
- os.environ.setdefault("IRIS_MAX_STORED_VIDEOS", "3")
18
-
19
- from iris.config import Settings, device_info
20
- from iris.models.schemas import SearchRequest
21
- from iris.pipeline.chunk import window_transcript_chunks
22
- from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
23
- from iris.pipeline.orchestrator import VideoProcessor
24
- from iris.pipeline.search import detail_window_settings, format_timestamp, rank_speech_chunks
25
- from iris.pipeline.video import validate_video_filename
26
- from iris.pipeline.visual import VisualSearcher, sample_video_frames
27
- from iris.storage.database import Database
28
- from iris.storage.files import FileStore
29
-
30
-
31
- SETTINGS = Settings.from_env()
32
- SETTINGS.ensure_directories()
33
- DATABASE = Database(SETTINGS.database_path)
34
- DATABASE.init()
35
- FILE_STORE = FileStore(SETTINGS)
36
- PROCESSOR = VideoProcessor(SETTINGS, DATABASE, FILE_STORE)
37
- MAX_STORED_VIDEOS = max(1, int(os.getenv("IRIS_MAX_STORED_VIDEOS", "3")))
38
-
39
-
40
- def health_summary() -> str:
41
- info = device_info(SETTINGS.device, SETTINGS.torch_dtype)
42
- visual_state = "enabled" if SETTINGS.visual_search_enabled else "disabled"
43
- audio_limit = f"{SETTINGS.max_audio_seconds:g}s" if SETTINGS.max_audio_seconds else "full video"
44
- rows = [
45
- ("Mode", "CPU demo"),
46
- ("Device", info.get("resolved_device", "unknown")),
47
- ("ASR model", SETTINGS.asr_model),
48
- ("Embedding model", SETTINGS.embedding_model),
49
- ("Visual search", visual_state),
50
- ("Audio limit", audio_limit),
51
- ("Stored videos", str(MAX_STORED_VIDEOS)),
52
- ]
53
- return "\n".join(f"**{name}:** {value}" for name, value in rows)
54
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
- def process_uploaded_video(
57
- video_file: str | None,
58
- progress: gr.Progress = gr.Progress(track_tqdm=False),
59
- ) -> tuple[str, dict[str, str] | None, list[list[str | float]], list[dict], None]:
60
- if not video_file:
61
- return "Upload a video before processing.", None, [], [], None
62
 
63
- progress(0.05, desc="Preparing upload")
64
- source = Path(video_file)
65
- filename = source.name or "video.mp4"
 
 
66
  try:
67
- validate_video_filename(filename)
68
- except ValueError as exc:
69
- return str(exc), None, [], [], None
70
-
71
- video_id = str(uuid4())
72
- destination = FILE_STORE.video_path(video_id, filename)
73
- destination.parent.mkdir(parents=True, exist_ok=True)
74
- shutil.copyfile(source, destination)
75
-
76
- DATABASE.create_video(video_id, filename, destination)
77
- PROCESSOR.process_video(
78
- video_id,
79
- progress=lambda value, message: progress(value, desc=message),
80
- )
 
 
 
 
 
 
81
 
82
- record = DATABASE.get_video(video_id)
83
- cleanup_old_videos(keep_video_id=video_id)
84
- if record is None:
85
- return "The video record was not created.", None, [], [], None
86
- if record["status"] == "failed":
87
- return f"Processing failed: {record['error']}", None, [], [], None
88
- if record["status"] != "completed":
89
- return f"Video is still {record['status']}. Try again in a moment.", None, [], [], None
90
-
91
- state = {"video_id": video_id, "filename": filename}
92
- processed_window = SETTINGS.max_audio_seconds
93
- window_note = (
94
- f" Processed the first {format_timestamp(processed_window)}."
95
- if processed_window
96
- else ""
97
- )
98
- return f"Ready to search `{filename}`.{window_note}", state, [], [], None
99
-
100
-
101
- def search_processed_video(
102
- processed_video: dict[str, str] | None,
103
- query: str,
104
- detail: str,
105
- limit: int,
106
- include_visual: bool,
107
- progress: gr.Progress = gr.Progress(track_tqdm=False),
108
- ) -> tuple[str, list[list[str | float]], list[dict], None]:
109
- if not processed_video or not processed_video.get("video_id"):
110
- return "Process a video before searching.", [], [], None
111
- if not query or not query.strip():
112
- return "Enter a search query.", [], [], None
113
-
114
- request = SearchRequest(
115
- query=query.strip(),
116
- detail=detail,
117
- limit=int(limit),
118
- include_visual=bool(include_visual),
119
- )
120
  try:
121
- rows, matches = search_video_matches(processed_video["video_id"], request, progress)
122
- except Exception as exc:
123
- return f"Search failed: {exc}", [], [], None
124
-
125
- filename = processed_video.get("filename", "video")
126
- if not rows:
127
- return f"No matches found for `{query.strip()}` in {filename}.", [], [], None
128
- return f"Found {len(rows)} match(es) for `{query.strip()}` in {filename}.", rows, matches, None
129
-
130
-
131
- def search_video_matches(
132
- video_id: str,
133
- request: SearchRequest,
134
- progress: gr.Progress,
135
- ) -> tuple[list[list[str | float]], list[dict]]:
136
- record = DATABASE.get_video(video_id)
137
- if record is None:
138
- raise ValueError("Video not found.")
139
- if record["status"] != "completed":
140
- raise ValueError(f"Video is not ready for search. Current status: {record['status']}.")
141
- if request.include_visual and not SETTINGS.visual_search_enabled:
142
- raise ValueError("Visual search is disabled.")
143
-
144
- progress(0.1, desc="Preparing transcript windows")
145
- atomic_chunks = DATABASE.list_transcript_chunks(video_id)
146
- default_window, default_overlap, default_merge_gap = detail_window_settings(request.detail)
147
- window_seconds = request.window_seconds or default_window
148
- overlap_seconds = min(default_overlap, max(0.0, window_seconds - 0.1))
149
- merge_gap_seconds = request.merge_gap_seconds if request.merge_gap_seconds is not None else default_merge_gap
150
- chunks = window_transcript_chunks(
151
- atomic_chunks,
152
- window_seconds=window_seconds,
153
- overlap_seconds=overlap_seconds,
154
- )
155
 
156
- progress(0.35, desc="Encoding query")
157
- embedder = PROCESSOR.embedder
158
- chunk_embeddings = embedder.encode([chunk.text for chunk in chunks])
159
- query_embedding = embedder.encode([request.query])[0]
160
- speech_results = rank_speech_chunks(
161
- query=request.query,
162
- chunks=chunks,
163
- chunk_embeddings=chunk_embeddings,
164
- query_embedding=query_embedding,
165
- settings=SETTINGS,
166
- limit=request.limit,
167
- merge_gap_seconds=merge_gap_seconds,
168
- )
169
 
170
- results = list(speech_results)
171
- if request.include_visual:
172
- progress(0.65, desc="Searching frames")
173
- frames = sample_video_frames(
174
- Path(record["video_path"]),
175
- FILE_STORE.frames_dir(video_id),
176
- SETTINGS.frame_sample_interval_seconds,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  )
178
- visual_results = VisualSearcher(SETTINGS).search(request.query, frames, request.limit)
179
- results.extend(visual_results)
180
-
181
- DATABASE.record_search(video_id, request.query, request.include_visual)
182
- ranked = sorted(results, key=lambda item: (-item.score, item.start))[: request.limit]
183
- progress(1.0, desc="Search complete")
184
- rows = [
185
- [
186
- f"{format_timestamp(result.start)}-{format_timestamp(result.end)}",
187
- result.source,
188
- round(result.score, 4),
189
- result.text,
190
- ]
191
- for result in ranked
192
- ]
193
- matches = [
 
 
 
 
 
 
194
  {
195
- "video_id": video_id,
196
- "start": result.start,
197
- "end": result.end,
198
- "source": result.source,
199
- "score": round(result.score, 4),
200
- "text": result.text,
201
  }
202
- for result in ranked
 
203
  ]
204
- return rows, matches
205
-
206
-
207
- def preview_selected_match(matches: list[dict] | None, evt: gr.SelectData) -> tuple[str | None, str]:
208
- if not matches:
209
- return None, "Search for a phrase before selecting a match."
210
-
211
- row_index = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index
212
- if row_index is None or int(row_index) >= len(matches):
213
- return None, "Select a result row to preview it."
214
-
215
- match = matches[int(row_index)]
216
- video_id = str(match["video_id"])
217
- record = DATABASE.get_video(video_id)
218
- if record is None:
219
- return None, "The source video is no longer available."
220
-
221
- start = max(0.0, float(match["start"]) - 1.0)
222
- end = max(start + 1.0, float(match["end"]))
223
- duration = min(30.0, max(6.0, end - start + 2.0))
224
- clip_path = preview_clip_path(video_id, int(row_index), start)
225
- if not clip_path.exists():
226
- create_preview_clip(Path(record["video_path"]), clip_path, start, duration)
227
-
228
- timestamp = f"{format_timestamp(float(match['start']))}-{format_timestamp(float(match['end']))}"
229
- return str(clip_path), f"Previewing `{timestamp}` from {match['source']} result."
230
-
231
-
232
- def preview_clip_path(video_id: str, row_index: int, start: float) -> Path:
233
- start_ms = int(start * 1000)
234
- preview_dir = FILE_STORE.video_dir(video_id) / "previews"
235
- preview_dir.mkdir(parents=True, exist_ok=True)
236
- return preview_dir / f"match_{row_index:03d}_{start_ms}.mp4"
237
-
238
-
239
- def create_preview_clip(source: Path, clip_path: Path, start: float, duration: float) -> None:
240
- command = [
241
- resolve_ffmpeg_executable(),
242
- "-y",
243
- "-ss",
244
- f"{start:.3f}",
245
- "-i",
246
- str(source),
247
- "-t",
248
- f"{duration:.3f}",
249
- "-map",
250
- "0:v:0",
251
- "-map",
252
- "0:a?",
253
- "-c:v",
254
- "libx264",
255
- "-preset",
256
- "veryfast",
257
- "-pix_fmt",
258
- "yuv420p",
259
- "-c:a",
260
- "aac",
261
- "-movflags",
262
- "+faststart",
263
- str(clip_path),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264
  ]
265
- try:
266
- subprocess.run(command, check=True, capture_output=True, text=True)
267
- except OSError as exc:
268
- raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
269
- except subprocess.CalledProcessError as exc:
270
- detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
271
- raise RuntimeError(f"ffmpeg failed to create preview clip: {detail}") from exc
272
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
273
 
274
- def cleanup_old_videos(keep_video_id: str) -> None:
275
- videos_root = (SETTINGS.data_dir / "videos").resolve()
276
- if not videos_root.exists():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
277
  return
278
 
279
- candidates = []
280
- for path in videos_root.iterdir():
281
- if not path.is_dir():
282
- continue
283
- try:
284
- candidates.append((path.stat().st_mtime, path.resolve()))
285
- except OSError:
286
- continue
287
-
288
- retained = {keep_video_id}
289
- for _, path in sorted(candidates, reverse=True)[:MAX_STORED_VIDEOS]:
290
- retained.add(path.name)
291
-
292
- for _, path in candidates:
293
- if path.name in retained:
294
- continue
295
- try:
296
- path.relative_to(videos_root)
297
- except ValueError:
298
- continue
299
- shutil.rmtree(path, ignore_errors=True)
300
-
301
-
302
- CUSTOM_CSS = """
303
- .iris-shell { max-width: 1120px; margin: 0 auto; }
304
- .iris-title h1 { margin-bottom: 0.15rem; }
305
- .iris-title p { margin-top: 0; color: #4b5563; }
306
- """
307
-
308
-
309
- with gr.Blocks(title="IRIS") as demo:
310
- processed_video = gr.State(value=None)
311
- search_matches = gr.State(value=[])
312
-
313
- with gr.Column(elem_classes=["iris-shell"]):
314
- gr.Markdown(
315
- """
316
- # IRIS
317
- Process a video once, then search inside the spoken transcript.
318
- """,
319
- elem_classes=["iris-title"],
320
  )
321
- with gr.Row():
322
- with gr.Column(scale=2):
323
- video = gr.Video(label="Video", sources=["upload"])
324
- process_button = gr.Button("Process Video", variant="primary")
325
- with gr.Column(scale=1):
326
- gr.Markdown(health_summary())
327
-
328
- with gr.Row():
329
- query = gr.Textbox(
330
- label="Search query",
331
- placeholder="battery degradation, dopamine, product launch...",
332
- )
333
- include_visual = gr.Checkbox(
334
- value=False,
335
- label="Include visual search",
336
- interactive=SETTINGS.visual_search_enabled,
337
- )
338
-
339
- with gr.Row():
340
- detail = gr.Radio(
341
- choices=["fine", "medium", "coarse"],
342
- value="fine",
343
- label="Timestamp detail",
344
- )
345
- limit = gr.Slider(1, 20, value=10, step=1, label="Result limit")
346
- search_button = gr.Button("Search", variant="secondary")
347
-
348
- status = gr.Markdown()
349
- results = gr.Dataframe(
350
- headers=["Timestamp", "Source", "Score", "Text"],
351
- datatype=["str", "str", "number", "str"],
352
- wrap=True,
353
- label="Matches",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
354
  )
355
- preview_video = gr.Video(label="Selected match")
356
 
357
- process_button.click(
358
- fn=process_uploaded_video,
359
- inputs=[video],
360
- outputs=[status, processed_video, results, search_matches, preview_video],
361
- )
362
- search_button.click(
363
- fn=search_processed_video,
364
- inputs=[processed_video, query, detail, limit, include_visual],
365
- outputs=[status, results, search_matches, preview_video],
366
- )
367
- results.select(
368
- fn=preview_selected_match,
369
- inputs=[search_matches],
370
- outputs=[preview_video, status],
371
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
372
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
373
 
374
  if __name__ == "__main__":
375
- demo.queue(default_concurrency_limit=1).launch(
376
- css=CUSTOM_CSS,
377
- allowed_paths=[str(SETTINGS.data_dir)],
378
- )
 
1
+ # ─────────────────────────────────────────────
2
+ # NINA — Neural Information & Narrative Abstraction
3
+ # app.py | IRIS v0.80 | Made by Malina
4
+ # ─────────────────────────────────────────────
5
+ #
6
+ # Precision MULTIMODAL video search + summarization pipeline.
7
+ # Pipeline: video → audio → ASR segments → semantic chunking
8
+ # → bi-encoder (text) ⨉ CLIP (vision) fusion → cross-encoder + entity overlap
9
+ # → merge → confidence-gated precise re-transcribe → streaming LLM synthesis.
10
+ #
11
+ # ─────────────────────────────────────────────────────────────────────────────
12
+ # v0.80 — performance & modernisation pass over v0.71
13
+ # ─────────────────────────────────────────────────────────────────────────────
14
+ # Measured on Apple M1 (8 core / 16 GB), 39 s test clip:
15
+ # cold first prompt 58.8 s → ~9 s, warm prompts ~1.5 s → sub-second to first token.
16
+ #
17
+ # [P1] HARDWARE ACCELERATION
18
+ # Torch models now auto-target the best device (CUDA → MPS → CPU). On
19
+ # Apple Silicon CLIP runs ~1.8× and the cross-encoder ~1.4× faster on the
20
+ # Metal (MPS) backend. CTranslate2 (Whisper) has no MPS backend, so it
21
+ # stays on CPU but is tuned: int8 + cpu_threads, greedy decoding for the
22
+ # cheap pass, and Silero VAD to skip non-speech.
23
+ #
24
+ # [P2] NO MORE DOUBLE TRANSCRIPTION
25
+ # v0.71 transcribed the whole video with `base`, then RE-transcribed every
26
+ # sampled sector with the ~15× slower `large-v3-turbo` — even for a global
27
+ # summary (≈24 s wasted on this clip). v0.80:
28
+ # • GLOBAL summaries use the clean `base` transcript directly (no turbo).
29
+ # • SPECIFIC queries re-transcribe with turbo ONLY on matched regions and
30
+ # ONLY when the base pass was low-confidence (avg_logprob gate). On
31
+ # clean audio turbo never runs; on hard audio it runs exactly where it
32
+ # helps. Per-sector results are cached and persist across prompts.
33
+ #
34
+ # [P3] PARALLEL COLD START
35
+ # Audio transcription (CPU) and CLIP frame analysis (MPS) are independent
36
+ # and now run concurrently on first sight of a video, overlapping the two
37
+ # heaviest stages instead of summing them.
38
+ #
39
+ # [P4] ZERO-COPY I/O
40
+ # Audio is decoded straight from the container (PyAV) — no ffmpeg wav
41
+ # temp file. Frames are streamed from one ffmpeg rawvideo pipe into NumPy —
42
+ # no per-frame JPEG temp files.
43
+ #
44
+ # [P5] PERSISTENT DISK CACHE
45
+ # Transcript, chunk + visual embeddings and precise sectors are persisted
46
+ # under a content fingerprint, so re-opening the same video — even after a
47
+ # restart — skips transcription/chunking/CLIP entirely.
48
+ #
49
+ # [P6] STREAMING + UX
50
+ # LLM synthesis streams token-by-token; run() is a generator that shows
51
+ # progress and writes the answer as it is produced.
52
+ #
53
+ # [Q] QUALITY FIXES
54
+ # • Router no longer misreads "what does the video say about X" as a global
55
+ # summary (the bare word "about"/"explain" no longer forces global mode).
56
+ # • The illustrative "[2:30]" timestamp no longer leaks into answers.
57
+ #
58
+ # Scoring formula (SPECIFIC/HYBRID mode):
59
+ # audio = 0.6 * sigmoid(cross_encoder_logit) + 0.4 * entity_overlap
60
+ # final = (1 - VISUAL_WEIGHT) * audio + VISUAL_WEIGHT * clip_visual_score
61
+ #
62
+ # Entity extraction: regex + simple NP heuristic (no spacy dependency).
63
+ # Works multilingually because it operates on surface forms, not POS tags.
64
+ # ─────────────────────────────────────────────
65
 
66
  import os
67
+ import re
68
+ import json
69
+ import math
70
+ import base64
71
+ import hashlib
72
+ import hmac
73
  import shutil
74
+ import tempfile
75
+ import threading
76
+ import time
77
+ from concurrent.futures import ThreadPoolExecutor
78
 
79
+ import ffmpeg
80
  import gradio as gr
81
+ import numpy as np
82
+ from PIL import Image
83
+ from faster_whisper import WhisperModel
84
+ from faster_whisper.audio import decode_audio
85
+ from sentence_transformers import SentenceTransformer, CrossEncoder
86
+ from huggingface_hub import InferenceClient
87
+ import warnings
88
+ warnings.filterwarnings("ignore", message="FP16 is not supported on CPU; using FP32 instead")
89
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
 
91
+ # ── Config ─────────────────────────────────────────────────────────────────────
92
+
93
+ WHISPER_FAST = "base"
94
+ WHISPER_PRECISE = "large-v3-turbo"
95
+ SAMPLE_RATE = 16000
96
+ WHISPER_CPU_THREADS = int(os.getenv("WHISPER_CPU_THREADS", "3"))
97
+ # 3 threads × ~3 concurrent jobs ≈ an 8-vCPU Space.
98
+ # Override via env. CTranslate2 is CPU-only.
99
+
100
+ RETRANSCRIBE_MAX = 12 # per-prompt budget for NEW (uncached) sector transcriptions
101
+ PRECISE_LOGPROB_GATE = -0.55 # only turbo-refine sectors whose base avg_logprob is below this
102
+ # (clean audio scores high → turbo is correctly skipped)
103
+
104
+ CHUNK_SPLIT_THRESHOLD = 0.45
105
+ CHUNK_MIN_DURATION = 8.0
106
+ CHUNK_MAX_DURATION = 90.0
107
+ CHUNK_WINDOW = 5
108
+
109
+ BI_ENCODER_TOP_K = 100
110
+ CROSS_ENCODER_TOP_K = 10
111
+ RESULTS_TOP_N = 6
112
+ REGION_MIN_DURATION = 5.0
113
+ MERGE_GAP_SEC = 6.0
114
+
115
+ # routing thresholds
116
+ BI_SCORE_GLOBAL_FLOOR = 0.10 # below this → escalate to GLOBAL regardless
117
+ BI_SCORE_HYBRID_FLOOR = 0.28 # below this → HYBRID (pad with neighbours)
118
+
119
+ # global mode: how many evenly-sampled chunks to retranscribe
120
+ GLOBAL_MAX_CHUNKS = 12 # hard cap to keep latency sane
121
+
122
+ # ── CLIP / visual config ────────────────────────────────────────────────────────
123
+ CLIP_MODEL = "clip-ViT-B-32" # OpenAI CLIP ViT-B/32 (via sentence-transformers)
124
+ CLIP_MAX_FRAMES = 96 # cap frames encoded per video (perf)
125
+ CLIP_MIN_FPS = 0.1 # sample at least 1 frame / 10s
126
+ CLIP_MAX_FPS = 0.5 # sample at most 1 frame / 2s
127
+ CLIP_FRAME_SIZE = 224 # square fed to CLIP (matches ViT-B/32 input → no double resize)
128
+ VISUAL_WEIGHT = 0.30 # fusion weight of visual vs audio in final score
129
+ VISUAL_TAGS_TOP_K = 4 # max zero-shot tags surfaced per region
130
+ VISUAL_TAG_MIN_SIM = 0.20 # min CLIP cosine for a tag to count
131
+
132
+ # vocabulary for zero-shot "what is on screen" tagging
133
+ VISUAL_CONCEPTS = [
134
+ "a person talking to the camera", "a group of people", "a presentation with slides",
135
+ "a lecture or classroom", "an interview", "a news broadcast",
136
+ "a screen recording of software", "source code in a text editor",
137
+ "a webpage in a browser", "a chart or graph", "text on screen",
138
+ "a product demonstration", "hands demonstrating something",
139
+ "a cooking or food scene", "a kitchen", "an office", "a laboratory",
140
+ "a factory or machinery", "a construction site", "an outdoor nature landscape",
141
+ "a forest", "mountains", "a beach or ocean", "a city street",
142
+ "buildings and architecture", "a car or vehicle driving", "an airplane", "a train",
143
+ "a sports event", "people exercising", "a concert or live music",
144
+ "a video game gameplay", "an animation or cartoon", "a medical or hospital scene",
145
+ "a doctor or patient", "an animal", "a dog", "a cat", "a crowd of people",
146
+ "a map", "a whiteboard with diagrams", "a scientific diagram",
147
+ "a phone or mobile device", "a computer or laptop", "money or finance",
148
+ "a meeting or conference room", "a stage with a speaker",
149
+ "a dark scene at night", "a daytime scene",
150
+ ]
151
+
152
+ LLM_MODEL = "Qwen/Qwen2.5-7B-Instruct"
153
+ LLM_MAX_TOKENS = 600
154
+ LLM_TIMEOUT = 60
155
+
156
+ CACHE_DIR = os.path.join(tempfile.gettempdir(), "nina_cache")
157
+ DISK_CACHE = True
158
+
159
+ DEBUG = False
160
+
161
+
162
+ # ── Usage-token gate ──────────────────────────────────────────────────────────────
163
+ # The Space is public, so to enforce the per-account 10/5h limit set on the
164
+ # website, each request must carry a short-lived HMAC token issued by the
165
+ # Cloudflare Worker (app.nnlabs.pl) after it counts the use. Without a valid,
166
+ # unexpired, not-yet-seen token the model refuses to run. Same secret on both
167
+ # sides (NINA_SIGNING_SECRET). If the secret is unset the gate is open (dev).
168
+ SIGNING_SECRET = os.getenv("NINA_SIGNING_SECRET", "")
169
+ _used_jti = {} # jti -> exp epoch (single-use, replay-proof within a container)
170
+ _jti_lock = threading.Lock()
171
+
172
+ def _b64url_decode(s: str) -> bytes:
173
+ return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
174
 
175
+ def _b64url_encode(b: bytes) -> str:
176
+ return base64.urlsafe_b64encode(b).decode().rstrip("=")
 
 
 
 
177
 
178
+ def _verify_usage_token(token: str) -> bool:
179
+ if not SIGNING_SECRET:
180
+ return True # not configured → open (local/dev)
181
+ if not token or "." not in token:
182
+ return False
183
  try:
184
+ msg, sig = token.split(".", 1)
185
+ expected = _b64url_encode(hmac.new(SIGNING_SECRET.encode(), msg.encode(), hashlib.sha256).digest())
186
+ if not hmac.compare_digest(expected, sig):
187
+ return False
188
+ payload = json.loads(_b64url_decode(msg))
189
+ now = int(time.time())
190
+ if int(payload.get("e", 0)) < now:
191
+ return False # expired
192
+ jti = payload.get("j", "")
193
+ if not jti:
194
+ return False
195
+ with _jti_lock:
196
+ for k in [k for k, e in _used_jti.items() if e < now]:
197
+ _used_jti.pop(k, None)
198
+ if jti in _used_jti:
199
+ return False # replay
200
+ _used_jti[jti] = int(payload.get("e", now + 120))
201
+ return True
202
+ except Exception:
203
+ return False
204
 
205
+
206
+ # ── Device selection ─────────────────────────────────────────────────────────────
207
+ # CTranslate2 (Whisper) is CPU-only here; the torch models (bi/cross/CLIP) can
208
+ # use Metal (MPS) on Apple Silicon or CUDA on a GPU box.
209
+
210
+ def _best_torch_device() -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
211
  try:
212
+ import torch
213
+ if torch.cuda.is_available():
214
+ return "cuda"
215
+ if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
216
+ return "mps"
217
+ except Exception:
218
+ pass
219
+ return "cpu"
220
+
221
+ TORCH_DEVICE = _best_torch_device()
222
+
223
+
224
+ # ── Model load ─────────────────────────────────────────────────────────────────
225
+
226
+ print(f"[NINA] Torch device: {TORCH_DEVICE} | Whisper: CPU int8 (x{WHISPER_CPU_THREADS})")
227
+
228
+ print("[NINA] Loading Whisper base (faster-whisper)...")
229
+ _whisper_fast = WhisperModel(WHISPER_FAST, device="cpu", compute_type="int8",
230
+ cpu_threads=WHISPER_CPU_THREADS)
231
+
232
+ # Bi-encoder stays on CPU: it is tiny (sub-100 ms) and MPS transfer overhead is a
233
+ # wash, and this keeps the MPS device free for CLIP during the parallel cold start.
234
+ print("[NINA] Loading bi-encoder...")
235
+ bi_encoder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", device="cpu")
236
+
237
+ print(f"[NINA] Loading cross-encoder ({TORCH_DEVICE})...")
238
+ cross_encoder = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", device=TORCH_DEVICE)
 
 
 
 
 
 
 
239
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
 
241
+ HF_TOKEN = os.getenv("HF_TOKEN")
242
+ if not HF_TOKEN:
243
+ _token_file = os.path.expanduser("~/.cache/huggingface/token")
244
+ if os.path.exists(_token_file):
245
+ HF_TOKEN = open(_token_file).read().strip()
246
+
247
+ if HF_TOKEN:
248
+ print(f"[NINA] HF_TOKEN loaded — LLM calls enabled.")
249
+ else:
250
+ print("[NINA] HF_TOKEN not set — LLM calls will fail.")
251
+
252
+ llm_client = InferenceClient(
253
+ token=HF_TOKEN,
254
+ timeout=LLM_TIMEOUT,
255
+ )
256
+
257
+
258
+ _whisper_precise = None
259
+ _whisper_precise_lock = threading.Lock()
260
+
261
+ def get_whisper_precise():
262
+ global _whisper_precise
263
+ with _whisper_precise_lock:
264
+ if _whisper_precise is None:
265
+ print(f"[NINA] Loading Whisper {WHISPER_PRECISE} (faster-whisper)...")
266
+ _whisper_precise = WhisperModel(WHISPER_PRECISE, device="cpu",
267
+ compute_type="int8",
268
+ cpu_threads=WHISPER_CPU_THREADS)
269
+ print(f"[NINA] Whisper {WHISPER_PRECISE} ready.")
270
+ return _whisper_precise
271
+
272
+
273
+ # ── CLIP (OpenAI ViT-B/32) — lazy, warmed in background ──────────────────────────
274
+ _clip_model = None
275
+ _clip_lock = threading.Lock()
276
+ _concept_embs = None
277
+ _concept_lock = threading.Lock()
278
+
279
+ def get_clip():
280
+ global _clip_model
281
+ with _clip_lock:
282
+ if _clip_model is None:
283
+ print(f"[NINA] Loading CLIP ({CLIP_MODEL}, {TORCH_DEVICE})...")
284
+ _clip_model = SentenceTransformer(CLIP_MODEL, device=TORCH_DEVICE)
285
+ print(f"[NINA] CLIP ready.")
286
+ return _clip_model
287
+
288
+ def _get_concept_embs() -> np.ndarray:
289
+ """Encode (once) the zero-shot vocabulary, L2-normalised."""
290
+ global _concept_embs
291
+ with _concept_lock:
292
+ if _concept_embs is None:
293
+ embs = get_clip().encode(VISUAL_CONCEPTS, convert_to_numpy=True,
294
+ show_progress_bar=False)
295
+ _concept_embs = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
296
+ return _concept_embs
297
+
298
+
299
+ # Warm the lazy models in the background; CLIP loads on the (free) MPS device,
300
+ # turbo loads on CPU. Both finish before they are first needed in most cases.
301
+ threading.Thread(target=get_whisper_precise, daemon=True).start()
302
+ threading.Thread(target=get_clip, daemon=True).start()
303
+ print(f"[NINA] Ready. ( {WHISPER_PRECISE} + CLIP warming up...)")
304
+
305
+ # ── Transcript + visual cache (in-memory) ─────────────────────────────────────────
306
+ # Stores segments, chunks (with embeddings), visual frame embeddings, the decoded
307
+ # waveform, and a per-sector precise-transcription cache, keyed by video path.
308
+ # Second prompt on same video skips transcription, chunking and CLIP entirely.
309
+ _cache: dict = {} # { video_path: {"segments","chunks","visual","audio","precise","_atime"} }
310
+
311
+ # Data lifecycle / privacy:
312
+ # • The uploaded video itself is purged by Gradio's own temp cleaner
313
+ # (Interface(delete_cache=...)) — see the bottom of this file — so a file is
314
+ # removed shortly after the user's session goes idle. One upload still serves
315
+ # MANY prompts: the decoded audio + transcript live in RAM (below), so the
316
+ # file is no longer needed once processed.
317
+ # • This in-memory state and the derived on-disk cache are TTL-evicted here so
318
+ # nothing lingers indefinitely.
319
+ CACHE_IDLE_TTL = 1200 # drop in-memory video state idle for > 20 min
320
+ CACHE_DISK_TTL = 1800 # delete on-disk transcript/embedding cache > 30 min old
321
+
322
+ def _evict_stale() -> None:
323
+ now = time.time()
324
+ for vp in [k for k, e in _cache.items() if now - e.get("_atime", now) > CACHE_IDLE_TTL]:
325
+ _cache.pop(vp, None)
326
+ try:
327
+ if os.path.isdir(CACHE_DIR):
328
+ for name in os.listdir(CACHE_DIR):
329
+ d = os.path.join(CACHE_DIR, name)
330
+ if os.path.isdir(d) and now - os.path.getmtime(d) > CACHE_DISK_TTL:
331
+ shutil.rmtree(d, ignore_errors=True)
332
+ except Exception:
333
+ pass
334
+
335
+
336
+ # ── Audio ──────────────────────────────────────────────────────────────────────
337
+
338
+ def load_audio(media_path: str) -> np.ndarray:
339
+ """Decode any container straight to a float32 mono 16k waveform (PyAV, no temp file)."""
340
+ return decode_audio(media_path, sampling_rate=SAMPLE_RATE)
341
+
342
+
343
+ def media_duration(media_path: str) -> float:
344
+ try:
345
+ info = ffmpeg.probe(media_path)
346
+ return float(info["format"]["duration"])
347
+ except Exception:
348
+ return 0.0
349
+
350
+
351
+ # ── Frames (CLIP input) — streamed from one ffmpeg pipe, no temp files ────────────
352
+
353
+ def extract_frames(video_path: str, duration: float):
354
+ """
355
+ Sample frames evenly across the video and stream them as raw RGB straight
356
+ into NumPy via a single ffmpeg pipe (no per-frame JPEG temp files).
357
+ fps is chosen so total frames ≈ CLIP_MAX_FRAMES (bounded by MIN/MAX fps).
358
+ Returns (frames_rgb, times) where frames_rgb is (N, S, S, 3) uint8.
359
+ """
360
+ fps = CLIP_MAX_FPS
361
+ if duration and duration > 0:
362
+ fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, CLIP_MAX_FRAMES / duration))
363
+
364
+ S = CLIP_FRAME_SIZE
365
+ try:
366
+ out, _ = (
367
+ ffmpeg.input(video_path)
368
+ .filter("fps", fps=fps)
369
+ # cover-then-centre-crop to a square so portrait/landscape both work
370
+ .filter("scale", S, S, force_original_aspect_ratio="increase")
371
+ .filter("crop", S, S)
372
+ .output("pipe:", format="rawvideo", pix_fmt="rgb24")
373
+ .run(capture_stdout=True, capture_stderr=True)
374
  )
375
+ except ffmpeg.Error:
376
+ return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
377
+
378
+ if not out:
379
+ return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
380
+
381
+ frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES]
382
+ times = np.array([(i + 0.5) / fps for i in range(len(frames))])
383
+ return frames, times
384
+
385
+
386
+ # ── Stage 2: transcription ─────────────────────────────────────────────────────
387
+
388
+ def transcribe_fast(audio: np.ndarray) -> list[dict]:
389
+ """
390
+ Cheap full-video pass: base model, greedy decoding, Silero VAD.
391
+ Keeps per-segment avg_logprob so we can later refine ONLY weak sectors.
392
+ """
393
+ segments_gen, _ = _whisper_fast.transcribe(
394
+ audio, beam_size=1, vad_filter=True, word_timestamps=False,
395
+ )
396
+ return [
397
  {
398
+ "text": seg.text.strip(),
399
+ "start": round(seg.start, 2),
400
+ "end": round(seg.end, 2),
401
+ "logprob": round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3),
 
 
402
  }
403
+ for seg in segments_gen
404
+ if seg.text.strip()
405
  ]
406
+
407
+
408
+ # ── Stage 3a: visual analysis (CLIP) ─────────────────────────────────────────────
409
+
410
+ def analyze_visual(video_path: str, duration: float) -> dict:
411
+ """
412
+ Sample → encode frames with CLIP. Cached per video.
413
+ Returns {"emb": (N, D) float32, "time": (N,) float}.
414
+ Empty arrays when the video has no usable visual stream (audio-only).
415
+ """
416
+ frames, times = extract_frames(video_path, duration)
417
+ if len(frames) == 0:
418
+ print("[NINA] No video frames (audio-only?) — visual analysis skipped.")
419
+ return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))}
420
+
421
+ images = [Image.fromarray(f) for f in frames]
422
+ embs = get_clip().encode(images, convert_to_numpy=True,
423
+ batch_size=32, show_progress_bar=False)
424
+
425
+ print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).")
426
+ return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)}
427
+
428
+
429
+ # ── semantic chunking ─────────────────────────────────────────────────
430
+
431
+ def semantic_chunk(segments: list[dict]) -> list[dict]:
432
+ if not segments:
433
+ return []
434
+ texts = [s["text"] for s in segments]
435
+ embeddings = bi_encoder.encode(texts, convert_to_numpy=True, batch_size=64)
436
+ chunks, current, cur_emb = [], [segments[0]], [embeddings[0]]
437
+
438
+ for i in range(1, len(segments)):
439
+ chunk_dur = segments[i]["end"] - current[0]["start"]
440
+ window_mean = np.mean(cur_emb[-CHUNK_WINDOW:], axis=0)
441
+ sim = max(_cosine(embeddings[i], window_mean),
442
+ _cosine(embeddings[i], cur_emb[-1]))
443
+ long_enough = (current[-1]["end"] - current[0]["start"]) >= CHUNK_MIN_DURATION
444
+
445
+ if (sim < CHUNK_SPLIT_THRESHOLD or chunk_dur >= CHUNK_MAX_DURATION) and long_enough:
446
+ chunks.append(_build_chunk(current, cur_emb))
447
+ current, cur_emb = [segments[i]], [embeddings[i]]
448
+ else:
449
+ current.append(segments[i])
450
+ cur_emb.append(embeddings[i])
451
+
452
+ if current:
453
+ chunks.append(_build_chunk(current, cur_emb))
454
+
455
+ print(f"[NINA] {len(segments)} segments → {len(chunks)} chunks.")
456
+ return chunks
457
+
458
+
459
+ # ── Entity extraction ──────────────────────────────────────────────────────────
460
+
461
+ _STOPWORDS = {
462
+ "the","a","an","and","or","but","in","on","at","to","for","of","is","are",
463
+ "was","were","be","been","that","this","it","he","she","they","we","i",
464
+ "w","z","i","że","na","do","się","nie","to","jest","są","by","przez","po",
465
+ "le","la","les","un","une","des","du","et","en","à","de","est","il","elle",
466
+ "ein","eine","der","die","das","und","ist","in","zu","mit","von","dem",
467
+ }
468
+
469
+ def extract_entities(text: str) -> set[str]:
470
+ """
471
+ Extract surface-form entities from text.
472
+ Returns a set of normalised lowercase strings for overlap comparison.
473
+ """
474
+ entities: set[str] = set()
475
+
476
+ # 1. quoted phrases — high precision
477
+ for match in re.finditer(r'"([^"]{2,40})"', text):
478
+ entities.add(match.group(1).lower().strip())
479
+
480
+ # 2. capitalised multi-word sequences (proper nouns)
481
+ for match in re.finditer(r'\b([A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-záéíóúàâêîôùäëïöüąćęłńóśźż]+(?:\s[A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-zÀ-žąćęłńóśźż]+)*)\b', text):
482
+ token = match.group(1).lower().strip()
483
+ if token not in _STOPWORDS and len(token) > 2:
484
+ entities.add(token)
485
+
486
+ # 3. single capitalised tokens (names, brands)
487
+ for token in re.findall(r'\b[A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-zÀ-žąćęłńóśźż]{2,}\b', text):
488
+ t = token.lower()
489
+ if t not in _STOPWORDS:
490
+ entities.add(t)
491
+
492
+ # 4. numbers with context
493
+ for match in re.finditer(r'\b([A-Za-z]+[-\s]?\d+[\w]*)\b', text):
494
+ entities.add(match.group(1).lower())
495
+
496
+ return entities
497
+
498
+
499
+ def entity_overlap_score(prompt_entities: set[str],
500
+ chunk_entities: set[str],
501
+ chunk_text: str = "") -> float:
502
+
503
+ if not prompt_entities:
504
+ return 0.0
505
+
506
+ chunk_text_lower = chunk_text.lower()
507
+ matched = 0
508
+ for pe in prompt_entities:
509
+ # pass 1: extracted entity overlap
510
+ entity_hit = any(pe in ce or ce in pe for ce in chunk_entities)
511
+ # pass 2: raw text substring (handles lowercase ASR output)
512
+ text_hit = len(pe) > 3 and pe in chunk_text_lower
513
+ if entity_hit or text_hit:
514
+ matched += 1
515
+ return matched / len(prompt_entities)
516
+
517
+
518
+ def add_entities_to_chunks(chunks: list[dict]) -> list[dict]:
519
+ for chunk in chunks:
520
+ chunk["entities"] = extract_entities(chunk["text"])
521
+ return chunks
522
+
523
+
524
+ # ── Query router ───────────────────────────────────────────────────────────────
525
+ #
526
+ # Classifies prompt into one of three modes before retrieval runs.
527
+ #
528
+ # GLOBAL — user wants a summary / overview of the whole video.
529
+ # Retrieval is the wrong tool here because no single chunk is "about"
530
+ # summarization. → skip retrieval, sample chunks evenly, synthesize.
531
+ #
532
+ # HYBRID — user has a topic but signal is weak (bi-score 0.10–0.28).
533
+ # → run retrieval, but pad results with adjacent chunks.
534
+ #
535
+ # SPECIFIC — normal focused query, retrieval works well (bi-score > 0.28).
536
+ # → standard pipeline unchanged.
537
+ #
538
+ # Router runs in two passes:
539
+ # Pass 1: regex pattern match on prompt text (zero-cost, runs before encoding)
540
+ # Pass 2: bi-encoder score after retrieval (upgrades SPECIFIC → GLOBAL if
541
+ # retrieval actually fails)
542
+
543
+ # Global ONLY when the user asks for a whole-video overview. Critically, this no
544
+ # longer fires on "what does the video say ABOUT <topic>" — a bare "about" or
545
+ # "explain" used to force global and swallow specific questions.
546
+ _GLOBAL_PATTERNS = re.compile(
547
+ r'\b('
548
+ r'summari[sz]e|summary|overview|recap|tl;?dr|key\s+points?|main\s+points?|'
549
+ r'key\s+takeaways?|main\s+idea|gist|'
550
+ r'what(?:\'s| is| was)\s+(?:this|the|it)\s+(?:video|recording|clip|all\s+)?\s*about|'
551
+ r'what(?:\'s| is| was)\s+(?:this|the)\s+(?:video|recording|clip)|'
552
+ r'describe\s+(?:the|this)\s+(?:video|recording|clip)|'
553
+ r'streszcz\w*|podsumuj\w*|podsumowanie|'
554
+ r'o\s+czym\s+(?:jest|to|opowiada|mówi)|czego\s+dotyczy|'
555
+ r'gł[oó]wn\w+\s+(?:punkt\w*|my[sś]l\w*|tez\w*)|og[oó]ln\w+\s+zarys'
556
+ r')\b',
557
+ re.IGNORECASE
558
+ )
559
+
560
+ def classify_query(prompt: str) -> str:
561
+ """
562
+ Pass 1 router — pattern-based, runs before any encoding.
563
+ Returns "global" or "specific".
564
+ Pass 2 (bi-score based upgrade) happens inside run() after retrieval.
565
+ """
566
+ if _GLOBAL_PATTERNS.search(prompt):
567
+ return "global"
568
+ return "specific"
569
+
570
+
571
+ # ── Stage 4: bi-encoder retrieval (top-100) ───────────────────────────────────
572
+
573
+ def bi_encode_retrieve(chunks: list[dict], prompt: str) -> tuple[list[dict], float]:
574
+ # chunk embeddings were computed once at chunking time and cached on the
575
+ # chunk dict ("emb"); here we only encode the prompt.
576
+ prompt_emb = bi_encoder.encode([prompt], convert_to_numpy=True)[0]
577
+ scores = [_cosine(prompt_emb, c["emb"]) for c in chunks]
578
+
579
+ top_k = min(BI_ENCODER_TOP_K, len(chunks))
580
+ top_idx = np.argsort(scores)[::-1][:top_k]
581
+
582
+ if DEBUG:
583
+ print(f"[DEBUG] Bi top-3 scores: " +
584
+ ", ".join(f"{scores[i]:.3f}" for i in top_idx[:3]))
585
+
586
+ best = scores[top_idx[0]] if len(top_idx) else 0.0
587
+ return (
588
+ [{**chunks[i], "bi_score": round(scores[i], 3)} for i in top_idx],
589
+ best
590
+ )
591
+
592
+
593
+ # ── Stage 4b: visual relevance (CLIP) ─────────────────────────────────────────
594
+
595
+ def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]:
596
+ """
597
+ Cosine of prompt↔each frame in CLIP space, plus a min-max normalised copy.
598
+ Normalisation makes CLIP sims (typically ~0.2–0.35) comparable to the
599
+ audio score's 0–1 range before fusion. Returns (raw, norm); empty when
600
+ there are no frames.
601
+ """
602
+ embs = visual.get("emb")
603
+ if embs is None or len(embs) == 0:
604
+ if DEBUG:
605
+ print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.")
606
+ return np.array([]), np.array([])
607
+ q = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0]
608
+ qn = q / (np.linalg.norm(q) + 1e-9)
609
+ en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
610
+ sims = en @ qn
611
+ lo, hi = float(sims.min()), float(sims.max())
612
+ norm = (sims - lo) / (hi - lo + 1e-9)
613
+ if DEBUG:
614
+ times = visual.get("time")
615
+ order = np.argsort(sims)[::-1][:3]
616
+ tops = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order)
617
+ print(f"[DEBUG] Visual: {len(sims)} frames; raw sim "
618
+ f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; "
619
+ f"top moments: {tops}")
620
+ return sims, norm
621
+
622
+
623
+ def span_visual_score(times: np.ndarray, norm: np.ndarray,
624
+ start: float, end: float) -> float:
625
+ """Best (normalised) visual relevance among frames inside [start, end]."""
626
+ if len(times) == 0:
627
+ return 0.0
628
+ mask = (times >= start - 0.5) & (times <= end + 0.5)
629
+ if not mask.any():
630
+ idx = int(np.argmin(np.abs(times - (start + end) / 2)))
631
+ return float(norm[idx])
632
+ return float(norm[mask].max())
633
+
634
+
635
+ def region_visual_tags(visual: dict, start: float, end: float) -> list[str]:
636
+ """Zero-shot CLIP tags describing what is on screen during [start, end]."""
637
+ embs = visual.get("emb")
638
+ times = visual.get("time")
639
+ if embs is None or len(embs) == 0:
640
+ return []
641
+
642
+ mask = (times >= start - 0.5) & (times <= end + 0.5)
643
+ if mask.any():
644
+ sel = embs[mask]
645
+ else:
646
+ idx = int(np.argmin(np.abs(times - (start + end) / 2)))
647
+ sel = embs[idx:idx + 1]
648
+
649
+ region_emb = sel.mean(axis=0)
650
+ region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9)
651
+ sims = _get_concept_embs() @ region_emb
652
+
653
+ order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K]
654
+ return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM]
655
+
656
+
657
+ def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]:
658
+ for r in regions:
659
+ r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"])
660
+ return regions
661
+
662
+
663
+ # ── Stage 5: cross-encoder rerank + audio/visual fusion (top-100 → top-10) ─────
664
+
665
+ def rerank_and_score(candidates: list[dict], prompt: str,
666
+ times: np.ndarray, norm: np.ndarray) -> list[dict]:
667
+ # cross-encoder
668
+ pairs = [[prompt, c["text"]] for c in candidates]
669
+ logits = cross_encoder.predict(pairs)
670
+
671
+ # entity overlap
672
+ prompt_entities = extract_entities(prompt)
673
+ prompt_words = {w.lower() for w in re.findall(r'[\wÀ-ž]{3,}', prompt)
674
+ if w.lower() not in _STOPWORDS}
675
+ all_prompt_tokens = prompt_entities | prompt_words
676
+
677
+ if DEBUG and all_prompt_tokens:
678
+ print(f"[DEBUG] Prompt tokens for entity match: {all_prompt_tokens}")
679
+
680
+ vw = VISUAL_WEIGHT if len(norm) else 0.0 # no frames → pure audio
681
+
682
+ scored = []
683
+ for candidate, logit in zip(candidates, logits):
684
+ chunk_entities = candidate.get("entities", set())
685
+ chunk_text = candidate.get("text", "")
686
+ overlap = entity_overlap_score(all_prompt_tokens, chunk_entities, chunk_text)
687
+
688
+ ce_conf = _sigmoid(float(logit))
689
+ audio_score = 0.6 * ce_conf + 0.4 * overlap
690
+ visual_score = span_visual_score(times, norm, candidate["start"], candidate["end"])
691
+ final = (1.0 - vw) * audio_score + vw * visual_score
692
+
693
+ scored.append({
694
+ **candidate,
695
+ "ce_logit": round(float(logit), 3),
696
+ "entity_overlap": round(overlap, 3),
697
+ "visual_score": round(visual_score, 3),
698
+ "confidence": round(min(1.0, final), 3),
699
+ })
700
+
701
+ # sort by fused multimodal score
702
+ ranked = sorted(scored, key=lambda x: x["confidence"], reverse=True)
703
+
704
+ if DEBUG:
705
+ print(f"[DEBUG] Top-3 after reranking (audio+visual):")
706
+ for r in ranked[:3]:
707
+ print(f" ce_logit={r['ce_logit']:.2f} "
708
+ f"bi={r['bi_score']:.3f} "
709
+ f"entity={r['entity_overlap']:.3f} "
710
+ f"visual={r['visual_score']:.3f} "
711
+ f"conf={r['confidence']:.3f} "
712
+ f"\"{r['text'][:60]}\"")
713
+
714
+ return ranked[:CROSS_ENCODER_TOP_K]
715
+
716
+
717
+ # ── Hybrid padding ─────────────────────────────────────────────────────────────
718
+
719
+ def pad_with_neighbours(results: list[dict], chunks: list[dict]) -> list[dict]:
720
+ """
721
+ HYBRID mode: for each retrieved chunk, include its immediate neighbours
722
+ from the full chunk list. Helps when the relevant content slightly
723
+ overflows the chunk boundary. Deduplicates by start timestamp.
724
+ """
725
+ result_starts = {r["start"] for r in results}
726
+ padded = list(results)
727
+
728
+ for r in results:
729
+ for i, c in enumerate(chunks):
730
+ if abs(c["start"] - r["start"]) < 0.5: # found matching chunk
731
+ for neighbour_i in [i - 1, i + 1]:
732
+ if 0 <= neighbour_i < len(chunks):
733
+ nb = chunks[neighbour_i]
734
+ if nb["start"] not in result_starts:
735
+ padded.append({
736
+ **nb,
737
+ "bi_score": 0.0,
738
+ "ce_logit": -9.0,
739
+ "entity_overlap": 0.0,
740
+ "visual_score": 0.0,
741
+ "confidence": nb.get("confidence", 0.1),
742
+ })
743
+ result_starts.add(nb["start"])
744
+ break
745
+
746
+ return sorted(padded, key=lambda x: x["start"])
747
+
748
+
749
+ # ── Global mode: even sampling ────────────────────────────────────────────────
750
+
751
+ def sample_chunks_for_global(chunks: list[dict], n_target: int) -> list[dict]:
752
+ """
753
+ For global summarization, pick `n_target` chunks evenly distributed across
754
+ the video, so the summary covers the whole thing, not just the beginning.
755
+ """
756
+ n_target = max(3, min(n_target, len(chunks)))
757
+ if n_target >= len(chunks):
758
+ return chunks
759
+
760
+ # evenly spaced indices
761
+ indices = [int(i * (len(chunks) - 1) / (n_target - 1)) for i in range(n_target)]
762
+ return [chunks[i] for i in indices]
763
+
764
+
765
+ # ── Stage 6: merge regions ─────────────────────────────────────────────────────
766
+
767
+ def merge_regions(results: list[dict]) -> list[dict]:
768
+ if not results:
769
+ return []
770
+ sorted_r = sorted(results, key=lambda x: x["start"])
771
+ merged, current = [], sorted_r[0].copy()
772
+ current["_frags"] = [sorted_r[0]]
773
+
774
+ for nxt in sorted_r[1:]:
775
+ if nxt["start"] - current["end"] <= MERGE_GAP_SEC:
776
+ current["end"] = max(current["end"], nxt["end"])
777
+ current["text"] += " " + nxt["text"]
778
+ current["_frags"].append(nxt)
779
+ else:
780
+ merged.append(_finalize_region(current))
781
+ current = nxt.copy()
782
+ current["_frags"] = [nxt]
783
+
784
+ merged.append(_finalize_region(current))
785
+ return [r for r in merged if (r["end"] - r["start"]) >= REGION_MIN_DURATION]
786
+
787
+
788
+ def _finalize_region(region: dict) -> dict:
789
+ frags = region.pop("_frags")
790
+ total_dur = sum(max(f["end"] - f["start"], 0.1) for f in frags)
791
+ w_conf = sum(f["confidence"] * max(f["end"] - f["start"], 0.1)
792
+ for f in frags) / total_dur
793
+ duration = region["end"] - region["start"]
794
+ w_conf *= min(1.0, duration / 20.0)
795
+ visual = max((f.get("visual_score", 0.0) for f in frags), default=0.0)
796
+ return {**region, "confidence": round(w_conf, 3), "visual_score": round(visual, 3)}
797
+
798
+
799
+ # ── Stage 7: precise re-transcription (confidence-gated) ──────────────────────
800
+
801
+ def retranscribe_region(audio: np.ndarray, region: dict) -> str:
802
+ """Slice the in-memory waveform (no per-region ffmpeg) and run precise ASR."""
803
+ s = max(0, int(region["start"] * SAMPLE_RATE))
804
+ e = min(len(audio), int(region["end"] * SAMPLE_RATE))
805
+ clip = audio[s:e]
806
+ if len(clip) < SAMPLE_RATE * 0.2: # < 0.2s → nothing to gain
807
+ return region.get("text", "")
808
+ model = get_whisper_precise()
809
+ segments_gen, _ = model.transcribe(clip, beam_size=1)
810
+ return " ".join(seg.text.strip() for seg in segments_gen).strip()
811
+
812
+
813
+ def fill_precise(audio: np.ndarray, regions: list[dict], all_chunks: list[dict],
814
+ precise_cache: dict, budget: int) -> int:
815
+ """
816
+ Ensure every region has `precise_text`. A sector is turbo-refined ONLY when:
817
+ • it is not already cached, AND
818
+ • the base pass for that sector was low-confidence (avg_logprob < gate), AND
819
+ • we are still within `budget` NEW transcriptions this call.
820
+ Clean/high-confidence sectors keep the (already accurate) base text for free,
821
+ so on clean audio the slow turbo model never runs. `precise_cache` persists
822
+ per video, so later prompts reuse earlier work. Returns remaining budget.
823
+ """
824
+ reused = new = skipped = 0
825
+ for region in regions:
826
+ # the chunk sectors this region spans (chunks are non-overlapping)
827
+ comp = [c for c in all_chunks
828
+ if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3]
829
+ if not comp:
830
+ comp = [region]
831
+
832
+ for ch in comp:
833
+ key = round(ch["start"], 2)
834
+ if key in precise_cache:
835
+ reused += 1
836
+ continue
837
+ weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE
838
+ if not weak:
839
+ skipped += 1 # base was confident → trust it
840
+ continue
841
+ if budget <= 0:
842
+ continue # over budget → fall back to fast text
843
+ print(f"[NINA] Re-transcribing weak sector "
844
+ f"[{fmt_time(ch['start'])} → {fmt_time(ch['end'])}] "
845
+ f"(logprob {ch.get('logprob', 0.0):.2f})")
846
+ precise_cache[key] = retranscribe_region(audio, ch)
847
+ if DEBUG:
848
+ snip = precise_cache[key]
849
+ print(f"[DEBUG] Precise sector [{fmt_time(ch['start'])}→{fmt_time(ch['end'])}]: "
850
+ f"{snip[:200]}{'…' if len(snip) > 200 else ''}")
851
+ new += 1
852
+ budget -= 1
853
+
854
+ region["precise_text"] = " ".join(
855
+ (precise_cache.get(round(ch["start"], 2)) or ch.get("text", ""))
856
+ for ch in comp
857
+ ).strip() or region.get("text", "")
858
+
859
+ print(f"[NINA] Precise sectors — {new} refined, {reused} cached, {skipped} trusted (base).")
860
+ return budget
861
+
862
+
863
+ # ── Stage 8: synthesis ─────────────────────────────────────────────────────────
864
+
865
+ # ── Output sizing — inferred from the PROMPT (no slider) ─────────────────────────
866
+
867
+ _BRIEF_CUES = (
868
+ "brief", "briefly", "short", "shortly", "quick", "quickly", "concise",
869
+ "in a sentence", "one sentence", "one-liner", "tl;dr", "tldr",
870
+ "krótko", "w skrócie", "zwięźle", "jednym zdaniem", "w jednym zdaniu",
871
+ )
872
+ _DETAIL_CUES = (
873
+ "in detail", "detailed", "in-depth", "in depth", "thorough", "thoroughly",
874
+ "comprehensive", "exhaustive", "everything", "full summary", "elaborate",
875
+ "step by step", "step-by-step", "deep dive", "as much as possible",
876
+ "szczegółowo", "dokładnie", "wyczerpująco", "wszystko", "krok po kroku",
877
+ "dogłębnie", "obszernie",
878
+ )
879
+ _FACTUAL_START = re.compile(
880
+ r'^\s*(what|who|whom|whose|when|where|which|how many|how much|how long|'
881
+ r'is|are|was|were|does|do|did|can|could|will|would|'
882
+ r'czy|kiedy|kto|kogo|gdzie|ile|jaki|jaka|jakie|który|która|które)\b',
883
+ re.IGNORECASE,
884
+ )
885
+
886
+ def _summary_tier(max_words: int) -> str:
887
+ """Human-readable label for the chosen output length."""
888
+ if max_words <= 60: return "brief / one-liner"
889
+ if max_words <= 110: return "short"
890
+ if max_words <= 250: return "standard"
891
+ if max_words <= 400: return "detailed"
892
+ return "exhaustive"
893
+
894
+
895
+ def _prompt_to_words(prompt: str) -> int:
896
+ """
897
+ Infer the answer's word budget from the prompt itself (replaces the slider):
898
+ - explicit brevity cues → ~40 words
899
+ - explicit detail cues → ~450 words
900
+ - factual wh-/yes-no Q → ~80 words
901
+ - otherwise scale gently with prompt length (longer ask → longer answer)
902
+ Clamped to [30, 500].
903
+ """
904
+ p = prompt.lower()
905
+ if any(c in p for c in _DETAIL_CUES):
906
+ base, reason = 450, "explicit detail cue"
907
+ elif any(c in p for c in _BRIEF_CUES):
908
+ base, reason = 40, "explicit brevity cue"
909
+ elif _FACTUAL_START.match(p):
910
+ base, reason = 80, "factual question"
911
+ else:
912
+ n_words = len(re.findall(r'\w+', p))
913
+ base, reason = 120 + min(180, n_words * 8), f"length-scaled ({n_words} prompt words)"
914
+
915
+ out = int(max(30, min(500, base)))
916
+ if DEBUG:
917
+ print(f"[DEBUG] Summary sizing: {reason} → {out} words [{_summary_tier(out)}]")
918
+ return out
919
+
920
+
921
+ def _words_to_global_chunks(max_words: int) -> int:
922
+ """More requested words → sample more chunks for the global overview."""
923
+ return max(3, min(GLOBAL_MAX_CHUNKS, round(max_words / 40)))
924
+
925
+
926
+ def _fallback_summary(regions: list[dict]) -> str:
927
+ snippets = [
928
+ f"[{fmt_time(r['start'])}] {r.get('precise_text', r['text'])[:200]}"
929
+ for r in regions[:2]
930
  ]
931
+ return "(LLM unavailable — raw excerpts:)\n" + "\n".join(snippets)
932
+
 
 
 
 
 
933
 
934
+ def _build_synth_prompt(regions: list[dict], prompt: str,
935
+ max_words: int, mode: str) -> str:
936
+ # Each fragment carries BOTH modalities: what is SAID (audio) and what is
937
+ # SHOWN (CLIP visual cues). The LLM is asked to fuse them.
938
+ blocks = []
939
+ for r in regions:
940
+ spoken = (r.get("precise_text", r["text"]) or "").strip()
941
+ tags = r.get("visual_tags", [])
942
+ block = (f"[{fmt_time(r['start'])} → {fmt_time(r['end'])}]\n"
943
+ f" Spoken (audio): {spoken or '(no speech)'}")
944
+ if tags:
945
+ block += f"\n On screen (video): {', '.join(tags)}"
946
+ blocks.append(block)
947
+ fragments = "\n\n".join(blocks)
948
 
949
+ if mode == "global":
950
+ task_line = "Task: Give a complete overview of what this video is about."
951
+ else:
952
+ task_line = f'Task: Summarize what the video says/shows about "{prompt}".'
953
+
954
+ return (
955
+ f"You are analyzing a video. Each fragment below contains what is SPOKEN "
956
+ f"(audio transcript) and what is SHOWN (visual cues detected in the frames).\n"
957
+ f'The user is looking for: "{prompt}"\n\n'
958
+ f"Fragments:\n\n{fragments}\n\n"
959
+ f"{task_line}\n\n"
960
+ f"Output format:\n"
961
+ f"- Combine BOTH the spoken content and the on-screen visuals into one coherent answer\n"
962
+ f"- After each statement, append the timestamp it came from, taken from the "
963
+ f"bracketed start time of the relevant fragment above (use that exact value)\n"
964
+ f"- Maximum {max_words} words total\n"
965
+ f"- IMPORTANT: Write in the same language as the prompt given by the user\n"
966
+ f"- Only use information from the fragments (audio + visual) — do not invent details\n"
967
+ f"- If fragments contain no real information about the topic, say so clearly, "
968
+ f"without further explanation\n"
969
+ )
970
+
971
+
972
+ def llm_synthesize_stream(regions: list[dict], prompt: str,
973
+ max_words: int = 150, mode: str = "specific"):
974
+ """
975
+ Generator: yields the answer incrementally as the LLM streams tokens.
976
+ On any error, yields a single fallback string.
977
+ """
978
+ if not regions:
979
+ yield "(No relevant regions to summarize.)"
980
  return
981
 
982
+ synth_prompt = _build_synth_prompt(regions, prompt, max_words, mode)
983
+
984
+ if DEBUG:
985
+ print(f"[DEBUG] Synthesis [{mode}] — {len(regions)} region(s), "
986
+ f"max_words={max_words} [{_summary_tier(max_words)}].")
987
+
988
+ try:
989
+ acc = ""
990
+ stream = llm_client.chat_completion(
991
+ model=LLM_MODEL,
992
+ messages=[{"role": "user", "content": synth_prompt}],
993
+ max_tokens=LLM_MAX_TOKENS,
994
+ temperature=0.3,
995
+ stream=True,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
996
  )
997
+ for ev in stream:
998
+ delta = ev.choices[0].delta.content or ""
999
+ if delta:
1000
+ acc += delta
1001
+ yield acc.strip()
1002
+ if not acc.strip():
1003
+ yield _fallback_summary(regions)
1004
+ except Exception as e:
1005
+ print(f"[NINA] Synthesis error: {e}")
1006
+ yield _fallback_summary(regions)
1007
+
1008
+
1009
+ def llm_synthesize(regions: list[dict], prompt: str,
1010
+ max_words: int = 150, mode: str = "specific") -> str:
1011
+ """Non-streaming convenience wrapper (kept for API compatibility / testing)."""
1012
+ out = ""
1013
+ for out in llm_synthesize_stream(regions, prompt, max_words, mode):
1014
+ pass
1015
+ return out
1016
+
1017
+
1018
+ # ── Helpers ────────────────────────────────────────────────────────────────────
1019
+
1020
+ def _build_chunk(segs: list[dict], embs: list[np.ndarray]) -> dict:
1021
+ total = sum(max(s["end"] - s["start"], 0.1) for s in segs)
1022
+ logp = sum(s.get("logprob", 0.0) * max(s["end"] - s["start"], 0.1)
1023
+ for s in segs) / total # duration-weighted base confidence
1024
+ return {"text": " ".join(s["text"] for s in segs),
1025
+ "start": segs[0]["start"],
1026
+ "end": segs[-1]["end"],
1027
+ "logprob": round(logp, 3),
1028
+ "emb": np.mean(np.stack(embs), axis=0)} # cached chunk embedding
1029
+
1030
+ def _cosine(a: np.ndarray, b: np.ndarray) -> float:
1031
+ return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9))
1032
+
1033
+ def _sigmoid(x: float) -> float:
1034
+ return 1.0 / (1.0 + math.exp(-float(x)))
1035
+
1036
+ def fmt_time(s: float) -> str:
1037
+ m = int(s) // 60
1038
+ return f"{m}:{s % 60:05.2f}"
1039
+
1040
+ def conf_bar(c: float) -> str:
1041
+ f = round(c * 10)
1042
+ return "█" * f + "░" * (10 - f)
1043
+
1044
+
1045
+ # ── Persistent disk cache ────────────────────────────────────────────────────────
1046
+ # Keyed by a content fingerprint (size + head + tail bytes), so the same video
1047
+ # re-uploaded to a fresh temp path — or opened after a restart — is recognised
1048
+ # and its transcript / embeddings / precise sectors are loaded instead of
1049
+ # recomputed. Best-effort: any failure silently falls back to recomputation.
1050
+
1051
+ def _fingerprint(path: str) -> str:
1052
+ h = hashlib.sha1()
1053
+ st = os.stat(path)
1054
+ h.update(str(st.st_size).encode())
1055
+ with open(path, "rb") as f:
1056
+ h.update(f.read(1 << 20)) # first 1 MB
1057
+ if st.st_size > (1 << 20):
1058
+ f.seek(-(1 << 20), os.SEEK_END)
1059
+ h.update(f.read(1 << 20)) # last 1 MB
1060
+ return h.hexdigest()[:16]
1061
+
1062
+
1063
+ def _disk_load(fp: str) -> dict | None:
1064
+ if not DISK_CACHE:
1065
+ return None
1066
+ d = os.path.join(CACHE_DIR, fp)
1067
+ try:
1068
+ with open(os.path.join(d, "meta.json"), encoding="utf-8") as f:
1069
+ meta = json.load(f)
1070
+ arr = np.load(os.path.join(d, "emb.npz"))
1071
+ chunk_emb = arr["chunk_emb"]
1072
+ chunks = []
1073
+ for i, c in enumerate(meta["chunks"]):
1074
+ chunks.append({
1075
+ "text": c["text"], "start": c["start"], "end": c["end"],
1076
+ "logprob": c.get("logprob", 0.0),
1077
+ "entities": set(c.get("entities", [])),
1078
+ "emb": chunk_emb[i],
1079
+ })
1080
+ visual = {"emb": arr["visual_emb"], "time": arr["visual_time"]}
1081
+ print(f"[NINA] Disk cache hit ({fp}) — loaded {len(chunks)} chunks.")
1082
+ return {"segments": meta["segments"], "chunks": chunks,
1083
+ "visual": visual, "precise": {float(k): v for k, v in meta["precise"].items()}}
1084
+ except Exception:
1085
+ return None
1086
+
1087
+
1088
+ def _disk_save(fp: str, entry: dict) -> None:
1089
+ if not DISK_CACHE:
1090
+ return
1091
+ d = os.path.join(CACHE_DIR, fp)
1092
+ try:
1093
+ os.makedirs(d, exist_ok=True)
1094
+ chunks = entry["chunks"]
1095
+ meta = {
1096
+ "segments": entry["segments"],
1097
+ "precise": entry["precise"],
1098
+ "chunks": [{"text": c["text"], "start": c["start"], "end": c["end"],
1099
+ "logprob": c.get("logprob", 0.0),
1100
+ "entities": sorted(c.get("entities", []))}
1101
+ for c in chunks],
1102
+ }
1103
+ chunk_emb = (np.stack([c["emb"] for c in chunks]) if chunks
1104
+ else np.zeros((0, 384), dtype=np.float32))
1105
+ with open(os.path.join(d, "meta.json"), "w", encoding="utf-8") as f:
1106
+ json.dump(meta, f)
1107
+ np.savez(os.path.join(d, "emb.npz"),
1108
+ chunk_emb=chunk_emb,
1109
+ visual_emb=entry["visual"]["emb"],
1110
+ visual_time=entry["visual"]["time"])
1111
+ except Exception as e:
1112
+ if DEBUG:
1113
+ print(f"[DEBUG] disk cache save failed: {e}")
1114
+
1115
+
1116
+ def _disk_update_precise(fp: str, precise: dict) -> None:
1117
+ """Persist newly-refined precise sectors without rewriting embeddings."""
1118
+ if not DISK_CACHE:
1119
+ return
1120
+ p = os.path.join(CACHE_DIR, fp, "meta.json")
1121
+ try:
1122
+ with open(p, encoding="utf-8") as f:
1123
+ meta = json.load(f)
1124
+ meta["precise"] = precise
1125
+ with open(p, "w", encoding="utf-8") as f:
1126
+ json.dump(meta, f)
1127
+ except Exception:
1128
+ pass
1129
+
1130
+
1131
+ # ── Formatter ──────────────────────────────────────────────────────────────────
1132
+
1133
+ def format_output(regions: list[dict], prompt: str, summary: str,
1134
+ mode: str = "specific") -> str:
1135
+ if not regions:
1136
+ return (
1137
+ f' Topic not found in this video.\n\n'
1138
+ f' Prompt : "{prompt}"\n\n'
1139
+ f' No relevant segments found.\n'
1140
+ f' Try a broader or differently worded prompt.'
1141
  )
 
1142
 
1143
+ mode_tag = {"global": "🌐 global", "hybrid": "⚡ hybrid", "specific": "🎯 specific"}.get(mode, "")
1144
+ lines = [f' {len(regions)} region(s) found for: "{prompt}" [{mode_tag} · audio+video]\n']
1145
+
1146
+ if DEBUG:
1147
+ for i, r in enumerate(regions, 1):
1148
+ dur = round(r["end"] - r["start"], 1)
1149
+ text = r.get("precise_text", r["text"])
1150
+ ent = r.get("entity_overlap", 0)
1151
+ bi = r.get("bi_score", 0)
1152
+ vis = r.get("visual_score", 0)
1153
+ tags = ", ".join(r.get("visual_tags", []))
1154
+ lines += [
1155
+ f'\n [{i}] {fmt_time(r["start"])} → {fmt_time(r["end"])} ({dur}s)',
1156
+ f' {conf_bar(r["confidence"])} {r["confidence"]:.0%} '
1157
+ f'(embed {bi:.0%} · entity {ent:.0%} · visual {vis:.0%})',
1158
+ f' 👁 {tags}' if tags else ' 👁 (no visual tags)',
1159
+ f' "{text[:220]}{"…" if len(text) > 220 else ""}"',
1160
+ ]
1161
+
1162
+ lines += [
1163
+ f' 🧠 Based on uploaded video (audio + video): — "{prompt}"',
1164
+ summary,
1165
+ f'\n Sources: ' + ', '.join(
1166
+ f'[{fmt_time(r["start"])} → {fmt_time(r["end"])}]' for r in regions
1167
+ ),
1168
+ ]
1169
+ return "\n".join(lines)
1170
+
1171
 
1172
+ # ── Cold-start ingestion (transcription + chunking + CLIP, in parallel) ───────────
1173
+
1174
+ def ingest_video(video_path: str) -> dict:
1175
+ """
1176
+ First sight of a video: decode audio, then run the two heavy independent
1177
+ stages concurrently —
1178
+ • CPU thread : base transcription → semantic chunking → entities
1179
+ • MPS thread : frame extraction → CLIP encoding
1180
+ Returns the cache entry dict.
1181
+ """
1182
+ audio = load_audio(video_path)
1183
+ duration = media_duration(video_path) or (len(audio) / SAMPLE_RATE)
1184
+
1185
+ result = {}
1186
+
1187
+ def _audio_branch():
1188
+ segments = transcribe_fast(audio)
1189
+ result["segments"] = segments
1190
+ if segments:
1191
+ result["chunks"] = add_entities_to_chunks(semantic_chunk(segments))
1192
+ else:
1193
+ result["chunks"] = []
1194
+
1195
+ def _visual_branch():
1196
+ result["visual"] = analyze_visual(video_path, duration)
1197
+
1198
+ with ThreadPoolExecutor(max_workers=2) as ex:
1199
+ fa = ex.submit(_audio_branch)
1200
+ fv = ex.submit(_visual_branch)
1201
+ fa.result()
1202
+ fv.result()
1203
+
1204
+ return {
1205
+ "segments": result["segments"],
1206
+ "chunks": result["chunks"],
1207
+ "visual": result["visual"],
1208
+ "audio": audio,
1209
+ "precise": {},
1210
+ }
1211
+
1212
+
1213
+ # ── Global path (shared by GLOBAL mode + every escalation/fallback) ───────────────
1214
+ # Global summaries run on the clean base transcript — no whole-video turbo pass.
1215
+
1216
+ def _run_global(chunks: list[dict], visual: dict,
1217
+ prompt: str, max_words: int) -> list[dict]:
1218
+ sampled = sample_chunks_for_global(chunks, _words_to_global_chunks(max_words))
1219
+ print(f"[NINA] Global mode — sampled {len(sampled)} chunks evenly (base transcript).")
1220
+ for c in sampled:
1221
+ c.setdefault("confidence", 0.5)
1222
+ c.setdefault("bi_score", 0.0)
1223
+ c.setdefault("entity_overlap", 0.0)
1224
+ c.setdefault("visual_score", 0.0)
1225
+ c["precise_text"] = c.get("text", "") # base text is enough for an overview
1226
+ return add_visual_tags(sampled, visual)
1227
+
1228
+
1229
+ # ── Main pipeline (generator: streams progress + answer) ─────────────────────────
1230
+
1231
+ def format_iris(regions, prompt, best_bi):
1232
+ if not regions:
1233
+ return (f'No clear moments about "{prompt}" were found in this video.\n\n'
1234
+ f'Try broader or differently-worded keywords.')
1235
+ lines = [f'IRIS - moments matching: "{prompt}"\n']
1236
+ for i, r in enumerate(sorted(regions, key=lambda x: x["start"]), 1):
1237
+ txt = (r.get("precise_text", r.get("text", "")) or "").strip()
1238
+ tags = ", ".join(r.get("visual_tags", []))
1239
+ lines.append(f' {i}. [{fmt_time(r["start"])} -> {fmt_time(r["end"])}] {r.get("confidence", 0):.0%} match')
1240
+ lines.append(f' "{txt[:260]}{"..." if len(txt) > 260 else ""}"')
1241
+ if tags:
1242
+ lines.append(f' on screen: {tags}')
1243
+ lines.append("")
1244
+ lines.append(f' {len(regions)} moment(s) - ranked by relevance, shown in order of appearance.')
1245
+ return "\n".join(lines)
1246
+
1247
+
1248
+ def run(video_path: str, prompt: str, token: str = ""):
1249
+ if not _verify_usage_token(token):
1250
+ yield ("This request is not authorized.\n\n"
1251
+ "Please use IRIS at https://app.nnlabs.pl/iris (sign in first). "
1252
+ "Direct API access is disabled, and each account has a usage limit.")
1253
+ return
1254
+ if video_path is None:
1255
+ yield "Please upload a video."
1256
+ return
1257
+ if not prompt or not prompt.strip():
1258
+ yield "Please type what you want to find."
1259
+ return
1260
+ prompt = prompt.strip()
1261
+ print(f'\n[IRIS] -- Find -- "{prompt}"')
1262
+ _evict_stale()
1263
+
1264
+ fp = None
1265
+ if video_path in _cache:
1266
+ entry = _cache[video_path]
1267
+ else:
1268
+ try: fp = _fingerprint(video_path)
1269
+ except Exception: fp = None
1270
+ disk = _disk_load(fp) if fp else None
1271
+ if disk is not None:
1272
+ disk["audio"] = load_audio(video_path)
1273
+ entry = disk; _cache[video_path] = entry
1274
+ else:
1275
+ yield "Reading video - transcribing audio and scanning frames..."
1276
+ entry = ingest_video(video_path)
1277
+ if not entry["segments"]:
1278
+ yield "No speech detected."
1279
+ return
1280
+ _cache[video_path] = entry
1281
+ if fp: _disk_save(fp, entry)
1282
+
1283
+ entry["_atime"] = time.time()
1284
+ segments, chunks, visual = entry["segments"], entry["chunks"], entry["visual"]
1285
+ precise_cache, audio = entry["precise"], entry["audio"]
1286
+ if fp is None:
1287
+ try: fp = _fingerprint(video_path)
1288
+ except Exception: fp = None
1289
+ if not chunks:
1290
+ yield "No speech detected."
1291
+ return
1292
+
1293
+ candidates, best_bi = bi_encode_retrieve(chunks, prompt)
1294
+ _, v_norm = visual_scores(visual, prompt)
1295
+ results = rerank_and_score(candidates, prompt, visual["time"], v_norm)
1296
+ regions = merge_regions(results)
1297
+ if not regions:
1298
+ yield format_iris([], prompt, best_bi)
1299
+ return
1300
+ before = dict(precise_cache)
1301
+ fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX)
1302
+ if fp and precise_cache != before:
1303
+ _disk_update_precise(fp, precise_cache)
1304
+ regions = add_visual_tags(regions, visual)
1305
+ regions = sorted(regions, key=lambda r: r.get("confidence", 0), reverse=True)[:RESULTS_TOP_N]
1306
+ print(f"[IRIS] Done - {len(regions)} moment(s).")
1307
+ yield format_iris(regions, prompt, best_bi)
1308
+
1309
+
1310
+ def _synthesize_streaming(regions: list[dict], prompt: str, max_words: int, mode: str):
1311
+ """Stream the formatted answer to the UI as the LLM produces it."""
1312
+ print(f"[NINA] Running synthesis [{mode}]...")
1313
+ yield format_output(regions, prompt, "✍️ …", mode)
1314
+ last = ""
1315
+ for partial in llm_synthesize_stream(regions, prompt, max_words, mode):
1316
+ last = partial
1317
+ yield format_output(regions, prompt, partial, mode)
1318
+ if not last:
1319
+ yield format_output(regions, prompt, _fallback_summary(regions), mode)
1320
+
1321
+
1322
+ # ── Gradio UI ──────────────────────────────────────────────────────────────────
1323
+
1324
+ demo = gr.Interface(
1325
+ fn=run,
1326
+ inputs=[
1327
+ gr.Video(label="Upload video "),
1328
+ gr.Textbox(
1329
+ label="What moment do you want to find?",
1330
+ placeholder='e.g. "What does the video say about...?" '
1331
+ '(add "briefly" or "in detail" to size the answer)',
1332
+ ),
1333
+ # Issued by app.nnlabs.pl after it counts the use; required to run.
1334
+ gr.Textbox(label="access token (set automatically by the website)", value=""),
1335
+ ],
1336
+ outputs=gr.Textbox(label="Results", lines=38),
1337
+ title="IRIS v0.80 — Intelligent Retrieval & Indexing (find the moment)",
1338
+ description=(
1339
+ "video -> find the exact moments a topic is mentioned or shown. "
1340
+ "Use it at https://app.nnlabs.pl/iris."
1341
+ ),
1342
+ flagging_mode="never",
1343
+ # Privacy: Gradio purges uploaded files older than 5 min, every 5 min. One
1344
+ # upload still serves many prompts (audio/transcript are cached in RAM), and
1345
+ # the file is cleaned shortly after the session goes idle.
1346
+ delete_cache=(300, 300),
1347
+ )
1348
+
1349
+ # Concurrency for ~5-10 simultaneous users on an 8-vCPU Space: run a few jobs in
1350
+ # parallel, queue the rest (CTranslate2 is CPU-bound). Tune with the env var.
1351
+ demo.queue(
1352
+ default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
1353
+ max_size=int(os.getenv("NINA_QUEUE_MAX", "40")),
1354
+ )
1355
 
1356
  if __name__ == "__main__":
1357
+ demo.launch()