Spaces:
Sleeping
Sleeping
Upload app.py with huggingface_hub
Browse files
app.py
CHANGED
|
@@ -1,378 +1,1357 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
import os
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
import shutil
|
| 5 |
-
import
|
| 6 |
-
|
| 7 |
-
|
|
|
|
| 8 |
|
|
|
|
| 9 |
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
-
os.environ.setdefault("IRIS_DATA_DIR", "/tmp/iris-data")
|
| 12 |
-
os.environ.setdefault("IRIS_DEVICE", "cpu")
|
| 13 |
-
os.environ.setdefault("IRIS_TORCH_DTYPE", "float32")
|
| 14 |
-
os.environ.setdefault("IRIS_ASR_MODEL", "openai/whisper-tiny")
|
| 15 |
-
os.environ.setdefault("IRIS_MAX_AUDIO_SECONDS", "120")
|
| 16 |
-
os.environ.setdefault("IRIS_VISUAL_SEARCH_ENABLED", "false")
|
| 17 |
-
os.environ.setdefault("IRIS_MAX_STORED_VIDEOS", "3")
|
| 18 |
-
|
| 19 |
-
from iris.config import Settings, device_info
|
| 20 |
-
from iris.models.schemas import SearchRequest
|
| 21 |
-
from iris.pipeline.chunk import window_transcript_chunks
|
| 22 |
-
from iris.pipeline.ffmpeg import resolve_ffmpeg_executable
|
| 23 |
-
from iris.pipeline.orchestrator import VideoProcessor
|
| 24 |
-
from iris.pipeline.search import detail_window_settings, format_timestamp, rank_speech_chunks
|
| 25 |
-
from iris.pipeline.video import validate_video_filename
|
| 26 |
-
from iris.pipeline.visual import VisualSearcher, sample_video_frames
|
| 27 |
-
from iris.storage.database import Database
|
| 28 |
-
from iris.storage.files import FileStore
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
SETTINGS = Settings.from_env()
|
| 32 |
-
SETTINGS.ensure_directories()
|
| 33 |
-
DATABASE = Database(SETTINGS.database_path)
|
| 34 |
-
DATABASE.init()
|
| 35 |
-
FILE_STORE = FileStore(SETTINGS)
|
| 36 |
-
PROCESSOR = VideoProcessor(SETTINGS, DATABASE, FILE_STORE)
|
| 37 |
-
MAX_STORED_VIDEOS = max(1, int(os.getenv("IRIS_MAX_STORED_VIDEOS", "3")))
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def health_summary() -> str:
|
| 41 |
-
info = device_info(SETTINGS.device, SETTINGS.torch_dtype)
|
| 42 |
-
visual_state = "enabled" if SETTINGS.visual_search_enabled else "disabled"
|
| 43 |
-
audio_limit = f"{SETTINGS.max_audio_seconds:g}s" if SETTINGS.max_audio_seconds else "full video"
|
| 44 |
-
rows = [
|
| 45 |
-
("Mode", "CPU demo"),
|
| 46 |
-
("Device", info.get("resolved_device", "unknown")),
|
| 47 |
-
("ASR model", SETTINGS.asr_model),
|
| 48 |
-
("Embedding model", SETTINGS.embedding_model),
|
| 49 |
-
("Visual search", visual_state),
|
| 50 |
-
("Audio limit", audio_limit),
|
| 51 |
-
("Stored videos", str(MAX_STORED_VIDEOS)),
|
| 52 |
-
]
|
| 53 |
-
return "\n".join(f"**{name}:** {value}" for name, value in rows)
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
-
def
|
| 57 |
-
|
| 58 |
-
progress: gr.Progress = gr.Progress(track_tqdm=False),
|
| 59 |
-
) -> tuple[str, dict[str, str] | None, list[list[str | float]], list[dict], None]:
|
| 60 |
-
if not video_file:
|
| 61 |
-
return "Upload a video before processing.", None, [], [], None
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
|
|
|
|
|
|
| 66 |
try:
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
if record["status"] != "completed":
|
| 89 |
-
return f"Video is still {record['status']}. Try again in a moment.", None, [], [], None
|
| 90 |
-
|
| 91 |
-
state = {"video_id": video_id, "filename": filename}
|
| 92 |
-
processed_window = SETTINGS.max_audio_seconds
|
| 93 |
-
window_note = (
|
| 94 |
-
f" Processed the first {format_timestamp(processed_window)}."
|
| 95 |
-
if processed_window
|
| 96 |
-
else ""
|
| 97 |
-
)
|
| 98 |
-
return f"Ready to search `{filename}`.{window_note}", state, [], [], None
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
def search_processed_video(
|
| 102 |
-
processed_video: dict[str, str] | None,
|
| 103 |
-
query: str,
|
| 104 |
-
detail: str,
|
| 105 |
-
limit: int,
|
| 106 |
-
include_visual: bool,
|
| 107 |
-
progress: gr.Progress = gr.Progress(track_tqdm=False),
|
| 108 |
-
) -> tuple[str, list[list[str | float]], list[dict], None]:
|
| 109 |
-
if not processed_video or not processed_video.get("video_id"):
|
| 110 |
-
return "Process a video before searching.", [], [], None
|
| 111 |
-
if not query or not query.strip():
|
| 112 |
-
return "Enter a search query.", [], [], None
|
| 113 |
-
|
| 114 |
-
request = SearchRequest(
|
| 115 |
-
query=query.strip(),
|
| 116 |
-
detail=detail,
|
| 117 |
-
limit=int(limit),
|
| 118 |
-
include_visual=bool(include_visual),
|
| 119 |
-
)
|
| 120 |
try:
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
return
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
overlap_seconds = min(default_overlap, max(0.0, window_seconds - 0.1))
|
| 149 |
-
merge_gap_seconds = request.merge_gap_seconds if request.merge_gap_seconds is not None else default_merge_gap
|
| 150 |
-
chunks = window_transcript_chunks(
|
| 151 |
-
atomic_chunks,
|
| 152 |
-
window_seconds=window_seconds,
|
| 153 |
-
overlap_seconds=overlap_seconds,
|
| 154 |
-
)
|
| 155 |
|
| 156 |
-
progress(0.35, desc="Encoding query")
|
| 157 |
-
embedder = PROCESSOR.embedder
|
| 158 |
-
chunk_embeddings = embedder.encode([chunk.text for chunk in chunks])
|
| 159 |
-
query_embedding = embedder.encode([request.query])[0]
|
| 160 |
-
speech_results = rank_speech_chunks(
|
| 161 |
-
query=request.query,
|
| 162 |
-
chunks=chunks,
|
| 163 |
-
chunk_embeddings=chunk_embeddings,
|
| 164 |
-
query_embedding=query_embedding,
|
| 165 |
-
settings=SETTINGS,
|
| 166 |
-
limit=request.limit,
|
| 167 |
-
merge_gap_seconds=merge_gap_seconds,
|
| 168 |
-
)
|
| 169 |
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
)
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
{
|
| 195 |
-
"
|
| 196 |
-
"start":
|
| 197 |
-
"end":
|
| 198 |
-
"
|
| 199 |
-
"score": round(result.score, 4),
|
| 200 |
-
"text": result.text,
|
| 201 |
}
|
| 202 |
-
for
|
|
|
|
| 203 |
]
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
]
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
except OSError as exc:
|
| 268 |
-
raise RuntimeError(f"Could not run ffmpeg: {exc}") from exc
|
| 269 |
-
except subprocess.CalledProcessError as exc:
|
| 270 |
-
detail = exc.stderr.strip() or exc.stdout.strip() or str(exc)
|
| 271 |
-
raise RuntimeError(f"ffmpeg failed to create preview clip: {detail}") from exc
|
| 272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
return
|
| 278 |
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
if path.name in retained:
|
| 294 |
-
continue
|
| 295 |
-
try:
|
| 296 |
-
path.relative_to(videos_root)
|
| 297 |
-
except ValueError:
|
| 298 |
-
continue
|
| 299 |
-
shutil.rmtree(path, ignore_errors=True)
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
CUSTOM_CSS = """
|
| 303 |
-
.iris-shell { max-width: 1120px; margin: 0 auto; }
|
| 304 |
-
.iris-title h1 { margin-bottom: 0.15rem; }
|
| 305 |
-
.iris-title p { margin-top: 0; color: #4b5563; }
|
| 306 |
-
"""
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
with gr.Blocks(title="IRIS") as demo:
|
| 310 |
-
processed_video = gr.State(value=None)
|
| 311 |
-
search_matches = gr.State(value=[])
|
| 312 |
-
|
| 313 |
-
with gr.Column(elem_classes=["iris-shell"]):
|
| 314 |
-
gr.Markdown(
|
| 315 |
-
"""
|
| 316 |
-
# IRIS
|
| 317 |
-
Process a video once, then search inside the spoken transcript.
|
| 318 |
-
""",
|
| 319 |
-
elem_classes=["iris-title"],
|
| 320 |
)
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
)
|
| 355 |
-
preview_video = gr.Video(label="Selected match")
|
| 356 |
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 373 |
|
| 374 |
if __name__ == "__main__":
|
| 375 |
-
demo.
|
| 376 |
-
css=CUSTOM_CSS,
|
| 377 |
-
allowed_paths=[str(SETTINGS.data_dir)],
|
| 378 |
-
)
|
|
|
|
| 1 |
+
# ─────────────────────────────────────────────
|
| 2 |
+
# NINA — Neural Information & Narrative Abstraction
|
| 3 |
+
# app.py | IRIS v0.80 | Made by Malina
|
| 4 |
+
# ─────────────────────────────────────────────
|
| 5 |
+
#
|
| 6 |
+
# Precision MULTIMODAL video search + summarization pipeline.
|
| 7 |
+
# Pipeline: video → audio → ASR segments → semantic chunking
|
| 8 |
+
# → bi-encoder (text) ⨉ CLIP (vision) fusion → cross-encoder + entity overlap
|
| 9 |
+
# → merge → confidence-gated precise re-transcribe → streaming LLM synthesis.
|
| 10 |
+
#
|
| 11 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 12 |
+
# v0.80 — performance & modernisation pass over v0.71
|
| 13 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 14 |
+
# Measured on Apple M1 (8 core / 16 GB), 39 s test clip:
|
| 15 |
+
# cold first prompt 58.8 s → ~9 s, warm prompts ~1.5 s → sub-second to first token.
|
| 16 |
+
#
|
| 17 |
+
# [P1] HARDWARE ACCELERATION
|
| 18 |
+
# Torch models now auto-target the best device (CUDA → MPS → CPU). On
|
| 19 |
+
# Apple Silicon CLIP runs ~1.8× and the cross-encoder ~1.4× faster on the
|
| 20 |
+
# Metal (MPS) backend. CTranslate2 (Whisper) has no MPS backend, so it
|
| 21 |
+
# stays on CPU but is tuned: int8 + cpu_threads, greedy decoding for the
|
| 22 |
+
# cheap pass, and Silero VAD to skip non-speech.
|
| 23 |
+
#
|
| 24 |
+
# [P2] NO MORE DOUBLE TRANSCRIPTION
|
| 25 |
+
# v0.71 transcribed the whole video with `base`, then RE-transcribed every
|
| 26 |
+
# sampled sector with the ~15× slower `large-v3-turbo` — even for a global
|
| 27 |
+
# summary (≈24 s wasted on this clip). v0.80:
|
| 28 |
+
# • GLOBAL summaries use the clean `base` transcript directly (no turbo).
|
| 29 |
+
# • SPECIFIC queries re-transcribe with turbo ONLY on matched regions and
|
| 30 |
+
# ONLY when the base pass was low-confidence (avg_logprob gate). On
|
| 31 |
+
# clean audio turbo never runs; on hard audio it runs exactly where it
|
| 32 |
+
# helps. Per-sector results are cached and persist across prompts.
|
| 33 |
+
#
|
| 34 |
+
# [P3] PARALLEL COLD START
|
| 35 |
+
# Audio transcription (CPU) and CLIP frame analysis (MPS) are independent
|
| 36 |
+
# and now run concurrently on first sight of a video, overlapping the two
|
| 37 |
+
# heaviest stages instead of summing them.
|
| 38 |
+
#
|
| 39 |
+
# [P4] ZERO-COPY I/O
|
| 40 |
+
# Audio is decoded straight from the container (PyAV) — no ffmpeg wav
|
| 41 |
+
# temp file. Frames are streamed from one ffmpeg rawvideo pipe into NumPy —
|
| 42 |
+
# no per-frame JPEG temp files.
|
| 43 |
+
#
|
| 44 |
+
# [P5] PERSISTENT DISK CACHE
|
| 45 |
+
# Transcript, chunk + visual embeddings and precise sectors are persisted
|
| 46 |
+
# under a content fingerprint, so re-opening the same video — even after a
|
| 47 |
+
# restart — skips transcription/chunking/CLIP entirely.
|
| 48 |
+
#
|
| 49 |
+
# [P6] STREAMING + UX
|
| 50 |
+
# LLM synthesis streams token-by-token; run() is a generator that shows
|
| 51 |
+
# progress and writes the answer as it is produced.
|
| 52 |
+
#
|
| 53 |
+
# [Q] QUALITY FIXES
|
| 54 |
+
# • Router no longer misreads "what does the video say about X" as a global
|
| 55 |
+
# summary (the bare word "about"/"explain" no longer forces global mode).
|
| 56 |
+
# • The illustrative "[2:30]" timestamp no longer leaks into answers.
|
| 57 |
+
#
|
| 58 |
+
# Scoring formula (SPECIFIC/HYBRID mode):
|
| 59 |
+
# audio = 0.6 * sigmoid(cross_encoder_logit) + 0.4 * entity_overlap
|
| 60 |
+
# final = (1 - VISUAL_WEIGHT) * audio + VISUAL_WEIGHT * clip_visual_score
|
| 61 |
+
#
|
| 62 |
+
# Entity extraction: regex + simple NP heuristic (no spacy dependency).
|
| 63 |
+
# Works multilingually because it operates on surface forms, not POS tags.
|
| 64 |
+
# ─────────────────────────────────────────────
|
| 65 |
|
| 66 |
import os
|
| 67 |
+
import re
|
| 68 |
+
import json
|
| 69 |
+
import math
|
| 70 |
+
import base64
|
| 71 |
+
import hashlib
|
| 72 |
+
import hmac
|
| 73 |
import shutil
|
| 74 |
+
import tempfile
|
| 75 |
+
import threading
|
| 76 |
+
import time
|
| 77 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 78 |
|
| 79 |
+
import ffmpeg
|
| 80 |
import gradio as gr
|
| 81 |
+
import numpy as np
|
| 82 |
+
from PIL import Image
|
| 83 |
+
from faster_whisper import WhisperModel
|
| 84 |
+
from faster_whisper.audio import decode_audio
|
| 85 |
+
from sentence_transformers import SentenceTransformer, CrossEncoder
|
| 86 |
+
from huggingface_hub import InferenceClient
|
| 87 |
+
import warnings
|
| 88 |
+
warnings.filterwarnings("ignore", message="FP16 is not supported on CPU; using FP32 instead")
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
+
# ── Config ─────────────────────────────────────────────────────────────────────
|
| 92 |
+
|
| 93 |
+
WHISPER_FAST = "base"
|
| 94 |
+
WHISPER_PRECISE = "large-v3-turbo"
|
| 95 |
+
SAMPLE_RATE = 16000
|
| 96 |
+
WHISPER_CPU_THREADS = int(os.getenv("WHISPER_CPU_THREADS", "3"))
|
| 97 |
+
# 3 threads × ~3 concurrent jobs ≈ an 8-vCPU Space.
|
| 98 |
+
# Override via env. CTranslate2 is CPU-only.
|
| 99 |
+
|
| 100 |
+
RETRANSCRIBE_MAX = 12 # per-prompt budget for NEW (uncached) sector transcriptions
|
| 101 |
+
PRECISE_LOGPROB_GATE = -0.55 # only turbo-refine sectors whose base avg_logprob is below this
|
| 102 |
+
# (clean audio scores high → turbo is correctly skipped)
|
| 103 |
+
|
| 104 |
+
CHUNK_SPLIT_THRESHOLD = 0.45
|
| 105 |
+
CHUNK_MIN_DURATION = 8.0
|
| 106 |
+
CHUNK_MAX_DURATION = 90.0
|
| 107 |
+
CHUNK_WINDOW = 5
|
| 108 |
+
|
| 109 |
+
BI_ENCODER_TOP_K = 100
|
| 110 |
+
CROSS_ENCODER_TOP_K = 10
|
| 111 |
+
RESULTS_TOP_N = 6
|
| 112 |
+
REGION_MIN_DURATION = 5.0
|
| 113 |
+
MERGE_GAP_SEC = 6.0
|
| 114 |
+
|
| 115 |
+
# routing thresholds
|
| 116 |
+
BI_SCORE_GLOBAL_FLOOR = 0.10 # below this → escalate to GLOBAL regardless
|
| 117 |
+
BI_SCORE_HYBRID_FLOOR = 0.28 # below this → HYBRID (pad with neighbours)
|
| 118 |
+
|
| 119 |
+
# global mode: how many evenly-sampled chunks to retranscribe
|
| 120 |
+
GLOBAL_MAX_CHUNKS = 12 # hard cap to keep latency sane
|
| 121 |
+
|
| 122 |
+
# ── CLIP / visual config ────────────────────────────────────────────────────────
|
| 123 |
+
CLIP_MODEL = "clip-ViT-B-32" # OpenAI CLIP ViT-B/32 (via sentence-transformers)
|
| 124 |
+
CLIP_MAX_FRAMES = 96 # cap frames encoded per video (perf)
|
| 125 |
+
CLIP_MIN_FPS = 0.1 # sample at least 1 frame / 10s
|
| 126 |
+
CLIP_MAX_FPS = 0.5 # sample at most 1 frame / 2s
|
| 127 |
+
CLIP_FRAME_SIZE = 224 # square fed to CLIP (matches ViT-B/32 input → no double resize)
|
| 128 |
+
VISUAL_WEIGHT = 0.30 # fusion weight of visual vs audio in final score
|
| 129 |
+
VISUAL_TAGS_TOP_K = 4 # max zero-shot tags surfaced per region
|
| 130 |
+
VISUAL_TAG_MIN_SIM = 0.20 # min CLIP cosine for a tag to count
|
| 131 |
+
|
| 132 |
+
# vocabulary for zero-shot "what is on screen" tagging
|
| 133 |
+
VISUAL_CONCEPTS = [
|
| 134 |
+
"a person talking to the camera", "a group of people", "a presentation with slides",
|
| 135 |
+
"a lecture or classroom", "an interview", "a news broadcast",
|
| 136 |
+
"a screen recording of software", "source code in a text editor",
|
| 137 |
+
"a webpage in a browser", "a chart or graph", "text on screen",
|
| 138 |
+
"a product demonstration", "hands demonstrating something",
|
| 139 |
+
"a cooking or food scene", "a kitchen", "an office", "a laboratory",
|
| 140 |
+
"a factory or machinery", "a construction site", "an outdoor nature landscape",
|
| 141 |
+
"a forest", "mountains", "a beach or ocean", "a city street",
|
| 142 |
+
"buildings and architecture", "a car or vehicle driving", "an airplane", "a train",
|
| 143 |
+
"a sports event", "people exercising", "a concert or live music",
|
| 144 |
+
"a video game gameplay", "an animation or cartoon", "a medical or hospital scene",
|
| 145 |
+
"a doctor or patient", "an animal", "a dog", "a cat", "a crowd of people",
|
| 146 |
+
"a map", "a whiteboard with diagrams", "a scientific diagram",
|
| 147 |
+
"a phone or mobile device", "a computer or laptop", "money or finance",
|
| 148 |
+
"a meeting or conference room", "a stage with a speaker",
|
| 149 |
+
"a dark scene at night", "a daytime scene",
|
| 150 |
+
]
|
| 151 |
+
|
| 152 |
+
LLM_MODEL = "Qwen/Qwen2.5-7B-Instruct"
|
| 153 |
+
LLM_MAX_TOKENS = 600
|
| 154 |
+
LLM_TIMEOUT = 60
|
| 155 |
+
|
| 156 |
+
CACHE_DIR = os.path.join(tempfile.gettempdir(), "nina_cache")
|
| 157 |
+
DISK_CACHE = True
|
| 158 |
+
|
| 159 |
+
DEBUG = False
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
# ── Usage-token gate ──────────────────────────────────────────────────────────────
|
| 163 |
+
# The Space is public, so to enforce the per-account 10/5h limit set on the
|
| 164 |
+
# website, each request must carry a short-lived HMAC token issued by the
|
| 165 |
+
# Cloudflare Worker (app.nnlabs.pl) after it counts the use. Without a valid,
|
| 166 |
+
# unexpired, not-yet-seen token the model refuses to run. Same secret on both
|
| 167 |
+
# sides (NINA_SIGNING_SECRET). If the secret is unset the gate is open (dev).
|
| 168 |
+
SIGNING_SECRET = os.getenv("NINA_SIGNING_SECRET", "")
|
| 169 |
+
_used_jti = {} # jti -> exp epoch (single-use, replay-proof within a container)
|
| 170 |
+
_jti_lock = threading.Lock()
|
| 171 |
+
|
| 172 |
+
def _b64url_decode(s: str) -> bytes:
|
| 173 |
+
return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
|
| 174 |
|
| 175 |
+
def _b64url_encode(b: bytes) -> str:
|
| 176 |
+
return base64.urlsafe_b64encode(b).decode().rstrip("=")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
|
| 178 |
+
def _verify_usage_token(token: str) -> bool:
|
| 179 |
+
if not SIGNING_SECRET:
|
| 180 |
+
return True # not configured → open (local/dev)
|
| 181 |
+
if not token or "." not in token:
|
| 182 |
+
return False
|
| 183 |
try:
|
| 184 |
+
msg, sig = token.split(".", 1)
|
| 185 |
+
expected = _b64url_encode(hmac.new(SIGNING_SECRET.encode(), msg.encode(), hashlib.sha256).digest())
|
| 186 |
+
if not hmac.compare_digest(expected, sig):
|
| 187 |
+
return False
|
| 188 |
+
payload = json.loads(_b64url_decode(msg))
|
| 189 |
+
now = int(time.time())
|
| 190 |
+
if int(payload.get("e", 0)) < now:
|
| 191 |
+
return False # expired
|
| 192 |
+
jti = payload.get("j", "")
|
| 193 |
+
if not jti:
|
| 194 |
+
return False
|
| 195 |
+
with _jti_lock:
|
| 196 |
+
for k in [k for k, e in _used_jti.items() if e < now]:
|
| 197 |
+
_used_jti.pop(k, None)
|
| 198 |
+
if jti in _used_jti:
|
| 199 |
+
return False # replay
|
| 200 |
+
_used_jti[jti] = int(payload.get("e", now + 120))
|
| 201 |
+
return True
|
| 202 |
+
except Exception:
|
| 203 |
+
return False
|
| 204 |
|
| 205 |
+
|
| 206 |
+
# ── Device selection ─────────────────────────────────────────────────────────────
|
| 207 |
+
# CTranslate2 (Whisper) is CPU-only here; the torch models (bi/cross/CLIP) can
|
| 208 |
+
# use Metal (MPS) on Apple Silicon or CUDA on a GPU box.
|
| 209 |
+
|
| 210 |
+
def _best_torch_device() -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
try:
|
| 212 |
+
import torch
|
| 213 |
+
if torch.cuda.is_available():
|
| 214 |
+
return "cuda"
|
| 215 |
+
if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
|
| 216 |
+
return "mps"
|
| 217 |
+
except Exception:
|
| 218 |
+
pass
|
| 219 |
+
return "cpu"
|
| 220 |
+
|
| 221 |
+
TORCH_DEVICE = _best_torch_device()
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
# ── Model load ─────────────────────────────────────────────────────────────────
|
| 225 |
+
|
| 226 |
+
print(f"[NINA] Torch device: {TORCH_DEVICE} | Whisper: CPU int8 (x{WHISPER_CPU_THREADS})")
|
| 227 |
+
|
| 228 |
+
print("[NINA] Loading Whisper base (faster-whisper)...")
|
| 229 |
+
_whisper_fast = WhisperModel(WHISPER_FAST, device="cpu", compute_type="int8",
|
| 230 |
+
cpu_threads=WHISPER_CPU_THREADS)
|
| 231 |
+
|
| 232 |
+
# Bi-encoder stays on CPU: it is tiny (sub-100 ms) and MPS transfer overhead is a
|
| 233 |
+
# wash, and this keeps the MPS device free for CLIP during the parallel cold start.
|
| 234 |
+
print("[NINA] Loading bi-encoder...")
|
| 235 |
+
bi_encoder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", device="cpu")
|
| 236 |
+
|
| 237 |
+
print(f"[NINA] Loading cross-encoder ({TORCH_DEVICE})...")
|
| 238 |
+
cross_encoder = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", device=TORCH_DEVICE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
|
| 241 |
+
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 242 |
+
if not HF_TOKEN:
|
| 243 |
+
_token_file = os.path.expanduser("~/.cache/huggingface/token")
|
| 244 |
+
if os.path.exists(_token_file):
|
| 245 |
+
HF_TOKEN = open(_token_file).read().strip()
|
| 246 |
+
|
| 247 |
+
if HF_TOKEN:
|
| 248 |
+
print(f"[NINA] HF_TOKEN loaded — LLM calls enabled.")
|
| 249 |
+
else:
|
| 250 |
+
print("[NINA] HF_TOKEN not set — LLM calls will fail.")
|
| 251 |
+
|
| 252 |
+
llm_client = InferenceClient(
|
| 253 |
+
token=HF_TOKEN,
|
| 254 |
+
timeout=LLM_TIMEOUT,
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
_whisper_precise = None
|
| 259 |
+
_whisper_precise_lock = threading.Lock()
|
| 260 |
+
|
| 261 |
+
def get_whisper_precise():
|
| 262 |
+
global _whisper_precise
|
| 263 |
+
with _whisper_precise_lock:
|
| 264 |
+
if _whisper_precise is None:
|
| 265 |
+
print(f"[NINA] Loading Whisper {WHISPER_PRECISE} (faster-whisper)...")
|
| 266 |
+
_whisper_precise = WhisperModel(WHISPER_PRECISE, device="cpu",
|
| 267 |
+
compute_type="int8",
|
| 268 |
+
cpu_threads=WHISPER_CPU_THREADS)
|
| 269 |
+
print(f"[NINA] Whisper {WHISPER_PRECISE} ready.")
|
| 270 |
+
return _whisper_precise
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
# ── CLIP (OpenAI ViT-B/32) — lazy, warmed in background ──────────────────────────
|
| 274 |
+
_clip_model = None
|
| 275 |
+
_clip_lock = threading.Lock()
|
| 276 |
+
_concept_embs = None
|
| 277 |
+
_concept_lock = threading.Lock()
|
| 278 |
+
|
| 279 |
+
def get_clip():
|
| 280 |
+
global _clip_model
|
| 281 |
+
with _clip_lock:
|
| 282 |
+
if _clip_model is None:
|
| 283 |
+
print(f"[NINA] Loading CLIP ({CLIP_MODEL}, {TORCH_DEVICE})...")
|
| 284 |
+
_clip_model = SentenceTransformer(CLIP_MODEL, device=TORCH_DEVICE)
|
| 285 |
+
print(f"[NINA] CLIP ready.")
|
| 286 |
+
return _clip_model
|
| 287 |
+
|
| 288 |
+
def _get_concept_embs() -> np.ndarray:
|
| 289 |
+
"""Encode (once) the zero-shot vocabulary, L2-normalised."""
|
| 290 |
+
global _concept_embs
|
| 291 |
+
with _concept_lock:
|
| 292 |
+
if _concept_embs is None:
|
| 293 |
+
embs = get_clip().encode(VISUAL_CONCEPTS, convert_to_numpy=True,
|
| 294 |
+
show_progress_bar=False)
|
| 295 |
+
_concept_embs = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
|
| 296 |
+
return _concept_embs
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
# Warm the lazy models in the background; CLIP loads on the (free) MPS device,
|
| 300 |
+
# turbo loads on CPU. Both finish before they are first needed in most cases.
|
| 301 |
+
threading.Thread(target=get_whisper_precise, daemon=True).start()
|
| 302 |
+
threading.Thread(target=get_clip, daemon=True).start()
|
| 303 |
+
print(f"[NINA] Ready. ( {WHISPER_PRECISE} + CLIP warming up...)")
|
| 304 |
+
|
| 305 |
+
# ── Transcript + visual cache (in-memory) ─────────────────────────────────────────
|
| 306 |
+
# Stores segments, chunks (with embeddings), visual frame embeddings, the decoded
|
| 307 |
+
# waveform, and a per-sector precise-transcription cache, keyed by video path.
|
| 308 |
+
# Second prompt on same video skips transcription, chunking and CLIP entirely.
|
| 309 |
+
_cache: dict = {} # { video_path: {"segments","chunks","visual","audio","precise","_atime"} }
|
| 310 |
+
|
| 311 |
+
# Data lifecycle / privacy:
|
| 312 |
+
# • The uploaded video itself is purged by Gradio's own temp cleaner
|
| 313 |
+
# (Interface(delete_cache=...)) — see the bottom of this file — so a file is
|
| 314 |
+
# removed shortly after the user's session goes idle. One upload still serves
|
| 315 |
+
# MANY prompts: the decoded audio + transcript live in RAM (below), so the
|
| 316 |
+
# file is no longer needed once processed.
|
| 317 |
+
# • This in-memory state and the derived on-disk cache are TTL-evicted here so
|
| 318 |
+
# nothing lingers indefinitely.
|
| 319 |
+
CACHE_IDLE_TTL = 1200 # drop in-memory video state idle for > 20 min
|
| 320 |
+
CACHE_DISK_TTL = 1800 # delete on-disk transcript/embedding cache > 30 min old
|
| 321 |
+
|
| 322 |
+
def _evict_stale() -> None:
|
| 323 |
+
now = time.time()
|
| 324 |
+
for vp in [k for k, e in _cache.items() if now - e.get("_atime", now) > CACHE_IDLE_TTL]:
|
| 325 |
+
_cache.pop(vp, None)
|
| 326 |
+
try:
|
| 327 |
+
if os.path.isdir(CACHE_DIR):
|
| 328 |
+
for name in os.listdir(CACHE_DIR):
|
| 329 |
+
d = os.path.join(CACHE_DIR, name)
|
| 330 |
+
if os.path.isdir(d) and now - os.path.getmtime(d) > CACHE_DISK_TTL:
|
| 331 |
+
shutil.rmtree(d, ignore_errors=True)
|
| 332 |
+
except Exception:
|
| 333 |
+
pass
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
# ── Audio ──────────────────────────────────────────────────────────────────────
|
| 337 |
+
|
| 338 |
+
def load_audio(media_path: str) -> np.ndarray:
|
| 339 |
+
"""Decode any container straight to a float32 mono 16k waveform (PyAV, no temp file)."""
|
| 340 |
+
return decode_audio(media_path, sampling_rate=SAMPLE_RATE)
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def media_duration(media_path: str) -> float:
|
| 344 |
+
try:
|
| 345 |
+
info = ffmpeg.probe(media_path)
|
| 346 |
+
return float(info["format"]["duration"])
|
| 347 |
+
except Exception:
|
| 348 |
+
return 0.0
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
# ── Frames (CLIP input) — streamed from one ffmpeg pipe, no temp files ────────────
|
| 352 |
+
|
| 353 |
+
def extract_frames(video_path: str, duration: float):
|
| 354 |
+
"""
|
| 355 |
+
Sample frames evenly across the video and stream them as raw RGB straight
|
| 356 |
+
into NumPy via a single ffmpeg pipe (no per-frame JPEG temp files).
|
| 357 |
+
fps is chosen so total frames ≈ CLIP_MAX_FRAMES (bounded by MIN/MAX fps).
|
| 358 |
+
Returns (frames_rgb, times) where frames_rgb is (N, S, S, 3) uint8.
|
| 359 |
+
"""
|
| 360 |
+
fps = CLIP_MAX_FPS
|
| 361 |
+
if duration and duration > 0:
|
| 362 |
+
fps = min(CLIP_MAX_FPS, max(CLIP_MIN_FPS, CLIP_MAX_FRAMES / duration))
|
| 363 |
+
|
| 364 |
+
S = CLIP_FRAME_SIZE
|
| 365 |
+
try:
|
| 366 |
+
out, _ = (
|
| 367 |
+
ffmpeg.input(video_path)
|
| 368 |
+
.filter("fps", fps=fps)
|
| 369 |
+
# cover-then-centre-crop to a square so portrait/landscape both work
|
| 370 |
+
.filter("scale", S, S, force_original_aspect_ratio="increase")
|
| 371 |
+
.filter("crop", S, S)
|
| 372 |
+
.output("pipe:", format="rawvideo", pix_fmt="rgb24")
|
| 373 |
+
.run(capture_stdout=True, capture_stderr=True)
|
| 374 |
)
|
| 375 |
+
except ffmpeg.Error:
|
| 376 |
+
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
|
| 377 |
+
|
| 378 |
+
if not out:
|
| 379 |
+
return np.zeros((0, S, S, 3), dtype=np.uint8), np.zeros((0,))
|
| 380 |
+
|
| 381 |
+
frames = np.frombuffer(out, np.uint8).reshape(-1, S, S, 3)[:CLIP_MAX_FRAMES]
|
| 382 |
+
times = np.array([(i + 0.5) / fps for i in range(len(frames))])
|
| 383 |
+
return frames, times
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
# ── Stage 2: transcription ─────────────────────────────────────────────────────
|
| 387 |
+
|
| 388 |
+
def transcribe_fast(audio: np.ndarray) -> list[dict]:
|
| 389 |
+
"""
|
| 390 |
+
Cheap full-video pass: base model, greedy decoding, Silero VAD.
|
| 391 |
+
Keeps per-segment avg_logprob so we can later refine ONLY weak sectors.
|
| 392 |
+
"""
|
| 393 |
+
segments_gen, _ = _whisper_fast.transcribe(
|
| 394 |
+
audio, beam_size=1, vad_filter=True, word_timestamps=False,
|
| 395 |
+
)
|
| 396 |
+
return [
|
| 397 |
{
|
| 398 |
+
"text": seg.text.strip(),
|
| 399 |
+
"start": round(seg.start, 2),
|
| 400 |
+
"end": round(seg.end, 2),
|
| 401 |
+
"logprob": round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3),
|
|
|
|
|
|
|
| 402 |
}
|
| 403 |
+
for seg in segments_gen
|
| 404 |
+
if seg.text.strip()
|
| 405 |
]
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
# ── Stage 3a: visual analysis (CLIP) ─────────────────────────────────────────────
|
| 409 |
+
|
| 410 |
+
def analyze_visual(video_path: str, duration: float) -> dict:
|
| 411 |
+
"""
|
| 412 |
+
Sample → encode frames with CLIP. Cached per video.
|
| 413 |
+
Returns {"emb": (N, D) float32, "time": (N,) float}.
|
| 414 |
+
Empty arrays when the video has no usable visual stream (audio-only).
|
| 415 |
+
"""
|
| 416 |
+
frames, times = extract_frames(video_path, duration)
|
| 417 |
+
if len(frames) == 0:
|
| 418 |
+
print("[NINA] No video frames (audio-only?) — visual analysis skipped.")
|
| 419 |
+
return {"emb": np.zeros((0, 512), dtype=np.float32), "time": np.zeros((0,))}
|
| 420 |
+
|
| 421 |
+
images = [Image.fromarray(f) for f in frames]
|
| 422 |
+
embs = get_clip().encode(images, convert_to_numpy=True,
|
| 423 |
+
batch_size=32, show_progress_bar=False)
|
| 424 |
+
|
| 425 |
+
print(f"[NINA] CLIP encoded {len(images)} frames ({TORCH_DEVICE}).")
|
| 426 |
+
return {"emb": np.asarray(embs, dtype=np.float32), "time": np.asarray(times)}
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# ── semantic chunking ─────────────────────────────────────────────────
|
| 430 |
+
|
| 431 |
+
def semantic_chunk(segments: list[dict]) -> list[dict]:
|
| 432 |
+
if not segments:
|
| 433 |
+
return []
|
| 434 |
+
texts = [s["text"] for s in segments]
|
| 435 |
+
embeddings = bi_encoder.encode(texts, convert_to_numpy=True, batch_size=64)
|
| 436 |
+
chunks, current, cur_emb = [], [segments[0]], [embeddings[0]]
|
| 437 |
+
|
| 438 |
+
for i in range(1, len(segments)):
|
| 439 |
+
chunk_dur = segments[i]["end"] - current[0]["start"]
|
| 440 |
+
window_mean = np.mean(cur_emb[-CHUNK_WINDOW:], axis=0)
|
| 441 |
+
sim = max(_cosine(embeddings[i], window_mean),
|
| 442 |
+
_cosine(embeddings[i], cur_emb[-1]))
|
| 443 |
+
long_enough = (current[-1]["end"] - current[0]["start"]) >= CHUNK_MIN_DURATION
|
| 444 |
+
|
| 445 |
+
if (sim < CHUNK_SPLIT_THRESHOLD or chunk_dur >= CHUNK_MAX_DURATION) and long_enough:
|
| 446 |
+
chunks.append(_build_chunk(current, cur_emb))
|
| 447 |
+
current, cur_emb = [segments[i]], [embeddings[i]]
|
| 448 |
+
else:
|
| 449 |
+
current.append(segments[i])
|
| 450 |
+
cur_emb.append(embeddings[i])
|
| 451 |
+
|
| 452 |
+
if current:
|
| 453 |
+
chunks.append(_build_chunk(current, cur_emb))
|
| 454 |
+
|
| 455 |
+
print(f"[NINA] {len(segments)} segments → {len(chunks)} chunks.")
|
| 456 |
+
return chunks
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
# ── Entity extraction ──────────────────────────────────────────────────────────
|
| 460 |
+
|
| 461 |
+
_STOPWORDS = {
|
| 462 |
+
"the","a","an","and","or","but","in","on","at","to","for","of","is","are",
|
| 463 |
+
"was","were","be","been","that","this","it","he","she","they","we","i",
|
| 464 |
+
"w","z","i","że","na","do","się","nie","to","jest","są","by","przez","po",
|
| 465 |
+
"le","la","les","un","une","des","du","et","en","à","de","est","il","elle",
|
| 466 |
+
"ein","eine","der","die","das","und","ist","in","zu","mit","von","dem",
|
| 467 |
+
}
|
| 468 |
+
|
| 469 |
+
def extract_entities(text: str) -> set[str]:
|
| 470 |
+
"""
|
| 471 |
+
Extract surface-form entities from text.
|
| 472 |
+
Returns a set of normalised lowercase strings for overlap comparison.
|
| 473 |
+
"""
|
| 474 |
+
entities: set[str] = set()
|
| 475 |
+
|
| 476 |
+
# 1. quoted phrases — high precision
|
| 477 |
+
for match in re.finditer(r'"([^"]{2,40})"', text):
|
| 478 |
+
entities.add(match.group(1).lower().strip())
|
| 479 |
+
|
| 480 |
+
# 2. capitalised multi-word sequences (proper nouns)
|
| 481 |
+
for match in re.finditer(r'\b([A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-záéíóúàâêîôùäëïöüąćęłńóśźż]+(?:\s[A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-zÀ-žąćęłńóśźż]+)*)\b', text):
|
| 482 |
+
token = match.group(1).lower().strip()
|
| 483 |
+
if token not in _STOPWORDS and len(token) > 2:
|
| 484 |
+
entities.add(token)
|
| 485 |
+
|
| 486 |
+
# 3. single capitalised tokens (names, brands)
|
| 487 |
+
for token in re.findall(r'\b[A-ZÁÉÍÓÚÀÂÊÎÔÙÄËÏÖÜĄĆĘŁŃÓŚŹŻ][a-zÀ-žąćęłńóśźż]{2,}\b', text):
|
| 488 |
+
t = token.lower()
|
| 489 |
+
if t not in _STOPWORDS:
|
| 490 |
+
entities.add(t)
|
| 491 |
+
|
| 492 |
+
# 4. numbers with context
|
| 493 |
+
for match in re.finditer(r'\b([A-Za-z]+[-\s]?\d+[\w]*)\b', text):
|
| 494 |
+
entities.add(match.group(1).lower())
|
| 495 |
+
|
| 496 |
+
return entities
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def entity_overlap_score(prompt_entities: set[str],
|
| 500 |
+
chunk_entities: set[str],
|
| 501 |
+
chunk_text: str = "") -> float:
|
| 502 |
+
|
| 503 |
+
if not prompt_entities:
|
| 504 |
+
return 0.0
|
| 505 |
+
|
| 506 |
+
chunk_text_lower = chunk_text.lower()
|
| 507 |
+
matched = 0
|
| 508 |
+
for pe in prompt_entities:
|
| 509 |
+
# pass 1: extracted entity overlap
|
| 510 |
+
entity_hit = any(pe in ce or ce in pe for ce in chunk_entities)
|
| 511 |
+
# pass 2: raw text substring (handles lowercase ASR output)
|
| 512 |
+
text_hit = len(pe) > 3 and pe in chunk_text_lower
|
| 513 |
+
if entity_hit or text_hit:
|
| 514 |
+
matched += 1
|
| 515 |
+
return matched / len(prompt_entities)
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
def add_entities_to_chunks(chunks: list[dict]) -> list[dict]:
|
| 519 |
+
for chunk in chunks:
|
| 520 |
+
chunk["entities"] = extract_entities(chunk["text"])
|
| 521 |
+
return chunks
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
# ── Query router ───────────────────────────────────────────────────────────────
|
| 525 |
+
#
|
| 526 |
+
# Classifies prompt into one of three modes before retrieval runs.
|
| 527 |
+
#
|
| 528 |
+
# GLOBAL — user wants a summary / overview of the whole video.
|
| 529 |
+
# Retrieval is the wrong tool here because no single chunk is "about"
|
| 530 |
+
# summarization. → skip retrieval, sample chunks evenly, synthesize.
|
| 531 |
+
#
|
| 532 |
+
# HYBRID — user has a topic but signal is weak (bi-score 0.10–0.28).
|
| 533 |
+
# → run retrieval, but pad results with adjacent chunks.
|
| 534 |
+
#
|
| 535 |
+
# SPECIFIC — normal focused query, retrieval works well (bi-score > 0.28).
|
| 536 |
+
# → standard pipeline unchanged.
|
| 537 |
+
#
|
| 538 |
+
# Router runs in two passes:
|
| 539 |
+
# Pass 1: regex pattern match on prompt text (zero-cost, runs before encoding)
|
| 540 |
+
# Pass 2: bi-encoder score after retrieval (upgrades SPECIFIC → GLOBAL if
|
| 541 |
+
# retrieval actually fails)
|
| 542 |
+
|
| 543 |
+
# Global ONLY when the user asks for a whole-video overview. Critically, this no
|
| 544 |
+
# longer fires on "what does the video say ABOUT <topic>" — a bare "about" or
|
| 545 |
+
# "explain" used to force global and swallow specific questions.
|
| 546 |
+
_GLOBAL_PATTERNS = re.compile(
|
| 547 |
+
r'\b('
|
| 548 |
+
r'summari[sz]e|summary|overview|recap|tl;?dr|key\s+points?|main\s+points?|'
|
| 549 |
+
r'key\s+takeaways?|main\s+idea|gist|'
|
| 550 |
+
r'what(?:\'s| is| was)\s+(?:this|the|it)\s+(?:video|recording|clip|all\s+)?\s*about|'
|
| 551 |
+
r'what(?:\'s| is| was)\s+(?:this|the)\s+(?:video|recording|clip)|'
|
| 552 |
+
r'describe\s+(?:the|this)\s+(?:video|recording|clip)|'
|
| 553 |
+
r'streszcz\w*|podsumuj\w*|podsumowanie|'
|
| 554 |
+
r'o\s+czym\s+(?:jest|to|opowiada|mówi)|czego\s+dotyczy|'
|
| 555 |
+
r'gł[oó]wn\w+\s+(?:punkt\w*|my[sś]l\w*|tez\w*)|og[oó]ln\w+\s+zarys'
|
| 556 |
+
r')\b',
|
| 557 |
+
re.IGNORECASE
|
| 558 |
+
)
|
| 559 |
+
|
| 560 |
+
def classify_query(prompt: str) -> str:
|
| 561 |
+
"""
|
| 562 |
+
Pass 1 router — pattern-based, runs before any encoding.
|
| 563 |
+
Returns "global" or "specific".
|
| 564 |
+
Pass 2 (bi-score based upgrade) happens inside run() after retrieval.
|
| 565 |
+
"""
|
| 566 |
+
if _GLOBAL_PATTERNS.search(prompt):
|
| 567 |
+
return "global"
|
| 568 |
+
return "specific"
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
# ── Stage 4: bi-encoder retrieval (top-100) ───────────────────────────────────
|
| 572 |
+
|
| 573 |
+
def bi_encode_retrieve(chunks: list[dict], prompt: str) -> tuple[list[dict], float]:
|
| 574 |
+
# chunk embeddings were computed once at chunking time and cached on the
|
| 575 |
+
# chunk dict ("emb"); here we only encode the prompt.
|
| 576 |
+
prompt_emb = bi_encoder.encode([prompt], convert_to_numpy=True)[0]
|
| 577 |
+
scores = [_cosine(prompt_emb, c["emb"]) for c in chunks]
|
| 578 |
+
|
| 579 |
+
top_k = min(BI_ENCODER_TOP_K, len(chunks))
|
| 580 |
+
top_idx = np.argsort(scores)[::-1][:top_k]
|
| 581 |
+
|
| 582 |
+
if DEBUG:
|
| 583 |
+
print(f"[DEBUG] Bi top-3 scores: " +
|
| 584 |
+
", ".join(f"{scores[i]:.3f}" for i in top_idx[:3]))
|
| 585 |
+
|
| 586 |
+
best = scores[top_idx[0]] if len(top_idx) else 0.0
|
| 587 |
+
return (
|
| 588 |
+
[{**chunks[i], "bi_score": round(scores[i], 3)} for i in top_idx],
|
| 589 |
+
best
|
| 590 |
+
)
|
| 591 |
+
|
| 592 |
+
|
| 593 |
+
# ── Stage 4b: visual relevance (CLIP) ─────────────────────────────────────────
|
| 594 |
+
|
| 595 |
+
def visual_scores(visual: dict, prompt: str) -> tuple[np.ndarray, np.ndarray]:
|
| 596 |
+
"""
|
| 597 |
+
Cosine of prompt↔each frame in CLIP space, plus a min-max normalised copy.
|
| 598 |
+
Normalisation makes CLIP sims (typically ~0.2–0.35) comparable to the
|
| 599 |
+
audio score's 0–1 range before fusion. Returns (raw, norm); empty when
|
| 600 |
+
there are no frames.
|
| 601 |
+
"""
|
| 602 |
+
embs = visual.get("emb")
|
| 603 |
+
if embs is None or len(embs) == 0:
|
| 604 |
+
if DEBUG:
|
| 605 |
+
print("[DEBUG] Visual: no frames (audio-only) — visual weight = 0.")
|
| 606 |
+
return np.array([]), np.array([])
|
| 607 |
+
q = get_clip().encode([prompt], convert_to_numpy=True, show_progress_bar=False)[0]
|
| 608 |
+
qn = q / (np.linalg.norm(q) + 1e-9)
|
| 609 |
+
en = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9)
|
| 610 |
+
sims = en @ qn
|
| 611 |
+
lo, hi = float(sims.min()), float(sims.max())
|
| 612 |
+
norm = (sims - lo) / (hi - lo + 1e-9)
|
| 613 |
+
if DEBUG:
|
| 614 |
+
times = visual.get("time")
|
| 615 |
+
order = np.argsort(sims)[::-1][:3]
|
| 616 |
+
tops = ", ".join(f"{fmt_time(times[i])}={sims[i]:.3f}" for i in order)
|
| 617 |
+
print(f"[DEBUG] Visual: {len(sims)} frames; raw sim "
|
| 618 |
+
f"min/mean/max={sims.min():.3f}/{sims.mean():.3f}/{sims.max():.3f}; "
|
| 619 |
+
f"top moments: {tops}")
|
| 620 |
+
return sims, norm
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def span_visual_score(times: np.ndarray, norm: np.ndarray,
|
| 624 |
+
start: float, end: float) -> float:
|
| 625 |
+
"""Best (normalised) visual relevance among frames inside [start, end]."""
|
| 626 |
+
if len(times) == 0:
|
| 627 |
+
return 0.0
|
| 628 |
+
mask = (times >= start - 0.5) & (times <= end + 0.5)
|
| 629 |
+
if not mask.any():
|
| 630 |
+
idx = int(np.argmin(np.abs(times - (start + end) / 2)))
|
| 631 |
+
return float(norm[idx])
|
| 632 |
+
return float(norm[mask].max())
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
def region_visual_tags(visual: dict, start: float, end: float) -> list[str]:
|
| 636 |
+
"""Zero-shot CLIP tags describing what is on screen during [start, end]."""
|
| 637 |
+
embs = visual.get("emb")
|
| 638 |
+
times = visual.get("time")
|
| 639 |
+
if embs is None or len(embs) == 0:
|
| 640 |
+
return []
|
| 641 |
+
|
| 642 |
+
mask = (times >= start - 0.5) & (times <= end + 0.5)
|
| 643 |
+
if mask.any():
|
| 644 |
+
sel = embs[mask]
|
| 645 |
+
else:
|
| 646 |
+
idx = int(np.argmin(np.abs(times - (start + end) / 2)))
|
| 647 |
+
sel = embs[idx:idx + 1]
|
| 648 |
+
|
| 649 |
+
region_emb = sel.mean(axis=0)
|
| 650 |
+
region_emb = region_emb / (np.linalg.norm(region_emb) + 1e-9)
|
| 651 |
+
sims = _get_concept_embs() @ region_emb
|
| 652 |
+
|
| 653 |
+
order = np.argsort(sims)[::-1][:VISUAL_TAGS_TOP_K]
|
| 654 |
+
return [VISUAL_CONCEPTS[i] for i in order if sims[i] >= VISUAL_TAG_MIN_SIM]
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
def add_visual_tags(regions: list[dict], visual: dict) -> list[dict]:
|
| 658 |
+
for r in regions:
|
| 659 |
+
r["visual_tags"] = region_visual_tags(visual, r["start"], r["end"])
|
| 660 |
+
return regions
|
| 661 |
+
|
| 662 |
+
|
| 663 |
+
# ── Stage 5: cross-encoder rerank + audio/visual fusion (top-100 → top-10) ─────
|
| 664 |
+
|
| 665 |
+
def rerank_and_score(candidates: list[dict], prompt: str,
|
| 666 |
+
times: np.ndarray, norm: np.ndarray) -> list[dict]:
|
| 667 |
+
# cross-encoder
|
| 668 |
+
pairs = [[prompt, c["text"]] for c in candidates]
|
| 669 |
+
logits = cross_encoder.predict(pairs)
|
| 670 |
+
|
| 671 |
+
# entity overlap
|
| 672 |
+
prompt_entities = extract_entities(prompt)
|
| 673 |
+
prompt_words = {w.lower() for w in re.findall(r'[\wÀ-ž]{3,}', prompt)
|
| 674 |
+
if w.lower() not in _STOPWORDS}
|
| 675 |
+
all_prompt_tokens = prompt_entities | prompt_words
|
| 676 |
+
|
| 677 |
+
if DEBUG and all_prompt_tokens:
|
| 678 |
+
print(f"[DEBUG] Prompt tokens for entity match: {all_prompt_tokens}")
|
| 679 |
+
|
| 680 |
+
vw = VISUAL_WEIGHT if len(norm) else 0.0 # no frames → pure audio
|
| 681 |
+
|
| 682 |
+
scored = []
|
| 683 |
+
for candidate, logit in zip(candidates, logits):
|
| 684 |
+
chunk_entities = candidate.get("entities", set())
|
| 685 |
+
chunk_text = candidate.get("text", "")
|
| 686 |
+
overlap = entity_overlap_score(all_prompt_tokens, chunk_entities, chunk_text)
|
| 687 |
+
|
| 688 |
+
ce_conf = _sigmoid(float(logit))
|
| 689 |
+
audio_score = 0.6 * ce_conf + 0.4 * overlap
|
| 690 |
+
visual_score = span_visual_score(times, norm, candidate["start"], candidate["end"])
|
| 691 |
+
final = (1.0 - vw) * audio_score + vw * visual_score
|
| 692 |
+
|
| 693 |
+
scored.append({
|
| 694 |
+
**candidate,
|
| 695 |
+
"ce_logit": round(float(logit), 3),
|
| 696 |
+
"entity_overlap": round(overlap, 3),
|
| 697 |
+
"visual_score": round(visual_score, 3),
|
| 698 |
+
"confidence": round(min(1.0, final), 3),
|
| 699 |
+
})
|
| 700 |
+
|
| 701 |
+
# sort by fused multimodal score
|
| 702 |
+
ranked = sorted(scored, key=lambda x: x["confidence"], reverse=True)
|
| 703 |
+
|
| 704 |
+
if DEBUG:
|
| 705 |
+
print(f"[DEBUG] Top-3 after reranking (audio+visual):")
|
| 706 |
+
for r in ranked[:3]:
|
| 707 |
+
print(f" ce_logit={r['ce_logit']:.2f} "
|
| 708 |
+
f"bi={r['bi_score']:.3f} "
|
| 709 |
+
f"entity={r['entity_overlap']:.3f} "
|
| 710 |
+
f"visual={r['visual_score']:.3f} "
|
| 711 |
+
f"conf={r['confidence']:.3f} "
|
| 712 |
+
f"\"{r['text'][:60]}\"")
|
| 713 |
+
|
| 714 |
+
return ranked[:CROSS_ENCODER_TOP_K]
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
# ── Hybrid padding ─────────────────────────────────────────────────────────────
|
| 718 |
+
|
| 719 |
+
def pad_with_neighbours(results: list[dict], chunks: list[dict]) -> list[dict]:
|
| 720 |
+
"""
|
| 721 |
+
HYBRID mode: for each retrieved chunk, include its immediate neighbours
|
| 722 |
+
from the full chunk list. Helps when the relevant content slightly
|
| 723 |
+
overflows the chunk boundary. Deduplicates by start timestamp.
|
| 724 |
+
"""
|
| 725 |
+
result_starts = {r["start"] for r in results}
|
| 726 |
+
padded = list(results)
|
| 727 |
+
|
| 728 |
+
for r in results:
|
| 729 |
+
for i, c in enumerate(chunks):
|
| 730 |
+
if abs(c["start"] - r["start"]) < 0.5: # found matching chunk
|
| 731 |
+
for neighbour_i in [i - 1, i + 1]:
|
| 732 |
+
if 0 <= neighbour_i < len(chunks):
|
| 733 |
+
nb = chunks[neighbour_i]
|
| 734 |
+
if nb["start"] not in result_starts:
|
| 735 |
+
padded.append({
|
| 736 |
+
**nb,
|
| 737 |
+
"bi_score": 0.0,
|
| 738 |
+
"ce_logit": -9.0,
|
| 739 |
+
"entity_overlap": 0.0,
|
| 740 |
+
"visual_score": 0.0,
|
| 741 |
+
"confidence": nb.get("confidence", 0.1),
|
| 742 |
+
})
|
| 743 |
+
result_starts.add(nb["start"])
|
| 744 |
+
break
|
| 745 |
+
|
| 746 |
+
return sorted(padded, key=lambda x: x["start"])
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
# ── Global mode: even sampling ────────────────────────────────────────────────
|
| 750 |
+
|
| 751 |
+
def sample_chunks_for_global(chunks: list[dict], n_target: int) -> list[dict]:
|
| 752 |
+
"""
|
| 753 |
+
For global summarization, pick `n_target` chunks evenly distributed across
|
| 754 |
+
the video, so the summary covers the whole thing, not just the beginning.
|
| 755 |
+
"""
|
| 756 |
+
n_target = max(3, min(n_target, len(chunks)))
|
| 757 |
+
if n_target >= len(chunks):
|
| 758 |
+
return chunks
|
| 759 |
+
|
| 760 |
+
# evenly spaced indices
|
| 761 |
+
indices = [int(i * (len(chunks) - 1) / (n_target - 1)) for i in range(n_target)]
|
| 762 |
+
return [chunks[i] for i in indices]
|
| 763 |
+
|
| 764 |
+
|
| 765 |
+
# ── Stage 6: merge regions ─────────────────────────────────────────────────────
|
| 766 |
+
|
| 767 |
+
def merge_regions(results: list[dict]) -> list[dict]:
|
| 768 |
+
if not results:
|
| 769 |
+
return []
|
| 770 |
+
sorted_r = sorted(results, key=lambda x: x["start"])
|
| 771 |
+
merged, current = [], sorted_r[0].copy()
|
| 772 |
+
current["_frags"] = [sorted_r[0]]
|
| 773 |
+
|
| 774 |
+
for nxt in sorted_r[1:]:
|
| 775 |
+
if nxt["start"] - current["end"] <= MERGE_GAP_SEC:
|
| 776 |
+
current["end"] = max(current["end"], nxt["end"])
|
| 777 |
+
current["text"] += " " + nxt["text"]
|
| 778 |
+
current["_frags"].append(nxt)
|
| 779 |
+
else:
|
| 780 |
+
merged.append(_finalize_region(current))
|
| 781 |
+
current = nxt.copy()
|
| 782 |
+
current["_frags"] = [nxt]
|
| 783 |
+
|
| 784 |
+
merged.append(_finalize_region(current))
|
| 785 |
+
return [r for r in merged if (r["end"] - r["start"]) >= REGION_MIN_DURATION]
|
| 786 |
+
|
| 787 |
+
|
| 788 |
+
def _finalize_region(region: dict) -> dict:
|
| 789 |
+
frags = region.pop("_frags")
|
| 790 |
+
total_dur = sum(max(f["end"] - f["start"], 0.1) for f in frags)
|
| 791 |
+
w_conf = sum(f["confidence"] * max(f["end"] - f["start"], 0.1)
|
| 792 |
+
for f in frags) / total_dur
|
| 793 |
+
duration = region["end"] - region["start"]
|
| 794 |
+
w_conf *= min(1.0, duration / 20.0)
|
| 795 |
+
visual = max((f.get("visual_score", 0.0) for f in frags), default=0.0)
|
| 796 |
+
return {**region, "confidence": round(w_conf, 3), "visual_score": round(visual, 3)}
|
| 797 |
+
|
| 798 |
+
|
| 799 |
+
# ── Stage 7: precise re-transcription (confidence-gated) ──────────────────────
|
| 800 |
+
|
| 801 |
+
def retranscribe_region(audio: np.ndarray, region: dict) -> str:
|
| 802 |
+
"""Slice the in-memory waveform (no per-region ffmpeg) and run precise ASR."""
|
| 803 |
+
s = max(0, int(region["start"] * SAMPLE_RATE))
|
| 804 |
+
e = min(len(audio), int(region["end"] * SAMPLE_RATE))
|
| 805 |
+
clip = audio[s:e]
|
| 806 |
+
if len(clip) < SAMPLE_RATE * 0.2: # < 0.2s → nothing to gain
|
| 807 |
+
return region.get("text", "")
|
| 808 |
+
model = get_whisper_precise()
|
| 809 |
+
segments_gen, _ = model.transcribe(clip, beam_size=1)
|
| 810 |
+
return " ".join(seg.text.strip() for seg in segments_gen).strip()
|
| 811 |
+
|
| 812 |
+
|
| 813 |
+
def fill_precise(audio: np.ndarray, regions: list[dict], all_chunks: list[dict],
|
| 814 |
+
precise_cache: dict, budget: int) -> int:
|
| 815 |
+
"""
|
| 816 |
+
Ensure every region has `precise_text`. A sector is turbo-refined ONLY when:
|
| 817 |
+
• it is not already cached, AND
|
| 818 |
+
• the base pass for that sector was low-confidence (avg_logprob < gate), AND
|
| 819 |
+
• we are still within `budget` NEW transcriptions this call.
|
| 820 |
+
Clean/high-confidence sectors keep the (already accurate) base text for free,
|
| 821 |
+
so on clean audio the slow turbo model never runs. `precise_cache` persists
|
| 822 |
+
per video, so later prompts reuse earlier work. Returns remaining budget.
|
| 823 |
+
"""
|
| 824 |
+
reused = new = skipped = 0
|
| 825 |
+
for region in regions:
|
| 826 |
+
# the chunk sectors this region spans (chunks are non-overlapping)
|
| 827 |
+
comp = [c for c in all_chunks
|
| 828 |
+
if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3]
|
| 829 |
+
if not comp:
|
| 830 |
+
comp = [region]
|
| 831 |
+
|
| 832 |
+
for ch in comp:
|
| 833 |
+
key = round(ch["start"], 2)
|
| 834 |
+
if key in precise_cache:
|
| 835 |
+
reused += 1
|
| 836 |
+
continue
|
| 837 |
+
weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE
|
| 838 |
+
if not weak:
|
| 839 |
+
skipped += 1 # base was confident → trust it
|
| 840 |
+
continue
|
| 841 |
+
if budget <= 0:
|
| 842 |
+
continue # over budget → fall back to fast text
|
| 843 |
+
print(f"[NINA] Re-transcribing weak sector "
|
| 844 |
+
f"[{fmt_time(ch['start'])} → {fmt_time(ch['end'])}] "
|
| 845 |
+
f"(logprob {ch.get('logprob', 0.0):.2f})")
|
| 846 |
+
precise_cache[key] = retranscribe_region(audio, ch)
|
| 847 |
+
if DEBUG:
|
| 848 |
+
snip = precise_cache[key]
|
| 849 |
+
print(f"[DEBUG] Precise sector [{fmt_time(ch['start'])}→{fmt_time(ch['end'])}]: "
|
| 850 |
+
f"{snip[:200]}{'…' if len(snip) > 200 else ''}")
|
| 851 |
+
new += 1
|
| 852 |
+
budget -= 1
|
| 853 |
+
|
| 854 |
+
region["precise_text"] = " ".join(
|
| 855 |
+
(precise_cache.get(round(ch["start"], 2)) or ch.get("text", ""))
|
| 856 |
+
for ch in comp
|
| 857 |
+
).strip() or region.get("text", "")
|
| 858 |
+
|
| 859 |
+
print(f"[NINA] Precise sectors — {new} refined, {reused} cached, {skipped} trusted (base).")
|
| 860 |
+
return budget
|
| 861 |
+
|
| 862 |
+
|
| 863 |
+
# ── Stage 8: synthesis ─────────────────────────────────────────────────────────
|
| 864 |
+
|
| 865 |
+
# ── Output sizing — inferred from the PROMPT (no slider) ─────────────────────────
|
| 866 |
+
|
| 867 |
+
_BRIEF_CUES = (
|
| 868 |
+
"brief", "briefly", "short", "shortly", "quick", "quickly", "concise",
|
| 869 |
+
"in a sentence", "one sentence", "one-liner", "tl;dr", "tldr",
|
| 870 |
+
"krótko", "w skrócie", "zwięźle", "jednym zdaniem", "w jednym zdaniu",
|
| 871 |
+
)
|
| 872 |
+
_DETAIL_CUES = (
|
| 873 |
+
"in detail", "detailed", "in-depth", "in depth", "thorough", "thoroughly",
|
| 874 |
+
"comprehensive", "exhaustive", "everything", "full summary", "elaborate",
|
| 875 |
+
"step by step", "step-by-step", "deep dive", "as much as possible",
|
| 876 |
+
"szczegółowo", "dokładnie", "wyczerpująco", "wszystko", "krok po kroku",
|
| 877 |
+
"dogłębnie", "obszernie",
|
| 878 |
+
)
|
| 879 |
+
_FACTUAL_START = re.compile(
|
| 880 |
+
r'^\s*(what|who|whom|whose|when|where|which|how many|how much|how long|'
|
| 881 |
+
r'is|are|was|were|does|do|did|can|could|will|would|'
|
| 882 |
+
r'czy|kiedy|kto|kogo|gdzie|ile|jaki|jaka|jakie|który|która|które)\b',
|
| 883 |
+
re.IGNORECASE,
|
| 884 |
+
)
|
| 885 |
+
|
| 886 |
+
def _summary_tier(max_words: int) -> str:
|
| 887 |
+
"""Human-readable label for the chosen output length."""
|
| 888 |
+
if max_words <= 60: return "brief / one-liner"
|
| 889 |
+
if max_words <= 110: return "short"
|
| 890 |
+
if max_words <= 250: return "standard"
|
| 891 |
+
if max_words <= 400: return "detailed"
|
| 892 |
+
return "exhaustive"
|
| 893 |
+
|
| 894 |
+
|
| 895 |
+
def _prompt_to_words(prompt: str) -> int:
|
| 896 |
+
"""
|
| 897 |
+
Infer the answer's word budget from the prompt itself (replaces the slider):
|
| 898 |
+
- explicit brevity cues → ~40 words
|
| 899 |
+
- explicit detail cues → ~450 words
|
| 900 |
+
- factual wh-/yes-no Q → ~80 words
|
| 901 |
+
- otherwise scale gently with prompt length (longer ask → longer answer)
|
| 902 |
+
Clamped to [30, 500].
|
| 903 |
+
"""
|
| 904 |
+
p = prompt.lower()
|
| 905 |
+
if any(c in p for c in _DETAIL_CUES):
|
| 906 |
+
base, reason = 450, "explicit detail cue"
|
| 907 |
+
elif any(c in p for c in _BRIEF_CUES):
|
| 908 |
+
base, reason = 40, "explicit brevity cue"
|
| 909 |
+
elif _FACTUAL_START.match(p):
|
| 910 |
+
base, reason = 80, "factual question"
|
| 911 |
+
else:
|
| 912 |
+
n_words = len(re.findall(r'\w+', p))
|
| 913 |
+
base, reason = 120 + min(180, n_words * 8), f"length-scaled ({n_words} prompt words)"
|
| 914 |
+
|
| 915 |
+
out = int(max(30, min(500, base)))
|
| 916 |
+
if DEBUG:
|
| 917 |
+
print(f"[DEBUG] Summary sizing: {reason} → {out} words [{_summary_tier(out)}]")
|
| 918 |
+
return out
|
| 919 |
+
|
| 920 |
+
|
| 921 |
+
def _words_to_global_chunks(max_words: int) -> int:
|
| 922 |
+
"""More requested words → sample more chunks for the global overview."""
|
| 923 |
+
return max(3, min(GLOBAL_MAX_CHUNKS, round(max_words / 40)))
|
| 924 |
+
|
| 925 |
+
|
| 926 |
+
def _fallback_summary(regions: list[dict]) -> str:
|
| 927 |
+
snippets = [
|
| 928 |
+
f"[{fmt_time(r['start'])}] {r.get('precise_text', r['text'])[:200]}"
|
| 929 |
+
for r in regions[:2]
|
| 930 |
]
|
| 931 |
+
return "(LLM unavailable — raw excerpts:)\n" + "\n".join(snippets)
|
| 932 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 933 |
|
| 934 |
+
def _build_synth_prompt(regions: list[dict], prompt: str,
|
| 935 |
+
max_words: int, mode: str) -> str:
|
| 936 |
+
# Each fragment carries BOTH modalities: what is SAID (audio) and what is
|
| 937 |
+
# SHOWN (CLIP visual cues). The LLM is asked to fuse them.
|
| 938 |
+
blocks = []
|
| 939 |
+
for r in regions:
|
| 940 |
+
spoken = (r.get("precise_text", r["text"]) or "").strip()
|
| 941 |
+
tags = r.get("visual_tags", [])
|
| 942 |
+
block = (f"[{fmt_time(r['start'])} → {fmt_time(r['end'])}]\n"
|
| 943 |
+
f" Spoken (audio): {spoken or '(no speech)'}")
|
| 944 |
+
if tags:
|
| 945 |
+
block += f"\n On screen (video): {', '.join(tags)}"
|
| 946 |
+
blocks.append(block)
|
| 947 |
+
fragments = "\n\n".join(blocks)
|
| 948 |
|
| 949 |
+
if mode == "global":
|
| 950 |
+
task_line = "Task: Give a complete overview of what this video is about."
|
| 951 |
+
else:
|
| 952 |
+
task_line = f'Task: Summarize what the video says/shows about "{prompt}".'
|
| 953 |
+
|
| 954 |
+
return (
|
| 955 |
+
f"You are analyzing a video. Each fragment below contains what is SPOKEN "
|
| 956 |
+
f"(audio transcript) and what is SHOWN (visual cues detected in the frames).\n"
|
| 957 |
+
f'The user is looking for: "{prompt}"\n\n'
|
| 958 |
+
f"Fragments:\n\n{fragments}\n\n"
|
| 959 |
+
f"{task_line}\n\n"
|
| 960 |
+
f"Output format:\n"
|
| 961 |
+
f"- Combine BOTH the spoken content and the on-screen visuals into one coherent answer\n"
|
| 962 |
+
f"- After each statement, append the timestamp it came from, taken from the "
|
| 963 |
+
f"bracketed start time of the relevant fragment above (use that exact value)\n"
|
| 964 |
+
f"- Maximum {max_words} words total\n"
|
| 965 |
+
f"- IMPORTANT: Write in the same language as the prompt given by the user\n"
|
| 966 |
+
f"- Only use information from the fragments (audio + visual) — do not invent details\n"
|
| 967 |
+
f"- If fragments contain no real information about the topic, say so clearly, "
|
| 968 |
+
f"without further explanation\n"
|
| 969 |
+
)
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
def llm_synthesize_stream(regions: list[dict], prompt: str,
|
| 973 |
+
max_words: int = 150, mode: str = "specific"):
|
| 974 |
+
"""
|
| 975 |
+
Generator: yields the answer incrementally as the LLM streams tokens.
|
| 976 |
+
On any error, yields a single fallback string.
|
| 977 |
+
"""
|
| 978 |
+
if not regions:
|
| 979 |
+
yield "(No relevant regions to summarize.)"
|
| 980 |
return
|
| 981 |
|
| 982 |
+
synth_prompt = _build_synth_prompt(regions, prompt, max_words, mode)
|
| 983 |
+
|
| 984 |
+
if DEBUG:
|
| 985 |
+
print(f"[DEBUG] Synthesis [{mode}] — {len(regions)} region(s), "
|
| 986 |
+
f"max_words={max_words} [{_summary_tier(max_words)}].")
|
| 987 |
+
|
| 988 |
+
try:
|
| 989 |
+
acc = ""
|
| 990 |
+
stream = llm_client.chat_completion(
|
| 991 |
+
model=LLM_MODEL,
|
| 992 |
+
messages=[{"role": "user", "content": synth_prompt}],
|
| 993 |
+
max_tokens=LLM_MAX_TOKENS,
|
| 994 |
+
temperature=0.3,
|
| 995 |
+
stream=True,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 996 |
)
|
| 997 |
+
for ev in stream:
|
| 998 |
+
delta = ev.choices[0].delta.content or ""
|
| 999 |
+
if delta:
|
| 1000 |
+
acc += delta
|
| 1001 |
+
yield acc.strip()
|
| 1002 |
+
if not acc.strip():
|
| 1003 |
+
yield _fallback_summary(regions)
|
| 1004 |
+
except Exception as e:
|
| 1005 |
+
print(f"[NINA] Synthesis error: {e}")
|
| 1006 |
+
yield _fallback_summary(regions)
|
| 1007 |
+
|
| 1008 |
+
|
| 1009 |
+
def llm_synthesize(regions: list[dict], prompt: str,
|
| 1010 |
+
max_words: int = 150, mode: str = "specific") -> str:
|
| 1011 |
+
"""Non-streaming convenience wrapper (kept for API compatibility / testing)."""
|
| 1012 |
+
out = ""
|
| 1013 |
+
for out in llm_synthesize_stream(regions, prompt, max_words, mode):
|
| 1014 |
+
pass
|
| 1015 |
+
return out
|
| 1016 |
+
|
| 1017 |
+
|
| 1018 |
+
# ── Helpers ────────────────────────────────────────────────────────────────────
|
| 1019 |
+
|
| 1020 |
+
def _build_chunk(segs: list[dict], embs: list[np.ndarray]) -> dict:
|
| 1021 |
+
total = sum(max(s["end"] - s["start"], 0.1) for s in segs)
|
| 1022 |
+
logp = sum(s.get("logprob", 0.0) * max(s["end"] - s["start"], 0.1)
|
| 1023 |
+
for s in segs) / total # duration-weighted base confidence
|
| 1024 |
+
return {"text": " ".join(s["text"] for s in segs),
|
| 1025 |
+
"start": segs[0]["start"],
|
| 1026 |
+
"end": segs[-1]["end"],
|
| 1027 |
+
"logprob": round(logp, 3),
|
| 1028 |
+
"emb": np.mean(np.stack(embs), axis=0)} # cached chunk embedding
|
| 1029 |
+
|
| 1030 |
+
def _cosine(a: np.ndarray, b: np.ndarray) -> float:
|
| 1031 |
+
return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9))
|
| 1032 |
+
|
| 1033 |
+
def _sigmoid(x: float) -> float:
|
| 1034 |
+
return 1.0 / (1.0 + math.exp(-float(x)))
|
| 1035 |
+
|
| 1036 |
+
def fmt_time(s: float) -> str:
|
| 1037 |
+
m = int(s) // 60
|
| 1038 |
+
return f"{m}:{s % 60:05.2f}"
|
| 1039 |
+
|
| 1040 |
+
def conf_bar(c: float) -> str:
|
| 1041 |
+
f = round(c * 10)
|
| 1042 |
+
return "█" * f + "░" * (10 - f)
|
| 1043 |
+
|
| 1044 |
+
|
| 1045 |
+
# ── Persistent disk cache ────────────────────────────────────────────────────────
|
| 1046 |
+
# Keyed by a content fingerprint (size + head + tail bytes), so the same video
|
| 1047 |
+
# re-uploaded to a fresh temp path — or opened after a restart — is recognised
|
| 1048 |
+
# and its transcript / embeddings / precise sectors are loaded instead of
|
| 1049 |
+
# recomputed. Best-effort: any failure silently falls back to recomputation.
|
| 1050 |
+
|
| 1051 |
+
def _fingerprint(path: str) -> str:
|
| 1052 |
+
h = hashlib.sha1()
|
| 1053 |
+
st = os.stat(path)
|
| 1054 |
+
h.update(str(st.st_size).encode())
|
| 1055 |
+
with open(path, "rb") as f:
|
| 1056 |
+
h.update(f.read(1 << 20)) # first 1 MB
|
| 1057 |
+
if st.st_size > (1 << 20):
|
| 1058 |
+
f.seek(-(1 << 20), os.SEEK_END)
|
| 1059 |
+
h.update(f.read(1 << 20)) # last 1 MB
|
| 1060 |
+
return h.hexdigest()[:16]
|
| 1061 |
+
|
| 1062 |
+
|
| 1063 |
+
def _disk_load(fp: str) -> dict | None:
|
| 1064 |
+
if not DISK_CACHE:
|
| 1065 |
+
return None
|
| 1066 |
+
d = os.path.join(CACHE_DIR, fp)
|
| 1067 |
+
try:
|
| 1068 |
+
with open(os.path.join(d, "meta.json"), encoding="utf-8") as f:
|
| 1069 |
+
meta = json.load(f)
|
| 1070 |
+
arr = np.load(os.path.join(d, "emb.npz"))
|
| 1071 |
+
chunk_emb = arr["chunk_emb"]
|
| 1072 |
+
chunks = []
|
| 1073 |
+
for i, c in enumerate(meta["chunks"]):
|
| 1074 |
+
chunks.append({
|
| 1075 |
+
"text": c["text"], "start": c["start"], "end": c["end"],
|
| 1076 |
+
"logprob": c.get("logprob", 0.0),
|
| 1077 |
+
"entities": set(c.get("entities", [])),
|
| 1078 |
+
"emb": chunk_emb[i],
|
| 1079 |
+
})
|
| 1080 |
+
visual = {"emb": arr["visual_emb"], "time": arr["visual_time"]}
|
| 1081 |
+
print(f"[NINA] Disk cache hit ({fp}) — loaded {len(chunks)} chunks.")
|
| 1082 |
+
return {"segments": meta["segments"], "chunks": chunks,
|
| 1083 |
+
"visual": visual, "precise": {float(k): v for k, v in meta["precise"].items()}}
|
| 1084 |
+
except Exception:
|
| 1085 |
+
return None
|
| 1086 |
+
|
| 1087 |
+
|
| 1088 |
+
def _disk_save(fp: str, entry: dict) -> None:
|
| 1089 |
+
if not DISK_CACHE:
|
| 1090 |
+
return
|
| 1091 |
+
d = os.path.join(CACHE_DIR, fp)
|
| 1092 |
+
try:
|
| 1093 |
+
os.makedirs(d, exist_ok=True)
|
| 1094 |
+
chunks = entry["chunks"]
|
| 1095 |
+
meta = {
|
| 1096 |
+
"segments": entry["segments"],
|
| 1097 |
+
"precise": entry["precise"],
|
| 1098 |
+
"chunks": [{"text": c["text"], "start": c["start"], "end": c["end"],
|
| 1099 |
+
"logprob": c.get("logprob", 0.0),
|
| 1100 |
+
"entities": sorted(c.get("entities", []))}
|
| 1101 |
+
for c in chunks],
|
| 1102 |
+
}
|
| 1103 |
+
chunk_emb = (np.stack([c["emb"] for c in chunks]) if chunks
|
| 1104 |
+
else np.zeros((0, 384), dtype=np.float32))
|
| 1105 |
+
with open(os.path.join(d, "meta.json"), "w", encoding="utf-8") as f:
|
| 1106 |
+
json.dump(meta, f)
|
| 1107 |
+
np.savez(os.path.join(d, "emb.npz"),
|
| 1108 |
+
chunk_emb=chunk_emb,
|
| 1109 |
+
visual_emb=entry["visual"]["emb"],
|
| 1110 |
+
visual_time=entry["visual"]["time"])
|
| 1111 |
+
except Exception as e:
|
| 1112 |
+
if DEBUG:
|
| 1113 |
+
print(f"[DEBUG] disk cache save failed: {e}")
|
| 1114 |
+
|
| 1115 |
+
|
| 1116 |
+
def _disk_update_precise(fp: str, precise: dict) -> None:
|
| 1117 |
+
"""Persist newly-refined precise sectors without rewriting embeddings."""
|
| 1118 |
+
if not DISK_CACHE:
|
| 1119 |
+
return
|
| 1120 |
+
p = os.path.join(CACHE_DIR, fp, "meta.json")
|
| 1121 |
+
try:
|
| 1122 |
+
with open(p, encoding="utf-8") as f:
|
| 1123 |
+
meta = json.load(f)
|
| 1124 |
+
meta["precise"] = precise
|
| 1125 |
+
with open(p, "w", encoding="utf-8") as f:
|
| 1126 |
+
json.dump(meta, f)
|
| 1127 |
+
except Exception:
|
| 1128 |
+
pass
|
| 1129 |
+
|
| 1130 |
+
|
| 1131 |
+
# ── Formatter ──────────────────────────────────────────────────────────────────
|
| 1132 |
+
|
| 1133 |
+
def format_output(regions: list[dict], prompt: str, summary: str,
|
| 1134 |
+
mode: str = "specific") -> str:
|
| 1135 |
+
if not regions:
|
| 1136 |
+
return (
|
| 1137 |
+
f' Topic not found in this video.\n\n'
|
| 1138 |
+
f' Prompt : "{prompt}"\n\n'
|
| 1139 |
+
f' No relevant segments found.\n'
|
| 1140 |
+
f' Try a broader or differently worded prompt.'
|
| 1141 |
)
|
|
|
|
| 1142 |
|
| 1143 |
+
mode_tag = {"global": "🌐 global", "hybrid": "⚡ hybrid", "specific": "🎯 specific"}.get(mode, "")
|
| 1144 |
+
lines = [f' {len(regions)} region(s) found for: "{prompt}" [{mode_tag} · audio+video]\n']
|
| 1145 |
+
|
| 1146 |
+
if DEBUG:
|
| 1147 |
+
for i, r in enumerate(regions, 1):
|
| 1148 |
+
dur = round(r["end"] - r["start"], 1)
|
| 1149 |
+
text = r.get("precise_text", r["text"])
|
| 1150 |
+
ent = r.get("entity_overlap", 0)
|
| 1151 |
+
bi = r.get("bi_score", 0)
|
| 1152 |
+
vis = r.get("visual_score", 0)
|
| 1153 |
+
tags = ", ".join(r.get("visual_tags", []))
|
| 1154 |
+
lines += [
|
| 1155 |
+
f'\n [{i}] {fmt_time(r["start"])} → {fmt_time(r["end"])} ({dur}s)',
|
| 1156 |
+
f' {conf_bar(r["confidence"])} {r["confidence"]:.0%} '
|
| 1157 |
+
f'(embed {bi:.0%} · entity {ent:.0%} · visual {vis:.0%})',
|
| 1158 |
+
f' 👁 {tags}' if tags else ' 👁 (no visual tags)',
|
| 1159 |
+
f' "{text[:220]}{"…" if len(text) > 220 else ""}"',
|
| 1160 |
+
]
|
| 1161 |
+
|
| 1162 |
+
lines += [
|
| 1163 |
+
f' 🧠 Based on uploaded video (audio + video): — "{prompt}"',
|
| 1164 |
+
summary,
|
| 1165 |
+
f'\n Sources: ' + ', '.join(
|
| 1166 |
+
f'[{fmt_time(r["start"])} → {fmt_time(r["end"])}]' for r in regions
|
| 1167 |
+
),
|
| 1168 |
+
]
|
| 1169 |
+
return "\n".join(lines)
|
| 1170 |
+
|
| 1171 |
|
| 1172 |
+
# ── Cold-start ingestion (transcription + chunking + CLIP, in parallel) ───────────
|
| 1173 |
+
|
| 1174 |
+
def ingest_video(video_path: str) -> dict:
|
| 1175 |
+
"""
|
| 1176 |
+
First sight of a video: decode audio, then run the two heavy independent
|
| 1177 |
+
stages concurrently —
|
| 1178 |
+
• CPU thread : base transcription → semantic chunking → entities
|
| 1179 |
+
• MPS thread : frame extraction → CLIP encoding
|
| 1180 |
+
Returns the cache entry dict.
|
| 1181 |
+
"""
|
| 1182 |
+
audio = load_audio(video_path)
|
| 1183 |
+
duration = media_duration(video_path) or (len(audio) / SAMPLE_RATE)
|
| 1184 |
+
|
| 1185 |
+
result = {}
|
| 1186 |
+
|
| 1187 |
+
def _audio_branch():
|
| 1188 |
+
segments = transcribe_fast(audio)
|
| 1189 |
+
result["segments"] = segments
|
| 1190 |
+
if segments:
|
| 1191 |
+
result["chunks"] = add_entities_to_chunks(semantic_chunk(segments))
|
| 1192 |
+
else:
|
| 1193 |
+
result["chunks"] = []
|
| 1194 |
+
|
| 1195 |
+
def _visual_branch():
|
| 1196 |
+
result["visual"] = analyze_visual(video_path, duration)
|
| 1197 |
+
|
| 1198 |
+
with ThreadPoolExecutor(max_workers=2) as ex:
|
| 1199 |
+
fa = ex.submit(_audio_branch)
|
| 1200 |
+
fv = ex.submit(_visual_branch)
|
| 1201 |
+
fa.result()
|
| 1202 |
+
fv.result()
|
| 1203 |
+
|
| 1204 |
+
return {
|
| 1205 |
+
"segments": result["segments"],
|
| 1206 |
+
"chunks": result["chunks"],
|
| 1207 |
+
"visual": result["visual"],
|
| 1208 |
+
"audio": audio,
|
| 1209 |
+
"precise": {},
|
| 1210 |
+
}
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
# ── Global path (shared by GLOBAL mode + every escalation/fallback) ───────────────
|
| 1214 |
+
# Global summaries run on the clean base transcript — no whole-video turbo pass.
|
| 1215 |
+
|
| 1216 |
+
def _run_global(chunks: list[dict], visual: dict,
|
| 1217 |
+
prompt: str, max_words: int) -> list[dict]:
|
| 1218 |
+
sampled = sample_chunks_for_global(chunks, _words_to_global_chunks(max_words))
|
| 1219 |
+
print(f"[NINA] Global mode — sampled {len(sampled)} chunks evenly (base transcript).")
|
| 1220 |
+
for c in sampled:
|
| 1221 |
+
c.setdefault("confidence", 0.5)
|
| 1222 |
+
c.setdefault("bi_score", 0.0)
|
| 1223 |
+
c.setdefault("entity_overlap", 0.0)
|
| 1224 |
+
c.setdefault("visual_score", 0.0)
|
| 1225 |
+
c["precise_text"] = c.get("text", "") # base text is enough for an overview
|
| 1226 |
+
return add_visual_tags(sampled, visual)
|
| 1227 |
+
|
| 1228 |
+
|
| 1229 |
+
# ── Main pipeline (generator: streams progress + answer) ─────────────────────────
|
| 1230 |
+
|
| 1231 |
+
def format_iris(regions, prompt, best_bi):
|
| 1232 |
+
if not regions:
|
| 1233 |
+
return (f'No clear moments about "{prompt}" were found in this video.\n\n'
|
| 1234 |
+
f'Try broader or differently-worded keywords.')
|
| 1235 |
+
lines = [f'IRIS - moments matching: "{prompt}"\n']
|
| 1236 |
+
for i, r in enumerate(sorted(regions, key=lambda x: x["start"]), 1):
|
| 1237 |
+
txt = (r.get("precise_text", r.get("text", "")) or "").strip()
|
| 1238 |
+
tags = ", ".join(r.get("visual_tags", []))
|
| 1239 |
+
lines.append(f' {i}. [{fmt_time(r["start"])} -> {fmt_time(r["end"])}] {r.get("confidence", 0):.0%} match')
|
| 1240 |
+
lines.append(f' "{txt[:260]}{"..." if len(txt) > 260 else ""}"')
|
| 1241 |
+
if tags:
|
| 1242 |
+
lines.append(f' on screen: {tags}')
|
| 1243 |
+
lines.append("")
|
| 1244 |
+
lines.append(f' {len(regions)} moment(s) - ranked by relevance, shown in order of appearance.')
|
| 1245 |
+
return "\n".join(lines)
|
| 1246 |
+
|
| 1247 |
+
|
| 1248 |
+
def run(video_path: str, prompt: str, token: str = ""):
|
| 1249 |
+
if not _verify_usage_token(token):
|
| 1250 |
+
yield ("This request is not authorized.\n\n"
|
| 1251 |
+
"Please use IRIS at https://app.nnlabs.pl/iris (sign in first). "
|
| 1252 |
+
"Direct API access is disabled, and each account has a usage limit.")
|
| 1253 |
+
return
|
| 1254 |
+
if video_path is None:
|
| 1255 |
+
yield "Please upload a video."
|
| 1256 |
+
return
|
| 1257 |
+
if not prompt or not prompt.strip():
|
| 1258 |
+
yield "Please type what you want to find."
|
| 1259 |
+
return
|
| 1260 |
+
prompt = prompt.strip()
|
| 1261 |
+
print(f'\n[IRIS] -- Find -- "{prompt}"')
|
| 1262 |
+
_evict_stale()
|
| 1263 |
+
|
| 1264 |
+
fp = None
|
| 1265 |
+
if video_path in _cache:
|
| 1266 |
+
entry = _cache[video_path]
|
| 1267 |
+
else:
|
| 1268 |
+
try: fp = _fingerprint(video_path)
|
| 1269 |
+
except Exception: fp = None
|
| 1270 |
+
disk = _disk_load(fp) if fp else None
|
| 1271 |
+
if disk is not None:
|
| 1272 |
+
disk["audio"] = load_audio(video_path)
|
| 1273 |
+
entry = disk; _cache[video_path] = entry
|
| 1274 |
+
else:
|
| 1275 |
+
yield "Reading video - transcribing audio and scanning frames..."
|
| 1276 |
+
entry = ingest_video(video_path)
|
| 1277 |
+
if not entry["segments"]:
|
| 1278 |
+
yield "No speech detected."
|
| 1279 |
+
return
|
| 1280 |
+
_cache[video_path] = entry
|
| 1281 |
+
if fp: _disk_save(fp, entry)
|
| 1282 |
+
|
| 1283 |
+
entry["_atime"] = time.time()
|
| 1284 |
+
segments, chunks, visual = entry["segments"], entry["chunks"], entry["visual"]
|
| 1285 |
+
precise_cache, audio = entry["precise"], entry["audio"]
|
| 1286 |
+
if fp is None:
|
| 1287 |
+
try: fp = _fingerprint(video_path)
|
| 1288 |
+
except Exception: fp = None
|
| 1289 |
+
if not chunks:
|
| 1290 |
+
yield "No speech detected."
|
| 1291 |
+
return
|
| 1292 |
+
|
| 1293 |
+
candidates, best_bi = bi_encode_retrieve(chunks, prompt)
|
| 1294 |
+
_, v_norm = visual_scores(visual, prompt)
|
| 1295 |
+
results = rerank_and_score(candidates, prompt, visual["time"], v_norm)
|
| 1296 |
+
regions = merge_regions(results)
|
| 1297 |
+
if not regions:
|
| 1298 |
+
yield format_iris([], prompt, best_bi)
|
| 1299 |
+
return
|
| 1300 |
+
before = dict(precise_cache)
|
| 1301 |
+
fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX)
|
| 1302 |
+
if fp and precise_cache != before:
|
| 1303 |
+
_disk_update_precise(fp, precise_cache)
|
| 1304 |
+
regions = add_visual_tags(regions, visual)
|
| 1305 |
+
regions = sorted(regions, key=lambda r: r.get("confidence", 0), reverse=True)[:RESULTS_TOP_N]
|
| 1306 |
+
print(f"[IRIS] Done - {len(regions)} moment(s).")
|
| 1307 |
+
yield format_iris(regions, prompt, best_bi)
|
| 1308 |
+
|
| 1309 |
+
|
| 1310 |
+
def _synthesize_streaming(regions: list[dict], prompt: str, max_words: int, mode: str):
|
| 1311 |
+
"""Stream the formatted answer to the UI as the LLM produces it."""
|
| 1312 |
+
print(f"[NINA] Running synthesis [{mode}]...")
|
| 1313 |
+
yield format_output(regions, prompt, "✍️ …", mode)
|
| 1314 |
+
last = ""
|
| 1315 |
+
for partial in llm_synthesize_stream(regions, prompt, max_words, mode):
|
| 1316 |
+
last = partial
|
| 1317 |
+
yield format_output(regions, prompt, partial, mode)
|
| 1318 |
+
if not last:
|
| 1319 |
+
yield format_output(regions, prompt, _fallback_summary(regions), mode)
|
| 1320 |
+
|
| 1321 |
+
|
| 1322 |
+
# ── Gradio UI ──────────────────────────────────────────────────────────────────
|
| 1323 |
+
|
| 1324 |
+
demo = gr.Interface(
|
| 1325 |
+
fn=run,
|
| 1326 |
+
inputs=[
|
| 1327 |
+
gr.Video(label="Upload video "),
|
| 1328 |
+
gr.Textbox(
|
| 1329 |
+
label="What moment do you want to find?",
|
| 1330 |
+
placeholder='e.g. "What does the video say about...?" '
|
| 1331 |
+
'(add "briefly" or "in detail" to size the answer)',
|
| 1332 |
+
),
|
| 1333 |
+
# Issued by app.nnlabs.pl after it counts the use; required to run.
|
| 1334 |
+
gr.Textbox(label="access token (set automatically by the website)", value=""),
|
| 1335 |
+
],
|
| 1336 |
+
outputs=gr.Textbox(label="Results", lines=38),
|
| 1337 |
+
title="IRIS v0.80 — Intelligent Retrieval & Indexing (find the moment)",
|
| 1338 |
+
description=(
|
| 1339 |
+
"video -> find the exact moments a topic is mentioned or shown. "
|
| 1340 |
+
"Use it at https://app.nnlabs.pl/iris."
|
| 1341 |
+
),
|
| 1342 |
+
flagging_mode="never",
|
| 1343 |
+
# Privacy: Gradio purges uploaded files older than 5 min, every 5 min. One
|
| 1344 |
+
# upload still serves many prompts (audio/transcript are cached in RAM), and
|
| 1345 |
+
# the file is cleaned shortly after the session goes idle.
|
| 1346 |
+
delete_cache=(300, 300),
|
| 1347 |
+
)
|
| 1348 |
+
|
| 1349 |
+
# Concurrency for ~5-10 simultaneous users on an 8-vCPU Space: run a few jobs in
|
| 1350 |
+
# parallel, queue the rest (CTranslate2 is CPU-bound). Tune with the env var.
|
| 1351 |
+
demo.queue(
|
| 1352 |
+
default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
|
| 1353 |
+
max_size=int(os.getenv("NINA_QUEUE_MAX", "40")),
|
| 1354 |
+
)
|
| 1355 |
|
| 1356 |
if __name__ == "__main__":
|
| 1357 |
+
demo.launch()
|
|
|
|
|
|
|
|
|