Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit Β·
710a0dd
1
Parent(s): d53485f
Stream real per-stage progress markers during analysis
Browse filesReplace the static "Reading videoβ¦" / "β³ β¦" keepalives with genuine
sequential stage markers (subtitles, vocal isolation, OCR, retrieval,
re-check) via a thread-safe _Stage holder polled every 2s, so the chat
UI can show what NINA is actually doing instead of one static line.
All markers stay π¬-prefixed so clients keep them out of the bubble.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- pipeline.py +42 -8
pipeline.py
CHANGED
|
@@ -49,9 +49,31 @@ def _ingest_lock(key: str) -> threading.Lock:
|
|
| 49 |
return lk
|
| 50 |
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
# ββ Cold-start ingestion βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 53 |
|
| 54 |
-
def ingest_video(video_path: str) -> dict:
|
| 55 |
"""
|
| 56 |
First sight of a video: run the WHISPER (audio) and CLIP (visual) branches
|
| 57 |
fully in parallel, then return once both finish so synthesis can fuse them.
|
|
@@ -60,7 +82,12 @@ def ingest_video(video_path: str) -> dict:
|
|
| 60 |
decode, and the visual branch re-decodes its own frames via ffmpeg, so neither
|
| 61 |
waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release
|
| 62 |
the GIL during their heavy native compute, so the two threads genuinely overlap.
|
|
|
|
|
|
|
|
|
|
| 63 |
"""
|
|
|
|
|
|
|
| 64 |
# Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
|
| 65 |
# already small) so audio decode + frame sampling run on a lean file.
|
| 66 |
video_path = ensure_compressed(video_path)
|
|
@@ -79,6 +106,7 @@ def ingest_video(video_path: str) -> dict:
|
|
| 79 |
if not rel["reliable"]:
|
| 80 |
used = False
|
| 81 |
if USE_SUBTITLES:
|
|
|
|
| 82 |
subs = extract_embedded_subs(video_path)
|
| 83 |
if subs:
|
| 84 |
print(f"[NINA] Transcript weak (score {rel['score']}) β using the "
|
|
@@ -86,6 +114,7 @@ def ingest_video(video_path: str) -> dict:
|
|
| 86 |
segments, used = subs, True
|
| 87 |
if not used and USE_VOCAL_ISOLATION:
|
| 88 |
print(f"[NINA] Transcript weak (score {rel['score']}) β trying vocal isolationβ¦")
|
|
|
|
| 89 |
vocals = isolate_vocals(audio)
|
| 90 |
if vocals is not None:
|
| 91 |
seg2 = transcribe_fast(vocals)
|
|
@@ -120,6 +149,7 @@ def ingest_video(video_path: str) -> dict:
|
|
| 120 |
if not rel["reliable"] and has_on_screen_text(result["visual"]):
|
| 121 |
print(f"[NINA] Transcript still weak (score {rel['score']}) + on-screen text "
|
| 122 |
f"detected β reading burned-in captions (OCR)β¦")
|
|
|
|
| 123 |
ocr_segs = ocr_burned_in(video_path, media_duration(video_path))
|
| 124 |
if ocr_segs:
|
| 125 |
result["segments"] = ocr_segs
|
|
@@ -239,7 +269,7 @@ def _run_core(video_path: str, prompt: str):
|
|
| 239 |
# the cached result.
|
| 240 |
lock = _ingest_lock(video_path)
|
| 241 |
while not lock.acquire(timeout=10):
|
| 242 |
-
yield "
|
| 243 |
try:
|
| 244 |
if video_path in _cache:
|
| 245 |
print("[NINA] Cache hit after wait β another request ingested it.")
|
|
@@ -251,20 +281,22 @@ def _run_core(video_path: str, prompt: str):
|
|
| 251 |
entry = disk
|
| 252 |
_cache[video_path] = entry
|
| 253 |
else:
|
| 254 |
-
yield "π¬
|
| 255 |
-
# Ingest in a background thread so we can
|
| 256 |
-
#
|
|
|
|
| 257 |
_res: dict = {}
|
| 258 |
_evt = threading.Event()
|
|
|
|
| 259 |
def _do_ingest():
|
| 260 |
try:
|
| 261 |
-
_res["v"] = ingest_video(video_path)
|
| 262 |
except Exception as e:
|
| 263 |
_res["err"] = e
|
| 264 |
_evt.set()
|
| 265 |
threading.Thread(target=_do_ingest, daemon=True).start()
|
| 266 |
-
while not _evt.wait(
|
| 267 |
-
yield
|
| 268 |
if "err" in _res:
|
| 269 |
yield "Error reading video."
|
| 270 |
return
|
|
@@ -326,6 +358,7 @@ def _run_core(video_path: str, prompt: str):
|
|
| 326 |
return
|
| 327 |
|
| 328 |
# SPECIFIC / HYBRID retrieval
|
|
|
|
| 329 |
candidates, best_bi = bi_encode_retrieve(chunks, prompt)
|
| 330 |
|
| 331 |
# Upgrade any candidate whose precise text is already cached (improves cross-encoder accuracy)
|
|
@@ -370,6 +403,7 @@ def _run_core(video_path: str, prompt: str):
|
|
| 370 |
# tagging (CLIP) are independent β one fills region["precise_text"], the other
|
| 371 |
# region["visual_tags"] β so we run them together and wait for both before the
|
| 372 |
# summary fuses audio + visual.
|
|
|
|
| 373 |
before = dict(precise_cache)
|
| 374 |
|
| 375 |
def _precise_branch():
|
|
|
|
| 49 |
return lk
|
| 50 |
|
| 51 |
|
| 52 |
+
class _Stage:
|
| 53 |
+
"""Thread-safe holder for the current ingest stage. The ingest branches run in
|
| 54 |
+
parallel in a background thread; _run_core polls this every couple of seconds and
|
| 55 |
+
streams the marker so the website can show what NINA is actually doing. The base
|
| 56 |
+
"π¬ β¦" means "the parallel pass is running" (the client rotates transcribe β₯
|
| 57 |
+
scenes β₯ speakers); a worded marker is a genuine sequential step (subtitles,
|
| 58 |
+
vocal isolation, OCR). All markers stay π¬-prefixed so clients keep them out of
|
| 59 |
+
the chat bubble."""
|
| 60 |
+
|
| 61 |
+
def __init__(self) -> None:
|
| 62 |
+
self._lock = threading.Lock()
|
| 63 |
+
self._marker = "π¬ β¦"
|
| 64 |
+
|
| 65 |
+
def set(self, marker: str) -> None:
|
| 66 |
+
with self._lock:
|
| 67 |
+
self._marker = marker
|
| 68 |
+
|
| 69 |
+
def get(self) -> str:
|
| 70 |
+
with self._lock:
|
| 71 |
+
return self._marker
|
| 72 |
+
|
| 73 |
+
|
| 74 |
# ββ Cold-start ingestion βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 75 |
|
| 76 |
+
def ingest_video(video_path: str, stage: "_Stage | None" = None) -> dict:
|
| 77 |
"""
|
| 78 |
First sight of a video: run the WHISPER (audio) and CLIP (visual) branches
|
| 79 |
fully in parallel, then return once both finish so synthesis can fuse them.
|
|
|
|
| 82 |
decode, and the visual branch re-decodes its own frames via ffmpeg, so neither
|
| 83 |
waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release
|
| 84 |
the GIL during their heavy native compute, so the two threads genuinely overlap.
|
| 85 |
+
|
| 86 |
+
`stage` (optional) is updated as the conditional, sequential steps fire so the
|
| 87 |
+
caller can stream real progress to the UI.
|
| 88 |
"""
|
| 89 |
+
if stage is None:
|
| 90 |
+
stage = _Stage()
|
| 91 |
# Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
|
| 92 |
# already small) so audio decode + frame sampling run on a lean file.
|
| 93 |
video_path = ensure_compressed(video_path)
|
|
|
|
| 106 |
if not rel["reliable"]:
|
| 107 |
used = False
|
| 108 |
if USE_SUBTITLES:
|
| 109 |
+
stage.set("π¬ Reading the subtitle trackβ¦")
|
| 110 |
subs = extract_embedded_subs(video_path)
|
| 111 |
if subs:
|
| 112 |
print(f"[NINA] Transcript weak (score {rel['score']}) β using the "
|
|
|
|
| 114 |
segments, used = subs, True
|
| 115 |
if not used and USE_VOCAL_ISOLATION:
|
| 116 |
print(f"[NINA] Transcript weak (score {rel['score']}) β trying vocal isolationβ¦")
|
| 117 |
+
stage.set("π¬ Isolating the vocalsβ¦")
|
| 118 |
vocals = isolate_vocals(audio)
|
| 119 |
if vocals is not None:
|
| 120 |
seg2 = transcribe_fast(vocals)
|
|
|
|
| 149 |
if not rel["reliable"] and has_on_screen_text(result["visual"]):
|
| 150 |
print(f"[NINA] Transcript still weak (score {rel['score']}) + on-screen text "
|
| 151 |
f"detected β reading burned-in captions (OCR)β¦")
|
| 152 |
+
stage.set("π¬ Reading on-screen textβ¦")
|
| 153 |
ocr_segs = ocr_burned_in(video_path, media_duration(video_path))
|
| 154 |
if ocr_segs:
|
| 155 |
result["segments"] = ocr_segs
|
|
|
|
| 269 |
# the cached result.
|
| 270 |
lock = _ingest_lock(video_path)
|
| 271 |
while not lock.acquire(timeout=10):
|
| 272 |
+
yield "π¬ β¦"
|
| 273 |
try:
|
| 274 |
if video_path in _cache:
|
| 275 |
print("[NINA] Cache hit after wait β another request ingested it.")
|
|
|
|
| 281 |
entry = disk
|
| 282 |
_cache[video_path] = entry
|
| 283 |
else:
|
| 284 |
+
yield "π¬ β¦"
|
| 285 |
+
# Ingest in a background thread so we can stream progress while the
|
| 286 |
+
# parallel pass runs (keeps the SSE alive β mobile drops after ~60 s
|
| 287 |
+
# of silence β and feeds the client the live stage).
|
| 288 |
_res: dict = {}
|
| 289 |
_evt = threading.Event()
|
| 290 |
+
_stage = _Stage()
|
| 291 |
def _do_ingest():
|
| 292 |
try:
|
| 293 |
+
_res["v"] = ingest_video(video_path, _stage)
|
| 294 |
except Exception as e:
|
| 295 |
_res["err"] = e
|
| 296 |
_evt.set()
|
| 297 |
threading.Thread(target=_do_ingest, daemon=True).start()
|
| 298 |
+
while not _evt.wait(2.0):
|
| 299 |
+
yield _stage.get() # live stage marker; client keeps it out of the bubble
|
| 300 |
if "err" in _res:
|
| 301 |
yield "Error reading video."
|
| 302 |
return
|
|
|
|
| 358 |
return
|
| 359 |
|
| 360 |
# SPECIFIC / HYBRID retrieval
|
| 361 |
+
yield "π¬ Finding the key momentsβ¦"
|
| 362 |
candidates, best_bi = bi_encode_retrieve(chunks, prompt)
|
| 363 |
|
| 364 |
# Upgrade any candidate whose precise text is already cached (improves cross-encoder accuracy)
|
|
|
|
| 403 |
# tagging (CLIP) are independent β one fills region["precise_text"], the other
|
| 404 |
# region["visual_tags"] β so we run them together and wait for both before the
|
| 405 |
# summary fuses audio + visual.
|
| 406 |
+
yield "π¬ Re-checking the detailsβ¦"
|
| 407 |
before = dict(precise_cache)
|
| 408 |
|
| 409 |
def _precise_branch():
|