Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit ·
264efa2
1
Parent(s): 0695bec
NINA: fully parallel Whisper ∥ CLIP, fused at summary
Browse files- ingest_video: both branches start at t=0 — move the audio decode into the
audio branch so the visual (CLIP) branch no longer waits on it; wait for both
before continuing
- run(): precise re-transcription (Whisper) and visual tagging (CLIP) now run in
parallel before synthesis, since they touch independent region fields
- synthesis fuses audio + visual only once both branches complete
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- pipeline.py +36 -16
pipeline.py
CHANGED
|
@@ -8,7 +8,7 @@ from cache import _cache, disk_load, disk_save, disk_update_precise, evict_stale
|
|
| 8 |
from compress import ensure_compressed, file_mb, too_large
|
| 9 |
from config import (
|
| 10 |
RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
|
| 11 |
-
MAX_UPLOAD_MB,
|
| 12 |
)
|
| 13 |
from retrieval import (
|
| 14 |
add_entities_to_chunks, bi_encode_retrieve, classify_query,
|
|
@@ -26,36 +26,46 @@ from visual import add_visual_tags, analyze_visual, visual_scores
|
|
| 26 |
|
| 27 |
def ingest_video(video_path: str) -> dict:
|
| 28 |
"""
|
| 29 |
-
First sight of a video:
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
"""
|
| 32 |
# Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
|
| 33 |
# already small) so audio decode + frame sampling run on a lean file.
|
| 34 |
video_path = ensure_compressed(video_path)
|
| 35 |
-
audio = load_audio(video_path)
|
| 36 |
-
duration = media_duration(video_path) or (len(audio) / SAMPLE_RATE)
|
| 37 |
|
| 38 |
result = {}
|
| 39 |
|
| 40 |
def _audio_branch():
|
|
|
|
|
|
|
|
|
|
| 41 |
segments = transcribe_fast(audio)
|
| 42 |
result["segments"] = segments
|
| 43 |
result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
|
| 44 |
|
| 45 |
def _visual_branch():
|
|
|
|
|
|
|
|
|
|
| 46 |
result["visual"] = analyze_visual(video_path, duration)
|
| 47 |
|
|
|
|
| 48 |
with ThreadPoolExecutor(max_workers=2) as ex:
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
|
| 54 |
return {
|
| 55 |
"segments": result["segments"],
|
| 56 |
"chunks": result["chunks"],
|
| 57 |
"visual": result["visual"],
|
| 58 |
-
"audio": audio,
|
| 59 |
"precise": {},
|
| 60 |
}
|
| 61 |
|
|
@@ -224,15 +234,25 @@ def run(video_path: str, prompt: str, token: str = ""):
|
|
| 224 |
yield from _synthesize_streaming(regions, prompt, max_words, "global")
|
| 225 |
return
|
| 226 |
|
| 227 |
-
# Stage 7: precise re-transcription
|
|
|
|
|
|
|
|
|
|
| 228 |
before = dict(precise_cache)
|
| 229 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
if fp and precise_cache != before:
|
| 231 |
disk_update_precise(fp, precise_cache)
|
| 232 |
|
| 233 |
-
# Stage
|
| 234 |
-
regions = add_visual_tags(regions, visual)
|
| 235 |
-
|
| 236 |
-
# Stage 8: LLM synthesis
|
| 237 |
yield from _synthesize_streaming(regions, prompt, max_words, mode)
|
| 238 |
print(f"[NINA] Done — {len(regions)} region(s) [{mode}].")
|
|
|
|
| 8 |
from compress import ensure_compressed, file_mb, too_large
|
| 9 |
from config import (
|
| 10 |
RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
|
| 11 |
+
MAX_UPLOAD_MB,
|
| 12 |
)
|
| 13 |
from retrieval import (
|
| 14 |
add_entities_to_chunks, bi_encode_retrieve, classify_query,
|
|
|
|
| 26 |
|
| 27 |
def ingest_video(video_path: str) -> dict:
|
| 28 |
"""
|
| 29 |
+
First sight of a video: run the WHISPER (audio) and CLIP (visual) branches
|
| 30 |
+
fully in parallel, then return once both finish so synthesis can fuse them.
|
| 31 |
+
|
| 32 |
+
Both branches start at t=0 — the audio branch owns the (potentially slow) audio
|
| 33 |
+
decode, and the visual branch re-decodes its own frames via ffmpeg, so neither
|
| 34 |
+
waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release
|
| 35 |
+
the GIL during their heavy native compute, so the two threads genuinely overlap.
|
| 36 |
"""
|
| 37 |
# Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
|
| 38 |
# already small) so audio decode + frame sampling run on a lean file.
|
| 39 |
video_path = ensure_compressed(video_path)
|
|
|
|
|
|
|
| 40 |
|
| 41 |
result = {}
|
| 42 |
|
| 43 |
def _audio_branch():
|
| 44 |
+
# WHISPER: decode waveform → fast transcription → semantic chunks.
|
| 45 |
+
audio = load_audio(video_path)
|
| 46 |
+
result["audio"] = audio
|
| 47 |
segments = transcribe_fast(audio)
|
| 48 |
result["segments"] = segments
|
| 49 |
result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
|
| 50 |
|
| 51 |
def _visual_branch():
|
| 52 |
+
# CLIP: sample frames → encode. Probes its own duration (a cheap ffprobe),
|
| 53 |
+
# so it never blocks on the audio decode.
|
| 54 |
+
duration = media_duration(video_path)
|
| 55 |
result["visual"] = analyze_visual(video_path, duration)
|
| 56 |
|
| 57 |
+
print("[NINA] Ingesting — Whisper (audio) ∥ CLIP (visual) running in parallel…")
|
| 58 |
with ThreadPoolExecutor(max_workers=2) as ex:
|
| 59 |
+
futures = [ex.submit(_audio_branch), ex.submit(_visual_branch)]
|
| 60 |
+
for f in futures:
|
| 61 |
+
f.result() # block until BOTH finish; re-raise the first branch error
|
| 62 |
+
print("[NINA] Ingest done — both branches complete.")
|
| 63 |
|
| 64 |
return {
|
| 65 |
"segments": result["segments"],
|
| 66 |
"chunks": result["chunks"],
|
| 67 |
"visual": result["visual"],
|
| 68 |
+
"audio": result["audio"],
|
| 69 |
"precise": {},
|
| 70 |
}
|
| 71 |
|
|
|
|
| 234 |
yield from _synthesize_streaming(regions, prompt, max_words, "global")
|
| 235 |
return
|
| 236 |
|
| 237 |
+
# Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual
|
| 238 |
+
# tagging (CLIP) are independent — one fills region["precise_text"], the other
|
| 239 |
+
# region["visual_tags"] — so we run them together and wait for both before the
|
| 240 |
+
# summary fuses audio + visual.
|
| 241 |
before = dict(precise_cache)
|
| 242 |
+
|
| 243 |
+
def _precise_branch():
|
| 244 |
+
fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX)
|
| 245 |
+
|
| 246 |
+
def _tags_branch():
|
| 247 |
+
add_visual_tags(regions, visual) # mutates regions in place
|
| 248 |
+
|
| 249 |
+
with ThreadPoolExecutor(max_workers=2) as ex:
|
| 250 |
+
for f in [ex.submit(_precise_branch), ex.submit(_tags_branch)]:
|
| 251 |
+
f.result()
|
| 252 |
+
|
| 253 |
if fp and precise_cache != before:
|
| 254 |
disk_update_precise(fp, precise_cache)
|
| 255 |
|
| 256 |
+
# Stage 8: LLM synthesis (fuses audio + visual once both branches are done)
|
|
|
|
|
|
|
|
|
|
| 257 |
yield from _synthesize_streaming(regions, prompt, max_words, mode)
|
| 258 |
print(f"[NINA] Done — {len(regions)} region(s) [{mode}].")
|