Malina Claude Opus 4.8 commited on
Commit
264efa2
·
1 Parent(s): 0695bec

NINA: fully parallel Whisper ∥ CLIP, fused at summary

Browse files

- ingest_video: both branches start at t=0 — move the audio decode into the
audio branch so the visual (CLIP) branch no longer waits on it; wait for both
before continuing
- run(): precise re-transcription (Whisper) and visual tagging (CLIP) now run in
parallel before synthesis, since they touch independent region fields
- synthesis fuses audio + visual only once both branches complete

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (1) hide show
  1. pipeline.py +36 -16
pipeline.py CHANGED
@@ -8,7 +8,7 @@ from cache import _cache, disk_load, disk_save, disk_update_precise, evict_stale
8
  from compress import ensure_compressed, file_mb, too_large
9
  from config import (
10
  RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
11
- MAX_UPLOAD_MB, SAMPLE_RATE,
12
  )
13
  from retrieval import (
14
  add_entities_to_chunks, bi_encode_retrieve, classify_query,
@@ -26,36 +26,46 @@ from visual import add_visual_tags, analyze_visual, visual_scores
26
 
27
  def ingest_video(video_path: str) -> dict:
28
  """
29
- First sight of a video: decode audio, then run audio and visual branches
30
- concurrently (CPU: transcription+chunking, MPS/GPU: CLIP).
 
 
 
 
 
31
  """
32
  # Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
33
  # already small) so audio decode + frame sampling run on a lean file.
34
  video_path = ensure_compressed(video_path)
35
- audio = load_audio(video_path)
36
- duration = media_duration(video_path) or (len(audio) / SAMPLE_RATE)
37
 
38
  result = {}
39
 
40
  def _audio_branch():
 
 
 
41
  segments = transcribe_fast(audio)
42
  result["segments"] = segments
43
  result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
44
 
45
  def _visual_branch():
 
 
 
46
  result["visual"] = analyze_visual(video_path, duration)
47
 
 
48
  with ThreadPoolExecutor(max_workers=2) as ex:
49
- fa = ex.submit(_audio_branch)
50
- fv = ex.submit(_visual_branch)
51
- fa.result()
52
- fv.result()
53
 
54
  return {
55
  "segments": result["segments"],
56
  "chunks": result["chunks"],
57
  "visual": result["visual"],
58
- "audio": audio,
59
  "precise": {},
60
  }
61
 
@@ -224,15 +234,25 @@ def run(video_path: str, prompt: str, token: str = ""):
224
  yield from _synthesize_streaming(regions, prompt, max_words, "global")
225
  return
226
 
227
- # Stage 7: precise re-transcription
 
 
 
228
  before = dict(precise_cache)
229
- fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX)
 
 
 
 
 
 
 
 
 
 
230
  if fp and precise_cache != before:
231
  disk_update_precise(fp, precise_cache)
232
 
233
- # Stage 7b: CLIP visual tags
234
- regions = add_visual_tags(regions, visual)
235
-
236
- # Stage 8: LLM synthesis
237
  yield from _synthesize_streaming(regions, prompt, max_words, mode)
238
  print(f"[NINA] Done — {len(regions)} region(s) [{mode}].")
 
8
  from compress import ensure_compressed, file_mb, too_large
9
  from config import (
10
  RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
11
+ MAX_UPLOAD_MB,
12
  )
13
  from retrieval import (
14
  add_entities_to_chunks, bi_encode_retrieve, classify_query,
 
26
 
27
  def ingest_video(video_path: str) -> dict:
28
  """
29
+ First sight of a video: run the WHISPER (audio) and CLIP (visual) branches
30
+ fully in parallel, then return once both finish so synthesis can fuse them.
31
+
32
+ Both branches start at t=0 — the audio branch owns the (potentially slow) audio
33
+ decode, and the visual branch re-decodes its own frames via ffmpeg, so neither
34
+ waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release
35
+ the GIL during their heavy native compute, so the two threads genuinely overlap.
36
  """
37
  # Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
38
  # already small) so audio decode + frame sampling run on a lean file.
39
  video_path = ensure_compressed(video_path)
 
 
40
 
41
  result = {}
42
 
43
  def _audio_branch():
44
+ # WHISPER: decode waveform → fast transcription → semantic chunks.
45
+ audio = load_audio(video_path)
46
+ result["audio"] = audio
47
  segments = transcribe_fast(audio)
48
  result["segments"] = segments
49
  result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
50
 
51
  def _visual_branch():
52
+ # CLIP: sample frames → encode. Probes its own duration (a cheap ffprobe),
53
+ # so it never blocks on the audio decode.
54
+ duration = media_duration(video_path)
55
  result["visual"] = analyze_visual(video_path, duration)
56
 
57
+ print("[NINA] Ingesting — Whisper (audio) ∥ CLIP (visual) running in parallel…")
58
  with ThreadPoolExecutor(max_workers=2) as ex:
59
+ futures = [ex.submit(_audio_branch), ex.submit(_visual_branch)]
60
+ for f in futures:
61
+ f.result() # block until BOTH finish; re-raise the first branch error
62
+ print("[NINA] Ingest done — both branches complete.")
63
 
64
  return {
65
  "segments": result["segments"],
66
  "chunks": result["chunks"],
67
  "visual": result["visual"],
68
+ "audio": result["audio"],
69
  "precise": {},
70
  }
71
 
 
234
  yield from _synthesize_streaming(regions, prompt, max_words, "global")
235
  return
236
 
237
+ # Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual
238
+ # tagging (CLIP) are independent — one fills region["precise_text"], the other
239
+ # region["visual_tags"] — so we run them together and wait for both before the
240
+ # summary fuses audio + visual.
241
  before = dict(precise_cache)
242
+
243
+ def _precise_branch():
244
+ fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX)
245
+
246
+ def _tags_branch():
247
+ add_visual_tags(regions, visual) # mutates regions in place
248
+
249
+ with ThreadPoolExecutor(max_workers=2) as ex:
250
+ for f in [ex.submit(_precise_branch), ex.submit(_tags_branch)]:
251
+ f.result()
252
+
253
  if fp and precise_cache != before:
254
  disk_update_precise(fp, precise_cache)
255
 
256
+ # Stage 8: LLM synthesis (fuses audio + visual once both branches are done)
 
 
 
257
  yield from _synthesize_streaming(regions, prompt, max_words, mode)
258
  print(f"[NINA] Done — {len(regions)} region(s) [{mode}].")