Malina Claude Opus 4.8 commited on
Commit
e2c96b8
Β·
1 Parent(s): 6acd603

NINA Phase 2: vocal isolation + speaker diarization + name attribution

Browse files

- enhance.py: torchaudio Hybrid Demucs vocal isolation (best-effort, chunked).
In ingest, when the first ASR pass is unreliable (musical/distorted), isolate
the voice and re-transcribe; keep it only if it scores better.
- diarize.py: pyannote speaker diarization β†’ per-segment speaker labels, friendly
Speaker 1/2 display names, multilingual self-introduction β†’ name roster, and
region speaker annotation. All best-effort (no-op if pyannote/token unavailable).
- pipeline: third parallel ingest branch (Whisper βˆ₯ CLIP βˆ₯ diarization); speakers
attached after join; region speakers + speaker note threaded to synthesis.
- synthesis: fragments tagged with speaker; system prompt attributes lines and uses
a self-introduced name for all of that speaker's lines (handles the gamer 'I won'
case) β€” done inside the single existing LLM call (no extra inference credits).
- requirements: torchaudio==2.12.0, pyannote.audio==3.3.2.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (6) hide show
  1. config.py +19 -0
  2. diarize.py +168 -0
  3. enhance.py +81 -0
  4. pipeline.py +62 -22
  5. requirements.txt +2 -0
  6. synthesis.py +20 -10
config.py CHANGED
@@ -95,6 +95,25 @@ COMPRESS_TARGET_HEIGHT = 720
95
  COMPRESS_VIDEO_BITRATE = "1200k"
96
  COMPRESS_AUDIO_BITRATE = "96k"
97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  CACHE_DIR = os.path.join(tempfile.gettempdir(), "nina_cache")
99
  DISK_CACHE = True
100
 
 
95
  COMPRESS_VIDEO_BITRATE = "1200k"
96
  COMPRESS_AUDIO_BITRATE = "96k"
97
 
98
+ # ── Vocal isolation (torchaudio Hybrid Demucs) ────────────────────────────────
99
+ # Pull the speaking voice out from under music/noise before re-transcribing, but
100
+ # only when the first ASR pass looks unreliable (musical/distorted) β€” it's CPU-heavy.
101
+ # Uses torchaudio's bundled Hybrid Demucs (no separate `demucs` package β†’ no torch
102
+ # version conflict). Best-effort: silently skipped if torchaudio is unavailable.
103
+ USE_VOCAL_ISOLATION = True
104
+ DEMUCS_BUNDLE = "HDEMUCS_HIGH_MUSDB_PLUS"
105
+ ISOLATION_MAX_SECONDS = 480 # don't isolate very long audio (CPU/memory)
106
+ ISOLATION_CHUNK_SEC = 12.0 # process in chunks to bound memory
107
+
108
+ # ── Speaker diarization (pyannote) + name attribution ─────────────────────────
109
+ # Identifies who-spoke-when; synthesis then attributes lines and maps a speaker to
110
+ # a name when they introduce themselves. Gated models β€” the Space's HF_TOKEN must
111
+ # have accepted pyannote/speaker-diarization-3.1 + pyannote/segmentation-3.0.
112
+ # Best-effort: skipped if pyannote is unavailable or the token lacks access.
113
+ USE_DIARIZATION = True
114
+ DIARIZATION_MODEL = "pyannote/speaker-diarization-3.1"
115
+ DIARIZATION_MAX_SECONDS = 1800
116
+
117
  CACHE_DIR = os.path.join(tempfile.gettempdir(), "nina_cache")
118
  DISK_CACHE = True
119
 
diarize.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ─────────────────────────────────────────────
2
+ # NINA β€” speaker diarization (pyannote) + name attribution
3
+ # ─────────────────────────────────────────────
4
+ # Diarization answers "who spoke when" so synthesis can attribute each line to a
5
+ # speaker, and β€” when someone introduces themselves anywhere in the video β€” use
6
+ # their NAME for all of their lines (e.g. two gamers who name themselves, then one
7
+ # shouts "I won"). Best-effort: if pyannote isn't installed or the HF token can't
8
+ # load the gated model, we return empty results and the pipeline runs without
9
+ # speaker labels.
10
+
11
+ import re
12
+
13
+ import numpy as np
14
+
15
+ from config import DIARIZATION_MAX_SECONDS, DIARIZATION_MODEL, SAMPLE_RATE
16
+ from models import HF_TOKEN
17
+
18
+ _pipeline = None
19
+ _failed = False
20
+
21
+
22
+ def _get_pipeline():
23
+ global _pipeline, _failed
24
+ if _failed:
25
+ return None
26
+ if _pipeline is None:
27
+ try:
28
+ import torch
29
+ from pyannote.audio import Pipeline
30
+ _pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, use_auth_token=HF_TOKEN)
31
+ if _pipeline is None: # happens when the token lacks gated access
32
+ raise RuntimeError("pipeline is None β€” has the HF token accepted the gated "
33
+ "pyannote terms?")
34
+ _pipeline.to(torch.device("cpu"))
35
+ print("[NINA] Diarization pipeline ready.")
36
+ except Exception as e:
37
+ _failed = True
38
+ print(f"[NINA] Diarization unavailable: {e} β€” continuing without speaker labels.")
39
+ return None
40
+ return _pipeline
41
+
42
+
43
+ def diarize(audio: np.ndarray, sr: int = SAMPLE_RATE) -> list[tuple]:
44
+ """Return speaker turns as a sorted list of (start, end, label), or [] on failure."""
45
+ try:
46
+ if audio is None or len(audio) == 0:
47
+ return []
48
+ if len(audio) / sr > DIARIZATION_MAX_SECONDS:
49
+ print("[NINA] Audio too long for diarization β€” skipping.")
50
+ return []
51
+ pipe = _get_pipeline()
52
+ if pipe is None:
53
+ return []
54
+
55
+ import torch
56
+ wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0)
57
+ annotation = pipe({"waveform": wav, "sample_rate": sr})
58
+ turns = [
59
+ (float(seg.start), float(seg.end), str(label))
60
+ for seg, _, label in annotation.itertracks(yield_label=True)
61
+ ]
62
+ turns.sort(key=lambda t: t[0])
63
+ n_spk = len({lbl for _, _, lbl in turns})
64
+ print(f"[NINA] Diarization: {n_spk} speaker(s) across {len(turns)} turns.")
65
+ return turns
66
+ except Exception as e:
67
+ print(f"[NINA] Diarization failed: {e}")
68
+ return []
69
+
70
+
71
+ def attach_speakers(segments: list[dict], turns: list[tuple]) -> None:
72
+ """Tag each transcript segment with the speaker label that overlaps it most."""
73
+ if not turns or not segments:
74
+ return
75
+ for s in segments:
76
+ ss, se = s["start"], s["end"]
77
+ best, best_ov = None, 0.0
78
+ for ts, te, lbl in turns:
79
+ ov = max(0.0, min(se, te) - max(ss, ts))
80
+ if ov > best_ov:
81
+ best_ov, best = ov, lbl
82
+ if best:
83
+ s["speaker"] = best
84
+
85
+
86
+ def speaker_display_map(segments: list[dict]) -> dict:
87
+ """Map raw pyannote labels (SPEAKER_00…) β†’ 'Speaker 1/2…' by first appearance."""
88
+ order: list[str] = []
89
+ for s in segments:
90
+ sp = s.get("speaker")
91
+ if sp and sp not in order:
92
+ order.append(sp)
93
+ return {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(order)}
94
+
95
+
96
+ # Self-introduction patterns (multilingual). Capture the name token after a cue.
97
+ _NAME = r"([A-ZΓ€-ΕΉΔ„Δ†Δ˜ΕΕƒΓ“ΕšΕΉΕ»][\w''\-]+)"
98
+ _INTRO_PATTERNS = [
99
+ re.compile(p, re.IGNORECASE) for p in (
100
+ rf"\bmy name(?:'s| is)\s+{_NAME}",
101
+ rf"\bi(?:['’]?m| am)\s+{_NAME}",
102
+ rf"\bthis is\s+{_NAME}\b",
103
+ rf"\bcall me\s+{_NAME}",
104
+ rf"\bnazywam si[eΔ™]\s+{_NAME}",
105
+ rf"\bjestem\s+{_NAME}",
106
+ rf"\bmam na imi[eΔ™]\s+{_NAME}",
107
+ rf"\bich (?:heiße|bin)\s+{_NAME}",
108
+ rf"\bme llamo\s+{_NAME}",
109
+ rf"\bje m'appelle\s+{_NAME}",
110
+ )
111
+ ]
112
+ # Words that look like names after "I'm…" but aren't.
113
+ _NOT_A_NAME = {
114
+ "here", "back", "going", "gonna", "just", "not", "so", "the", "a", "an",
115
+ "good", "fine", "sorry", "sure", "ready", "live", "playing", "streaming",
116
+ "tutaj", "dobry", "gotowy",
117
+ }
118
+
119
+
120
+ def name_roster(segments: list[dict], display: dict) -> dict:
121
+ """Scan each speaker's lines for a self-introduction β†’ {display_name: real_name}."""
122
+ roster: dict = {}
123
+ for s in segments:
124
+ sp = s.get("speaker")
125
+ if not sp:
126
+ continue
127
+ disp = display.get(sp, sp)
128
+ if disp in roster:
129
+ continue
130
+ text = s.get("text", "")
131
+ for pat in _INTRO_PATTERNS:
132
+ m = pat.search(text)
133
+ if m:
134
+ name = m.group(1).strip()
135
+ if name and len(name) > 1 and name.lower() not in _NOT_A_NAME:
136
+ roster[disp] = name
137
+ break
138
+ return roster
139
+
140
+
141
+ def speaker_note(display: dict, roster: dict) -> str:
142
+ """One-line summary of speakers + any discovered names, fed to synthesis."""
143
+ if len(display) <= 1:
144
+ return ""
145
+ names = ", ".join(display.values())
146
+ note = f"This video has multiple speakers ({names})."
147
+ if roster:
148
+ intro = "; ".join(f"{disp} introduced themselves as \"{name}\""
149
+ for disp, name in roster.items())
150
+ note += f" Self-introductions detected: {intro}."
151
+ return note
152
+
153
+
154
+ def annotate_region_speakers(regions: list[dict], segments: list[dict], display: dict) -> None:
155
+ """Tag each region with the display-name(s) of the speakers talking during it."""
156
+ if not display or not regions:
157
+ return
158
+ spans = [
159
+ (s["start"], s["end"], display.get(s.get("speaker")))
160
+ for s in segments if s.get("speaker")
161
+ ]
162
+ for r in regions:
163
+ names: list[str] = []
164
+ for ss, se, nm in spans:
165
+ if nm and ss < r["end"] and se > r["start"] and nm not in names:
166
+ names.append(nm)
167
+ if names:
168
+ r["speakers"] = names
enhance.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ─────────────────────────────────────────────
2
+ # NINA β€” vocal isolation (torchaudio Hybrid Demucs)
3
+ # ─────────────────────────────────────────────
4
+ # Separates the singing/speaking voice from music + noise so Whisper can read
5
+ # speech that's otherwise masked (e.g. a concert recording). Uses torchaudio's
6
+ # bundled Hybrid Demucs so we don't pull the standalone `demucs` package (which
7
+ # pins older torch). Everything is best-effort: any failure returns None and the
8
+ # caller keeps the original audio.
9
+
10
+ import numpy as np
11
+
12
+ from config import (
13
+ DEMUCS_BUNDLE, ISOLATION_CHUNK_SEC, ISOLATION_MAX_SECONDS, SAMPLE_RATE,
14
+ )
15
+
16
+ _model = None
17
+ _model_sr = None
18
+ _failed = False
19
+
20
+
21
+ def _get_model():
22
+ global _model, _model_sr, _failed
23
+ if _failed:
24
+ return None
25
+ if _model is None:
26
+ try:
27
+ import torch # noqa: F401
28
+ import torchaudio
29
+ bundle = getattr(torchaudio.pipelines, DEMUCS_BUNDLE)
30
+ _model = bundle.get_model().to("cpu").eval()
31
+ _model_sr = bundle.sample_rate
32
+ print(f"[NINA] Vocal isolation ready ({DEMUCS_BUNDLE}, sr={_model_sr}).")
33
+ except Exception as e:
34
+ _failed = True
35
+ print(f"[NINA] Vocal isolation unavailable: {e} β€” keeping original audio.")
36
+ return None
37
+ return _model
38
+
39
+
40
+ def isolate_vocals(audio: np.ndarray, sr: int = SAMPLE_RATE):
41
+ """Return a vocals-only mono waveform at `sr`, or None if unavailable/failed.
42
+
43
+ Timing is preserved (same length/timeline as the input), so transcript
44
+ timestamps and diarization stay aligned.
45
+ """
46
+ try:
47
+ if audio is None or len(audio) == 0:
48
+ return None
49
+ if len(audio) / sr > ISOLATION_MAX_SECONDS:
50
+ print("[NINA] Audio too long for vocal isolation β€” skipping.")
51
+ return None
52
+ model = _get_model()
53
+ if model is None:
54
+ return None
55
+
56
+ import torch
57
+ import torchaudio
58
+
59
+ with torch.no_grad():
60
+ wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0)
61
+ wav = torchaudio.functional.resample(wav, sr, _model_sr)
62
+ wav = wav.repeat(2, 1) # Hybrid Demucs expects stereo
63
+ ref = wav.mean(0)
64
+ mean, std = ref.mean(), ref.std() + 1e-8
65
+ wav = (wav - mean) / std # normalize, per Demucs
66
+
67
+ voc_idx = model.sources.index("vocals")
68
+ chunk = max(1, int(ISOLATION_CHUNK_SEC * _model_sr))
69
+ pieces = []
70
+ for s in range(0, wav.shape[1], chunk): # chunk to bound memory
71
+ seg = wav[:, s:s + chunk]
72
+ out = model(seg.unsqueeze(0))[0] # (sources, channels, length)
73
+ pieces.append(out[voc_idx]) # (channels, length)
74
+
75
+ vocals = torch.cat(pieces, dim=1) * std + mean
76
+ vocals = vocals.mean(0, keepdim=True) # β†’ mono
77
+ vocals = torchaudio.functional.resample(vocals, _model_sr, sr)
78
+ return vocals.squeeze(0).cpu().numpy().astype(np.float32)
79
+ except Exception as e:
80
+ print(f"[NINA] Vocal isolation failed: {e}")
81
+ return None
pipeline.py CHANGED
@@ -10,8 +10,13 @@ from cache import _cache, disk_load, disk_save, disk_update_precise, evict_stale
10
  from compress import ensure_compressed, file_mb, too_large
11
  from config import (
12
  RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
13
- MAX_UPLOAD_MB,
14
  )
 
 
 
 
 
15
  from retrieval import (
16
  add_entities_to_chunks, bi_encode_retrieve, classify_query,
17
  merge_regions, pad_with_neighbours, rerank_and_score,
@@ -40,34 +45,50 @@ def ingest_video(video_path: str) -> dict:
40
  # already small) so audio decode + frame sampling run on a lean file.
41
  video_path = ensure_compressed(video_path)
42
 
43
- result = {}
 
 
44
 
45
  def _audio_branch():
46
- # WHISPER: decode waveform β†’ fast transcription β†’ semantic chunks.
47
- audio = load_audio(video_path)
48
- result["audio"] = audio
49
  segments = transcribe_fast(audio)
 
 
 
 
 
 
 
 
 
 
 
50
  result["segments"] = segments
51
  result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
52
 
53
  def _visual_branch():
54
- # CLIP: sample frames β†’ encode. Probes its own duration (a cheap ffprobe),
55
- # so it never blocks on the audio decode.
56
- duration = media_duration(video_path)
57
- result["visual"] = analyze_visual(video_path, duration)
58
 
59
- print("[NINA] Ingesting β€” Whisper (audio) βˆ₯ CLIP (visual) running in parallel…")
60
- with ThreadPoolExecutor(max_workers=2) as ex:
61
- futures = [ex.submit(_audio_branch), ex.submit(_visual_branch)]
 
 
 
 
62
  for f in futures:
63
- f.result() # block until BOTH finish; re-raise the first branch error
64
- print("[NINA] Ingest done β€” both branches complete.")
 
 
 
65
 
66
  return {
67
  "segments": result["segments"],
68
  "chunks": result["chunks"],
69
  "visual": result["visual"],
70
- "audio": result["audio"],
71
  "precise": {},
72
  }
73
 
@@ -90,11 +111,13 @@ def _run_global(chunks: list[dict], visual: dict,
90
  # ── Streaming synthesis wrapper ────────────────────────────────────────────────
91
 
92
  def _synthesize_streaming(regions: list[dict], prompt: str,
93
- max_words: int, mode: str, reliable: bool = True):
 
94
  print(f"[NINA] Running synthesis [{mode}]...")
95
  yield format_output(regions, prompt, "✍️ …", mode)
96
  last = ""
97
- for partial in llm_synthesize_stream(regions, prompt, max_words, mode, reliable=reliable):
 
98
  last = partial
99
  yield format_output(regions, prompt, partial, mode)
100
  if not last:
@@ -198,13 +221,24 @@ def run(video_path: str, prompt: str, token: str = ""):
198
  if not reliable:
199
  print(f"[NINA] ⚠ Transcript looks UNRELIABLE {reliability} β€” synthesis will hedge.")
200
 
 
 
 
 
 
 
 
 
 
201
  # Pass 1 routing
202
  mode = classify_query(prompt)
203
  print(f"[NINA] Query mode (pass 1): {mode}")
204
 
205
  if mode == "global":
206
  regions = _run_global(chunks, visual, prompt, max_words)
207
- yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=reliable)
 
 
208
  print(f"[NINA] Done β€” {len(regions)} region(s) [global].")
209
  return
210
 
@@ -223,7 +257,9 @@ def run(video_path: str, prompt: str, token: str = ""):
223
  if best_bi < BI_SCORE_GLOBAL_FLOOR:
224
  print(f"[NINA] bi-score {best_bi:.3f} below floor β†’ escalating to global.")
225
  regions = _run_global(chunks, visual, prompt, max_words)
226
- yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=reliable)
 
 
227
  print(f"[NINA] Done β€” {len(regions)} region(s) [escalated global].")
228
  return
229
  elif best_bi < BI_SCORE_HYBRID_FLOOR:
@@ -240,7 +276,9 @@ def run(video_path: str, prompt: str, token: str = ""):
240
  if not regions:
241
  print("[NINA] No regions after merge β†’ falling back to global.")
242
  regions = _run_global(chunks, visual, prompt, max_words)
243
- yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=reliable)
 
 
244
  return
245
 
246
  # Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual
@@ -262,6 +300,8 @@ def run(video_path: str, prompt: str, token: str = ""):
262
  if fp and precise_cache != before:
263
  disk_update_precise(fp, precise_cache)
264
 
265
- # Stage 8: LLM synthesis (fuses audio + visual once both branches are done)
266
- yield from _synthesize_streaming(regions, prompt, max_words, mode, reliable=reliable)
 
 
267
  print(f"[NINA] Done β€” {len(regions)} region(s) [{mode}].")
 
10
  from compress import ensure_compressed, file_mb, too_large
11
  from config import (
12
  RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
13
+ MAX_UPLOAD_MB, USE_DIARIZATION, USE_VOCAL_ISOLATION,
14
  )
15
+ from diarize import (
16
+ annotate_region_speakers, attach_speakers, diarize, name_roster,
17
+ speaker_display_map, speaker_note,
18
+ )
19
+ from enhance import isolate_vocals
20
  from retrieval import (
21
  add_entities_to_chunks, bi_encode_retrieve, classify_query,
22
  merge_regions, pad_with_neighbours, rerank_and_score,
 
45
  # already small) so audio decode + frame sampling run on a lean file.
46
  video_path = ensure_compressed(video_path)
47
 
48
+ # Decode once up front β€” transcription AND diarization both need the waveform.
49
+ audio = load_audio(video_path)
50
+ result = {"audio": audio}
51
 
52
  def _audio_branch():
53
+ # WHISPER: fast transcription β†’ (conditional vocal isolation) β†’ chunks.
 
 
54
  segments = transcribe_fast(audio)
55
+ if USE_VOCAL_ISOLATION and segments:
56
+ rel = transcript_reliability(segments)
57
+ if not rel["reliable"]:
58
+ # Musical/distorted audio β€” pull the voice out and try again.
59
+ print(f"[NINA] Transcript weak (score {rel['score']}) β€” trying vocal isolation…")
60
+ vocals = isolate_vocals(audio)
61
+ if vocals is not None:
62
+ seg2 = transcribe_fast(vocals)
63
+ if seg2 and transcript_reliability(seg2)["score"] > rel["score"]:
64
+ print("[NINA] Vocal isolation improved the transcript β€” using it.")
65
+ segments = seg2
66
  result["segments"] = segments
67
  result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
68
 
69
  def _visual_branch():
70
+ # CLIP: sample frames β†’ encode.
71
+ result["visual"] = analyze_visual(video_path, media_duration(video_path))
 
 
72
 
73
+ def _diar_branch():
74
+ # pyannote: who-spoke-when (best-effort; [] if unavailable).
75
+ result["turns"] = diarize(audio) if USE_DIARIZATION else []
76
+
77
+ print("[NINA] Ingesting β€” Whisper (audio) βˆ₯ CLIP (visual) βˆ₯ diarization in parallel…")
78
+ with ThreadPoolExecutor(max_workers=3) as ex:
79
+ futures = [ex.submit(_audio_branch), ex.submit(_visual_branch), ex.submit(_diar_branch)]
80
  for f in futures:
81
+ f.result() # block until ALL finish; re-raise the first branch error
82
+
83
+ # Tag the final transcript with speaker labels (needs both branches done).
84
+ attach_speakers(result["segments"], result.get("turns") or [])
85
+ print("[NINA] Ingest done β€” audio, visual and speakers ready.")
86
 
87
  return {
88
  "segments": result["segments"],
89
  "chunks": result["chunks"],
90
  "visual": result["visual"],
91
+ "audio": audio,
92
  "precise": {},
93
  }
94
 
 
111
  # ── Streaming synthesis wrapper ────────────────────────────────────────────────
112
 
113
  def _synthesize_streaming(regions: list[dict], prompt: str,
114
+ max_words: int, mode: str, reliable: bool = True,
115
+ speaker_note: str = ""):
116
  print(f"[NINA] Running synthesis [{mode}]...")
117
  yield format_output(regions, prompt, "✍️ …", mode)
118
  last = ""
119
+ for partial in llm_synthesize_stream(regions, prompt, max_words, mode,
120
+ reliable=reliable, speaker_note=speaker_note):
121
  last = partial
122
  yield format_output(regions, prompt, partial, mode)
123
  if not last:
 
221
  if not reliable:
222
  print(f"[NINA] ⚠ Transcript looks UNRELIABLE {reliability} β€” synthesis will hedge.")
223
 
224
+ # Speaker labels (from diarization) β†’ friendly display names + a roster mapping a
225
+ # speaker to a name when they introduced themselves anywhere in the video. Both
226
+ # are fed to synthesis so it can attribute lines and resolve names.
227
+ spk_display = speaker_display_map(segments)
228
+ spk_roster = name_roster(segments, spk_display) if spk_display else {}
229
+ spk_note = speaker_note(spk_display, spk_roster)
230
+ if spk_note:
231
+ print(f"[NINA] {spk_note}")
232
+
233
  # Pass 1 routing
234
  mode = classify_query(prompt)
235
  print(f"[NINA] Query mode (pass 1): {mode}")
236
 
237
  if mode == "global":
238
  regions = _run_global(chunks, visual, prompt, max_words)
239
+ annotate_region_speakers(regions, segments, spk_display)
240
+ yield from _synthesize_streaming(regions, prompt, max_words, "global",
241
+ reliable=reliable, speaker_note=spk_note)
242
  print(f"[NINA] Done β€” {len(regions)} region(s) [global].")
243
  return
244
 
 
257
  if best_bi < BI_SCORE_GLOBAL_FLOOR:
258
  print(f"[NINA] bi-score {best_bi:.3f} below floor β†’ escalating to global.")
259
  regions = _run_global(chunks, visual, prompt, max_words)
260
+ annotate_region_speakers(regions, segments, spk_display)
261
+ yield from _synthesize_streaming(regions, prompt, max_words, "global",
262
+ reliable=reliable, speaker_note=spk_note)
263
  print(f"[NINA] Done β€” {len(regions)} region(s) [escalated global].")
264
  return
265
  elif best_bi < BI_SCORE_HYBRID_FLOOR:
 
276
  if not regions:
277
  print("[NINA] No regions after merge β†’ falling back to global.")
278
  regions = _run_global(chunks, visual, prompt, max_words)
279
+ annotate_region_speakers(regions, segments, spk_display)
280
+ yield from _synthesize_streaming(regions, prompt, max_words, "global",
281
+ reliable=reliable, speaker_note=spk_note)
282
  return
283
 
284
  # Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual
 
300
  if fp and precise_cache != before:
301
  disk_update_precise(fp, precise_cache)
302
 
303
+ # Stage 8: LLM synthesis (fuses audio + visual + speakers once branches are done)
304
+ annotate_region_speakers(regions, segments, spk_display)
305
+ yield from _synthesize_streaming(regions, prompt, max_words, mode,
306
+ reliable=reliable, speaker_note=spk_note)
307
  print(f"[NINA] Done β€” {len(regions)} region(s) [{mode}].")
requirements.txt CHANGED
@@ -13,6 +13,8 @@ langdetect==1.0.9 # robust prompt-language detection (answer in user's
13
 
14
  # ── ML / inference ──
15
  torch==2.12.0
 
 
16
  sentence-transformers==5.5.1
17
  transformers==5.8.1
18
  huggingface-hub==1.15.0
 
13
 
14
  # ── ML / inference ──
15
  torch==2.12.0
16
+ torchaudio==2.12.0 # bundled Hybrid Demucs (vocal isolation) + pyannote backend
17
+ pyannote.audio==3.3.2 # speaker diarization (gated: accept the model terms on HF)
18
  sentence-transformers==5.5.1
19
  transformers==5.8.1
20
  huggingface-hub==1.15.0
synthesis.py CHANGED
@@ -119,7 +119,14 @@ def _detect_language(text: str) -> str:
119
 
120
 
121
  def _make_system_msg(language: str, max_words: int, tier: str,
122
- reliable: bool = True) -> str:
 
 
 
 
 
 
 
123
  audio_quality = "" if reliable else (
124
  "\n\nAUDIO QUALITY (important) β€” This transcript was auto-extracted from "
125
  "low-quality, noisy or distorted audio and is partly UNRELIABLE. Some fragments "
@@ -145,6 +152,7 @@ def _make_system_msg(language: str, max_words: int, tier: str,
145
  "complete partial words, names or numbers. If the fragments don't address the "
146
  "question, say so plainly (in the user's language) instead of speculating. Don't "
147
  "fix grammar inside quoted speech."
 
148
  f"{audio_quality}\n\n"
149
  "FORMAT β€” Write a natural, flowing answer in prose (sentences or short paragraphs). "
150
  "Do NOT begin lines with timestamps. After the answer, add exactly one final line "
@@ -157,12 +165,13 @@ _STRONG_CONF = 0.45
157
 
158
 
159
  def _render_region(r: dict) -> str:
160
- spoken = (r.get("precise_text", r["text"]) or "").strip()
161
- tags = r.get("visual_tags", [])
162
- block = (
163
- f"[{fmt_time(r['start'])} β†’ {fmt_time(r['end'])}]\n"
164
- f" Spoken: {spoken or '(no speech)'}"
165
- )
 
166
  if tags:
167
  block += f"\n On screen: {', '.join(tags)}"
168
  return block
@@ -211,12 +220,13 @@ def _build_synth_prompt(regions: list[dict], prompt: str, max_words: int, mode:
211
 
212
  def llm_synthesize_stream(
213
  regions: list[dict], prompt: str, max_words: int = 150, mode: str = "specific",
214
- reliable: bool = True,
215
  ):
216
  """Generator: yields the answer incrementally as the LLM streams tokens.
217
 
218
  `reliable=False` (poor ASR audio) makes the system prompt tell the model to flag
219
- unclear audio instead of treating garbled fragments as real speech.
 
220
  """
221
  if not regions:
222
  yield "(No relevant regions to summarize.)"
@@ -240,7 +250,7 @@ def llm_synthesize_stream(
240
  messages=[
241
  {"role": "system",
242
  "content": _make_system_msg(language, max_words, _summary_tier(max_words),
243
- reliable=reliable)},
244
  {"role": "user", "content": synth_prompt},
245
  ],
246
  max_tokens=LLM_MAX_TOKENS,
 
119
 
120
 
121
  def _make_system_msg(language: str, max_words: int, tier: str,
122
+ reliable: bool = True, speaker_note: str = "") -> str:
123
+ speakers = "" if not speaker_note else (
124
+ "\n\nSPEAKERS β€” " + speaker_note + " Fragments are tagged with their speaker. "
125
+ "Attribute statements to the correct speaker. If a speaker stated their own name "
126
+ "(a self-introduction), use that NAME for EVERY line by that speaker β€” including "
127
+ "lines where the name isn't repeated. When the question asks who did or said "
128
+ "something, answer with the specific speaker/name (e.g. who won, who claimed what)."
129
+ )
130
  audio_quality = "" if reliable else (
131
  "\n\nAUDIO QUALITY (important) β€” This transcript was auto-extracted from "
132
  "low-quality, noisy or distorted audio and is partly UNRELIABLE. Some fragments "
 
152
  "complete partial words, names or numbers. If the fragments don't address the "
153
  "question, say so plainly (in the user's language) instead of speculating. Don't "
154
  "fix grammar inside quoted speech."
155
+ f"{speakers}"
156
  f"{audio_quality}\n\n"
157
  "FORMAT β€” Write a natural, flowing answer in prose (sentences or short paragraphs). "
158
  "Do NOT begin lines with timestamps. After the answer, add exactly one final line "
 
165
 
166
 
167
  def _render_region(r: dict) -> str:
168
+ spoken = (r.get("precise_text", r["text"]) or "").strip()
169
+ tags = r.get("visual_tags", [])
170
+ speakers = r.get("speakers", [])
171
+ header = f"[{fmt_time(r['start'])} β†’ {fmt_time(r['end'])}]"
172
+ if speakers:
173
+ header += f" ({' / '.join(speakers)})"
174
+ block = f"{header}\n Spoken: {spoken or '(no speech)'}"
175
  if tags:
176
  block += f"\n On screen: {', '.join(tags)}"
177
  return block
 
220
 
221
  def llm_synthesize_stream(
222
  regions: list[dict], prompt: str, max_words: int = 150, mode: str = "specific",
223
+ reliable: bool = True, speaker_note: str = "",
224
  ):
225
  """Generator: yields the answer incrementally as the LLM streams tokens.
226
 
227
  `reliable=False` (poor ASR audio) makes the system prompt tell the model to flag
228
+ unclear audio instead of treating garbled fragments as real speech. `speaker_note`
229
+ (when multiple speakers were diarized) drives speaker/name attribution.
230
  """
231
  if not regions:
232
  yield "(No relevant regions to summarize.)"
 
250
  messages=[
251
  {"role": "system",
252
  "content": _make_system_msg(language, max_words, _summary_tier(max_words),
253
+ reliable=reliable, speaker_note=speaker_note)},
254
  {"role": "user", "content": synth_prompt},
255
  ],
256
  max_tokens=LLM_MAX_TOKENS,