# ───────────────────────────────────────────── # NINA — speaker diarization (pyannote) + name attribution # ───────────────────────────────────────────── # Diarization answers "who spoke when" so synthesis can attribute each line to a # speaker, and — when someone introduces themselves anywhere in the video — use # their NAME for all of their lines (e.g. two gamers who name themselves, then one # shouts "I won"). Best-effort: if pyannote isn't installed or the HF token can't # load the gated model, we return empty results and the pipeline runs without # speaker labels. import re import numpy as np from config import DIARIZATION_MAX_SECONDS, DIARIZATION_MODEL, SAMPLE_RATE from models import HF_TOKEN _pipeline = None _failed = False _shimmed = False def _shim_torchaudio(torchaudio) -> None: """pyannote 3.3.2 was written against torchaudio's old I/O backend, which torchaudio 2.11 removed (AudioMetaData, list/get/set_audio_backend). We feed pyannote an in-memory tensor (no file I/O), so stubbing these missing top-level symbols lets it import and run without touching the removed code paths.""" global _shimmed if _shimmed: return if not hasattr(torchaudio, "AudioMetaData"): _AMD = None for _mp in ("torchaudio.backend.common", "torchaudio._backend.common"): try: _AMD = getattr(__import__(_mp, fromlist=["AudioMetaData"]), "AudioMetaData", None) if _AMD: break except Exception: pass if _AMD is None: from dataclasses import dataclass @dataclass class _AMD: sample_rate: float = 0.0 num_frames: int = 0 num_channels: int = 0 bits_per_sample: int = 0 encoding: str = "PCM_S" torchaudio.AudioMetaData = _AMD for _name, _stub in ( ("list_audio_backends", lambda: ["soundfile"]), ("get_audio_backend", lambda: "soundfile"), ("set_audio_backend", lambda *a, **k: None), ): if not hasattr(torchaudio, _name): setattr(torchaudio, _name, _stub) _shimmed = True def _shim_hf_hub() -> None: """pyannote 3.3.2 internally calls hf_hub_download(use_auth_token=...) — a kwarg that huggingface-hub 1.x removed (it's now `token`). Translate it so the gated model still loads. Must run BEFORE `from pyannote.audio import Pipeline` so pyannote binds the patched function.""" import huggingface_hub as _hf for _name in ("hf_hub_download", "snapshot_download"): _orig = getattr(_hf, _name, None) if _orig is None or getattr(_orig, "_nina_shim", False): continue def _make(orig): def _wrapped(*a, **k): if "use_auth_token" in k: k.setdefault("token", k.pop("use_auth_token")) return orig(*a, **k) _wrapped._nina_shim = True return _wrapped setattr(_hf, _name, _make(_orig)) def _get_pipeline(): global _pipeline, _failed if _failed: return None if _pipeline is None: try: import torch import torchaudio _shim_torchaudio(torchaudio) # paper over torchaudio 2.11's removed I/O API _shim_hf_hub() # paper over huggingface-hub's use_auth_token removal from pyannote.audio import Pipeline # huggingface-hub renamed use_auth_token → token; support both. try: _pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, token=HF_TOKEN) except TypeError: _pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, use_auth_token=HF_TOKEN) if _pipeline is None: # happens when the token lacks gated access raise RuntimeError("pipeline is None — has the HF token accepted the gated " "pyannote terms?") _pipeline.to(torch.device("cpu")) print("[NINA] Diarization pipeline ready.") except Exception as e: _failed = True print(f"[NINA] Diarization unavailable: {e} — continuing without speaker labels.") return None return _pipeline def diarize(audio: np.ndarray, sr: int = SAMPLE_RATE) -> list[tuple]: """Return speaker turns as a sorted list of (start, end, label), or [] on failure.""" try: if audio is None or len(audio) == 0: return [] if len(audio) / sr > DIARIZATION_MAX_SECONDS: print("[NINA] Audio too long for diarization — skipping.") return [] pipe = _get_pipeline() if pipe is None: return [] import torch wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0) annotation = pipe({"waveform": wav, "sample_rate": sr}) turns = [ (float(seg.start), float(seg.end), str(label)) for seg, _, label in annotation.itertracks(yield_label=True) ] turns.sort(key=lambda t: t[0]) n_spk = len({lbl for _, _, lbl in turns}) print(f"[NINA] Diarization: {n_spk} speaker(s) across {len(turns)} turns.") return turns except Exception as e: print(f"[NINA] Diarization failed: {e}") return [] def attach_speakers(segments: list[dict], turns: list[tuple]) -> None: """Tag each transcript segment with the speaker label that overlaps it most.""" if not turns or not segments: return for s in segments: ss, se = s["start"], s["end"] best, best_ov = None, 0.0 for ts, te, lbl in turns: ov = max(0.0, min(se, te) - max(ss, ts)) if ov > best_ov: best_ov, best = ov, lbl if best: s["speaker"] = best def speaker_display_map(segments: list[dict]) -> dict: """Map raw pyannote labels (SPEAKER_00…) → 'Speaker 1/2…' by first appearance.""" order: list[str] = [] for s in segments: sp = s.get("speaker") if sp and sp not in order: order.append(sp) return {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(order)} # Self-introduction patterns (multilingual). Capture the name token after a cue. _NAME = r"([A-ZÀ-ŹĄĆĘŁŃÓŚŹŻ][\w''\-]+)" _INTRO_PATTERNS = [ re.compile(p, re.IGNORECASE) for p in ( rf"\bmy name(?:'s| is)\s+{_NAME}", rf"\bi(?:['’]?m| am)\s+{_NAME}", rf"\bthis is\s+{_NAME}\b", rf"\bcall me\s+{_NAME}", rf"\bnazywam si[eę]\s+{_NAME}", rf"\bjestem\s+{_NAME}", rf"\bmam na imi[eę]\s+{_NAME}", rf"\bich (?:heiße|bin)\s+{_NAME}", rf"\bme llamo\s+{_NAME}", rf"\bje m'appelle\s+{_NAME}", ) ] # Words that look like names after "I'm…" but aren't. _NOT_A_NAME = { "here", "back", "going", "gonna", "just", "not", "so", "the", "a", "an", "good", "fine", "sorry", "sure", "ready", "live", "playing", "streaming", "tutaj", "dobry", "gotowy", } def name_roster(segments: list[dict], display: dict) -> dict: """Scan each speaker's lines for a self-introduction → {display_name: real_name}.""" roster: dict = {} for s in segments: sp = s.get("speaker") if not sp: continue disp = display.get(sp, sp) if disp in roster: continue text = s.get("text", "") for pat in _INTRO_PATTERNS: m = pat.search(text) if m: name = m.group(1).strip() if name and len(name) > 1 and name.lower() not in _NOT_A_NAME: roster[disp] = name break return roster def speaker_note(display: dict, roster: dict) -> str: """One-line summary of speakers + any discovered names, fed to synthesis.""" if len(display) <= 1: return "" names = ", ".join(display.values()) note = f"This video has multiple speakers ({names})." if roster: intro = "; ".join(f"{disp} introduced themselves as \"{name}\"" for disp, name in roster.items()) note += f" Self-introductions detected: {intro}." return note def annotate_region_speakers(regions: list[dict], segments: list[dict], display: dict) -> None: """Tag each region with the display-name(s) of the speakers talking during it.""" if not display or not regions: return spans = [ (s["start"], s["end"], display.get(s.get("speaker"))) for s in segments if s.get("speaker") ] for r in regions: names: list[str] = [] for ss, se, nm in spans: if nm and ss < r["end"] and se > r["start"] and nm not in names: names.append(nm) if names: r["speakers"] = names