NINA / diarize.py
Malina
Claude Opus 4.8
Fix diarization hub kwarg, yt-dlp cookies, stronger output formatting
d0ff022
Raw History Blame
9.13 kB
# ─────────────────────────────────────────────
# NINA β€” speaker diarization (pyannote) + name attribution
# ─────────────────────────────────────────────
# Diarization answers "who spoke when" so synthesis can attribute each line to a
# speaker, and β€” when someone introduces themselves anywhere in the video β€” use
# their NAME for all of their lines (e.g. two gamers who name themselves, then one
# shouts "I won"). Best-effort: if pyannote isn't installed or the HF token can't
# load the gated model, we return empty results and the pipeline runs without
# speaker labels.
import re
import numpy as np
from config import DIARIZATION_MAX_SECONDS, DIARIZATION_MODEL, SAMPLE_RATE
from models import HF_TOKEN
_pipeline = None
_failed = False
_shimmed = False
def _shim_torchaudio(torchaudio) -> None:
"""pyannote 3.3.2 was written against torchaudio's old I/O backend, which torchaudio
2.11 removed (AudioMetaData, list/get/set_audio_backend). We feed pyannote an
in-memory tensor (no file I/O), so stubbing these missing top-level symbols lets it
import and run without touching the removed code paths."""
global _shimmed
if _shimmed:
return
if not hasattr(torchaudio, "AudioMetaData"):
_AMD = None
for _mp in ("torchaudio.backend.common", "torchaudio._backend.common"):
try:
_AMD = getattr(__import__(_mp, fromlist=["AudioMetaData"]), "AudioMetaData", None)
if _AMD:
break
except Exception:
pass
if _AMD is None:
from dataclasses import dataclass
@dataclass
class _AMD:
sample_rate: float = 0.0
num_frames: int = 0
num_channels: int = 0
bits_per_sample: int = 0
encoding: str = "PCM_S"
torchaudio.AudioMetaData = _AMD
for _name, _stub in (
("list_audio_backends", lambda: ["soundfile"]),
("get_audio_backend", lambda: "soundfile"),
("set_audio_backend", lambda *a, **k: None),
):
if not hasattr(torchaudio, _name):
setattr(torchaudio, _name, _stub)
_shimmed = True
def _shim_hf_hub() -> None:
"""pyannote 3.3.2 internally calls hf_hub_download(use_auth_token=...) β€” a kwarg
that huggingface-hub 1.x removed (it's now `token`). Translate it so the gated
model still loads. Must run BEFORE `from pyannote.audio import Pipeline` so
pyannote binds the patched function."""
import huggingface_hub as _hf
for _name in ("hf_hub_download", "snapshot_download"):
_orig = getattr(_hf, _name, None)
if _orig is None or getattr(_orig, "_nina_shim", False):
continue
def _make(orig):
def _wrapped(*a, **k):
if "use_auth_token" in k:
k.setdefault("token", k.pop("use_auth_token"))
return orig(*a, **k)
_wrapped._nina_shim = True
return _wrapped
setattr(_hf, _name, _make(_orig))
def _get_pipeline():
global _pipeline, _failed
if _failed:
return None
if _pipeline is None:
try:
import torch
import torchaudio
_shim_torchaudio(torchaudio) # paper over torchaudio 2.11's removed I/O API
_shim_hf_hub() # paper over huggingface-hub's use_auth_token removal
from pyannote.audio import Pipeline
# huggingface-hub renamed use_auth_token β†’ token; support both.
try:
_pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, token=HF_TOKEN)
except TypeError:
_pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, use_auth_token=HF_TOKEN)
if _pipeline is None: # happens when the token lacks gated access
raise RuntimeError("pipeline is None β€” has the HF token accepted the gated "
"pyannote terms?")
_pipeline.to(torch.device("cpu"))
print("[NINA] Diarization pipeline ready.")
except Exception as e:
_failed = True
print(f"[NINA] Diarization unavailable: {e} β€” continuing without speaker labels.")
return None
return _pipeline
def diarize(audio: np.ndarray, sr: int = SAMPLE_RATE) -> list[tuple]:
"""Return speaker turns as a sorted list of (start, end, label), or [] on failure."""
try:
if audio is None or len(audio) == 0:
return []
if len(audio) / sr > DIARIZATION_MAX_SECONDS:
print("[NINA] Audio too long for diarization β€” skipping.")
return []
pipe = _get_pipeline()
if pipe is None:
return []
import torch
wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0)
annotation = pipe({"waveform": wav, "sample_rate": sr})
turns = [
(float(seg.start), float(seg.end), str(label))
for seg, _, label in annotation.itertracks(yield_label=True)
]
turns.sort(key=lambda t: t[0])
n_spk = len({lbl for _, _, lbl in turns})
print(f"[NINA] Diarization: {n_spk} speaker(s) across {len(turns)} turns.")
return turns
except Exception as e:
print(f"[NINA] Diarization failed: {e}")
return []
def attach_speakers(segments: list[dict], turns: list[tuple]) -> None:
"""Tag each transcript segment with the speaker label that overlaps it most."""
if not turns or not segments:
return
for s in segments:
ss, se = s["start"], s["end"]
best, best_ov = None, 0.0
for ts, te, lbl in turns:
ov = max(0.0, min(se, te) - max(ss, ts))
if ov > best_ov:
best_ov, best = ov, lbl
if best:
s["speaker"] = best
def speaker_display_map(segments: list[dict]) -> dict:
"""Map raw pyannote labels (SPEAKER_00…) β†’ 'Speaker 1/2…' by first appearance."""
order: list[str] = []
for s in segments:
sp = s.get("speaker")
if sp and sp not in order:
order.append(sp)
return {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(order)}
# Self-introduction patterns (multilingual). Capture the name token after a cue.
_NAME = r"([A-ZΓ€-ΕΉΔ„Δ†Δ˜ΕΕƒΓ“ΕšΕΉΕ»][\w''\-]+)"
_INTRO_PATTERNS = [
re.compile(p, re.IGNORECASE) for p in (
rf"\bmy name(?:'s| is)\s+{_NAME}",
rf"\bi(?:['’]?m| am)\s+{_NAME}",
rf"\bthis is\s+{_NAME}\b",
rf"\bcall me\s+{_NAME}",
rf"\bnazywam si[eΔ™]\s+{_NAME}",
rf"\bjestem\s+{_NAME}",
rf"\bmam na imi[eΔ™]\s+{_NAME}",
rf"\bich (?:heiße|bin)\s+{_NAME}",
rf"\bme llamo\s+{_NAME}",
rf"\bje m'appelle\s+{_NAME}",
)
]
# Words that look like names after "I'm…" but aren't.
_NOT_A_NAME = {
"here", "back", "going", "gonna", "just", "not", "so", "the", "a", "an",
"good", "fine", "sorry", "sure", "ready", "live", "playing", "streaming",
"tutaj", "dobry", "gotowy",
}
def name_roster(segments: list[dict], display: dict) -> dict:
"""Scan each speaker's lines for a self-introduction β†’ {display_name: real_name}."""
roster: dict = {}
for s in segments:
sp = s.get("speaker")
if not sp:
continue
disp = display.get(sp, sp)
if disp in roster:
continue
text = s.get("text", "")
for pat in _INTRO_PATTERNS:
m = pat.search(text)
if m:
name = m.group(1).strip()
if name and len(name) > 1 and name.lower() not in _NOT_A_NAME:
roster[disp] = name
break
return roster
def speaker_note(display: dict, roster: dict) -> str:
"""One-line summary of speakers + any discovered names, fed to synthesis."""
if len(display) <= 1:
return ""
names = ", ".join(display.values())
note = f"This video has multiple speakers ({names})."
if roster:
intro = "; ".join(f"{disp} introduced themselves as \"{name}\""
for disp, name in roster.items())
note += f" Self-introductions detected: {intro}."
return note
def annotate_region_speakers(regions: list[dict], segments: list[dict], display: dict) -> None:
"""Tag each region with the display-name(s) of the speakers talking during it."""
if not display or not regions:
return
spans = [
(s["start"], s["end"], display.get(s.get("speaker")))
for s in segments if s.get("speaker")
]
for r in regions:
names: list[str] = []
for ss, se, nm in spans:
if nm and ss < r["end"] and se > r["start"] and nm not in names:
names.append(nm)
if names:
r["speakers"] = names