Spaces:
Paused
Paused
File size: 9,132 Bytes
e2c96b8 6dd7da1 e2c96b8 d0ff022 e2c96b8 7342244 6dd7da1 d0ff022 e2c96b8 57dbdb2 e2c96b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | # βββββββββββββββββββββββββββββββββββββββββββββ
# NINA β speaker diarization (pyannote) + name attribution
# βββββββββββββββββββββββββββββββββββββββββββββ
# Diarization answers "who spoke when" so synthesis can attribute each line to a
# speaker, and β when someone introduces themselves anywhere in the video β use
# their NAME for all of their lines (e.g. two gamers who name themselves, then one
# shouts "I won"). Best-effort: if pyannote isn't installed or the HF token can't
# load the gated model, we return empty results and the pipeline runs without
# speaker labels.
import re
import numpy as np
from config import DIARIZATION_MAX_SECONDS, DIARIZATION_MODEL, SAMPLE_RATE
from models import HF_TOKEN
_pipeline = None
_failed = False
_shimmed = False
def _shim_torchaudio(torchaudio) -> None:
"""pyannote 3.3.2 was written against torchaudio's old I/O backend, which torchaudio
2.11 removed (AudioMetaData, list/get/set_audio_backend). We feed pyannote an
in-memory tensor (no file I/O), so stubbing these missing top-level symbols lets it
import and run without touching the removed code paths."""
global _shimmed
if _shimmed:
return
if not hasattr(torchaudio, "AudioMetaData"):
_AMD = None
for _mp in ("torchaudio.backend.common", "torchaudio._backend.common"):
try:
_AMD = getattr(__import__(_mp, fromlist=["AudioMetaData"]), "AudioMetaData", None)
if _AMD:
break
except Exception:
pass
if _AMD is None:
from dataclasses import dataclass
@dataclass
class _AMD:
sample_rate: float = 0.0
num_frames: int = 0
num_channels: int = 0
bits_per_sample: int = 0
encoding: str = "PCM_S"
torchaudio.AudioMetaData = _AMD
for _name, _stub in (
("list_audio_backends", lambda: ["soundfile"]),
("get_audio_backend", lambda: "soundfile"),
("set_audio_backend", lambda *a, **k: None),
):
if not hasattr(torchaudio, _name):
setattr(torchaudio, _name, _stub)
_shimmed = True
def _shim_hf_hub() -> None:
"""pyannote 3.3.2 internally calls hf_hub_download(use_auth_token=...) β a kwarg
that huggingface-hub 1.x removed (it's now `token`). Translate it so the gated
model still loads. Must run BEFORE `from pyannote.audio import Pipeline` so
pyannote binds the patched function."""
import huggingface_hub as _hf
for _name in ("hf_hub_download", "snapshot_download"):
_orig = getattr(_hf, _name, None)
if _orig is None or getattr(_orig, "_nina_shim", False):
continue
def _make(orig):
def _wrapped(*a, **k):
if "use_auth_token" in k:
k.setdefault("token", k.pop("use_auth_token"))
return orig(*a, **k)
_wrapped._nina_shim = True
return _wrapped
setattr(_hf, _name, _make(_orig))
def _get_pipeline():
global _pipeline, _failed
if _failed:
return None
if _pipeline is None:
try:
import torch
import torchaudio
_shim_torchaudio(torchaudio) # paper over torchaudio 2.11's removed I/O API
_shim_hf_hub() # paper over huggingface-hub's use_auth_token removal
from pyannote.audio import Pipeline
# huggingface-hub renamed use_auth_token β token; support both.
try:
_pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, token=HF_TOKEN)
except TypeError:
_pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, use_auth_token=HF_TOKEN)
if _pipeline is None: # happens when the token lacks gated access
raise RuntimeError("pipeline is None β has the HF token accepted the gated "
"pyannote terms?")
_pipeline.to(torch.device("cpu"))
print("[NINA] Diarization pipeline ready.")
except Exception as e:
_failed = True
print(f"[NINA] Diarization unavailable: {e} β continuing without speaker labels.")
return None
return _pipeline
def diarize(audio: np.ndarray, sr: int = SAMPLE_RATE) -> list[tuple]:
"""Return speaker turns as a sorted list of (start, end, label), or [] on failure."""
try:
if audio is None or len(audio) == 0:
return []
if len(audio) / sr > DIARIZATION_MAX_SECONDS:
print("[NINA] Audio too long for diarization β skipping.")
return []
pipe = _get_pipeline()
if pipe is None:
return []
import torch
wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0)
annotation = pipe({"waveform": wav, "sample_rate": sr})
turns = [
(float(seg.start), float(seg.end), str(label))
for seg, _, label in annotation.itertracks(yield_label=True)
]
turns.sort(key=lambda t: t[0])
n_spk = len({lbl for _, _, lbl in turns})
print(f"[NINA] Diarization: {n_spk} speaker(s) across {len(turns)} turns.")
return turns
except Exception as e:
print(f"[NINA] Diarization failed: {e}")
return []
def attach_speakers(segments: list[dict], turns: list[tuple]) -> None:
"""Tag each transcript segment with the speaker label that overlaps it most."""
if not turns or not segments:
return
for s in segments:
ss, se = s["start"], s["end"]
best, best_ov = None, 0.0
for ts, te, lbl in turns:
ov = max(0.0, min(se, te) - max(ss, ts))
if ov > best_ov:
best_ov, best = ov, lbl
if best:
s["speaker"] = best
def speaker_display_map(segments: list[dict]) -> dict:
"""Map raw pyannote labels (SPEAKER_00β¦) β 'Speaker 1/2β¦' by first appearance."""
order: list[str] = []
for s in segments:
sp = s.get("speaker")
if sp and sp not in order:
order.append(sp)
return {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(order)}
# Self-introduction patterns (multilingual). Capture the name token after a cue.
_NAME = r"([A-ZΓ-ΕΉΔΔΔΕΕΓΕΕΉΕ»][\w''\-]+)"
_INTRO_PATTERNS = [
re.compile(p, re.IGNORECASE) for p in (
rf"\bmy name(?:'s| is)\s+{_NAME}",
rf"\bi(?:['β]?m| am)\s+{_NAME}",
rf"\bthis is\s+{_NAME}\b",
rf"\bcall me\s+{_NAME}",
rf"\bnazywam si[eΔ]\s+{_NAME}",
rf"\bjestem\s+{_NAME}",
rf"\bmam na imi[eΔ]\s+{_NAME}",
rf"\bich (?:heiΓe|bin)\s+{_NAME}",
rf"\bme llamo\s+{_NAME}",
rf"\bje m'appelle\s+{_NAME}",
)
]
# Words that look like names after "I'mβ¦" but aren't.
_NOT_A_NAME = {
"here", "back", "going", "gonna", "just", "not", "so", "the", "a", "an",
"good", "fine", "sorry", "sure", "ready", "live", "playing", "streaming",
"tutaj", "dobry", "gotowy",
}
def name_roster(segments: list[dict], display: dict) -> dict:
"""Scan each speaker's lines for a self-introduction β {display_name: real_name}."""
roster: dict = {}
for s in segments:
sp = s.get("speaker")
if not sp:
continue
disp = display.get(sp, sp)
if disp in roster:
continue
text = s.get("text", "")
for pat in _INTRO_PATTERNS:
m = pat.search(text)
if m:
name = m.group(1).strip()
if name and len(name) > 1 and name.lower() not in _NOT_A_NAME:
roster[disp] = name
break
return roster
def speaker_note(display: dict, roster: dict) -> str:
"""One-line summary of speakers + any discovered names, fed to synthesis."""
if len(display) <= 1:
return ""
names = ", ".join(display.values())
note = f"This video has multiple speakers ({names})."
if roster:
intro = "; ".join(f"{disp} introduced themselves as \"{name}\""
for disp, name in roster.items())
note += f" Self-introductions detected: {intro}."
return note
def annotate_region_speakers(regions: list[dict], segments: list[dict], display: dict) -> None:
"""Tag each region with the display-name(s) of the speakers talking during it."""
if not display or not regions:
return
spans = [
(s["start"], s["end"], display.get(s.get("speaker")))
for s in segments if s.get("speaker")
]
for r in regions:
names: list[str] = []
for ss, se, nm in spans:
if nm and ss < r["end"] and se > r["start"] and nm not in names:
names.append(nm)
if names:
r["speakers"] = names
|