File size: 9,132 Bytes
e2c96b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6dd7da1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e2c96b8
 
d0ff022
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e2c96b8
 
 
 
 
 
 
7342244
6dd7da1
d0ff022
e2c96b8
57dbdb2
 
 
 
 
e2c96b8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
# ─────────────────────────────────────────────
#  NINA β€” speaker diarization (pyannote) + name attribution
# ─────────────────────────────────────────────
#  Diarization answers "who spoke when" so synthesis can attribute each line to a
#  speaker, and β€” when someone introduces themselves anywhere in the video β€” use
#  their NAME for all of their lines (e.g. two gamers who name themselves, then one
#  shouts "I won").  Best-effort: if pyannote isn't installed or the HF token can't
#  load the gated model, we return empty results and the pipeline runs without
#  speaker labels.

import re

import numpy as np

from config import DIARIZATION_MAX_SECONDS, DIARIZATION_MODEL, SAMPLE_RATE
from models import HF_TOKEN

_pipeline = None
_failed = False
_shimmed = False


def _shim_torchaudio(torchaudio) -> None:
    """pyannote 3.3.2 was written against torchaudio's old I/O backend, which torchaudio
    2.11 removed (AudioMetaData, list/get/set_audio_backend). We feed pyannote an
    in-memory tensor (no file I/O), so stubbing these missing top-level symbols lets it
    import and run without touching the removed code paths."""
    global _shimmed
    if _shimmed:
        return
    if not hasattr(torchaudio, "AudioMetaData"):
        _AMD = None
        for _mp in ("torchaudio.backend.common", "torchaudio._backend.common"):
            try:
                _AMD = getattr(__import__(_mp, fromlist=["AudioMetaData"]), "AudioMetaData", None)
                if _AMD:
                    break
            except Exception:
                pass
        if _AMD is None:
            from dataclasses import dataclass

            @dataclass
            class _AMD:
                sample_rate: float = 0.0
                num_frames: int = 0
                num_channels: int = 0
                bits_per_sample: int = 0
                encoding: str = "PCM_S"
        torchaudio.AudioMetaData = _AMD

    for _name, _stub in (
        ("list_audio_backends", lambda: ["soundfile"]),
        ("get_audio_backend",   lambda: "soundfile"),
        ("set_audio_backend",   lambda *a, **k: None),
    ):
        if not hasattr(torchaudio, _name):
            setattr(torchaudio, _name, _stub)
    _shimmed = True


def _shim_hf_hub() -> None:
    """pyannote 3.3.2 internally calls hf_hub_download(use_auth_token=...) β€” a kwarg
    that huggingface-hub 1.x removed (it's now `token`). Translate it so the gated
    model still loads. Must run BEFORE `from pyannote.audio import Pipeline` so
    pyannote binds the patched function."""
    import huggingface_hub as _hf
    for _name in ("hf_hub_download", "snapshot_download"):
        _orig = getattr(_hf, _name, None)
        if _orig is None or getattr(_orig, "_nina_shim", False):
            continue
        def _make(orig):
            def _wrapped(*a, **k):
                if "use_auth_token" in k:
                    k.setdefault("token", k.pop("use_auth_token"))
                return orig(*a, **k)
            _wrapped._nina_shim = True
            return _wrapped
        setattr(_hf, _name, _make(_orig))


def _get_pipeline():
    global _pipeline, _failed
    if _failed:
        return None
    if _pipeline is None:
        try:
            import torch
            import torchaudio
            _shim_torchaudio(torchaudio)   # paper over torchaudio 2.11's removed I/O API
            _shim_hf_hub()                 # paper over huggingface-hub's use_auth_token removal
            from pyannote.audio import Pipeline
            # huggingface-hub renamed use_auth_token β†’ token; support both.
            try:
                _pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, token=HF_TOKEN)
            except TypeError:
                _pipeline = Pipeline.from_pretrained(DIARIZATION_MODEL, use_auth_token=HF_TOKEN)
            if _pipeline is None:                 # happens when the token lacks gated access
                raise RuntimeError("pipeline is None β€” has the HF token accepted the gated "
                                   "pyannote terms?")
            _pipeline.to(torch.device("cpu"))
            print("[NINA] Diarization pipeline ready.")
        except Exception as e:
            _failed = True
            print(f"[NINA] Diarization unavailable: {e} β€” continuing without speaker labels.")
            return None
    return _pipeline


def diarize(audio: np.ndarray, sr: int = SAMPLE_RATE) -> list[tuple]:
    """Return speaker turns as a sorted list of (start, end, label), or [] on failure."""
    try:
        if audio is None or len(audio) == 0:
            return []
        if len(audio) / sr > DIARIZATION_MAX_SECONDS:
            print("[NINA] Audio too long for diarization β€” skipping.")
            return []
        pipe = _get_pipeline()
        if pipe is None:
            return []

        import torch
        wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0)
        annotation = pipe({"waveform": wav, "sample_rate": sr})
        turns = [
            (float(seg.start), float(seg.end), str(label))
            for seg, _, label in annotation.itertracks(yield_label=True)
        ]
        turns.sort(key=lambda t: t[0])
        n_spk = len({lbl for _, _, lbl in turns})
        print(f"[NINA] Diarization: {n_spk} speaker(s) across {len(turns)} turns.")
        return turns
    except Exception as e:
        print(f"[NINA] Diarization failed: {e}")
        return []


def attach_speakers(segments: list[dict], turns: list[tuple]) -> None:
    """Tag each transcript segment with the speaker label that overlaps it most."""
    if not turns or not segments:
        return
    for s in segments:
        ss, se = s["start"], s["end"]
        best, best_ov = None, 0.0
        for ts, te, lbl in turns:
            ov = max(0.0, min(se, te) - max(ss, ts))
            if ov > best_ov:
                best_ov, best = ov, lbl
        if best:
            s["speaker"] = best


def speaker_display_map(segments: list[dict]) -> dict:
    """Map raw pyannote labels (SPEAKER_00…) β†’ 'Speaker 1/2…' by first appearance."""
    order: list[str] = []
    for s in segments:
        sp = s.get("speaker")
        if sp and sp not in order:
            order.append(sp)
    return {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(order)}


# Self-introduction patterns (multilingual). Capture the name token after a cue.
_NAME = r"([A-ZΓ€-ΕΉΔ„Δ†Δ˜ΕΕƒΓ“ΕšΕΉΕ»][\w''\-]+)"
_INTRO_PATTERNS = [
    re.compile(p, re.IGNORECASE) for p in (
        rf"\bmy name(?:'s| is)\s+{_NAME}",
        rf"\bi(?:['’]?m| am)\s+{_NAME}",
        rf"\bthis is\s+{_NAME}\b",
        rf"\bcall me\s+{_NAME}",
        rf"\bnazywam si[eΔ™]\s+{_NAME}",
        rf"\bjestem\s+{_NAME}",
        rf"\bmam na imi[eΔ™]\s+{_NAME}",
        rf"\bich (?:heiße|bin)\s+{_NAME}",
        rf"\bme llamo\s+{_NAME}",
        rf"\bje m'appelle\s+{_NAME}",
    )
]
# Words that look like names after "I'm…" but aren't.
_NOT_A_NAME = {
    "here", "back", "going", "gonna", "just", "not", "so", "the", "a", "an",
    "good", "fine", "sorry", "sure", "ready", "live", "playing", "streaming",
    "tutaj", "dobry", "gotowy",
}


def name_roster(segments: list[dict], display: dict) -> dict:
    """Scan each speaker's lines for a self-introduction β†’ {display_name: real_name}."""
    roster: dict = {}
    for s in segments:
        sp = s.get("speaker")
        if not sp:
            continue
        disp = display.get(sp, sp)
        if disp in roster:
            continue
        text = s.get("text", "")
        for pat in _INTRO_PATTERNS:
            m = pat.search(text)
            if m:
                name = m.group(1).strip()
                if name and len(name) > 1 and name.lower() not in _NOT_A_NAME:
                    roster[disp] = name
                    break
    return roster


def speaker_note(display: dict, roster: dict) -> str:
    """One-line summary of speakers + any discovered names, fed to synthesis."""
    if len(display) <= 1:
        return ""
    names = ", ".join(display.values())
    note = f"This video has multiple speakers ({names})."
    if roster:
        intro = "; ".join(f"{disp} introduced themselves as \"{name}\""
                          for disp, name in roster.items())
        note += f" Self-introductions detected: {intro}."
    return note


def annotate_region_speakers(regions: list[dict], segments: list[dict], display: dict) -> None:
    """Tag each region with the display-name(s) of the speakers talking during it."""
    if not display or not regions:
        return
    spans = [
        (s["start"], s["end"], display.get(s.get("speaker")))
        for s in segments if s.get("speaker")
    ]
    for r in regions:
        names: list[str] = []
        for ss, se, nm in spans:
            if nm and ss < r["end"] and se > r["start"] and nm not in names:
                names.append(nm)
        if names:
            r["speakers"] = names