ediar / app.py
multimodalart's picture
multimodalart HF Staff
Tighten GPU duration to 60s based on measured inference (7s for 60s streaming, 0.5s offline)
5bfb0b9 verified
Raw History Blame Contribute Delete
9.44 kB
"""eDIAR: Training-Free Open-Speaker Streaming Diarization — interactive demo.
Upload a meeting recording → get back who spoke when: a color-coded speaker-turn
timeline, a per-speaker summary, and standard RTTM output.
Method (training-free, open-speaker):
* NVIDIA Nemotron-3-Diarization slot head provides frame-level slot posteriors
* WeSpeaker ResNet34-LM provides 256-d speaker embeddings
* official e-means (faiss contrib) clusters embeddings within slots, creating
new speakers on demand — no fixed speaker count
* streaming mode: bounded 90 s window re-clustered every chunk, labels revised
for 10 s, permanent speaker ids anchored to slots (Hungarian matching)
Code: Kyudan/eDIAR (Apache-2.0). Weights are fetched at runtime from their own
repos: nvidia/Nemotron-3-Diarization (openmdw-1.1) and
Wespeaker/wespeaker-voxceleb-resnet34-LM (CC-BY-4.0).
"""
import os
os.environ.setdefault("NUMBA_DISABLE_CUDA", "1") # numba must not touch CUDA before spaces patches torch
import time
import wave
import contextlib
import numpy as np
import spaces # noqa: E402 (must come before torch)
import gradio as gr
import soundfile as sf
from emeans_diar import SlotEMeansDiarizer
MAX_SECONDS = 180.0 # demo cap; the method itself handles hour-long meetings
SPEAKER_COLORS = [
"#f97316", "#3b82f6", "#10b981", "#ef4444", "#8b5cf6", "#eab308",
"#06b6d4", "#ec4899", "#84cc16", "#f43f5e", "#6366f1", "#14b8a6",
]
print("Loading eDIAR (Nemotron-3-Diarization slot head + WeSpeaker ResNet34-LM)...")
t0 = time.perf_counter()
STREAM_DIA = SlotEMeansDiarizer(device="cuda", streaming=True, latency="low_latency")
OFFLINE_DIA = SlotEMeansDiarizer(device="cuda", streaming=False)
print(f"eDIAR ready in {time.perf_counter() - t0:.1f}s")
def _load_mono(path):
"""Read any audio file as mono float32 in [-1, 1] at its native rate."""
data, sr = sf.read(path, dtype="float32", always_2d=False)
if getattr(data, "ndim", 1) > 1:
data = data.mean(axis=1)
return np.asarray(data, np.float32), sr
def _fmt(t):
m, s = divmod(int(t), 60)
return f"{m:02d}:{s:02d}"
def _timeline_html(segments, total_s, elapsed_s):
"""Color-coded speaker-turn timeline: one lane per speaker."""
if not segments:
return "<div style='padding:14px;color:#888'>No speech detected.</div>"
speakers = sorted({s for _, _, s in segments})
color = {spk: SPEAKER_COLORS[i % len(SPEAKER_COLORS)] for i, spk in enumerate(speakers)}
dur = max(total_s, max(e for _, e, _ in segments), 1e-6)
lanes, legend = [], []
for spk in speakers:
segs = [(s, e) for s, e, g in segments if g == spk]
speech = sum(e - s for s, e in segs)
bars = "".join(
f"<div title='{_fmt(s)} → {_fmt(e)} ({e - s:.1f}s)' style='position:absolute;"
f"left:{s / dur * 100:.3f}%;width:{max((e - s) / dur * 100, 0.35):.3f}%;"
f"top:22%;height:56%;background:{color[spk]};border-radius:3px;"
f"box-shadow:0 1px 3px rgba(0,0,0,.35)'></div>"
for s, e in segs
)
lanes.append(
f"<div style='display:flex;align-items:center;gap:8px;margin:7px 0'>"
f"<div style='flex:0 0 92px;font-size:13px;font-weight:600;color:{color[spk]}'>"
f"Speaker {chr(65 + spk) if spk < 26 else spk}</div>"
f"<div style='flex:1;position:relative;height:34px;background:rgba(127,127,127,.12);"
f"border-radius:6px;overflow:hidden'>{bars}</div>"
f"<div style='flex:0 0 74px;font-size:11px;color:#888;text-align:right'>{speech:.1f}s</div></div>"
)
legend.append(
f"<span style='color:{color[spk]};font-weight:600;margin-right:10px'>■ {chr(65 + spk) if spk < 26 else spk}</span>"
)
ticks = "".join(
f"<div style='position:absolute;left:{t / dur * 100:.2f}%;top:0;transform:translateX(-50%);"
f"font-size:10px;color:#777'>{_fmt(t)}</div>"
for t in np.linspace(0, dur, min(7, int(dur // 30) + 2))
)
return (
"<div style='font-family:ui-sans-serif,system-ui,sans-serif'>"
f"<div style='margin:4px 0 8px;font-size:12px;color:#888'>"
f"{len(speakers)} speakers · {len(segments)} turns · processed in {elapsed_s:.1f}s</div>"
f"<div style='position:relative;height:16px'>{ticks}</div>"
+ "".join(lanes)
+ f"<div style='margin-top:10px;font-size:12px'>{''.join(legend)}</div></div>"
)
def _rttm(segments, name="audio"):
return "\n".join(
f"SPEAKER {name} 1 {s:.3f} {e - s:.3f} <NA> <NA> spk{spk} <NA> <NA>"
for s, e, spk in segments
)
@spaces.GPU(duration=60)
def diarize(audio, mode="streaming", progress=gr.Progress()):
"""Run eDIAR on an uploaded recording.
audio: filepath or (sr, np.ndarray) tuple from gr.Audio.
mode: "streaming" (causal, labels may be revised for 10 s — the paper's
setting) or "offline" (whole recording at once, best accuracy).
Returns (timeline html, rttm text, summary text).
"""
t0 = time.perf_counter()
if audio is None:
raise gr.Error("Please upload or record a meeting clip first (WAV/MP3/FLAC…).")
if isinstance(audio, (tuple, list)): # gr.Audio(type="numpy"): int16-scale or float
wav, sr = audio[1], audio[0]
if getattr(wav, "ndim", 1) > 1:
wav = wav.mean(axis=1)
wav = np.asarray(wav, np.float32)
if np.issubdtype(wav.dtype, np.integer) or np.abs(wav).max() > 1.5:
wav = wav / 32768.0 # Gradio hands back int16-scale samples, not [-1, 1] floats
else:
wav, sr = _load_mono(audio)
if len(wav) < sr * 1.0:
raise gr.Error("Clip too short — need at least 1 second of audio.")
if len(wav) > sr * MAX_SECONDS:
wav = wav[: int(sr * MAX_SECONDS)]
note = f" (trimmed to {MAX_SECONDS:.0f}s demo cap)"
else:
note = ""
dia = STREAM_DIA if mode == "streaming" else OFFLINE_DIA
segments = dia.diarize(wav, sr)
elapsed = time.perf_counter() - t0
total_s = len(wav) / sr
speakers = sorted({s for _, _, s in segments})
lines = [
f"**{len(speakers)} speakers** detected · {len(segments)} turns · "
f"{total_s:.1f}s audio in **{elapsed:.1f}s** ({total_s / max(elapsed, 1e-6):.1f}× real-time){note}",
"",
]
for spk in speakers:
segs = [(s, e) for s, e, g in segments if g == spk]
speech = sum(e - s for s, e in segs)
lines.append(f"- **Speaker {chr(65 + spk) if spk < 26 else spk}**: {len(segs)} turns, {speech:.1f}s of speech ({100 * speech / total_s:.0f}%)")
return (
_timeline_html(segments, total_s, elapsed),
_rttm(segments),
"\n".join(lines),
)
CSS = """
#col { max-width: 1050px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="eDIAR · Streaming Speaker Diarization") as demo:
gr.HTML(
"<div style='text-align:center'>"
"<h1 style='margin-bottom:4px'>🎧 eDIAR — Open-Speaker Streaming Diarization</h1>"
"<p style='color:#9ca3af;margin-top:0'>Training-free diarization of meetings with an "
"<b>unknown number of speakers</b>, via e-means over Nemotron-3 Diarization slots + WeSpeaker embeddings. "
"<a href='https://huggingface.co/papers/2611.18529' target='_blank'>Paper</a> · "
"<a href='https://huggingface.co/Kyudan/eDIAR' target='_blank'>Code</a> · "
"<a href='https://kyudan1.github.io/eDIAR/' target='_blank'>Project page</a></p></div>"
)
with gr.Column(elem_id="col"):
with gr.Row():
audio_in = gr.Audio(
label="Meeting / recording audio (16 kHz mono is ideal; stereo is mixed down)",
type="filepath",
sources=["upload", "microphone"],
format="wav",
)
with gr.Row():
mode = gr.Radio(
["streaming", "offline"],
value="streaming",
label="Mode",
info="streaming: causal, audio arrives in 100 ms chunks, labels may be revised for 10 s (the paper's live setting). "
"offline: whole recording at once — best accuracy but not causal.",
)
run = gr.Button("Diarize", variant="primary", scale=1)
timeline = gr.HTML(label="Speaker-turn timeline", value="<div style='padding:14px;color:#888'>Upload audio and press Diarize.</div>")
summary = gr.Markdown("")
with gr.Accordion("RTTM output", open=False):
rttm = gr.Textbox(label="RTTM", lines=10, max_lines=20, buttons=["copy"])
gr.Examples(
examples=[
["samples/sample1_4spk_30s.wav", "streaming"],
["samples/sample2_10spk_60s.wav", "streaming"],
["samples/sample3_10spk_60s.wav", "streaming"],
["samples/sample4_10spk_60s.wav", "offline"],
],
inputs=[audio_in, mode],
outputs=[timeline, rttm, summary],
fn=diarize,
cache_examples=True,
cache_mode="lazy",
label="Examples (synthetic LibriSpeech conversations from the eDIAR project page)",
)
run.click(diarize, inputs=[audio_in, mode], outputs=[timeline, rttm, summary])
demo.launch(ssr_mode=False, theme=gr.themes.Citrus(), css=CSS)