Automatic Speech Recognition
Transformers
Safetensors
English
asr_model
feature-extraction
asr
speech-recognition
speech-to-text
audio
speech-llm
word-timestamps
speaker-diarization
qwen
granite-speech
custom_code
Instructions to use mazesmazes/tiny-audio with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use mazesmazes/tiny-audio with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("automatic-speech-recognition", model="mazesmazes/tiny-audio", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("mazesmazes/tiny-audio", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Update custom model files, README, and requirements
Browse files- diarization.py +57 -12
diarization.py
CHANGED
|
@@ -10,6 +10,15 @@ import warnings
|
|
| 10 |
import numpy as np
|
| 11 |
import torch
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
def _get_device() -> torch.device:
|
| 15 |
"""Get best available device for inference."""
|
|
@@ -80,7 +89,10 @@ class SpectralCluster:
|
|
| 80 |
|
| 81 |
def get_sim_mat(self, embeddings: np.ndarray) -> np.ndarray:
|
| 82 |
"""Compute cosine similarity matrix."""
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
| 84 |
|
| 85 |
return cosine_similarity(embeddings, embeddings)
|
| 86 |
|
|
@@ -99,7 +111,10 @@ class SpectralCluster:
|
|
| 99 |
|
| 100 |
def get_laplacian(self, sim_mat: np.ndarray) -> np.ndarray:
|
| 101 |
"""Compute unnormalized Laplacian matrix."""
|
| 102 |
-
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
np.fill_diagonal(sim_mat, 0)
|
| 105 |
return laplacian(sim_mat, normed=False)
|
|
@@ -108,7 +123,10 @@ class SpectralCluster:
|
|
| 108 |
self, laplacian: np.ndarray, k_oracle: int | None = None
|
| 109 |
) -> tuple[np.ndarray, int]:
|
| 110 |
"""Extract spectral embeddings from Laplacian."""
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
lambdas, eig_vecs = eigh(laplacian)
|
| 114 |
|
|
@@ -125,7 +143,10 @@ class SpectralCluster:
|
|
| 125 |
|
| 126 |
def cluster_embs(self, emb: np.ndarray, k: int) -> np.ndarray:
|
| 127 |
"""Cluster spectral embeddings using k-means."""
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
_, labels, _ = k_means(emb, k, n_init=10)
|
| 131 |
return labels
|
|
@@ -187,7 +208,10 @@ class SpeakerClusterer:
|
|
| 187 |
return np.zeros(embeddings.shape[0], dtype=int)
|
| 188 |
|
| 189 |
# Normalize embeddings and replace NaN/inf
|
| 190 |
-
|
|
|
|
|
|
|
|
|
|
| 191 |
|
| 192 |
embeddings = np.nan_to_num(embeddings, nan=0.0, posinf=0.0, neginf=0.0)
|
| 193 |
embeddings = normalize(embeddings)
|
|
@@ -220,8 +244,14 @@ class SpeakerClusterer:
|
|
| 220 |
|
| 221 |
def _merge_by_cos(self, labels: np.ndarray, embs: np.ndarray, cos_thr: float) -> np.ndarray:
|
| 222 |
"""Merge similar speakers by cosine similarity of centroids."""
|
| 223 |
-
|
| 224 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
|
| 226 |
unique_labels, inverse = np.unique(labels, return_inverse=True)
|
| 227 |
if len(unique_labels) <= 1:
|
|
@@ -300,7 +330,10 @@ class SpeakerDiarizer:
|
|
| 300 |
def _get_ten_vad_model(cls):
|
| 301 |
"""Lazy-load TEN-VAD model (singleton)."""
|
| 302 |
if cls._ten_vad_model is None:
|
| 303 |
-
|
|
|
|
|
|
|
|
|
|
| 304 |
|
| 305 |
cls._ten_vad_model = TenVad(hop_size=256, threshold=cls.VAD_THRESHOLD)
|
| 306 |
return cls._ten_vad_model
|
|
@@ -319,7 +352,10 @@ class SpeakerDiarizer:
|
|
| 319 |
# Suppress torchaudio deprecation warning from SpeechBrain
|
| 320 |
with warnings.catch_warnings():
|
| 321 |
warnings.filterwarnings("ignore", message="torchaudio._backend")
|
| 322 |
-
|
|
|
|
|
|
|
|
|
|
| 323 |
|
| 324 |
device = cls._get_device()
|
| 325 |
cls._ecapa_model = EncoderClassifier.from_hparams(
|
|
@@ -353,13 +389,19 @@ class SpeakerDiarizer:
|
|
| 353 |
"""
|
| 354 |
# Handle file path input
|
| 355 |
if isinstance(audio, str):
|
| 356 |
-
|
|
|
|
|
|
|
|
|
|
| 357 |
|
| 358 |
audio, sample_rate = librosa.load(audio, sr=16000)
|
| 359 |
|
| 360 |
# Ensure correct sample rate
|
| 361 |
if sample_rate != 16000:
|
| 362 |
-
|
|
|
|
|
|
|
|
|
|
| 363 |
|
| 364 |
audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000)
|
| 365 |
sample_rate = 16000
|
|
@@ -518,7 +560,10 @@ class SpeakerDiarizer:
|
|
| 518 |
|
| 519 |
# Normalize all embeddings at once
|
| 520 |
if embeddings:
|
| 521 |
-
|
|
|
|
|
|
|
|
|
|
| 522 |
|
| 523 |
return normalize(np.array(embeddings)), window_segments
|
| 524 |
return np.array([]), []
|
|
|
|
| 10 |
import numpy as np
|
| 11 |
import torch
|
| 12 |
|
| 13 |
+
# The ECAPA SpeakerDiarizer's packages, imported inside try blocks: transformers
|
| 14 |
+
# scans every bundled file's imports before loading the model and refuses to load
|
| 15 |
+
# if one is missing, and it skips imports inside `try`. The pipeline diarizes with
|
| 16 |
+
# NemotronDiarizer, so these stay optional.
|
| 17 |
+
_ECAPA_EXTRAS = (
|
| 18 |
+
"SpeakerDiarizer (ECAPA) needs extra packages: "
|
| 19 |
+
"pip install speechbrain ten-vad scikit-learn scipy librosa"
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
|
| 23 |
def _get_device() -> torch.device:
|
| 24 |
"""Get best available device for inference."""
|
|
|
|
| 89 |
|
| 90 |
def get_sim_mat(self, embeddings: np.ndarray) -> np.ndarray:
|
| 91 |
"""Compute cosine similarity matrix."""
|
| 92 |
+
try:
|
| 93 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
| 94 |
+
except ImportError as e:
|
| 95 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 96 |
|
| 97 |
return cosine_similarity(embeddings, embeddings)
|
| 98 |
|
|
|
|
| 111 |
|
| 112 |
def get_laplacian(self, sim_mat: np.ndarray) -> np.ndarray:
|
| 113 |
"""Compute unnormalized Laplacian matrix."""
|
| 114 |
+
try:
|
| 115 |
+
from scipy.sparse.csgraph import laplacian
|
| 116 |
+
except ImportError as e:
|
| 117 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 118 |
|
| 119 |
np.fill_diagonal(sim_mat, 0)
|
| 120 |
return laplacian(sim_mat, normed=False)
|
|
|
|
| 123 |
self, laplacian: np.ndarray, k_oracle: int | None = None
|
| 124 |
) -> tuple[np.ndarray, int]:
|
| 125 |
"""Extract spectral embeddings from Laplacian."""
|
| 126 |
+
try:
|
| 127 |
+
from scipy.linalg import eigh
|
| 128 |
+
except ImportError as e:
|
| 129 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 130 |
|
| 131 |
lambdas, eig_vecs = eigh(laplacian)
|
| 132 |
|
|
|
|
| 143 |
|
| 144 |
def cluster_embs(self, emb: np.ndarray, k: int) -> np.ndarray:
|
| 145 |
"""Cluster spectral embeddings using k-means."""
|
| 146 |
+
try:
|
| 147 |
+
from sklearn.cluster._kmeans import k_means
|
| 148 |
+
except ImportError as e:
|
| 149 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 150 |
|
| 151 |
_, labels, _ = k_means(emb, k, n_init=10)
|
| 152 |
return labels
|
|
|
|
| 208 |
return np.zeros(embeddings.shape[0], dtype=int)
|
| 209 |
|
| 210 |
# Normalize embeddings and replace NaN/inf
|
| 211 |
+
try:
|
| 212 |
+
from sklearn.preprocessing import normalize
|
| 213 |
+
except ImportError as e:
|
| 214 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 215 |
|
| 216 |
embeddings = np.nan_to_num(embeddings, nan=0.0, posinf=0.0, neginf=0.0)
|
| 217 |
embeddings = normalize(embeddings)
|
|
|
|
| 244 |
|
| 245 |
def _merge_by_cos(self, labels: np.ndarray, embs: np.ndarray, cos_thr: float) -> np.ndarray:
|
| 246 |
"""Merge similar speakers by cosine similarity of centroids."""
|
| 247 |
+
try:
|
| 248 |
+
from sklearn.cluster import AgglomerativeClustering
|
| 249 |
+
except ImportError as e:
|
| 250 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 251 |
+
try:
|
| 252 |
+
from sklearn.preprocessing import normalize
|
| 253 |
+
except ImportError as e:
|
| 254 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 255 |
|
| 256 |
unique_labels, inverse = np.unique(labels, return_inverse=True)
|
| 257 |
if len(unique_labels) <= 1:
|
|
|
|
| 330 |
def _get_ten_vad_model(cls):
|
| 331 |
"""Lazy-load TEN-VAD model (singleton)."""
|
| 332 |
if cls._ten_vad_model is None:
|
| 333 |
+
try:
|
| 334 |
+
from ten_vad import TenVad
|
| 335 |
+
except ImportError as e:
|
| 336 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 337 |
|
| 338 |
cls._ten_vad_model = TenVad(hop_size=256, threshold=cls.VAD_THRESHOLD)
|
| 339 |
return cls._ten_vad_model
|
|
|
|
| 352 |
# Suppress torchaudio deprecation warning from SpeechBrain
|
| 353 |
with warnings.catch_warnings():
|
| 354 |
warnings.filterwarnings("ignore", message="torchaudio._backend")
|
| 355 |
+
try:
|
| 356 |
+
from speechbrain.inference.speaker import EncoderClassifier
|
| 357 |
+
except ImportError as e:
|
| 358 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 359 |
|
| 360 |
device = cls._get_device()
|
| 361 |
cls._ecapa_model = EncoderClassifier.from_hparams(
|
|
|
|
| 389 |
"""
|
| 390 |
# Handle file path input
|
| 391 |
if isinstance(audio, str):
|
| 392 |
+
try:
|
| 393 |
+
import librosa
|
| 394 |
+
except ImportError as e:
|
| 395 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 396 |
|
| 397 |
audio, sample_rate = librosa.load(audio, sr=16000)
|
| 398 |
|
| 399 |
# Ensure correct sample rate
|
| 400 |
if sample_rate != 16000:
|
| 401 |
+
try:
|
| 402 |
+
import librosa
|
| 403 |
+
except ImportError as e:
|
| 404 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 405 |
|
| 406 |
audio = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000)
|
| 407 |
sample_rate = 16000
|
|
|
|
| 560 |
|
| 561 |
# Normalize all embeddings at once
|
| 562 |
if embeddings:
|
| 563 |
+
try:
|
| 564 |
+
from sklearn.preprocessing import normalize
|
| 565 |
+
except ImportError as e:
|
| 566 |
+
raise ImportError(_ECAPA_EXTRAS) from e
|
| 567 |
|
| 568 |
return normalize(np.array(embeddings)), window_segments
|
| 569 |
return np.array([]), []
|