Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 12,194 Bytes
ec01e2d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | """Run the MLX port of VeriSafe-Tanglish-STT-2 on audio files. No PyTorch needed.
python run_mlx.py CLIP.wav [CLIP2.mp3 ...] [--weights FILE] [--mode codemix|tamil] [--streaming] [--head rnnt|ctc]
[--ctc-assist | --no-ctc-assist] [--json OUT.jsonl]
The rows of ../code/transcribe.py, ported. Defaults (the code-mix recipe): --mode codemix; offline = RNN-T with --ctc-assist,
the J(0.3, 0) rescoring of the RNN-T beam-4 n-best and the CTC greedy hypothesis, on the reader output rounded to bf16 as the
evaluated decode read it; --streaming (480 ms block-causal mask) = RNN-T greedy with the start penalty 1 and the start cue from
the mode's CTC head. --no-ctc-assist gives offline greedy, --head ctc the CTC head, --mode tamil the Tamil heads (greedy by default also
offline; --ctc-assist opts in). The default is CTC_ASSIST_DEFAULT for CTC_ASSIST_DEFAULT_MODES below (keep both equal to
transcribe.py's). Reads stt2_model.mlx.safetensors, layout.json and preprocessor_config.json from this folder, and spm.model
from this folder or, if absent, ../tokenizer/. --weights stt2_model.mlx8.safetensors (a name in this folder, or a path) reads
the 8-bit file instead: mlx_model.load_weights dequantizes its 2-D weights to float32 once on load, the rest is unchanged.
Without --weights: stt2_model.mlx.safetensors if this folder has it, else stt2_model.mlx8.safetensors (a copy that ships only
the 8-bit file); neither, or a --weights file that does not exist, exits with a message naming the files.
STT-2: --mode codemix reads the code-mix reader fork (fork_cm.*, mlx_model.encode(stack=FORK)), --mode tamil the Tamil stack, as
transcribe.py does. layout.json's "codemix_fork": true requires every fork_cm.* tensor and both code-mix heads; fork_cm.* tensors
without it are refused. On a fork file MlxModel.encode and ctc_text/rnnt_text/assist_text need mode= (None raises), as
transcribe.py's Release.mix/read/encode and greedy/ctc_assist do.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import mlx.core as mx
import numpy as np
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from mlx_model import FORK, IndicSTT, fork_names, load_weights # noqa: E402
from text_norm import normalize # noqa: E402 (indic_stt/text.py's normalize, copied so no PyTorch is needed)
WEIGHTS = "stt2_model.mlx.safetensors"
WEIGHTS_8BIT = "stt2_model.mlx8.safetensors" # --weights: the 8-bit file (README.md)
SAMPLE_RATE = 16000
MODES = {"codemix": ("transducer_cm", "ctc_head_cm"), "tamil": ("transducer", "ctc_head")} # mode -> (RNN-T head, CTC head)
START_PENALTY = 1.0 # eval_dump.py --start-penalty 1 --start-cue, both modes
LAMBDA, BETA, BEAM = 0.3, 0.0, 4 # --ctc-assist: decode_ctc_assist.py --joint 0.3:0, beam 4
CTC_ASSIST_DEFAULT = True # the offline RNN-T default; same value as ../code/transcribe.py CTC_ASSIST_DEFAULT
CTC_ASSIST_DEFAULT_MODES = ("codemix",) # modes the default applies to; same as transcribe.py (tamil: greedy unless --ctc-assist)
def find_tokenizer(folder: Path) -> Path:
for p in (folder / "spm.model", folder.parent / "tokenizer" / "spm.model"):
if p.exists():
return p
raise SystemExit(f"spm.model not found in {folder} or {folder.parent / 'tokenizer'}")
def weights_path(folder: Path, weights: str | Path | None) -> Path:
"""None -> folder/WEIGHTS if it is there, else folder/WEIGHTS_8BIT; a bare file name -> folder/name if it is there; any other
path as given. A missing file exits with a message here (mx.load of a missing file ends the process without one)."""
if weights is None:
for name in (WEIGHTS, WEIGHTS_8BIT):
if (folder / name).is_file():
return folder / name
raise SystemExit(f"no weights file in {folder}: neither {WEIGHTS} (full size) nor {WEIGHTS_8BIT} (8-bit) is there; "
"put one of them in this folder, or pass --weights FILE")
p = Path(weights)
p = folder / p if p.name == str(weights) and (folder / p).is_file() else p
if not p.is_file():
raise SystemExit(f"weights file not found: {p} (this release's files: {WEIGHTS} or {WEIGHTS_8BIT}, in {folder})")
return p
class MlxModel:
def __init__(self, folder: Path = HERE, weights: str | Path | None = None):
"""weights: None = stt2_model.mlx.safetensors in `folder`, or WEIGHTS_8BIT if only that is there; WEIGHTS_8BIT (or a path
to it) = the 8-bit file; weights_path's rules, a missing file exits with a message. The layout,
tokenizer and checks are the same for both (load_weights returns the 8-bit file's tensors dequantized to float32)."""
import sentencepiece as sm
folder = Path(folder)
self.sp = sm.SentencePieceProcessor(model_file=str(find_tokenizer(folder)))
path = weights_path(folder, weights)
self.name = path.name
weights, self.quantization = load_weights(path)
layout = json.loads((folder / "layout.json").read_text(encoding="utf-8"))
have, want = fork_names(weights)
flag = layout.get("codemix_fork", False) is True
if bool(have) != flag:
raise SystemExit(f"{self.name}: {'holds' if have else 'has no'} {FORK}* tensors, but layout.json says codemix_fork = "
f"{layout.get('codemix_fork', 'absent')}: refused")
if flag and have != want:
raise SystemExit(f"{self.name}: codemix_fork needs all {len(want)} {FORK}* tensors: missing {len(want - have)}, "
f"unexpected {len(have - want)}")
heads = {h: any(k.startswith(f"{h}.") for k in weights) for h in MODES["codemix"]}
if flag and not all(heads.values()): # transcribe.py refuses the same file (R3 F2)
raise SystemExit(f"{self.name}: the code-mix fork without both code-mix heads {heads}: refused")
self.stack = {"codemix": FORK if flag else "", "tamil": ""} # mode -> post-tower stack (transcribe.py Release.forked)
self.m = IndicSTT(weights)
def text(self, ids: list[int]) -> str:
return self.sp.decode([i - 1 for i in ids if i > 0]).strip()
def encode(self, feats: np.ndarray, streaming: bool = False, mode: str | None = None):
"""Reader output (1, T, 768) of `mode`'s stack. mode is required on a fork file (None raises: the output does not record
which stack made it, so the decode could read the wrong heads; R3 B1); None on a file without the fork = the one stack."""
if mode is None:
if self.stack["codemix"]:
raise ValueError(f"{self.name} has the code-mix reader fork ({FORK}*): pass mode='codemix' or 'tamil' to encode, "
"and the same mode to ctc_text/rnnt_text/assist_text")
mode = "tamil"
return self.m.encode(mx.array(feats)[None], streaming, self.stack[mode])
def decode_mode(self, mode: str | None) -> str:
"""ctc_text/rnnt_text/assist_text's mode: required on a fork file, as for encode (None raises); None on a file without the
fork is "codemix", STT-1.2's default for these three."""
if mode is None:
if self.stack["codemix"]:
raise ValueError(f"{self.name} has the code-mix reader fork ({FORK}*): pass mode='codemix' or 'tamil' to "
"ctc_text/rnnt_text/assist_text, the same mode as to encode")
return "codemix"
return mode
def ctc_text(self, enc, mode: str | None = None) -> str:
best = np.array(mx.argmax(self.m.ctc_logits(enc, MODES[self.decode_mode(mode)][1])[0], axis=-1))
ids, prev = [], -1
for i in best.tolist():
if i != prev and i != 0:
ids.append(i)
prev = i
return self.text(ids)
def rnnt_text(self, enc, mode: str | None = None) -> str:
"""RNN-T greedy with the start penalty, gated by the start cue: from the first frame the mode's CTC head leaves blank."""
rnnt, ctc = MODES[self.decode_mode(mode)]
best = np.array(mx.argmax(self.m.ctc_logits(enc, ctc)[0], axis=-1))
cue = np.cumsum(best != 0) > 0
return self.text(self.m.rnnt_greedy(enc, rnnt, START_PENALTY, cue.tolist()))
def assist_text(self, enc, mode: str | None = None) -> str:
"""--ctc-assist: J(LAMBDA, BETA) on the bf16-rounded reader output (T, D)."""
rnnt, ctc = MODES[self.decode_mode(mode)]
e = enc[0].astype(mx.bfloat16).astype(mx.float32)
mx.eval(e)
return self.text(self.m.joint_rescore(e, rnnt, ctc, LAMBDA, BETA, BEAM)[0])
def transcribe(self, feats: np.ndarray, mode: str = "codemix", head: str = "rnnt", streaming: bool = False,
ctc_assist: bool = False) -> str:
if ctc_assist and (streaming or head != "rnnt"):
raise ValueError("--ctc-assist rescores the RNN-T head offline: drop --streaming and --head ctc")
enc = self.encode(feats, streaming, mode=mode)
text = self.assist_text(enc, mode) if ctc_assist else self.ctc_text(enc, mode) if head == "ctc" else self.rnnt_text(enc, mode)
return normalize(text) # as transcribe.py prints
def load_audio(path: str) -> np.ndarray:
"""Mono float32 at 16 kHz; other rates resampled with librosa, as the PyTorch runner does."""
import soundfile as sf
wav, sr = sf.read(path, dtype="float32", always_2d=True)
wav = wav.mean(axis=1)
if sr != SAMPLE_RATE:
import librosa
wav = librosa.resample(wav, orig_sr=sr, target_sr=SAMPLE_RATE)
return wav
def features(fe, wav: np.ndarray) -> np.ndarray:
o = fe([wav], sampling_rate=SAMPLE_RATE, return_tensors="np", truncation=False)
return o["input_features"][0][: int(o["input_features_mask"][0].sum())].astype(np.float32)
def main() -> None:
from transformers import AutoFeatureExtractor
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("audio", nargs="+")
ap.add_argument("--weights", help=f"{WEIGHTS} or {WEIGHTS_8BIT} (8-bit): a file name in this folder, or a path (default: "
f"{WEIGHTS} if this folder has it, else {WEIGHTS_8BIT})")
ap.add_argument("--mode", choices=list(MODES), default="codemix", help="codemix (default): code-mix heads; tamil: Tamil heads")
ap.add_argument("--streaming", action="store_true", help="the 480 ms block-causal mask (default: offline)")
ap.add_argument("--head", choices=["rnnt", "ctc"], default="rnnt")
ap.add_argument("--ctc-assist", action=argparse.BooleanOptionalAction, default=None,
help=f"offline RNN-T: J(0.3, 0) CTC-assisted rescoring (default {'on' if CTC_ASSIST_DEFAULT else 'off'} "
f"for --mode {', '.join(CTC_ASSIST_DEFAULT_MODES)}, off otherwise)")
ap.add_argument("--json", help="also write one JSON line per clip to this file")
a = ap.parse_args()
sys.stdout.reconfigure(encoding="utf-8") # Tamil text: Windows writes a redirected stdout in the ANSI code page otherwise
if a.ctc_assist and (a.streaming or a.head == "ctc"):
ap.error("--ctc-assist rescores the RNN-T head offline: drop --streaming and --head ctc")
assist = (a.ctc_assist if a.ctc_assist is not None
else CTC_ASSIST_DEFAULT and a.mode in CTC_ASSIST_DEFAULT_MODES and not a.streaming and a.head == "rnnt")
fe = AutoFeatureExtractor.from_pretrained(str(HERE))
model = MlxModel(HERE, a.weights)
out = open(a.json, "w", encoding="utf-8") if a.json else None
for clip in a.audio:
wav = load_audio(clip)
text = model.transcribe(features(fe, wav), a.mode, a.head, a.streaming, assist)
print(f"{clip}\t{text}", flush=True)
if out:
out.write(json.dumps({"clip": clip, "secs": round(len(wav) / SAMPLE_RATE, 2), "mode": a.mode, "head": a.head,
"streaming": a.streaming, "ctc_assist": assist, "hyp": text}, ensure_ascii=False) + "\n")
if out:
out.close()
if __name__ == "__main__":
main()
|