Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download code/export/run_mlx.py from junafinity/VeriSafe-Tanglish-1-STT: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/export/run_mlx.py
- Command line
-
hf download hf://junafinity/VeriSafe-Tanglish-1-STT/code/export/run_mlx.py
-
curl -L -o run_mlx.py https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/export/run_mlx.py
12.2 kB
| """Run the MLX port of VeriSafe-Tanglish-STT-2 on audio files. No PyTorch needed. | |
| python run_mlx.py CLIP.wav [CLIP2.mp3 ...] [--weights FILE] [--mode codemix|tamil] [--streaming] [--head rnnt|ctc] | |
| [--ctc-assist | --no-ctc-assist] [--json OUT.jsonl] | |
| The rows of ../code/transcribe.py, ported. Defaults (the code-mix recipe): --mode codemix; offline = RNN-T with --ctc-assist, | |
| the J(0.3, 0) rescoring of the RNN-T beam-4 n-best and the CTC greedy hypothesis, on the reader output rounded to bf16 as the | |
| evaluated decode read it; --streaming (480 ms block-causal mask) = RNN-T greedy with the start penalty 1 and the start cue from | |
| the mode's CTC head. --no-ctc-assist gives offline greedy, --head ctc the CTC head, --mode tamil the Tamil heads (greedy by default also | |
| offline; --ctc-assist opts in). The default is CTC_ASSIST_DEFAULT for CTC_ASSIST_DEFAULT_MODES below (keep both equal to | |
| transcribe.py's). Reads stt2_model.mlx.safetensors, layout.json and preprocessor_config.json from this folder, and spm.model | |
| from this folder or, if absent, ../tokenizer/. --weights stt2_model.mlx8.safetensors (a name in this folder, or a path) reads | |
| the 8-bit file instead: mlx_model.load_weights dequantizes its 2-D weights to float32 once on load, the rest is unchanged. | |
| Without --weights: stt2_model.mlx.safetensors if this folder has it, else stt2_model.mlx8.safetensors (a copy that ships only | |
| the 8-bit file); neither, or a --weights file that does not exist, exits with a message naming the files. | |
| STT-2: --mode codemix reads the code-mix reader fork (fork_cm.*, mlx_model.encode(stack=FORK)), --mode tamil the Tamil stack, as | |
| transcribe.py does. layout.json's "codemix_fork": true requires every fork_cm.* tensor and both code-mix heads; fork_cm.* tensors | |
| without it are refused. On a fork file MlxModel.encode and ctc_text/rnnt_text/assist_text need mode= (None raises), as | |
| transcribe.py's Release.mix/read/encode and greedy/ctc_assist do. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import mlx.core as mx | |
| import numpy as np | |
| HERE = Path(__file__).resolve().parent | |
| sys.path.insert(0, str(HERE)) | |
| from mlx_model import FORK, IndicSTT, fork_names, load_weights # noqa: E402 | |
| from text_norm import normalize # noqa: E402 (indic_stt/text.py's normalize, copied so no PyTorch is needed) | |
| WEIGHTS = "stt2_model.mlx.safetensors" | |
| WEIGHTS_8BIT = "stt2_model.mlx8.safetensors" # --weights: the 8-bit file (README.md) | |
| SAMPLE_RATE = 16000 | |
| MODES = {"codemix": ("transducer_cm", "ctc_head_cm"), "tamil": ("transducer", "ctc_head")} # mode -> (RNN-T head, CTC head) | |
| START_PENALTY = 1.0 # eval_dump.py --start-penalty 1 --start-cue, both modes | |
| LAMBDA, BETA, BEAM = 0.3, 0.0, 4 # --ctc-assist: decode_ctc_assist.py --joint 0.3:0, beam 4 | |
| CTC_ASSIST_DEFAULT = True # the offline RNN-T default; same value as ../code/transcribe.py CTC_ASSIST_DEFAULT | |
| CTC_ASSIST_DEFAULT_MODES = ("codemix",) # modes the default applies to; same as transcribe.py (tamil: greedy unless --ctc-assist) | |
| def find_tokenizer(folder: Path) -> Path: | |
| for p in (folder / "spm.model", folder.parent / "tokenizer" / "spm.model"): | |
| if p.exists(): | |
| return p | |
| raise SystemExit(f"spm.model not found in {folder} or {folder.parent / 'tokenizer'}") | |
| def weights_path(folder: Path, weights: str | Path | None) -> Path: | |
| """None -> folder/WEIGHTS if it is there, else folder/WEIGHTS_8BIT; a bare file name -> folder/name if it is there; any other | |
| path as given. A missing file exits with a message here (mx.load of a missing file ends the process without one).""" | |
| if weights is None: | |
| for name in (WEIGHTS, WEIGHTS_8BIT): | |
| if (folder / name).is_file(): | |
| return folder / name | |
| raise SystemExit(f"no weights file in {folder}: neither {WEIGHTS} (full size) nor {WEIGHTS_8BIT} (8-bit) is there; " | |
| "put one of them in this folder, or pass --weights FILE") | |
| p = Path(weights) | |
| p = folder / p if p.name == str(weights) and (folder / p).is_file() else p | |
| if not p.is_file(): | |
| raise SystemExit(f"weights file not found: {p} (this release's files: {WEIGHTS} or {WEIGHTS_8BIT}, in {folder})") | |
| return p | |
| class MlxModel: | |
| def __init__(self, folder: Path = HERE, weights: str | Path | None = None): | |
| """weights: None = stt2_model.mlx.safetensors in `folder`, or WEIGHTS_8BIT if only that is there; WEIGHTS_8BIT (or a path | |
| to it) = the 8-bit file; weights_path's rules, a missing file exits with a message. The layout, | |
| tokenizer and checks are the same for both (load_weights returns the 8-bit file's tensors dequantized to float32).""" | |
| import sentencepiece as sm | |
| folder = Path(folder) | |
| self.sp = sm.SentencePieceProcessor(model_file=str(find_tokenizer(folder))) | |
| path = weights_path(folder, weights) | |
| self.name = path.name | |
| weights, self.quantization = load_weights(path) | |
| layout = json.loads((folder / "layout.json").read_text(encoding="utf-8")) | |
| have, want = fork_names(weights) | |
| flag = layout.get("codemix_fork", False) is True | |
| if bool(have) != flag: | |
| raise SystemExit(f"{self.name}: {'holds' if have else 'has no'} {FORK}* tensors, but layout.json says codemix_fork = " | |
| f"{layout.get('codemix_fork', 'absent')}: refused") | |
| if flag and have != want: | |
| raise SystemExit(f"{self.name}: codemix_fork needs all {len(want)} {FORK}* tensors: missing {len(want - have)}, " | |
| f"unexpected {len(have - want)}") | |
| heads = {h: any(k.startswith(f"{h}.") for k in weights) for h in MODES["codemix"]} | |
| if flag and not all(heads.values()): # transcribe.py refuses the same file (R3 F2) | |
| raise SystemExit(f"{self.name}: the code-mix fork without both code-mix heads {heads}: refused") | |
| self.stack = {"codemix": FORK if flag else "", "tamil": ""} # mode -> post-tower stack (transcribe.py Release.forked) | |
| self.m = IndicSTT(weights) | |
| def text(self, ids: list[int]) -> str: | |
| return self.sp.decode([i - 1 for i in ids if i > 0]).strip() | |
| def encode(self, feats: np.ndarray, streaming: bool = False, mode: str | None = None): | |
| """Reader output (1, T, 768) of `mode`'s stack. mode is required on a fork file (None raises: the output does not record | |
| which stack made it, so the decode could read the wrong heads; R3 B1); None on a file without the fork = the one stack.""" | |
| if mode is None: | |
| if self.stack["codemix"]: | |
| raise ValueError(f"{self.name} has the code-mix reader fork ({FORK}*): pass mode='codemix' or 'tamil' to encode, " | |
| "and the same mode to ctc_text/rnnt_text/assist_text") | |
| mode = "tamil" | |
| return self.m.encode(mx.array(feats)[None], streaming, self.stack[mode]) | |
| def decode_mode(self, mode: str | None) -> str: | |
| """ctc_text/rnnt_text/assist_text's mode: required on a fork file, as for encode (None raises); None on a file without the | |
| fork is "codemix", STT-1.2's default for these three.""" | |
| if mode is None: | |
| if self.stack["codemix"]: | |
| raise ValueError(f"{self.name} has the code-mix reader fork ({FORK}*): pass mode='codemix' or 'tamil' to " | |
| "ctc_text/rnnt_text/assist_text, the same mode as to encode") | |
| return "codemix" | |
| return mode | |
| def ctc_text(self, enc, mode: str | None = None) -> str: | |
| best = np.array(mx.argmax(self.m.ctc_logits(enc, MODES[self.decode_mode(mode)][1])[0], axis=-1)) | |
| ids, prev = [], -1 | |
| for i in best.tolist(): | |
| if i != prev and i != 0: | |
| ids.append(i) | |
| prev = i | |
| return self.text(ids) | |
| def rnnt_text(self, enc, mode: str | None = None) -> str: | |
| """RNN-T greedy with the start penalty, gated by the start cue: from the first frame the mode's CTC head leaves blank.""" | |
| rnnt, ctc = MODES[self.decode_mode(mode)] | |
| best = np.array(mx.argmax(self.m.ctc_logits(enc, ctc)[0], axis=-1)) | |
| cue = np.cumsum(best != 0) > 0 | |
| return self.text(self.m.rnnt_greedy(enc, rnnt, START_PENALTY, cue.tolist())) | |
| def assist_text(self, enc, mode: str | None = None) -> str: | |
| """--ctc-assist: J(LAMBDA, BETA) on the bf16-rounded reader output (T, D).""" | |
| rnnt, ctc = MODES[self.decode_mode(mode)] | |
| e = enc[0].astype(mx.bfloat16).astype(mx.float32) | |
| mx.eval(e) | |
| return self.text(self.m.joint_rescore(e, rnnt, ctc, LAMBDA, BETA, BEAM)[0]) | |
| def transcribe(self, feats: np.ndarray, mode: str = "codemix", head: str = "rnnt", streaming: bool = False, | |
| ctc_assist: bool = False) -> str: | |
| if ctc_assist and (streaming or head != "rnnt"): | |
| raise ValueError("--ctc-assist rescores the RNN-T head offline: drop --streaming and --head ctc") | |
| enc = self.encode(feats, streaming, mode=mode) | |
| text = self.assist_text(enc, mode) if ctc_assist else self.ctc_text(enc, mode) if head == "ctc" else self.rnnt_text(enc, mode) | |
| return normalize(text) # as transcribe.py prints | |
| def load_audio(path: str) -> np.ndarray: | |
| """Mono float32 at 16 kHz; other rates resampled with librosa, as the PyTorch runner does.""" | |
| import soundfile as sf | |
| wav, sr = sf.read(path, dtype="float32", always_2d=True) | |
| wav = wav.mean(axis=1) | |
| if sr != SAMPLE_RATE: | |
| import librosa | |
| wav = librosa.resample(wav, orig_sr=sr, target_sr=SAMPLE_RATE) | |
| return wav | |
| def features(fe, wav: np.ndarray) -> np.ndarray: | |
| o = fe([wav], sampling_rate=SAMPLE_RATE, return_tensors="np", truncation=False) | |
| return o["input_features"][0][: int(o["input_features_mask"][0].sum())].astype(np.float32) | |
| def main() -> None: | |
| from transformers import AutoFeatureExtractor | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("audio", nargs="+") | |
| ap.add_argument("--weights", help=f"{WEIGHTS} or {WEIGHTS_8BIT} (8-bit): a file name in this folder, or a path (default: " | |
| f"{WEIGHTS} if this folder has it, else {WEIGHTS_8BIT})") | |
| ap.add_argument("--mode", choices=list(MODES), default="codemix", help="codemix (default): code-mix heads; tamil: Tamil heads") | |
| ap.add_argument("--streaming", action="store_true", help="the 480 ms block-causal mask (default: offline)") | |
| ap.add_argument("--head", choices=["rnnt", "ctc"], default="rnnt") | |
| ap.add_argument("--ctc-assist", action=argparse.BooleanOptionalAction, default=None, | |
| help=f"offline RNN-T: J(0.3, 0) CTC-assisted rescoring (default {'on' if CTC_ASSIST_DEFAULT else 'off'} " | |
| f"for --mode {', '.join(CTC_ASSIST_DEFAULT_MODES)}, off otherwise)") | |
| ap.add_argument("--json", help="also write one JSON line per clip to this file") | |
| a = ap.parse_args() | |
| sys.stdout.reconfigure(encoding="utf-8") # Tamil text: Windows writes a redirected stdout in the ANSI code page otherwise | |
| if a.ctc_assist and (a.streaming or a.head == "ctc"): | |
| ap.error("--ctc-assist rescores the RNN-T head offline: drop --streaming and --head ctc") | |
| assist = (a.ctc_assist if a.ctc_assist is not None | |
| else CTC_ASSIST_DEFAULT and a.mode in CTC_ASSIST_DEFAULT_MODES and not a.streaming and a.head == "rnnt") | |
| fe = AutoFeatureExtractor.from_pretrained(str(HERE)) | |
| model = MlxModel(HERE, a.weights) | |
| out = open(a.json, "w", encoding="utf-8") if a.json else None | |
| for clip in a.audio: | |
| wav = load_audio(clip) | |
| text = model.transcribe(features(fe, wav), a.mode, a.head, a.streaming, assist) | |
| print(f"{clip}\t{text}", flush=True) | |
| if out: | |
| out.write(json.dumps({"clip": clip, "secs": round(len(wav) / SAMPLE_RATE, 2), "mode": a.mode, "head": a.head, | |
| "streaming": a.streaming, "ctc_assist": assist, "hyp": text}, ensure_ascii=False) + "\n") | |
| if out: | |
| out.close() | |
| if __name__ == "__main__": | |
| main() | |