junafinity's picture
claude opus 5.5
VeriSafe-Tanglish-1-STT (STT-2)
ec01e2d
Raw History Blame Contribute Delete
12.2 kB
"""Run the MLX port of VeriSafe-Tanglish-STT-2 on audio files. No PyTorch needed.
python run_mlx.py CLIP.wav [CLIP2.mp3 ...] [--weights FILE] [--mode codemix|tamil] [--streaming] [--head rnnt|ctc]
[--ctc-assist | --no-ctc-assist] [--json OUT.jsonl]
The rows of ../code/transcribe.py, ported. Defaults (the code-mix recipe): --mode codemix; offline = RNN-T with --ctc-assist,
the J(0.3, 0) rescoring of the RNN-T beam-4 n-best and the CTC greedy hypothesis, on the reader output rounded to bf16 as the
evaluated decode read it; --streaming (480 ms block-causal mask) = RNN-T greedy with the start penalty 1 and the start cue from
the mode's CTC head. --no-ctc-assist gives offline greedy, --head ctc the CTC head, --mode tamil the Tamil heads (greedy by default also
offline; --ctc-assist opts in). The default is CTC_ASSIST_DEFAULT for CTC_ASSIST_DEFAULT_MODES below (keep both equal to
transcribe.py's). Reads stt2_model.mlx.safetensors, layout.json and preprocessor_config.json from this folder, and spm.model
from this folder or, if absent, ../tokenizer/. --weights stt2_model.mlx8.safetensors (a name in this folder, or a path) reads
the 8-bit file instead: mlx_model.load_weights dequantizes its 2-D weights to float32 once on load, the rest is unchanged.
Without --weights: stt2_model.mlx.safetensors if this folder has it, else stt2_model.mlx8.safetensors (a copy that ships only
the 8-bit file); neither, or a --weights file that does not exist, exits with a message naming the files.
STT-2: --mode codemix reads the code-mix reader fork (fork_cm.*, mlx_model.encode(stack=FORK)), --mode tamil the Tamil stack, as
transcribe.py does. layout.json's "codemix_fork": true requires every fork_cm.* tensor and both code-mix heads; fork_cm.* tensors
without it are refused. On a fork file MlxModel.encode and ctc_text/rnnt_text/assist_text need mode= (None raises), as
transcribe.py's Release.mix/read/encode and greedy/ctc_assist do.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import mlx.core as mx
import numpy as np
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from mlx_model import FORK, IndicSTT, fork_names, load_weights # noqa: E402
from text_norm import normalize # noqa: E402 (indic_stt/text.py's normalize, copied so no PyTorch is needed)
WEIGHTS = "stt2_model.mlx.safetensors"
WEIGHTS_8BIT = "stt2_model.mlx8.safetensors" # --weights: the 8-bit file (README.md)
SAMPLE_RATE = 16000
MODES = {"codemix": ("transducer_cm", "ctc_head_cm"), "tamil": ("transducer", "ctc_head")} # mode -> (RNN-T head, CTC head)
START_PENALTY = 1.0 # eval_dump.py --start-penalty 1 --start-cue, both modes
LAMBDA, BETA, BEAM = 0.3, 0.0, 4 # --ctc-assist: decode_ctc_assist.py --joint 0.3:0, beam 4
CTC_ASSIST_DEFAULT = True # the offline RNN-T default; same value as ../code/transcribe.py CTC_ASSIST_DEFAULT
CTC_ASSIST_DEFAULT_MODES = ("codemix",) # modes the default applies to; same as transcribe.py (tamil: greedy unless --ctc-assist)
def find_tokenizer(folder: Path) -> Path:
for p in (folder / "spm.model", folder.parent / "tokenizer" / "spm.model"):
if p.exists():
return p
raise SystemExit(f"spm.model not found in {folder} or {folder.parent / 'tokenizer'}")
def weights_path(folder: Path, weights: str | Path | None) -> Path:
"""None -> folder/WEIGHTS if it is there, else folder/WEIGHTS_8BIT; a bare file name -> folder/name if it is there; any other
path as given. A missing file exits with a message here (mx.load of a missing file ends the process without one)."""
if weights is None:
for name in (WEIGHTS, WEIGHTS_8BIT):
if (folder / name).is_file():
return folder / name
raise SystemExit(f"no weights file in {folder}: neither {WEIGHTS} (full size) nor {WEIGHTS_8BIT} (8-bit) is there; "
"put one of them in this folder, or pass --weights FILE")
p = Path(weights)
p = folder / p if p.name == str(weights) and (folder / p).is_file() else p
if not p.is_file():
raise SystemExit(f"weights file not found: {p} (this release's files: {WEIGHTS} or {WEIGHTS_8BIT}, in {folder})")
return p
class MlxModel:
def __init__(self, folder: Path = HERE, weights: str | Path | None = None):
"""weights: None = stt2_model.mlx.safetensors in `folder`, or WEIGHTS_8BIT if only that is there; WEIGHTS_8BIT (or a path
to it) = the 8-bit file; weights_path's rules, a missing file exits with a message. The layout,
tokenizer and checks are the same for both (load_weights returns the 8-bit file's tensors dequantized to float32)."""
import sentencepiece as sm
folder = Path(folder)
self.sp = sm.SentencePieceProcessor(model_file=str(find_tokenizer(folder)))
path = weights_path(folder, weights)
self.name = path.name
weights, self.quantization = load_weights(path)
layout = json.loads((folder / "layout.json").read_text(encoding="utf-8"))
have, want = fork_names(weights)
flag = layout.get("codemix_fork", False) is True
if bool(have) != flag:
raise SystemExit(f"{self.name}: {'holds' if have else 'has no'} {FORK}* tensors, but layout.json says codemix_fork = "
f"{layout.get('codemix_fork', 'absent')}: refused")
if flag and have != want:
raise SystemExit(f"{self.name}: codemix_fork needs all {len(want)} {FORK}* tensors: missing {len(want - have)}, "
f"unexpected {len(have - want)}")
heads = {h: any(k.startswith(f"{h}.") for k in weights) for h in MODES["codemix"]}
if flag and not all(heads.values()): # transcribe.py refuses the same file (R3 F2)
raise SystemExit(f"{self.name}: the code-mix fork without both code-mix heads {heads}: refused")
self.stack = {"codemix": FORK if flag else "", "tamil": ""} # mode -> post-tower stack (transcribe.py Release.forked)
self.m = IndicSTT(weights)
def text(self, ids: list[int]) -> str:
return self.sp.decode([i - 1 for i in ids if i > 0]).strip()
def encode(self, feats: np.ndarray, streaming: bool = False, mode: str | None = None):
"""Reader output (1, T, 768) of `mode`'s stack. mode is required on a fork file (None raises: the output does not record
which stack made it, so the decode could read the wrong heads; R3 B1); None on a file without the fork = the one stack."""
if mode is None:
if self.stack["codemix"]:
raise ValueError(f"{self.name} has the code-mix reader fork ({FORK}*): pass mode='codemix' or 'tamil' to encode, "
"and the same mode to ctc_text/rnnt_text/assist_text")
mode = "tamil"
return self.m.encode(mx.array(feats)[None], streaming, self.stack[mode])
def decode_mode(self, mode: str | None) -> str:
"""ctc_text/rnnt_text/assist_text's mode: required on a fork file, as for encode (None raises); None on a file without the
fork is "codemix", STT-1.2's default for these three."""
if mode is None:
if self.stack["codemix"]:
raise ValueError(f"{self.name} has the code-mix reader fork ({FORK}*): pass mode='codemix' or 'tamil' to "
"ctc_text/rnnt_text/assist_text, the same mode as to encode")
return "codemix"
return mode
def ctc_text(self, enc, mode: str | None = None) -> str:
best = np.array(mx.argmax(self.m.ctc_logits(enc, MODES[self.decode_mode(mode)][1])[0], axis=-1))
ids, prev = [], -1
for i in best.tolist():
if i != prev and i != 0:
ids.append(i)
prev = i
return self.text(ids)
def rnnt_text(self, enc, mode: str | None = None) -> str:
"""RNN-T greedy with the start penalty, gated by the start cue: from the first frame the mode's CTC head leaves blank."""
rnnt, ctc = MODES[self.decode_mode(mode)]
best = np.array(mx.argmax(self.m.ctc_logits(enc, ctc)[0], axis=-1))
cue = np.cumsum(best != 0) > 0
return self.text(self.m.rnnt_greedy(enc, rnnt, START_PENALTY, cue.tolist()))
def assist_text(self, enc, mode: str | None = None) -> str:
"""--ctc-assist: J(LAMBDA, BETA) on the bf16-rounded reader output (T, D)."""
rnnt, ctc = MODES[self.decode_mode(mode)]
e = enc[0].astype(mx.bfloat16).astype(mx.float32)
mx.eval(e)
return self.text(self.m.joint_rescore(e, rnnt, ctc, LAMBDA, BETA, BEAM)[0])
def transcribe(self, feats: np.ndarray, mode: str = "codemix", head: str = "rnnt", streaming: bool = False,
ctc_assist: bool = False) -> str:
if ctc_assist and (streaming or head != "rnnt"):
raise ValueError("--ctc-assist rescores the RNN-T head offline: drop --streaming and --head ctc")
enc = self.encode(feats, streaming, mode=mode)
text = self.assist_text(enc, mode) if ctc_assist else self.ctc_text(enc, mode) if head == "ctc" else self.rnnt_text(enc, mode)
return normalize(text) # as transcribe.py prints
def load_audio(path: str) -> np.ndarray:
"""Mono float32 at 16 kHz; other rates resampled with librosa, as the PyTorch runner does."""
import soundfile as sf
wav, sr = sf.read(path, dtype="float32", always_2d=True)
wav = wav.mean(axis=1)
if sr != SAMPLE_RATE:
import librosa
wav = librosa.resample(wav, orig_sr=sr, target_sr=SAMPLE_RATE)
return wav
def features(fe, wav: np.ndarray) -> np.ndarray:
o = fe([wav], sampling_rate=SAMPLE_RATE, return_tensors="np", truncation=False)
return o["input_features"][0][: int(o["input_features_mask"][0].sum())].astype(np.float32)
def main() -> None:
from transformers import AutoFeatureExtractor
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("audio", nargs="+")
ap.add_argument("--weights", help=f"{WEIGHTS} or {WEIGHTS_8BIT} (8-bit): a file name in this folder, or a path (default: "
f"{WEIGHTS} if this folder has it, else {WEIGHTS_8BIT})")
ap.add_argument("--mode", choices=list(MODES), default="codemix", help="codemix (default): code-mix heads; tamil: Tamil heads")
ap.add_argument("--streaming", action="store_true", help="the 480 ms block-causal mask (default: offline)")
ap.add_argument("--head", choices=["rnnt", "ctc"], default="rnnt")
ap.add_argument("--ctc-assist", action=argparse.BooleanOptionalAction, default=None,
help=f"offline RNN-T: J(0.3, 0) CTC-assisted rescoring (default {'on' if CTC_ASSIST_DEFAULT else 'off'} "
f"for --mode {', '.join(CTC_ASSIST_DEFAULT_MODES)}, off otherwise)")
ap.add_argument("--json", help="also write one JSON line per clip to this file")
a = ap.parse_args()
sys.stdout.reconfigure(encoding="utf-8") # Tamil text: Windows writes a redirected stdout in the ANSI code page otherwise
if a.ctc_assist and (a.streaming or a.head == "ctc"):
ap.error("--ctc-assist rescores the RNN-T head offline: drop --streaming and --head ctc")
assist = (a.ctc_assist if a.ctc_assist is not None
else CTC_ASSIST_DEFAULT and a.mode in CTC_ASSIST_DEFAULT_MODES and not a.streaming and a.head == "rnnt")
fe = AutoFeatureExtractor.from_pretrained(str(HERE))
model = MlxModel(HERE, a.weights)
out = open(a.json, "w", encoding="utf-8") if a.json else None
for clip in a.audio:
wav = load_audio(clip)
text = model.transcribe(features(fe, wav), a.mode, a.head, a.streaming, assist)
print(f"{clip}\t{text}", flush=True)
if out:
out.write(json.dumps({"clip": clip, "secs": round(len(wav) / SAMPLE_RATE, 2), "mode": a.mode, "head": a.head,
"streaming": a.streaming, "ctc_assist": assist, "hyp": text}, ensure_ascii=False) + "\n")
if out:
out.close()
if __name__ == "__main__":
main()