VeriSafe-Tanglish-1-STT / code /export /export_release.py
junafinity's picture
claude opus 5.5
VeriSafe-Tanglish-1-STT (STT-2)
ec01e2d
Raw History Blame Contribute Delete
19 kB
"""Build the VeriSafe-Tanglish-STT-2 release formats from a push-8 fork candidate, and verify each against the one before it.
python -X utf8 export_release.py safetensors CANDIDATE.pt TOWER_REST.safetensors RELEASE_DIR STT1.safetensors
python -X utf8 export_release.py gguf RELEASE_DIR f32|f16|q8_0
python -X utf8 export_release.py mlx RELEASE_DIR
python -X utf8 export_release.py mlx8 RELEASE_DIR
python -X utf8 export_release.py check-stt1 RELEASE_DIR STT1.safetensors
safetensors: the candidate's 457 tensors (the project's scripts/build_fork.py output: cm-main ema.pt's 363 non-code-mix tensors,
step 84,000, unchanged = the tower and the Tamil stack and heads; the 13 code-mix head tensors transducer_cm + ctc_head_cm and
the code-mix reader fork, 81 fork_cm.* tensors, from the push-8 run) + the 482 tower tensors training never saved
(EmbeddingGemma 2's 480 clip bounds and output_proj, from extract_tower_rest.py) -> safetensors/stt2_model.safetensors,
939 float32 tensors, plus safetensors/stt_config.json (the settings transcribe.py builds the model with; "codemix_fork": true
makes transcribe.py require every fork_cm.* tensor). STT1.safetensors (STT-1's file) is required: check-stt1 runs after the
write (without it the command exits non-zero before writing anything).
gguf: the safetensors file -> gguf/stt2_model.FTYPE.gguf through indic_stt/gguf_io.py (tokenizer and configs inside).
f32 is checked byte-identical to the safetensors; f16 and q8_0 are checked for names, shapes and largest error.
mlx: the safetensors file -> mlx/stt2_model.mlx.safetensors with the 16 conv kernels (15 + the fork's position conv) in MLX's
channels-last layout, plus the port (mlx_model.py), its runner (run_mlx.py), text_norm.py and preprocessor_config.json;
layout.json records "codemix_fork". Checked byte-identical after the documented layout change. Writing it needs numpy and
safetensors, not MLX.
mlx8: mlx/stt2_model.mlx.safetensors -> mlx/stt2_model.mlx8.safetensors (needs MLX): the 198 2-D tensors (every matmul weight,
both embeddings; gguf_io's q8_0 rule with MLX's group of 64: 2-D, last dim a multiple of 64) quantized with mx.quantize,
affine, 8 bits, group 64: the packed uint32 under the tensor's name, float32 scales and biases as <stem>.scales and
<stem>.biases (mlx_model.quant_stem); the 16 conv kernels (MLX layout, as in the full file) and the 725 1-D tensors and
scalars float32, unchanged. Metadata "quantization" (mlx_model.QUANT) says so; layout.json applies unchanged. Checked: every
unquantized tensor byte-identical to the full MLX file, every quantized one equal to a fresh mx.quantize, and
mlx_model.load_weights reads the file back to the 939 names and shapes, float32.
check-stt1: every tensor of RELEASE_DIR's safetensors outside the code-mix heads and fork_cm.* is byte-equal (name, dtype, shape,
bytes) to STT-1's, and the file holds exactly STT-1's names plus the 81 fork_cm.* (push-8 rule T-EQ, second half).
"""
import hashlib
import json
import re
import shutil
import sys
from pathlib import Path
import numpy as np
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent))
NAME = "stt2_model"
ST_NAME = f"{NAME}.safetensors"
MLX_NAME = f"{NAME}.mlx.safetensors"
MLX8_NAME = f"{NAME}.mlx8.safetensors"
MLX8_BITS, MLX8_GROUP = 8, 64
BASE_STEP = 84000 # cm-main ema.pt: the tower, the Tamil stack and the Tamil heads
BASE_SHA256 = "2e1dec78d078a2e04b622b1e545fadadbeb4a18e9e945c4c0931327b0ebf4b49" # cm-main ema.pt (build_fork.py refuses any other)
FORK_TENSORS = 81 # fork_cm.*: layer_logits, in_norm, in_proj, position, 6 reader layers, out_norm
TENSORS = 858 + FORK_TENSORS
RUN = "cm-main"
CODEMIX = ("transducer_cm.", "ctc_head_cm.")
CONV2D = re.compile(r"tower\.subsample_conv_projection\.layer\d\.conv\.weight")
CONV1D = re.compile(r"tower\.layers\.\d+\.lconv1d\.depthwise_conv1d\.weight|(fork_cm\.)?position\.conv\.weight")
def sha256(p: Path) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
while b := f.read(1 << 24):
h.update(b)
return h.hexdigest()
def cmd_safetensors(cand: str, rest: str, rel: str, stt1: str | None = None) -> None:
if not stt1 or not Path(stt1).is_file(): # R3 F1: T-EQ's byte half is not optional; refuse before writing anything
raise SystemExit(f"safetensors needs STT1.safetensors (STT-1's file) as its 5th argument, for check-stt1 (T-EQ byte half); "
f"got {stt1!r}: T-EQ byte half NOT run, nothing written")
import torch
from safetensors import safe_open
from safetensors.torch import load_file, save_file
cand, rel = Path(cand), Path(rel)
ck = torch.load(cand, map_location="cpu", weights_only=False, mmap=True)
state = dict(ck["model"])
cfg = ck["cfg"]
prov = cfg.get("codemix_fork")
assert isinstance(prov, dict), f"{cand.name}: no cfg['codemix_fork'] provenance: not a scripts/build_fork.py candidate"
assert prov["base_sha256"] == BASE_SHA256 and prov["base_step"] == BASE_STEP, (prov["base_sha256"], prov["base_step"])
from indic_stt.model import FORK, FORK_MODULES, codemix_fork_in, codemix_heads_in
assert codemix_fork_in(state), f"{cand.name} holds no {FORK}.* tensor"
fork = {k for k in state if k.startswith(FORK + ".")}
want = {f"{FORK}.{k}" for k in state if k.split(".")[0] in FORK_MODULES} # the fork copies the stack: the same names
assert fork == want and len(fork) == FORK_TENSORS, (len(fork), sorted(fork ^ want)[:5])
assert all(state[k].shape == state[k[len(FORK) + 1:]].shape for k in fork), "a fork tensor's shape differs from its twin's"
more = load_file(rest)
assert not set(state) & set(more), "the two sources overlap"
state.update(more)
assert len(state) == TENSORS and all(v.dtype == torch.float32 for v in state.values()), len(state)
heads = codemix_heads_in(state)
assert all(heads.values()), heads
digest = sha256(cand)
src_file = re.split(r"[\\/]", prov["src_ckpt"])[-1]
src = f"{prov['src_run']}/{src_file}" # run/file only: no local paths in the release
fork_info = {"kind": prov["kind"], "blend_alpha": prov["blend_alpha"], "base": f"{RUN}/ema.pt", "base_step": prov["base_step"],
"base_sha256": prov["base_sha256"], "src": src, "src_step": prov["src_step"], "src_sha256": prov["src_sha256"]}
out = rel / "safetensors" / ST_NAME
out.parent.mkdir(parents=True, exist_ok=True)
save_file({k: v.contiguous() for k, v in state.items()}, str(out), metadata={
"format": "pt", "source": f"{cand.name} (push-8 fork candidate, step {ck['step']}) sha256 {digest}",
"fork": json.dumps(fork_info, sort_keys=True),
"tower_rest": "google/embeddinggemma-2 snapshot f9972ac22e696a105c482202646979e33e4b5a1e: 480 clip bounds + output_proj"})
stt = {"vocab_size": int(state["ctc_head.weight"].shape[0]), "train_tower_layers": cfg["train_tower_layers"],
"reader_layers": cfg["reader_layers"], "stream_chunk_frames": cfg["stream_chunk_frames"], "transducer": cfg["transducer"],
"codemix_heads": heads, "codemix_fork": True, "blank_id": 0, "sample_rate": 16000, "step": ck["step"],
"checkpoint": cand.name, "checkpoint_sha256": digest, "fork": fork_info}
(out.parent / "stt_config.json").write_text(json.dumps(stt, indent=1) + "\n", encoding="utf-8", newline="\n")
with safe_open(str(out), "pt") as f:
names = set(f.keys())
assert names == set(state), "names differ"
bad = [k for k in names if not torch.equal(f.get_tensor(k), state[k])]
assert not bad, f"{len(bad)} tensors differ: {bad[:5]}"
print(f"{cand.name} sha256 {digest}; {len(state)} tensors ({len(fork)} {FORK}.*), {sum(v.numel() for v in state.values())} "
f"values -> {out} ({out.stat().st_size} bytes); read back equal; stt_config {stt}")
cmd_check_stt1(str(rel), stt1) # always (checked above): an assert here exits non-zero
def cmd_check_stt1(rel: str, stt1: str) -> None:
"""T-EQ, second half: every tensor outside the code-mix heads and fork_cm.* byte-equal to STT-1's safetensors."""
from safetensors import safe_open
out = Path(rel) / "safetensors" / ST_NAME
with safe_open(str(out), "np") as f, safe_open(stt1, "np") as g:
mine, ref = set(f.keys()), set(g.keys())
fork = {k for k in mine if k.startswith("fork_cm.")}
assert mine - fork == ref, f"names: {sorted((mine - fork) ^ ref)[:5]}"
assert len(fork) == FORK_TENSORS, len(fork)
same = [k for k in sorted(ref) if not k.startswith(CODEMIX)]
bad = []
for k in same:
a, b = f.get_tensor(k), g.get_tensor(k)
if a.dtype != b.dtype or a.shape != b.shape or a.tobytes() != b.tobytes():
bad.append(k)
cm_equal = sum(f.get_tensor(k).tobytes() == g.get_tensor(k).tobytes() for k in ref if k.startswith(CODEMIX))
assert not bad, f"{len(bad)} of {len(same)} tensors differ from STT-1's: {bad[:5]}"
print(f"check-stt1: {len(same)} of {len(same)} tensors outside the code-mix heads and fork_cm.* byte-equal to {Path(stt1).name}; "
f"names = STT-1's {len(ref)} + {len(fork)} fork_cm.*; code-mix head tensors equal to STT-1's: {cm_equal} of "
f"{len(ref) - len(same)} (information only)")
def cmd_gguf(rel: str, ftype: str) -> None:
from safetensors import safe_open
from safetensors.numpy import load_file
from indic_stt.gguf_io import read_gguf, write_gguf
rel = Path(rel)
src = rel / "safetensors" / ST_NAME
stt = json.loads((rel / "safetensors" / "stt_config.json").read_text(encoding="utf-8"))
with safe_open(str(src), "np") as f:
source = f.metadata()["source"]
state = load_file(str(src))
out = rel / "gguf" / f"{NAME}.{ftype}.gguf"
out.parent.mkdir(exist_ok=True)
meta = {"name": f"VeriSafe-Tanglish-STT-2 ({stt['checkpoint']}, step {stt['step']}), {ftype}", "step": stt["step"], "source": source,
"description": "Tamil + English streaming STT: EmbeddingGemma 2 audio tower + two post-tower stacks (layer mix + 6-layer "
"reader each: Tamil, and the code-mix fork fork_cm) + two head sets, each CTC + RNN-T (Tamil: ctc_head, "
"transducer; code-mix: ctc_head_cm, transducer_cm, which read the fork). Custom architecture: runs only with this release's own runner (code/transcribe.py, PyTorch); "
"no llama.cpp or whisper.cpp graph exists for it.",
"stt_config": {k: stt[k] for k in ("vocab_size", "train_tower_layers", "reader_layers", "stream_chunk_frames", "transducer",
"codemix_heads", "codemix_fork")},
"eg2_config": (rel / "safetensors" / "eg2" / "config.json").read_text(encoding="utf-8"),
"preprocessor_config": (rel / "safetensors" / "eg2" / "preprocessor_config.json").read_text(encoding="utf-8"),
"spm_model": (rel / "tokenizer" / "spm.model").read_bytes()}
counts = write_gguf(state, out, ftype, meta)
back, m = read_gguf(out)
assert set(back) == set(state), f"names differ: {sorted(set(back) ^ set(state))[:5]}"
assert all(back[k].shape == state[k].shape for k in state), "shapes differ"
assert m["spm_model"] == meta["spm_model"] and m["eg2_config"] == meta["eg2_config"] and m["preprocessor_config"] == meta["preprocessor_config"]
assert m["stt_config"] == meta["stt_config"] and m["stt_config"]["codemix_fork"] is True, m["stt_config"] # transcribe.py's fork rule
err = {k: float(np.abs(back[k] - state[k]).max()) for k in state}
worst = max(err, key=err.get)
if ftype == "f32":
bad = [k for k in state if back[k].tobytes() != state[k].tobytes()]
assert not bad, f"{len(bad)} tensors differ: {bad[:5]}"
rel_err = float(np.abs(back[worst] - state[worst]).max() / max(np.abs(state[worst]).max(), 1e-12))
print(f"{out} ({out.stat().st_size} bytes): {len(state)} tensors, types {counts}; configs and tokenizer bytes equal; "
f"largest abs error {err[worst]:.3g} in {worst} (relative to its max |w|: {rel_err:.3g})"
+ ("; byte-identical to the safetensors" if ftype == "f32" else ""))
def to_mlx(name: str, t: np.ndarray) -> np.ndarray:
if CONV2D.fullmatch(name):
return np.ascontiguousarray(t.transpose(0, 2, 3, 1)) # safetensors writes a view's raw memory, not its transposed values
if CONV1D.fullmatch(name):
return np.ascontiguousarray(t.transpose(0, 2, 1))
return t
def cmd_mlx(rel: str) -> None:
from safetensors.numpy import load_file, save_file
rel = Path(rel)
out = rel / "mlx"
out.mkdir(exist_ok=True)
state = load_file(str(rel / "safetensors" / ST_NAME))
stt = json.loads((rel / "safetensors" / "stt_config.json").read_text(encoding="utf-8"))
fork = sum(k.startswith("fork_cm.") for k in state)
assert fork == (FORK_TENSORS if stt.get("codemix_fork") else 0), (fork, stt.get("codemix_fork"))
conv = sorted(k for k in state if CONV2D.fullmatch(k) or CONV1D.fullmatch(k))
assert len(conv) == 15 + bool(fork), conv # the fork's position conv is the 16th
save_file({k: to_mlx(k, v) for k, v in state.items()}, str(out / MLX_NAME), metadata={"format": "mlx"})
(out / "layout.json").write_text(json.dumps({"changed_layout": conv, "conv2d": "(out,in,kT,kF)->(out,kT,kF,in)",
"conv1d": "(out,in,k)->(out,k,in)", "tensors": len(state),
"codemix_fork": bool(stt.get("codemix_fork"))}, indent=1),
encoding="utf-8", newline="\n")
shutil.copy2(rel / "safetensors" / "eg2" / "preprocessor_config.json", out / "preprocessor_config.json")
for f in ["mlx_model.py", "run_mlx.py", "text_norm.py"]:
shutil.copy2(HERE / f, out / f)
del state
ref = load_file(str(rel / "safetensors" / ST_NAME))
got = load_file(str(out / MLX_NAME))
assert set(ref) == set(got), f"names differ: {sorted(set(ref) ^ set(got))[:5]}"
bad = [k for k in ref if to_mlx(k, ref[k]).tobytes() != got[k].tobytes() or to_mlx(k, ref[k]).shape != got[k].shape or got[k].dtype != np.float32]
assert not bad, f"{len(bad)} tensors differ: {bad[:5]}"
# text_norm.py must stay equal to indic_stt/text.py's normalize
sys.path.insert(0, str(out))
import text_norm
from indic_stt.text import normalize
probe = ["Hello, World! it's “fine” — ok", "வணக்கம்। நான் don't-know…", " क़ुतुब मीनार ", "A.B.C; 12:30"]
assert all(text_norm.normalize(s) == normalize(s) for s in probe), "text_norm.normalize differs from indic_stt.text.normalize"
print(f"{len(ref)} tensors, {len(conv)} with changed layout -> {out / MLX_NAME} ({(out / MLX_NAME).stat().st_size} bytes); "
"every tensor byte-identical after the layout change, all float32; text_norm equal on the probes")
def cmd_mlx8(rel: str) -> None:
import mlx.core as mx
sys.path.insert(0, str(HERE))
from mlx_model import QUANT, load_weights, quant_stem
out = Path(rel) / "mlx"
src, dst = out / MLX_NAME, out / MLX8_NAME
layout = json.loads((out / "layout.json").read_text(encoding="utf-8"))
state = mx.load(str(src))
assert len(state) == layout["tensors"] == TENSORS and all(v.dtype == mx.float32 for v in state.values()), len(state)
assert not [k for k in state if k.endswith((".scales", ".biases"))], "a source name ends in .scales or .biases"
quant = sorted(k for k, v in state.items() if v.ndim == 2 and v.shape[-1] % MLX8_GROUP == 0)
assert len(quant) == sum(v.ndim == 2 for v in state.values()) == 198, len(quant) # every 2-D tensor, as q8_0's Q8_0 set
assert not set(quant) & set(layout["changed_layout"]), "a relaid conv kernel would be quantized"
stems = {quant_stem(k) for k in quant}
assert len(stems) == len(quant) and not {f"{s}.{x}" for s in stems for x in ("scales", "biases")} & set(state)
arrays = {}
for k, v in state.items():
if k in quant:
wq, s, b = mx.quantize(v, group_size=MLX8_GROUP, bits=MLX8_BITS, mode="affine")
arrays[k], arrays[f"{quant_stem(k)}.scales"], arrays[f"{quant_stem(k)}.biases"] = wq, s, b
else:
arrays[k] = v
mx.eval(list(arrays.values()))
q = {"mode": "affine", "bits": MLX8_BITS, "group_size": MLX8_GROUP, "tensors": len(quant)}
meta = {"format": "mlx", QUANT: json.dumps(q, sort_keys=True), "source": f"{MLX_NAME} sha256 {sha256(src)}"}
mx.save_safetensors(str(dst), arrays, metadata=meta)
# read back: the stored tensors, then load_weights' float32 view against the full file
got, got_meta = mx.load(str(dst), return_metadata=True)
assert set(got) == set(arrays) and len(got) == TENSORS + 2 * len(quant) and got_meta == meta, (len(got), got_meta)
bad = [k for k in arrays if got[k].dtype != arrays[k].dtype or got[k].shape != arrays[k].shape
or np.array(got[k]).tobytes() != np.array(arrays[k]).tobytes()]
assert not bad, f"{len(bad)} stored tensors differ from what was written: {bad[:5]}"
again = [k for k in quant if any(np.array(a).tobytes() != np.array(got[n]).tobytes() for a, n in zip(
mx.quantize(state[k], group_size=MLX8_GROUP, bits=MLX8_BITS, mode="affine"),
(k, f"{quant_stem(k)}.scales", f"{quant_stem(k)}.biases")))]
assert not again, f"{len(again)} tensors differ from a fresh mx.quantize: {again[:5]}"
deq, info = load_weights(dst)
assert info == q and set(deq) == set(state), "load_weights: names or quantization differ"
assert all(deq[k].dtype == mx.float32 and deq[k].shape == state[k].shape for k in state), "load_weights: dtype or shape"
same = [k for k in state if k not in quant and np.array(deq[k]).tobytes() == np.array(state[k]).tobytes()]
assert len(same) == len(state) - len(quant), f"{len(state) - len(quant) - len(same)} unquantized tensors differ"
err = {k: float(mx.abs(deq[k] - state[k]).max()) for k in quant}
worst = max(err, key=err.get)
rel_err = err[worst] / max(float(mx.abs(state[worst]).max()), 1e-12)
print(f"{dst} ({dst.stat().st_size} bytes): {len(got)} stored tensors = {len(state) - len(quant)} float32 unchanged "
f"(byte-identical to {MLX_NAME}: 16 conv kernels + {len(state) - len(quant) - 16} 1-D and scalar) + {len(quant)} "
f"quantized x (packed uint32, scales, biases); metadata {meta}; load_weights -> {len(deq)} float32 tensors, names and "
f"shapes equal; largest abs error {err[worst]:.3g} in {worst} (relative to its max |w|: {rel_err:.3g}); "
f"equal to a fresh mx.quantize")
if __name__ == "__main__":
{"safetensors": cmd_safetensors, "gguf": cmd_gguf, "mlx": cmd_mlx, "mlx8": cmd_mlx8, "check-stt1": cmd_check_stt1}[
sys.argv[1]](*sys.argv[2:])