"""Small, reproducible Malayalam frozen-encoder learning test. Run through the project's recorded run recipe. Reads only the selected FLEURS language and writes durable manifests, predictions, metrics and reader weights. """ import argparse from collections import Counter import gc import hashlib import io import json import math from pathlib import Path import platform import random import time import datasets import jiwer import numpy as np import soundfile as sf import torch from torch.nn import functional as F from transformers import AutoFeatureExtractor, AutoModel, AutoTokenizer from reader import Reader from text_utils import collapse_ctc, minimum_ctc_frames, normalize def emit(event, **values): print(json.dumps({"event": event, **values}, ensure_ascii=False), flush=True) def save_json(path, value): path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") def select_data(cfg, tokenizer, output): result, audit = {}, {} seen_text, seen_audio = set(), set() for split, count in cfg["split_sizes"].items(): emit("data_start", split=split, target=count) stream = datasets.load_dataset(cfg["dataset_id"], cfg["dataset_config"], revision=cfg["dataset_revision"], split=split, streaming=True, batch_size=16) stream = stream.cast_column("audio", datasets.Audio(decode=False)) # A bounded shuffled buffer prevents simply taking the first ordered rows. stream = stream.shuffle(seed=cfg["seed"], buffer_size=128) selected, rejected = [], Counter() scanned = 0 for row in stream: scanned += 1 if scanned > cfg["scan_limit"]: break duration = row["num_samples"] / 16000 if not cfg["min_seconds"] <= duration <= cfg["max_seconds"]: rejected["duration"] += 1 continue text = normalize(row["raw_transcription"]) if not text or not any("\u0d00" <= c <= "\u0d7f" for c in text): rejected["not_malayalam"] += 1 continue if text in seen_text: rejected["duplicate_text"] += 1 continue token_ids = tokenizer(" " + text, add_special_tokens=False).input_ids # Preliminary conservative screen; actual encoder lengths checked later. if minimum_ctc_frames(token_ids) > math.floor(duration * 25) - 2: rejected["estimated_ctc_impossible"] += 1 continue audio = row["audio"] blob = audio["bytes"] if blob is None: blob = Path(audio["path"]).read_bytes() audio_hash = hashlib.sha256(blob).hexdigest() if audio_hash in seen_audio: rejected["duplicate_audio"] += 1 continue wav, rate = sf.read(io.BytesIO(blob), dtype="float32") if rate != 16000 or wav.ndim != 1 or not np.isfinite(wav).all(): raise ValueError("Unexpected FLEURS audio format") duration = len(wav) / rate selected.append({"id": str(row["id"]), "path": row["path"], "text": text, "raw_text": row["raw_transcription"], "seconds": duration, "gender": int(row["gender"]), "audio_sha256": audio_hash, "gemma_ids": token_ids, "wav": wav}) seen_text.add(text) seen_audio.add(audio_hash) if len(selected) % 16 == 0: emit("data_progress", split=split, selected=len(selected), scanned=scanned) if len(selected) == count: break if len(selected) != count: raise RuntimeError(f"Only {len(selected)} suitable examples for {split}; wanted {count}") result[split] = selected audit[split] = {"clips": count, "seconds": sum(r["seconds"] for r in selected), "scanned": scanned, "rejected": dict(rejected)} emit("data_ready", split=split, **audit[split]) manifest = {split: [{k: v for k, v in r.items() if k != "wav"} for r in rows] for split, rows in result.items()} save_json(output / "manifest.json", manifest) return result, audit def cache_encoder(cfg, data, gemma_ids, device, output): emit("model_load_start", model=cfg["model_id"], revision=cfg["model_revision"]) full = AutoModel.from_pretrained(cfg["model_id"], revision=cfg["model_revision"], dtype=torch.float32, vision_config=None) embeddings = full.language_model.embed_tokens.weight[gemma_ids].detach().float().clone() encoder = full.audio_tower del full gc.collect() encoder.requires_grad_(False).eval().to(device) feature_extractor = AutoFeatureExtractor.from_pretrained(cfg["model_id"], revision=cfg["model_revision"]) encoder_parameters = sum(p.numel() for p in encoder.parameters()) emit("encoder_ready", parameters=encoder_parameters, trainable=0, device=str(device)) acts = [] def capture(module, args, value): acts.append(value[0] if isinstance(value, tuple) else value) handles = [encoder.subsample_conv_projection.register_forward_hook(capture)] handles += [layer.register_forward_hook(capture) for layer in encoder.layers] start, done = time.monotonic(), 0 for split, rows in data.items(): for row in rows: # Explicit batch avoids the single-ndarray shape bug in Transformers 5.19. inputs = feature_extractor([row["wav"]], sampling_rate=16000, return_tensors="pt") acts.clear() with torch.no_grad(): out = encoder(inputs["input_features"].to(device), inputs["input_features_mask"].to(device), return_dict=True) length = int(out.attention_mask.sum().item()) if len(acts) != 13: raise RuntimeError(f"Expected 13 encoder states, got {len(acts)}") states = torch.stack([a[0, :length].detach().cpu() for a in acts]) if not torch.isfinite(states).all(): raise RuntimeError("Non-finite encoder features") if minimum_ctc_frames(row["target"]) > length: raise RuntimeError("Actual CTC length insufficient; do not silently zero loss") row["states"] = states.half() row["frames"] = length del row["wav"] done += 1 if done % 8 == 0: emit("feature_progress", clips=done, elapsed_seconds=round(time.monotonic() - start, 2)) for handle in handles: handle.remove() acts.clear() encoder_dim = encoder.config.hidden_size del encoder, out, states gc.collect() if device.type == "mps": torch.mps.empty_cache() torch.save({"data": data, "embeddings": embeddings, "config": cfg}, output / "features.pt") return embeddings, encoder_dim, encoder_parameters def batch_tensors(rows, device): max_frames = max(r["frames"] for r in rows) shape = rows[0]["states"].shape if "states" in rows[0] else rows[0]["state_shape"] states = torch.zeros(len(rows), shape[0], max_frames, shape[2]) for i, row in enumerate(rows): if "states" in row: states[i, :, :row["frames"]] = row["states"].float() else: cached = np.load(row["states_path"], mmap_mode="r", allow_pickle=False) states[i, :, :row["frames"]] = torch.from_numpy(np.array(cached, copy=True)).float() lengths = torch.tensor([r["frames"] for r in rows], dtype=torch.long) return states.to(device), lengths.to(device) def ctc_loss(logits, lengths, rows): targets = torch.tensor([t for r in rows for t in r["target"]], dtype=torch.long) target_lengths = torch.tensor([len(r["target"]) for r in rows], dtype=torch.long) # MPS lacks native CTC: this differentiable CPU copy keeps the reader on GPU. log_probs = logits.float().log_softmax(-1).transpose(0, 1).cpu() return F.ctc_loss(log_probs, targets, lengths.cpu(), target_lengths, blank=0, reduction="sum", zero_infinity=False) / target_lengths.sum() @torch.no_grad() def evaluate(model, rows, device, tokenizer, gemma_ids, batch_size): model.eval() predictions, losses = [], [] for begin in range(0, len(rows), batch_size): batch = rows[begin:begin + batch_size] states, lengths = batch_tensors(batch, device) logits = model(states, lengths) loss = ctc_loss(logits, lengths, batch) losses.append((float(loss), sum(len(r["target"]) for r in batch))) best = logits.argmax(-1).cpu().tolist() for row, predicted in zip(batch, best): units = collapse_ctc(predicted[:row["frames"]]) # Retain visible unknowns instead of silently dropping them. pieces, token_buffer = [], [] for unit in units: if unit == 1: pieces.append(tokenizer.decode(token_buffer)) token_buffer = [] pieces.append(" � ") else: token_buffer.append(gemma_ids[unit - 2]) pieces.append(tokenizer.decode(token_buffer)) # Replacement symbols must remain scored as errors. raw_hyp = "".join(pieces) hyp = " � ".join(normalize(p) for p in raw_hyp.split("�")).strip() predictions.append({"id": row["id"], "reference": row["text"], "hypothesis": hyp}) refs = [p["reference"] for p in predictions] hyps = [p["hypothesis"] for p in predictions] return {"wer": jiwer.wer(refs, hyps), "cer": jiwer.cer(refs, hyps), "ctc_loss_per_token": sum(v * n for v, n in losses) / sum(n for _, n in losses), "empty_hypotheses": sum(not h for h in hyps), "clips": len(rows)}, predictions def main(): parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) args = parser.parse_args() cfg = json.loads(Path(args.config).read_text()) if cfg.get("study") == "synthetic_diagnostic": from synthetic_diagnostic import run run(cfg) return if cfg.get("mode") == "kathbath_pilot": from kathbath_train import run run(cfg) return if cfg.get("mode") == "full_character": from full_train import run run(cfg) return if cfg.get("mode") == "paired_vocabulary": from paired_vocabulary import run run(cfg) return if cfg.get("mode") == "overfit": from overfit import run run(cfg) return start = time.monotonic() output = Path(cfg["artifact_root"]).expanduser() / time.strftime("%Y%m%d-%H%M%S") output.mkdir(parents=True, exist_ok=False) random.seed(cfg["seed"]) np.random.seed(cfg["seed"]) torch.manual_seed(cfg["seed"]) torch.set_num_threads(cfg["cpu_threads"]) device = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu") emit("configuration", config=cfg, output=str(output), device=str(device), python=platform.python_version(), torch=torch.__version__) save_json(output / "config.json", cfg) tokenizer = AutoTokenizer.from_pretrained(cfg["model_id"], revision=cfg["model_revision"]) cached = cfg.get("mode") == "cached_128" if cached: from cached_data import load_training_validation, vocabulary_audit data, embeddings, gemma_ids = load_training_validation(cfg) if cfg.get("coverage_vocabulary"): from coverage_vocab import apply_coverage data, embeddings, gemma_ids = apply_coverage(cfg, data, embeddings, gemma_ids, tokenizer, output) audit = vocabulary_audit(data, gemma_ids, tokenizer) save_json(output / "manifest.json", { split: [{k: v for k, v in row.items() if k != "states"} for row in rows] for split, rows in data.items()}) else: data, audit = select_data(cfg, tokenizer, output) # Preserve inputs before model initialization so a later failure is inspectable. torch.save({"data": data, "audit": audit, "config": cfg}, output / "prepared-data.pt") gemma_ids = sorted({token for row in data["train"] for token in row["gemma_ids"]}) to_unit = {token: i + 2 for i, token in enumerate(gemma_ids)} for split, rows in data.items(): unknown, total = 0, 0 for row in rows: row["target"] = [to_unit.get(token, 1) for token in row["gemma_ids"]] unknown += row["target"].count(1) total += len(row["target"]) audit[split]["unknown_token_rate"] = unknown / total audit[split]["target_tokens"] = total save_json(output / "units.json", {"gemma_ids": gemma_ids, "blank": 0, "unk": 1}) save_json(output / "data_audit.json", audit) emit("data_audit", splits=audit, vocab_size=len(gemma_ids) + 2) if cached: encoder_dim = data["train"][0]["states"].shape[-1] encoder_parameters = cfg["source_encoder_parameters"] emit("encoder_cache_ready", frozen_parameters_from_source=encoder_parameters) # Pin initialization independently of tokenizer/cache-loading internals. torch.manual_seed(cfg["seed"]) else: embeddings, encoder_dim, encoder_parameters = cache_encoder(cfg, data, gemma_ids, device, output) model = Reader(encoder_dim, 13, embeddings, cfg).to(device) trainable_parameters = sum(p.numel() for p in model.parameters()) emit("reader_ready", trainable_parameters=trainable_parameters) initial = {} history = [] metrics_file = (output / "metrics.jsonl").open("w") def record(step, split, metrics): rec = {"step": step, "split": split, **metrics} history.append(rec) metrics_file.write(json.dumps(rec) + "\n") metrics_file.flush() emit("evaluation", **rec) # The cached duration experiment deliberately excludes the old test split. initial_state = ({k: v.detach().cpu().clone() for k, v in model.state_dict().items()} if "test" in data else None) for split in ("train", "validation"): initial[split], _ = evaluate(model, data[split], device, tokenizer, gemma_ids, cfg["batch_size"]) record(0, split, initial[split]) optimizer = torch.optim.AdamW(model.parameters(), lr=cfg["learning_rate"], betas=(0.9, 0.98), weight_decay=cfg.get("weight_decay", 0.01)) best_cer, best_step = float("inf"), 0 last_log = time.monotonic() for step in range(1, cfg["steps"] + 1): model.train() rows = random.sample(data["train"], cfg["batch_size"]) states, lengths = batch_tensors(rows, device) lr_factor = min(step / cfg["warmup_steps"], 1.0) for group in optimizer.param_groups: group["lr"] = cfg["learning_rate"] * lr_factor optimizer.zero_grad(set_to_none=True) loss = ctc_loss(model(states, lengths), lengths, rows) if not torch.isfinite(loss): raise RuntimeError(f"Nonfinite loss at step {step}") loss.backward() grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) optimizer.step() if step % 20 == 0: emit("training", step=step, loss=float(loss.detach()), grad_norm=float(grad_norm), seconds_per_step=(time.monotonic() - last_log) / 20) last_log = time.monotonic() if step % cfg["eval_every"] == 0 or step == cfg["steps"]: for split in ("train", "validation"): metrics, predictions = evaluate(model, data[split], device, tokenizer, gemma_ids, cfg["batch_size"]) record(step, split, metrics) if cached: with (output / f"predictions-step-{step}-{split}.jsonl").open("w") as stream: for prediction in predictions: stream.write(json.dumps(prediction, ensure_ascii=False) + "\n") if split == "validation" and metrics["cer"] < best_cer: best_cer, best_step = metrics["cer"], step torch.save({"reader": model.state_dict(), "config": cfg, "gemma_ids": gemma_ids, "step": step, "validation": metrics}, output / "best-reader.pt") if cached and step in (400, cfg["steps"]): torch.save({"reader": model.state_dict(), "config": cfg, "gemma_ids": gemma_ids, "step": step}, output / f"reader-step-{step}.pt") if "test" in data: model.load_state_dict(initial_state) initial["test"], _ = evaluate(model, data["test"], device, tokenizer, gemma_ids, cfg["batch_size"]) record(0, "test", initial["test"]) ckpt = torch.load(output / "best-reader.pt", map_location=device, weights_only=True) model.load_state_dict(ckpt["reader"]) final = {} with (output / "predictions.jsonl").open("w") as predictions_file: for split, rows in data.items(): final[split], predictions = evaluate(model, rows, device, tokenizer, gemma_ids, cfg["batch_size"]) for prediction in predictions: predictions_file.write(json.dumps({"split": split, **prediction}, ensure_ascii=False) + "\n") emit("selected_checkpoint_evaluation", step=best_step, split=split, **final[split]) emit("example", split=split, **predictions[0]) summary = {"initial": initial, "final": final, "selected_step": best_step, "encoder_parameters": encoder_parameters, "trainable_parameters": trainable_parameters, "layer_weights": model.layer_logits.softmax(0).detach().cpu().tolist(), "data_audit": audit, "elapsed_seconds": time.monotonic() - start, "device": str(device), "output": str(output), "limitations": ["Tiny read-speech subsets; not competitive-ASR evidence", "No SpecAugment; frozen features cached in FP16, reader computes FP32", "Vocabulary from train only; held-out OOV tokens reported", "MPS CTC loss computed on CPU", "No Whisper/Koyal baseline in this learning test"]} if cached: summary.update(completed_steps=cfg["steps"], source_run=cfg["source_run"], step_400={r["split"]: r for r in history if r["step"] == 400}, last_step={r["split"]: r for r in history if r["step"] == cfg["steps"]}, evaluated_splits=list(data)) summary["limitations"] += ["No test evaluation; validation used for checkpoint selection", "Compare CUDA arms within this run; historical MPS is not a paired control", "Mapped-reference reconstruction is a diagnostic, not an achievable-error bound"] if cfg.get("coverage_vocabulary"): summary["limitations"] = [item for item in summary["limitations"] if item != "Vocabulary from train only; held-out OOV tokens reported"] summary["limitations"] += ["Character vocabulary uses fixed alphabet plus training characters; no validation vocabulary mining", "Tokenization, target lengths, and output-head size change together"] save_json(output / "summary.json", summary) metrics_file.close() emit("FINAL_SUMMARY", **summary) if __name__ == "__main__": main()