Download prototype.py from whiletruelearn/dulu-malayalam-ctc-778h: direct link, hf CLI and curl.
- Browser
- Download file 19.7 kB
-
https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/prototype.py
- Command line
-
hf download hf://whiletruelearn/dulu-malayalam-ctc-778h/prototype.py
-
curl -L -o prototype.py https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/prototype.py
19.7 kB
| """Small, reproducible Malayalam frozen-encoder learning test. | |
| Run through the project's recorded run recipe. Reads only the selected FLEURS | |
| language and writes durable manifests, predictions, metrics and reader weights. | |
| """ | |
| import argparse | |
| from collections import Counter | |
| import gc | |
| import hashlib | |
| import io | |
| import json | |
| import math | |
| from pathlib import Path | |
| import platform | |
| import random | |
| import time | |
| import datasets | |
| import jiwer | |
| import numpy as np | |
| import soundfile as sf | |
| import torch | |
| from torch.nn import functional as F | |
| from transformers import AutoFeatureExtractor, AutoModel, AutoTokenizer | |
| from reader import Reader | |
| from text_utils import collapse_ctc, minimum_ctc_frames, normalize | |
| def emit(event, **values): | |
| print(json.dumps({"event": event, **values}, ensure_ascii=False), flush=True) | |
| def save_json(path, value): | |
| path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") | |
| def select_data(cfg, tokenizer, output): | |
| result, audit = {}, {} | |
| seen_text, seen_audio = set(), set() | |
| for split, count in cfg["split_sizes"].items(): | |
| emit("data_start", split=split, target=count) | |
| stream = datasets.load_dataset(cfg["dataset_id"], cfg["dataset_config"], | |
| revision=cfg["dataset_revision"], split=split, streaming=True, | |
| batch_size=16) | |
| stream = stream.cast_column("audio", datasets.Audio(decode=False)) | |
| # A bounded shuffled buffer prevents simply taking the first ordered rows. | |
| stream = stream.shuffle(seed=cfg["seed"], buffer_size=128) | |
| selected, rejected = [], Counter() | |
| scanned = 0 | |
| for row in stream: | |
| scanned += 1 | |
| if scanned > cfg["scan_limit"]: | |
| break | |
| duration = row["num_samples"] / 16000 | |
| if not cfg["min_seconds"] <= duration <= cfg["max_seconds"]: | |
| rejected["duration"] += 1 | |
| continue | |
| text = normalize(row["raw_transcription"]) | |
| if not text or not any("\u0d00" <= c <= "\u0d7f" for c in text): | |
| rejected["not_malayalam"] += 1 | |
| continue | |
| if text in seen_text: | |
| rejected["duplicate_text"] += 1 | |
| continue | |
| token_ids = tokenizer(" " + text, add_special_tokens=False).input_ids | |
| # Preliminary conservative screen; actual encoder lengths checked later. | |
| if minimum_ctc_frames(token_ids) > math.floor(duration * 25) - 2: | |
| rejected["estimated_ctc_impossible"] += 1 | |
| continue | |
| audio = row["audio"] | |
| blob = audio["bytes"] | |
| if blob is None: | |
| blob = Path(audio["path"]).read_bytes() | |
| audio_hash = hashlib.sha256(blob).hexdigest() | |
| if audio_hash in seen_audio: | |
| rejected["duplicate_audio"] += 1 | |
| continue | |
| wav, rate = sf.read(io.BytesIO(blob), dtype="float32") | |
| if rate != 16000 or wav.ndim != 1 or not np.isfinite(wav).all(): | |
| raise ValueError("Unexpected FLEURS audio format") | |
| duration = len(wav) / rate | |
| selected.append({"id": str(row["id"]), "path": row["path"], "text": text, | |
| "raw_text": row["raw_transcription"], "seconds": duration, | |
| "gender": int(row["gender"]), "audio_sha256": audio_hash, | |
| "gemma_ids": token_ids, "wav": wav}) | |
| seen_text.add(text) | |
| seen_audio.add(audio_hash) | |
| if len(selected) % 16 == 0: | |
| emit("data_progress", split=split, selected=len(selected), scanned=scanned) | |
| if len(selected) == count: | |
| break | |
| if len(selected) != count: | |
| raise RuntimeError(f"Only {len(selected)} suitable examples for {split}; wanted {count}") | |
| result[split] = selected | |
| audit[split] = {"clips": count, "seconds": sum(r["seconds"] for r in selected), | |
| "scanned": scanned, "rejected": dict(rejected)} | |
| emit("data_ready", split=split, **audit[split]) | |
| manifest = {split: [{k: v for k, v in r.items() if k != "wav"} for r in rows] | |
| for split, rows in result.items()} | |
| save_json(output / "manifest.json", manifest) | |
| return result, audit | |
| def cache_encoder(cfg, data, gemma_ids, device, output): | |
| emit("model_load_start", model=cfg["model_id"], revision=cfg["model_revision"]) | |
| full = AutoModel.from_pretrained(cfg["model_id"], revision=cfg["model_revision"], | |
| dtype=torch.float32, vision_config=None) | |
| embeddings = full.language_model.embed_tokens.weight[gemma_ids].detach().float().clone() | |
| encoder = full.audio_tower | |
| del full | |
| gc.collect() | |
| encoder.requires_grad_(False).eval().to(device) | |
| feature_extractor = AutoFeatureExtractor.from_pretrained(cfg["model_id"], revision=cfg["model_revision"]) | |
| encoder_parameters = sum(p.numel() for p in encoder.parameters()) | |
| emit("encoder_ready", parameters=encoder_parameters, trainable=0, device=str(device)) | |
| acts = [] | |
| def capture(module, args, value): | |
| acts.append(value[0] if isinstance(value, tuple) else value) | |
| handles = [encoder.subsample_conv_projection.register_forward_hook(capture)] | |
| handles += [layer.register_forward_hook(capture) for layer in encoder.layers] | |
| start, done = time.monotonic(), 0 | |
| for split, rows in data.items(): | |
| for row in rows: | |
| # Explicit batch avoids the single-ndarray shape bug in Transformers 5.19. | |
| inputs = feature_extractor([row["wav"]], sampling_rate=16000, return_tensors="pt") | |
| acts.clear() | |
| with torch.no_grad(): | |
| out = encoder(inputs["input_features"].to(device), | |
| inputs["input_features_mask"].to(device), return_dict=True) | |
| length = int(out.attention_mask.sum().item()) | |
| if len(acts) != 13: | |
| raise RuntimeError(f"Expected 13 encoder states, got {len(acts)}") | |
| states = torch.stack([a[0, :length].detach().cpu() for a in acts]) | |
| if not torch.isfinite(states).all(): | |
| raise RuntimeError("Non-finite encoder features") | |
| if minimum_ctc_frames(row["target"]) > length: | |
| raise RuntimeError("Actual CTC length insufficient; do not silently zero loss") | |
| row["states"] = states.half() | |
| row["frames"] = length | |
| del row["wav"] | |
| done += 1 | |
| if done % 8 == 0: | |
| emit("feature_progress", clips=done, elapsed_seconds=round(time.monotonic() - start, 2)) | |
| for handle in handles: | |
| handle.remove() | |
| acts.clear() | |
| encoder_dim = encoder.config.hidden_size | |
| del encoder, out, states | |
| gc.collect() | |
| if device.type == "mps": | |
| torch.mps.empty_cache() | |
| torch.save({"data": data, "embeddings": embeddings, "config": cfg}, output / "features.pt") | |
| return embeddings, encoder_dim, encoder_parameters | |
| def batch_tensors(rows, device): | |
| max_frames = max(r["frames"] for r in rows) | |
| shape = rows[0]["states"].shape if "states" in rows[0] else rows[0]["state_shape"] | |
| states = torch.zeros(len(rows), shape[0], max_frames, shape[2]) | |
| for i, row in enumerate(rows): | |
| if "states" in row: | |
| states[i, :, :row["frames"]] = row["states"].float() | |
| else: | |
| cached = np.load(row["states_path"], mmap_mode="r", allow_pickle=False) | |
| states[i, :, :row["frames"]] = torch.from_numpy(np.array(cached, copy=True)).float() | |
| lengths = torch.tensor([r["frames"] for r in rows], dtype=torch.long) | |
| return states.to(device), lengths.to(device) | |
| def ctc_loss(logits, lengths, rows): | |
| targets = torch.tensor([t for r in rows for t in r["target"]], dtype=torch.long) | |
| target_lengths = torch.tensor([len(r["target"]) for r in rows], dtype=torch.long) | |
| # MPS lacks native CTC: this differentiable CPU copy keeps the reader on GPU. | |
| log_probs = logits.float().log_softmax(-1).transpose(0, 1).cpu() | |
| return F.ctc_loss(log_probs, targets, lengths.cpu(), target_lengths, | |
| blank=0, reduction="sum", zero_infinity=False) / target_lengths.sum() | |
| def evaluate(model, rows, device, tokenizer, gemma_ids, batch_size): | |
| model.eval() | |
| predictions, losses = [], [] | |
| for begin in range(0, len(rows), batch_size): | |
| batch = rows[begin:begin + batch_size] | |
| states, lengths = batch_tensors(batch, device) | |
| logits = model(states, lengths) | |
| loss = ctc_loss(logits, lengths, batch) | |
| losses.append((float(loss), sum(len(r["target"]) for r in batch))) | |
| best = logits.argmax(-1).cpu().tolist() | |
| for row, predicted in zip(batch, best): | |
| units = collapse_ctc(predicted[:row["frames"]]) | |
| # Retain visible unknowns instead of silently dropping them. | |
| pieces, token_buffer = [], [] | |
| for unit in units: | |
| if unit == 1: | |
| pieces.append(tokenizer.decode(token_buffer)) | |
| token_buffer = [] | |
| pieces.append(" � ") | |
| else: | |
| token_buffer.append(gemma_ids[unit - 2]) | |
| pieces.append(tokenizer.decode(token_buffer)) | |
| # Replacement symbols must remain scored as errors. | |
| raw_hyp = "".join(pieces) | |
| hyp = " � ".join(normalize(p) for p in raw_hyp.split("�")).strip() | |
| predictions.append({"id": row["id"], "reference": row["text"], "hypothesis": hyp}) | |
| refs = [p["reference"] for p in predictions] | |
| hyps = [p["hypothesis"] for p in predictions] | |
| return {"wer": jiwer.wer(refs, hyps), "cer": jiwer.cer(refs, hyps), | |
| "ctc_loss_per_token": sum(v * n for v, n in losses) / sum(n for _, n in losses), | |
| "empty_hypotheses": sum(not h for h in hyps), "clips": len(rows)}, predictions | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config", required=True) | |
| args = parser.parse_args() | |
| cfg = json.loads(Path(args.config).read_text()) | |
| if cfg.get("study") == "synthetic_diagnostic": | |
| from synthetic_diagnostic import run | |
| run(cfg) | |
| return | |
| if cfg.get("mode") == "kathbath_pilot": | |
| from kathbath_train import run | |
| run(cfg) | |
| return | |
| if cfg.get("mode") == "full_character": | |
| from full_train import run | |
| run(cfg) | |
| return | |
| if cfg.get("mode") == "paired_vocabulary": | |
| from paired_vocabulary import run | |
| run(cfg) | |
| return | |
| if cfg.get("mode") == "overfit": | |
| from overfit import run | |
| run(cfg) | |
| return | |
| start = time.monotonic() | |
| output = Path(cfg["artifact_root"]).expanduser() / time.strftime("%Y%m%d-%H%M%S") | |
| output.mkdir(parents=True, exist_ok=False) | |
| random.seed(cfg["seed"]) | |
| np.random.seed(cfg["seed"]) | |
| torch.manual_seed(cfg["seed"]) | |
| torch.set_num_threads(cfg["cpu_threads"]) | |
| device = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu") | |
| emit("configuration", config=cfg, output=str(output), device=str(device), | |
| python=platform.python_version(), torch=torch.__version__) | |
| save_json(output / "config.json", cfg) | |
| tokenizer = AutoTokenizer.from_pretrained(cfg["model_id"], revision=cfg["model_revision"]) | |
| cached = cfg.get("mode") == "cached_128" | |
| if cached: | |
| from cached_data import load_training_validation, vocabulary_audit | |
| data, embeddings, gemma_ids = load_training_validation(cfg) | |
| if cfg.get("coverage_vocabulary"): | |
| from coverage_vocab import apply_coverage | |
| data, embeddings, gemma_ids = apply_coverage(cfg, data, embeddings, gemma_ids, tokenizer, output) | |
| audit = vocabulary_audit(data, gemma_ids, tokenizer) | |
| save_json(output / "manifest.json", { | |
| split: [{k: v for k, v in row.items() if k != "states"} for row in rows] | |
| for split, rows in data.items()}) | |
| else: | |
| data, audit = select_data(cfg, tokenizer, output) | |
| # Preserve inputs before model initialization so a later failure is inspectable. | |
| torch.save({"data": data, "audit": audit, "config": cfg}, output / "prepared-data.pt") | |
| gemma_ids = sorted({token for row in data["train"] for token in row["gemma_ids"]}) | |
| to_unit = {token: i + 2 for i, token in enumerate(gemma_ids)} | |
| for split, rows in data.items(): | |
| unknown, total = 0, 0 | |
| for row in rows: | |
| row["target"] = [to_unit.get(token, 1) for token in row["gemma_ids"]] | |
| unknown += row["target"].count(1) | |
| total += len(row["target"]) | |
| audit[split]["unknown_token_rate"] = unknown / total | |
| audit[split]["target_tokens"] = total | |
| save_json(output / "units.json", {"gemma_ids": gemma_ids, "blank": 0, "unk": 1}) | |
| save_json(output / "data_audit.json", audit) | |
| emit("data_audit", splits=audit, vocab_size=len(gemma_ids) + 2) | |
| if cached: | |
| encoder_dim = data["train"][0]["states"].shape[-1] | |
| encoder_parameters = cfg["source_encoder_parameters"] | |
| emit("encoder_cache_ready", frozen_parameters_from_source=encoder_parameters) | |
| # Pin initialization independently of tokenizer/cache-loading internals. | |
| torch.manual_seed(cfg["seed"]) | |
| else: | |
| embeddings, encoder_dim, encoder_parameters = cache_encoder(cfg, data, gemma_ids, device, output) | |
| model = Reader(encoder_dim, 13, embeddings, cfg).to(device) | |
| trainable_parameters = sum(p.numel() for p in model.parameters()) | |
| emit("reader_ready", trainable_parameters=trainable_parameters) | |
| initial = {} | |
| history = [] | |
| metrics_file = (output / "metrics.jsonl").open("w") | |
| def record(step, split, metrics): | |
| rec = {"step": step, "split": split, **metrics} | |
| history.append(rec) | |
| metrics_file.write(json.dumps(rec) + "\n") | |
| metrics_file.flush() | |
| emit("evaluation", **rec) | |
| # The cached duration experiment deliberately excludes the old test split. | |
| initial_state = ({k: v.detach().cpu().clone() for k, v in model.state_dict().items()} | |
| if "test" in data else None) | |
| for split in ("train", "validation"): | |
| initial[split], _ = evaluate(model, data[split], device, tokenizer, gemma_ids, cfg["batch_size"]) | |
| record(0, split, initial[split]) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=cfg["learning_rate"], betas=(0.9, 0.98), | |
| weight_decay=cfg.get("weight_decay", 0.01)) | |
| best_cer, best_step = float("inf"), 0 | |
| last_log = time.monotonic() | |
| for step in range(1, cfg["steps"] + 1): | |
| model.train() | |
| rows = random.sample(data["train"], cfg["batch_size"]) | |
| states, lengths = batch_tensors(rows, device) | |
| lr_factor = min(step / cfg["warmup_steps"], 1.0) | |
| for group in optimizer.param_groups: | |
| group["lr"] = cfg["learning_rate"] * lr_factor | |
| optimizer.zero_grad(set_to_none=True) | |
| loss = ctc_loss(model(states, lengths), lengths, rows) | |
| if not torch.isfinite(loss): | |
| raise RuntimeError(f"Nonfinite loss at step {step}") | |
| loss.backward() | |
| grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) | |
| optimizer.step() | |
| if step % 20 == 0: | |
| emit("training", step=step, loss=float(loss.detach()), grad_norm=float(grad_norm), | |
| seconds_per_step=(time.monotonic() - last_log) / 20) | |
| last_log = time.monotonic() | |
| if step % cfg["eval_every"] == 0 or step == cfg["steps"]: | |
| for split in ("train", "validation"): | |
| metrics, predictions = evaluate(model, data[split], device, tokenizer, gemma_ids, cfg["batch_size"]) | |
| record(step, split, metrics) | |
| if cached: | |
| with (output / f"predictions-step-{step}-{split}.jsonl").open("w") as stream: | |
| for prediction in predictions: | |
| stream.write(json.dumps(prediction, ensure_ascii=False) + "\n") | |
| if split == "validation" and metrics["cer"] < best_cer: | |
| best_cer, best_step = metrics["cer"], step | |
| torch.save({"reader": model.state_dict(), "config": cfg, "gemma_ids": gemma_ids, | |
| "step": step, "validation": metrics}, output / "best-reader.pt") | |
| if cached and step in (400, cfg["steps"]): | |
| torch.save({"reader": model.state_dict(), "config": cfg, "gemma_ids": gemma_ids, | |
| "step": step}, output / f"reader-step-{step}.pt") | |
| if "test" in data: | |
| model.load_state_dict(initial_state) | |
| initial["test"], _ = evaluate(model, data["test"], device, tokenizer, gemma_ids, cfg["batch_size"]) | |
| record(0, "test", initial["test"]) | |
| ckpt = torch.load(output / "best-reader.pt", map_location=device, weights_only=True) | |
| model.load_state_dict(ckpt["reader"]) | |
| final = {} | |
| with (output / "predictions.jsonl").open("w") as predictions_file: | |
| for split, rows in data.items(): | |
| final[split], predictions = evaluate(model, rows, device, tokenizer, gemma_ids, cfg["batch_size"]) | |
| for prediction in predictions: | |
| predictions_file.write(json.dumps({"split": split, **prediction}, ensure_ascii=False) + "\n") | |
| emit("selected_checkpoint_evaluation", step=best_step, split=split, **final[split]) | |
| emit("example", split=split, **predictions[0]) | |
| summary = {"initial": initial, "final": final, "selected_step": best_step, | |
| "encoder_parameters": encoder_parameters, "trainable_parameters": trainable_parameters, | |
| "layer_weights": model.layer_logits.softmax(0).detach().cpu().tolist(), | |
| "data_audit": audit, "elapsed_seconds": time.monotonic() - start, "device": str(device), | |
| "output": str(output), | |
| "limitations": ["Tiny read-speech subsets; not competitive-ASR evidence", "No SpecAugment; frozen features cached in FP16, reader computes FP32", "Vocabulary from train only; held-out OOV tokens reported", "MPS CTC loss computed on CPU", "No Whisper/Koyal baseline in this learning test"]} | |
| if cached: | |
| summary.update(completed_steps=cfg["steps"], source_run=cfg["source_run"], | |
| step_400={r["split"]: r for r in history if r["step"] == 400}, | |
| last_step={r["split"]: r for r in history if r["step"] == cfg["steps"]}, | |
| evaluated_splits=list(data)) | |
| summary["limitations"] += ["No test evaluation; validation used for checkpoint selection", | |
| "Compare CUDA arms within this run; historical MPS is not a paired control", | |
| "Mapped-reference reconstruction is a diagnostic, not an achievable-error bound"] | |
| if cfg.get("coverage_vocabulary"): | |
| summary["limitations"] = [item for item in summary["limitations"] | |
| if item != "Vocabulary from train only; held-out OOV tokens reported"] | |
| summary["limitations"] += ["Character vocabulary uses fixed alphabet plus training characters; no validation vocabulary mining", | |
| "Tokenization, target lengths, and output-head size change together"] | |
| save_json(output / "summary.json", summary) | |
| metrics_file.close() | |
| emit("FINAL_SUMMARY", **summary) | |
| if __name__ == "__main__": | |
| main() | |