dulu-malayalam-ctc-778h / prototype.py
whiletruelearn's picture
Archive selected epoch 9 combined-data CTC checkpoint and evaluation
750e4eb verified
Raw History Blame Contribute Delete
19.7 kB
"""Small, reproducible Malayalam frozen-encoder learning test.
Run through the project's recorded run recipe. Reads only the selected FLEURS
language and writes durable manifests, predictions, metrics and reader weights.
"""
import argparse
from collections import Counter
import gc
import hashlib
import io
import json
import math
from pathlib import Path
import platform
import random
import time
import datasets
import jiwer
import numpy as np
import soundfile as sf
import torch
from torch.nn import functional as F
from transformers import AutoFeatureExtractor, AutoModel, AutoTokenizer
from reader import Reader
from text_utils import collapse_ctc, minimum_ctc_frames, normalize
def emit(event, **values):
print(json.dumps({"event": event, **values}, ensure_ascii=False), flush=True)
def save_json(path, value):
path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
def select_data(cfg, tokenizer, output):
result, audit = {}, {}
seen_text, seen_audio = set(), set()
for split, count in cfg["split_sizes"].items():
emit("data_start", split=split, target=count)
stream = datasets.load_dataset(cfg["dataset_id"], cfg["dataset_config"],
revision=cfg["dataset_revision"], split=split, streaming=True,
batch_size=16)
stream = stream.cast_column("audio", datasets.Audio(decode=False))
# A bounded shuffled buffer prevents simply taking the first ordered rows.
stream = stream.shuffle(seed=cfg["seed"], buffer_size=128)
selected, rejected = [], Counter()
scanned = 0
for row in stream:
scanned += 1
if scanned > cfg["scan_limit"]:
break
duration = row["num_samples"] / 16000
if not cfg["min_seconds"] <= duration <= cfg["max_seconds"]:
rejected["duration"] += 1
continue
text = normalize(row["raw_transcription"])
if not text or not any("\u0d00" <= c <= "\u0d7f" for c in text):
rejected["not_malayalam"] += 1
continue
if text in seen_text:
rejected["duplicate_text"] += 1
continue
token_ids = tokenizer(" " + text, add_special_tokens=False).input_ids
# Preliminary conservative screen; actual encoder lengths checked later.
if minimum_ctc_frames(token_ids) > math.floor(duration * 25) - 2:
rejected["estimated_ctc_impossible"] += 1
continue
audio = row["audio"]
blob = audio["bytes"]
if blob is None:
blob = Path(audio["path"]).read_bytes()
audio_hash = hashlib.sha256(blob).hexdigest()
if audio_hash in seen_audio:
rejected["duplicate_audio"] += 1
continue
wav, rate = sf.read(io.BytesIO(blob), dtype="float32")
if rate != 16000 or wav.ndim != 1 or not np.isfinite(wav).all():
raise ValueError("Unexpected FLEURS audio format")
duration = len(wav) / rate
selected.append({"id": str(row["id"]), "path": row["path"], "text": text,
"raw_text": row["raw_transcription"], "seconds": duration,
"gender": int(row["gender"]), "audio_sha256": audio_hash,
"gemma_ids": token_ids, "wav": wav})
seen_text.add(text)
seen_audio.add(audio_hash)
if len(selected) % 16 == 0:
emit("data_progress", split=split, selected=len(selected), scanned=scanned)
if len(selected) == count:
break
if len(selected) != count:
raise RuntimeError(f"Only {len(selected)} suitable examples for {split}; wanted {count}")
result[split] = selected
audit[split] = {"clips": count, "seconds": sum(r["seconds"] for r in selected),
"scanned": scanned, "rejected": dict(rejected)}
emit("data_ready", split=split, **audit[split])
manifest = {split: [{k: v for k, v in r.items() if k != "wav"} for r in rows]
for split, rows in result.items()}
save_json(output / "manifest.json", manifest)
return result, audit
def cache_encoder(cfg, data, gemma_ids, device, output):
emit("model_load_start", model=cfg["model_id"], revision=cfg["model_revision"])
full = AutoModel.from_pretrained(cfg["model_id"], revision=cfg["model_revision"],
dtype=torch.float32, vision_config=None)
embeddings = full.language_model.embed_tokens.weight[gemma_ids].detach().float().clone()
encoder = full.audio_tower
del full
gc.collect()
encoder.requires_grad_(False).eval().to(device)
feature_extractor = AutoFeatureExtractor.from_pretrained(cfg["model_id"], revision=cfg["model_revision"])
encoder_parameters = sum(p.numel() for p in encoder.parameters())
emit("encoder_ready", parameters=encoder_parameters, trainable=0, device=str(device))
acts = []
def capture(module, args, value):
acts.append(value[0] if isinstance(value, tuple) else value)
handles = [encoder.subsample_conv_projection.register_forward_hook(capture)]
handles += [layer.register_forward_hook(capture) for layer in encoder.layers]
start, done = time.monotonic(), 0
for split, rows in data.items():
for row in rows:
# Explicit batch avoids the single-ndarray shape bug in Transformers 5.19.
inputs = feature_extractor([row["wav"]], sampling_rate=16000, return_tensors="pt")
acts.clear()
with torch.no_grad():
out = encoder(inputs["input_features"].to(device),
inputs["input_features_mask"].to(device), return_dict=True)
length = int(out.attention_mask.sum().item())
if len(acts) != 13:
raise RuntimeError(f"Expected 13 encoder states, got {len(acts)}")
states = torch.stack([a[0, :length].detach().cpu() for a in acts])
if not torch.isfinite(states).all():
raise RuntimeError("Non-finite encoder features")
if minimum_ctc_frames(row["target"]) > length:
raise RuntimeError("Actual CTC length insufficient; do not silently zero loss")
row["states"] = states.half()
row["frames"] = length
del row["wav"]
done += 1
if done % 8 == 0:
emit("feature_progress", clips=done, elapsed_seconds=round(time.monotonic() - start, 2))
for handle in handles:
handle.remove()
acts.clear()
encoder_dim = encoder.config.hidden_size
del encoder, out, states
gc.collect()
if device.type == "mps":
torch.mps.empty_cache()
torch.save({"data": data, "embeddings": embeddings, "config": cfg}, output / "features.pt")
return embeddings, encoder_dim, encoder_parameters
def batch_tensors(rows, device):
max_frames = max(r["frames"] for r in rows)
shape = rows[0]["states"].shape if "states" in rows[0] else rows[0]["state_shape"]
states = torch.zeros(len(rows), shape[0], max_frames, shape[2])
for i, row in enumerate(rows):
if "states" in row:
states[i, :, :row["frames"]] = row["states"].float()
else:
cached = np.load(row["states_path"], mmap_mode="r", allow_pickle=False)
states[i, :, :row["frames"]] = torch.from_numpy(np.array(cached, copy=True)).float()
lengths = torch.tensor([r["frames"] for r in rows], dtype=torch.long)
return states.to(device), lengths.to(device)
def ctc_loss(logits, lengths, rows):
targets = torch.tensor([t for r in rows for t in r["target"]], dtype=torch.long)
target_lengths = torch.tensor([len(r["target"]) for r in rows], dtype=torch.long)
# MPS lacks native CTC: this differentiable CPU copy keeps the reader on GPU.
log_probs = logits.float().log_softmax(-1).transpose(0, 1).cpu()
return F.ctc_loss(log_probs, targets, lengths.cpu(), target_lengths,
blank=0, reduction="sum", zero_infinity=False) / target_lengths.sum()
@torch.no_grad()
def evaluate(model, rows, device, tokenizer, gemma_ids, batch_size):
model.eval()
predictions, losses = [], []
for begin in range(0, len(rows), batch_size):
batch = rows[begin:begin + batch_size]
states, lengths = batch_tensors(batch, device)
logits = model(states, lengths)
loss = ctc_loss(logits, lengths, batch)
losses.append((float(loss), sum(len(r["target"]) for r in batch)))
best = logits.argmax(-1).cpu().tolist()
for row, predicted in zip(batch, best):
units = collapse_ctc(predicted[:row["frames"]])
# Retain visible unknowns instead of silently dropping them.
pieces, token_buffer = [], []
for unit in units:
if unit == 1:
pieces.append(tokenizer.decode(token_buffer))
token_buffer = []
pieces.append(" � ")
else:
token_buffer.append(gemma_ids[unit - 2])
pieces.append(tokenizer.decode(token_buffer))
# Replacement symbols must remain scored as errors.
raw_hyp = "".join(pieces)
hyp = " � ".join(normalize(p) for p in raw_hyp.split("�")).strip()
predictions.append({"id": row["id"], "reference": row["text"], "hypothesis": hyp})
refs = [p["reference"] for p in predictions]
hyps = [p["hypothesis"] for p in predictions]
return {"wer": jiwer.wer(refs, hyps), "cer": jiwer.cer(refs, hyps),
"ctc_loss_per_token": sum(v * n for v, n in losses) / sum(n for _, n in losses),
"empty_hypotheses": sum(not h for h in hyps), "clips": len(rows)}, predictions
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
args = parser.parse_args()
cfg = json.loads(Path(args.config).read_text())
if cfg.get("study") == "synthetic_diagnostic":
from synthetic_diagnostic import run
run(cfg)
return
if cfg.get("mode") == "kathbath_pilot":
from kathbath_train import run
run(cfg)
return
if cfg.get("mode") == "full_character":
from full_train import run
run(cfg)
return
if cfg.get("mode") == "paired_vocabulary":
from paired_vocabulary import run
run(cfg)
return
if cfg.get("mode") == "overfit":
from overfit import run
run(cfg)
return
start = time.monotonic()
output = Path(cfg["artifact_root"]).expanduser() / time.strftime("%Y%m%d-%H%M%S")
output.mkdir(parents=True, exist_ok=False)
random.seed(cfg["seed"])
np.random.seed(cfg["seed"])
torch.manual_seed(cfg["seed"])
torch.set_num_threads(cfg["cpu_threads"])
device = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu")
emit("configuration", config=cfg, output=str(output), device=str(device),
python=platform.python_version(), torch=torch.__version__)
save_json(output / "config.json", cfg)
tokenizer = AutoTokenizer.from_pretrained(cfg["model_id"], revision=cfg["model_revision"])
cached = cfg.get("mode") == "cached_128"
if cached:
from cached_data import load_training_validation, vocabulary_audit
data, embeddings, gemma_ids = load_training_validation(cfg)
if cfg.get("coverage_vocabulary"):
from coverage_vocab import apply_coverage
data, embeddings, gemma_ids = apply_coverage(cfg, data, embeddings, gemma_ids, tokenizer, output)
audit = vocabulary_audit(data, gemma_ids, tokenizer)
save_json(output / "manifest.json", {
split: [{k: v for k, v in row.items() if k != "states"} for row in rows]
for split, rows in data.items()})
else:
data, audit = select_data(cfg, tokenizer, output)
# Preserve inputs before model initialization so a later failure is inspectable.
torch.save({"data": data, "audit": audit, "config": cfg}, output / "prepared-data.pt")
gemma_ids = sorted({token for row in data["train"] for token in row["gemma_ids"]})
to_unit = {token: i + 2 for i, token in enumerate(gemma_ids)}
for split, rows in data.items():
unknown, total = 0, 0
for row in rows:
row["target"] = [to_unit.get(token, 1) for token in row["gemma_ids"]]
unknown += row["target"].count(1)
total += len(row["target"])
audit[split]["unknown_token_rate"] = unknown / total
audit[split]["target_tokens"] = total
save_json(output / "units.json", {"gemma_ids": gemma_ids, "blank": 0, "unk": 1})
save_json(output / "data_audit.json", audit)
emit("data_audit", splits=audit, vocab_size=len(gemma_ids) + 2)
if cached:
encoder_dim = data["train"][0]["states"].shape[-1]
encoder_parameters = cfg["source_encoder_parameters"]
emit("encoder_cache_ready", frozen_parameters_from_source=encoder_parameters)
# Pin initialization independently of tokenizer/cache-loading internals.
torch.manual_seed(cfg["seed"])
else:
embeddings, encoder_dim, encoder_parameters = cache_encoder(cfg, data, gemma_ids, device, output)
model = Reader(encoder_dim, 13, embeddings, cfg).to(device)
trainable_parameters = sum(p.numel() for p in model.parameters())
emit("reader_ready", trainable_parameters=trainable_parameters)
initial = {}
history = []
metrics_file = (output / "metrics.jsonl").open("w")
def record(step, split, metrics):
rec = {"step": step, "split": split, **metrics}
history.append(rec)
metrics_file.write(json.dumps(rec) + "\n")
metrics_file.flush()
emit("evaluation", **rec)
# The cached duration experiment deliberately excludes the old test split.
initial_state = ({k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
if "test" in data else None)
for split in ("train", "validation"):
initial[split], _ = evaluate(model, data[split], device, tokenizer, gemma_ids, cfg["batch_size"])
record(0, split, initial[split])
optimizer = torch.optim.AdamW(model.parameters(), lr=cfg["learning_rate"], betas=(0.9, 0.98),
weight_decay=cfg.get("weight_decay", 0.01))
best_cer, best_step = float("inf"), 0
last_log = time.monotonic()
for step in range(1, cfg["steps"] + 1):
model.train()
rows = random.sample(data["train"], cfg["batch_size"])
states, lengths = batch_tensors(rows, device)
lr_factor = min(step / cfg["warmup_steps"], 1.0)
for group in optimizer.param_groups:
group["lr"] = cfg["learning_rate"] * lr_factor
optimizer.zero_grad(set_to_none=True)
loss = ctc_loss(model(states, lengths), lengths, rows)
if not torch.isfinite(loss):
raise RuntimeError(f"Nonfinite loss at step {step}")
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True)
optimizer.step()
if step % 20 == 0:
emit("training", step=step, loss=float(loss.detach()), grad_norm=float(grad_norm),
seconds_per_step=(time.monotonic() - last_log) / 20)
last_log = time.monotonic()
if step % cfg["eval_every"] == 0 or step == cfg["steps"]:
for split in ("train", "validation"):
metrics, predictions = evaluate(model, data[split], device, tokenizer, gemma_ids, cfg["batch_size"])
record(step, split, metrics)
if cached:
with (output / f"predictions-step-{step}-{split}.jsonl").open("w") as stream:
for prediction in predictions:
stream.write(json.dumps(prediction, ensure_ascii=False) + "\n")
if split == "validation" and metrics["cer"] < best_cer:
best_cer, best_step = metrics["cer"], step
torch.save({"reader": model.state_dict(), "config": cfg, "gemma_ids": gemma_ids,
"step": step, "validation": metrics}, output / "best-reader.pt")
if cached and step in (400, cfg["steps"]):
torch.save({"reader": model.state_dict(), "config": cfg, "gemma_ids": gemma_ids,
"step": step}, output / f"reader-step-{step}.pt")
if "test" in data:
model.load_state_dict(initial_state)
initial["test"], _ = evaluate(model, data["test"], device, tokenizer, gemma_ids, cfg["batch_size"])
record(0, "test", initial["test"])
ckpt = torch.load(output / "best-reader.pt", map_location=device, weights_only=True)
model.load_state_dict(ckpt["reader"])
final = {}
with (output / "predictions.jsonl").open("w") as predictions_file:
for split, rows in data.items():
final[split], predictions = evaluate(model, rows, device, tokenizer, gemma_ids, cfg["batch_size"])
for prediction in predictions:
predictions_file.write(json.dumps({"split": split, **prediction}, ensure_ascii=False) + "\n")
emit("selected_checkpoint_evaluation", step=best_step, split=split, **final[split])
emit("example", split=split, **predictions[0])
summary = {"initial": initial, "final": final, "selected_step": best_step,
"encoder_parameters": encoder_parameters, "trainable_parameters": trainable_parameters,
"layer_weights": model.layer_logits.softmax(0).detach().cpu().tolist(),
"data_audit": audit, "elapsed_seconds": time.monotonic() - start, "device": str(device),
"output": str(output),
"limitations": ["Tiny read-speech subsets; not competitive-ASR evidence", "No SpecAugment; frozen features cached in FP16, reader computes FP32", "Vocabulary from train only; held-out OOV tokens reported", "MPS CTC loss computed on CPU", "No Whisper/Koyal baseline in this learning test"]}
if cached:
summary.update(completed_steps=cfg["steps"], source_run=cfg["source_run"],
step_400={r["split"]: r for r in history if r["step"] == 400},
last_step={r["split"]: r for r in history if r["step"] == cfg["steps"]},
evaluated_splits=list(data))
summary["limitations"] += ["No test evaluation; validation used for checkpoint selection",
"Compare CUDA arms within this run; historical MPS is not a paired control",
"Mapped-reference reconstruction is a diagnostic, not an achievable-error bound"]
if cfg.get("coverage_vocabulary"):
summary["limitations"] = [item for item in summary["limitations"]
if item != "Vocabulary from train only; held-out OOV tokens reported"]
summary["limitations"] += ["Character vocabulary uses fixed alphabet plus training characters; no validation vocabulary mining",
"Tokenization, target lengths, and output-head size change together"]
save_json(output / "summary.json", summary)
metrics_file.close()
emit("FINAL_SUMMARY", **summary)
if __name__ == "__main__":
main()