dulu-malayalam-ctc-778h / cached_data.py
whiletruelearn's picture
Archive selected epoch 9 combined-data CTC checkpoint and evaluation
750e4eb verified
Raw History Blame Contribute Delete
3.41 kB
"""Reuse verified parent features; audit fixed vocabulary without changing it."""
import json
import torch
import jiwer
from overfit import verified_path
from text_utils import minimum_ctc_frames, normalize
def training_validation_only(cache):
return {split: cache["data"][split] for split in ("train", "validation")}
def load_training_validation(cfg):
cache = torch.load(verified_path(cfg["source_features"]), map_location="cpu", weights_only=True)
units = json.loads(verified_path(cfg["source_units"]).read_text())
for key in ("model_id", "model_revision", "dataset_id", "dataset_revision", "dataset_config"):
if cache["config"][key] != cfg[key]:
raise ValueError(f"Cache configuration mismatch: {key}")
data = training_validation_only(cache)
gemma_ids = units["gemma_ids"]
if sorted({t for r in data["train"] for t in r["gemma_ids"]}) != gemma_ids:
raise ValueError("Vocabulary must equal the original training-token union")
mapping = {t: i + 2 for i, t in enumerate(gemma_ids)}
seen_audio, seen_text = set(), set()
for split, rows in data.items():
if len(rows) != cfg["split_sizes"][split]:
raise ValueError("Wrong cached split size")
for row in rows:
if row["audio_sha256"] in seen_audio or row["text"] in seen_text:
raise ValueError("Duplicate audio or transcript across train/validation")
seen_audio.add(row["audio_sha256"])
seen_text.add(row["text"])
if row["target"] != [mapping.get(t, 1) for t in row["gemma_ids"]]:
raise ValueError("Cached targets do not match vocabulary")
if minimum_ctc_frames(row["target"]) > row["frames"]:
raise ValueError("Impossible CTC target")
if row["states"].shape[:2] != (13, row["frames"]) or not torch.isfinite(row["states"]).all():
raise ValueError("Invalid cached feature shape or values")
if cache["embeddings"].shape[0] != len(gemma_ids):
raise ValueError("Embedding rows do not match vocabulary")
return data, cache["embeddings"], gemma_ids
def reconstruct_mapped_reference(ids, known, tokenizer):
pieces, buffer = [], []
for token in ids:
if token in known:
buffer.append(token)
else:
pieces += [tokenizer.decode(buffer), " � "]
buffer = []
pieces.append(tokenizer.decode(buffer))
return " � ".join(normalize(p) for p in "".join(pieces).split("�")).strip()
def vocabulary_audit(data, gemma_ids, tokenizer):
known, result = set(gemma_ids), {}
for split, rows in data.items():
refs = [r["text"] for r in rows]
reconstructed = [reconstruct_mapped_reference(r["gemma_ids"], known, tokenizer) for r in rows]
counts = [sum(t not in known for t in r["gemma_ids"]) for r in rows]
result[split] = {"clips": len(rows), "seconds": sum(r["seconds"] for r in rows),
"unknown_tokens": sum(counts), "fully_covered_clips": counts.count(0),
"mapped_reference_wer": jiwer.wer(refs, reconstructed),
"mapped_reference_cer": jiwer.cer(refs, reconstructed),
"reconstruction_note": "Original reference tokenization with OOV replaced by unknown; not an optimal tokenization or lower error bound"}
return result