"""Reuse verified parent features; audit fixed vocabulary without changing it.""" import json import torch import jiwer from overfit import verified_path from text_utils import minimum_ctc_frames, normalize def training_validation_only(cache): return {split: cache["data"][split] for split in ("train", "validation")} def load_training_validation(cfg): cache = torch.load(verified_path(cfg["source_features"]), map_location="cpu", weights_only=True) units = json.loads(verified_path(cfg["source_units"]).read_text()) for key in ("model_id", "model_revision", "dataset_id", "dataset_revision", "dataset_config"): if cache["config"][key] != cfg[key]: raise ValueError(f"Cache configuration mismatch: {key}") data = training_validation_only(cache) gemma_ids = units["gemma_ids"] if sorted({t for r in data["train"] for t in r["gemma_ids"]}) != gemma_ids: raise ValueError("Vocabulary must equal the original training-token union") mapping = {t: i + 2 for i, t in enumerate(gemma_ids)} seen_audio, seen_text = set(), set() for split, rows in data.items(): if len(rows) != cfg["split_sizes"][split]: raise ValueError("Wrong cached split size") for row in rows: if row["audio_sha256"] in seen_audio or row["text"] in seen_text: raise ValueError("Duplicate audio or transcript across train/validation") seen_audio.add(row["audio_sha256"]) seen_text.add(row["text"]) if row["target"] != [mapping.get(t, 1) for t in row["gemma_ids"]]: raise ValueError("Cached targets do not match vocabulary") if minimum_ctc_frames(row["target"]) > row["frames"]: raise ValueError("Impossible CTC target") if row["states"].shape[:2] != (13, row["frames"]) or not torch.isfinite(row["states"]).all(): raise ValueError("Invalid cached feature shape or values") if cache["embeddings"].shape[0] != len(gemma_ids): raise ValueError("Embedding rows do not match vocabulary") return data, cache["embeddings"], gemma_ids def reconstruct_mapped_reference(ids, known, tokenizer): pieces, buffer = [], [] for token in ids: if token in known: buffer.append(token) else: pieces += [tokenizer.decode(buffer), " � "] buffer = [] pieces.append(tokenizer.decode(buffer)) return " � ".join(normalize(p) for p in "".join(pieces).split("�")).strip() def vocabulary_audit(data, gemma_ids, tokenizer): known, result = set(gemma_ids), {} for split, rows in data.items(): refs = [r["text"] for r in rows] reconstructed = [reconstruct_mapped_reference(r["gemma_ids"], known, tokenizer) for r in rows] counts = [sum(t not in known for t in r["gemma_ids"]) for r in rows] result[split] = {"clips": len(rows), "seconds": sum(r["seconds"] for r in rows), "unknown_tokens": sum(counts), "fully_covered_clips": counts.count(0), "mapped_reference_wer": jiwer.wer(refs, reconstructed), "mapped_reference_cer": jiwer.cer(refs, reconstructed), "reconstruction_note": "Original reference tokenization with OOV replaced by unknown; not an optimal tokenization or lower error bound"} return result