Download cached_data.py from whiletruelearn/dulu-malayalam-ctc-778h: direct link, hf CLI and curl.
- Browser
- Download file 3.41 kB
-
https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/cached_data.py
- Command line
-
hf download hf://whiletruelearn/dulu-malayalam-ctc-778h/cached_data.py
-
curl -L -o cached_data.py https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/cached_data.py
3.41 kB
| """Reuse verified parent features; audit fixed vocabulary without changing it.""" | |
| import json | |
| import torch | |
| import jiwer | |
| from overfit import verified_path | |
| from text_utils import minimum_ctc_frames, normalize | |
| def training_validation_only(cache): | |
| return {split: cache["data"][split] for split in ("train", "validation")} | |
| def load_training_validation(cfg): | |
| cache = torch.load(verified_path(cfg["source_features"]), map_location="cpu", weights_only=True) | |
| units = json.loads(verified_path(cfg["source_units"]).read_text()) | |
| for key in ("model_id", "model_revision", "dataset_id", "dataset_revision", "dataset_config"): | |
| if cache["config"][key] != cfg[key]: | |
| raise ValueError(f"Cache configuration mismatch: {key}") | |
| data = training_validation_only(cache) | |
| gemma_ids = units["gemma_ids"] | |
| if sorted({t for r in data["train"] for t in r["gemma_ids"]}) != gemma_ids: | |
| raise ValueError("Vocabulary must equal the original training-token union") | |
| mapping = {t: i + 2 for i, t in enumerate(gemma_ids)} | |
| seen_audio, seen_text = set(), set() | |
| for split, rows in data.items(): | |
| if len(rows) != cfg["split_sizes"][split]: | |
| raise ValueError("Wrong cached split size") | |
| for row in rows: | |
| if row["audio_sha256"] in seen_audio or row["text"] in seen_text: | |
| raise ValueError("Duplicate audio or transcript across train/validation") | |
| seen_audio.add(row["audio_sha256"]) | |
| seen_text.add(row["text"]) | |
| if row["target"] != [mapping.get(t, 1) for t in row["gemma_ids"]]: | |
| raise ValueError("Cached targets do not match vocabulary") | |
| if minimum_ctc_frames(row["target"]) > row["frames"]: | |
| raise ValueError("Impossible CTC target") | |
| if row["states"].shape[:2] != (13, row["frames"]) or not torch.isfinite(row["states"]).all(): | |
| raise ValueError("Invalid cached feature shape or values") | |
| if cache["embeddings"].shape[0] != len(gemma_ids): | |
| raise ValueError("Embedding rows do not match vocabulary") | |
| return data, cache["embeddings"], gemma_ids | |
| def reconstruct_mapped_reference(ids, known, tokenizer): | |
| pieces, buffer = [], [] | |
| for token in ids: | |
| if token in known: | |
| buffer.append(token) | |
| else: | |
| pieces += [tokenizer.decode(buffer), " � "] | |
| buffer = [] | |
| pieces.append(tokenizer.decode(buffer)) | |
| return " � ".join(normalize(p) for p in "".join(pieces).split("�")).strip() | |
| def vocabulary_audit(data, gemma_ids, tokenizer): | |
| known, result = set(gemma_ids), {} | |
| for split, rows in data.items(): | |
| refs = [r["text"] for r in rows] | |
| reconstructed = [reconstruct_mapped_reference(r["gemma_ids"], known, tokenizer) for r in rows] | |
| counts = [sum(t not in known for t in r["gemma_ids"]) for r in rows] | |
| result[split] = {"clips": len(rows), "seconds": sum(r["seconds"] for r in rows), | |
| "unknown_tokens": sum(counts), "fully_covered_clips": counts.count(0), | |
| "mapped_reference_wer": jiwer.wer(refs, reconstructed), | |
| "mapped_reference_cer": jiwer.cer(refs, reconstructed), | |
| "reconstruction_note": "Original reference tokenization with OOV replaced by unknown; not an optimal tokenization or lower error bound"} | |
| return result | |