dulu-malayalam-ctc-778h / full_data.py
whiletruelearn's picture
Archive selected epoch 9 combined-data CTC checkpoint and evaluation
750e4eb verified
Raw History Blame Contribute Delete
13.9 kB
"""Full official train/validation selection and per-recording FP16 feature cache."""
from collections import Counter
import gc
import hashlib
import io
import json
from pathlib import Path
import shutil
import time
import datasets
import numpy as np
import soundfile as sf
import torch
from transformers import AutoFeatureExtractor, AutoModel
from huggingface_hub import hf_hub_download
from safetensors import safe_open
from coverage_vocab import coverage_ids, segment_piece
from prototype import emit, save_json
from text_utils import minimum_ctc_frames, normalize
def file_hash(path):
with Path(path).open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def exclusion_reason(row, protected, seen_audio):
if row["text"] in protected["text"]:
return "heldout_transcript_overlap"
if row["audio_sha256"] in protected["audio"] or row["pcm_sha256"] in protected["pcm"]:
return "heldout_audio_overlap"
if row["audio_sha256"] in seen_audio or row["pcm_sha256"] in seen_audio:
return "duplicate_training_audio"
return None
def encode_characters(text, tokenizer, ids, allowed=None):
if allowed is None:
known = set(ids)
allowed = {piece: token for piece, token in tokenizer.get_vocab().items() if token in known}
tokens = []
for char in (" " + text).replace(" ", "▁"):
tokens.extend(segment_piece(char, allowed))
if normalize(tokenizer.decode(tokens)) != text:
raise ValueError("Character representation does not round-trip")
mapping = {token: i + 2 for i, token in enumerate(ids)}
return tokens, [mapping[token] for token in tokens]
def prepare_audio(cfg, cache, prior):
"""Stream only validation and train; preserve all distinct training recordings."""
manifest = cache / "prepared.json"
if manifest.exists():
prepared = json.loads(manifest.read_text())
emit("prepared_cache_reused", path=str(manifest), audit=prepared["audit"])
return prepared
audio_dir = cache / "audio"
audio_dir.mkdir(exist_ok=True)
protected = {"text": {r["text"] for r in prior["previously_inspected_test"]},
"audio": {r["audio_sha256"] for r in prior["previously_inspected_test"]}, "pcm": set()}
data, audit, rejected = {}, {}, []
for split in ("validation", "train"):
emit("data_start", split=split, expected=cfg["expected_split_sizes"][split])
stream = datasets.load_dataset(cfg["dataset_id"], cfg["dataset_config"],
revision=cfg["dataset_revision"], split=split, streaming=True, batch_size=16)
stream = stream.cast_column("audio", datasets.Audio(decode=False))
rows, counts, seen_audio = [], Counter(), set()
scanned = 0
for source in stream:
scanned += 1
text = normalize(source["raw_transcription"])
blob = source["audio"]["bytes"]
if blob is None:
blob = Path(source["audio"]["path"]).read_bytes()
raw_hash = hashlib.sha256(blob).hexdigest()
try:
wav, rate = sf.read(io.BytesIO(blob), dtype="float32")
valid = rate == 16000 and wav.ndim == 1 and len(wav) > 0 and np.isfinite(wav).all()
except (ValueError, RuntimeError):
valid = False
if not text or not valid:
reason = "empty_normalized_text" if not text else "invalid_audio"
counts[reason] += 1
rejected.append({"split": split, "id": str(source["id"]), "audio_sha256": raw_hash, "reason": reason})
continue
pcm_hash = hashlib.sha256(wav.tobytes()).hexdigest()
row = {"id": str(source["id"]), "path": source["path"], "text": text,
"raw_text": source["raw_transcription"], "seconds": len(wav) / rate,
"gender": int(source["gender"]), "audio_sha256": raw_hash,
"pcm_sha256": pcm_hash, "num_samples": len(wav)}
reason = exclusion_reason(row, protected, seen_audio) if split == "train" else None
if reason:
counts[reason] += 1
rejected.append({"split": split, "id": row["id"], "audio_sha256": raw_hash, "reason": reason})
continue
audio_path = audio_dir / f"{split}-{raw_hash}.npy"
np.save(audio_path, wav, allow_pickle=False)
row["audio_path"] = str(audio_path)
rows.append(row)
seen_audio.update((raw_hash, pcm_hash))
if scanned % 100 == 0:
emit("data_progress", split=split, scanned=scanned, accepted=len(rows))
if scanned != cfg["expected_split_sizes"][split] or not rows:
raise ValueError(f"Unexpected {split} source count: {scanned}")
data[split] = rows
audit[split] = {"source_recordings": scanned, "accepted_recordings": len(rows),
"hours": sum(r["seconds"] for r in rows) / 3600,
"unique_transcripts": len({r["text"] for r in rows}),
"min_seconds": min(r["seconds"] for r in rows),
"max_seconds": max(r["seconds"] for r in rows), "rejected": dict(counts)}
if split == "validation":
for row in rows:
protected["text"].add(row["text"])
protected["audio"].add(row["audio_sha256"])
protected["pcm"].add(row["pcm_sha256"])
emit("data_ready", split=split, **audit[split])
prepared = {"data": data, "audit": audit, "rejected": rejected}
save_json(manifest, prepared)
return prepared
def cache_features(cfg, data, cache, device):
"""Extract whole recordings one at a time; only one example's states in RAM."""
feature_dir = cache / "features"
feature_dir.mkdir(exist_ok=True)
rows = [r for split in ("validation", "train") for r in data[split]]
indexed = cache / "feature-index.json"
index = json.loads(indexed.read_text()) if indexed.exists() else {}
needed = []
for row in rows:
key = row["audio_sha256"]
path = feature_dir / f"{key}.npy"
if key in index and path.exists():
if file_hash(path) != index[key]["sha256"]:
raise ValueError(f"Corrupt cached features: {key}")
row.update(index[key], states_path=str(path))
else:
needed.append(row)
emit("feature_cache_scan", total=len(rows), cached=len(rows)-len(needed), pending=len(needed))
if not needed:
return
# Conservative 26 fps estimate. Completed audio arrays are discarded as
# their verified feature files are written, bounding peak storage.
estimate = sum((r["seconds"] * 26 + 8) * 13 * 1024 * 2 for r in needed)
reclaim = sum(Path(r["audio_path"]).stat().st_size for r in needed)
reserve = cfg["minimum_free_disk_gib"] * 2**30
free = shutil.disk_usage(cache).free
emit("storage_preflight", free_gib=free/2**30, estimated_features_gib=estimate/2**30,
reclaimable_audio_gib=reclaim/2**30, reserve_gib=reserve/2**30)
if free + reclaim < estimate + reserve:
raise RuntimeError("Insufficient disk for complete feature cache and reserve")
full = AutoModel.from_pretrained(cfg["model_id"], revision=cfg["model_revision"],
dtype=torch.float32, vision_config=None)
encoder = full.audio_tower
del full
gc.collect()
encoder.requires_grad_(False).eval().to(device)
extractor = AutoFeatureExtractor.from_pretrained(cfg["model_id"], revision=cfg["model_revision"])
states = []
def capture(module, args, value):
states.append(value[0] if isinstance(value, tuple) else value)
handles = [encoder.subsample_conv_projection.register_forward_hook(capture)]
handles += [layer.register_forward_hook(capture) for layer in encoder.layers]
emit("encoder_ready", parameters=sum(p.numel() for p in encoder.parameters()),
trainable=0, truncation=False, device=str(device))
start = time.monotonic()
for n, row in enumerate(needed, 1):
key = row["audio_sha256"]
if key in index:
# Official validation may contain duplicate audio. Preserve its
# evaluation weighting while extracting the recording only once.
row.update(index[key], states_path=str(feature_dir / f"{key}.npy"))
continue
wav = np.load(row["audio_path"], allow_pickle=False)
# The library defaults to truncating at 30 seconds: explicitly disable.
inputs = extractor([wav], sampling_rate=16000, return_tensors="pt", truncation=False, max_length=None)
states.clear()
with torch.no_grad():
encoded = encoder(inputs["input_features"].to(device),
inputs["input_features_mask"].to(device), return_dict=True)
frames = int(encoded.attention_mask.sum().item())
if len(states) != 13 or frames <= 0 or frames < row["seconds"] * 24 - 2:
raise ValueError("Unexpected encoder states or truncated audio")
array = torch.stack([s[0, :frames].detach().cpu() for s in states]).half().numpy()
if not np.isfinite(array).all():
raise ValueError("Nonfinite encoder feature")
if shutil.disk_usage(cache).free < array.nbytes + reserve:
raise RuntimeError("Feature cache reached reserved free-disk threshold")
key = row["audio_sha256"]
path = feature_dir / f"{key}.npy"
temp = feature_dir / f"{key}.partial.npy"
np.save(temp, array, allow_pickle=False)
temp.replace(path)
info = {"frames": frames, "state_shape": list(array.shape), "sha256": file_hash(path)}
index[key] = info
# Index before removing the reproducible temporary waveform, for restart.
save_json(indexed.with_suffix(".tmp"), index)
indexed.with_suffix(".tmp").replace(indexed)
row.update(info, states_path=str(path))
Path(row["audio_path"]).unlink(missing_ok=True)
del array, wav, inputs, encoded
states.clear()
if n % 25 == 0 or n == len(needed):
emit("feature_progress", completed=n, pending_at_start=len(needed),
elapsed_seconds=time.monotonic()-start, free_gib=shutil.disk_usage(cache).free/2**30)
for handle in handles:
handle.remove()
del encoder
gc.collect()
torch.cuda.empty_cache()
def prepare(cfg, tokenizer, output, device):
prior_path = Path(cfg["prior_evaluation_manifest"])
prior = json.loads(prior_path.read_text())
contract = {k: cfg[k] for k in ("dataset_id", "dataset_revision", "dataset_config", "model_id", "model_revision")}
contract.update(protocol="full-character-v1", prior_manifest_sha256=file_hash(prior_path))
fingerprint = hashlib.sha256(json.dumps(contract, sort_keys=True).encode()).hexdigest()[:16]
cache = Path(cfg["feature_cache_root"]).expanduser() / fingerprint
cache.mkdir(parents=True, exist_ok=True)
save_json(output / "cache-contract.json", {**contract, "path": str(cache)})
prepared = prepare_audio(cfg, cache, prior)
data = prepared["data"]
train_chars = set("".join(r["text"].replace(" ", "▁") for r in data["train"]))
ids = coverage_ids(tokenizer.get_vocab(), train_chars)
known = set(ids)
allowed = {piece: token for piece, token in tokenizer.get_vocab().items() if token in known}
for split, rows in data.items():
for row in rows:
row["gemma_ids"], row["target"] = encode_characters(row["text"], tokenizer, ids, allowed)
save_json(output / "units.json", {"gemma_ids": ids, "blank": 0, "unk": 1,
"vocabulary_uses_validation_text": False})
emit("vocabulary_ready", units=len(ids)+2, training_characters=len(train_chars), unknown_targets=0)
cache_features(cfg, data, cache, device)
impossible = [r for r in data["train"] if minimum_ctc_frames(r["target"]) > r["frames"]]
data["train"] = [r for r in data["train"] if minimum_ctc_frames(r["target"]) <= r["frames"]]
for row in impossible:
prepared["rejected"].append({"split": "train", "id": row["id"], "audio_sha256": row["audio_sha256"], "reason": "ctc_infeasible"})
if not data["train"]:
raise RuntimeError("No CTC-feasible training data")
original_hashes = {r["audio_sha256"] for r in prior["validation"]}
data["validation_24"] = [r for r in data["validation"] if r["audio_sha256"] in original_hashes]
if len(data["validation_24"]) != len(original_hashes):
raise ValueError("Original validation subset is not completely recoverable")
audit = {"selection": prepared["audit"], "training_ctc_rejections": len(impossible),
"training_recordings": len(data["train"]), "training_hours": sum(r["seconds"] for r in data["train"])/3600,
"validation_recordings": len(data["validation"]), "validation_24_recordings": len(data["validation_24"]),
"validation_ctc_infeasible": sum(minimum_ctc_frames(r["target"]) > r["frames"] for r in data["validation"]),
"test_loaded": False, "previously_inspected_test_recordings": len(prior["previously_inspected_test"]),
"vocabulary_units": len(ids)+2, "cache_path": str(cache)}
save_json(output / "data-audit.json", audit)
save_json(output / "rejected-recordings.json", prepared["rejected"])
save_json(output / "manifest.json", data)
emit("data_audit", **audit)
weights_path = hf_hub_download(cfg["model_id"], "model.safetensors", revision=cfg["model_revision"])
with safe_open(weights_path, framework="pt", device="cpu") as checkpoint:
weights = checkpoint.get_slice("language_model.embed_tokens.weight")
embeddings = torch.stack([weights[i:i+1].float()[0] for i in ids])
return data, embeddings, ids, audit