"""Full official train/validation selection and per-recording FP16 feature cache.""" from collections import Counter import gc import hashlib import io import json from pathlib import Path import shutil import time import datasets import numpy as np import soundfile as sf import torch from transformers import AutoFeatureExtractor, AutoModel from huggingface_hub import hf_hub_download from safetensors import safe_open from coverage_vocab import coverage_ids, segment_piece from prototype import emit, save_json from text_utils import minimum_ctc_frames, normalize def file_hash(path): with Path(path).open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest() def exclusion_reason(row, protected, seen_audio): if row["text"] in protected["text"]: return "heldout_transcript_overlap" if row["audio_sha256"] in protected["audio"] or row["pcm_sha256"] in protected["pcm"]: return "heldout_audio_overlap" if row["audio_sha256"] in seen_audio or row["pcm_sha256"] in seen_audio: return "duplicate_training_audio" return None def encode_characters(text, tokenizer, ids, allowed=None): if allowed is None: known = set(ids) allowed = {piece: token for piece, token in tokenizer.get_vocab().items() if token in known} tokens = [] for char in (" " + text).replace(" ", "▁"): tokens.extend(segment_piece(char, allowed)) if normalize(tokenizer.decode(tokens)) != text: raise ValueError("Character representation does not round-trip") mapping = {token: i + 2 for i, token in enumerate(ids)} return tokens, [mapping[token] for token in tokens] def prepare_audio(cfg, cache, prior): """Stream only validation and train; preserve all distinct training recordings.""" manifest = cache / "prepared.json" if manifest.exists(): prepared = json.loads(manifest.read_text()) emit("prepared_cache_reused", path=str(manifest), audit=prepared["audit"]) return prepared audio_dir = cache / "audio" audio_dir.mkdir(exist_ok=True) protected = {"text": {r["text"] for r in prior["previously_inspected_test"]}, "audio": {r["audio_sha256"] for r in prior["previously_inspected_test"]}, "pcm": set()} data, audit, rejected = {}, {}, [] for split in ("validation", "train"): emit("data_start", split=split, expected=cfg["expected_split_sizes"][split]) stream = datasets.load_dataset(cfg["dataset_id"], cfg["dataset_config"], revision=cfg["dataset_revision"], split=split, streaming=True, batch_size=16) stream = stream.cast_column("audio", datasets.Audio(decode=False)) rows, counts, seen_audio = [], Counter(), set() scanned = 0 for source in stream: scanned += 1 text = normalize(source["raw_transcription"]) blob = source["audio"]["bytes"] if blob is None: blob = Path(source["audio"]["path"]).read_bytes() raw_hash = hashlib.sha256(blob).hexdigest() try: wav, rate = sf.read(io.BytesIO(blob), dtype="float32") valid = rate == 16000 and wav.ndim == 1 and len(wav) > 0 and np.isfinite(wav).all() except (ValueError, RuntimeError): valid = False if not text or not valid: reason = "empty_normalized_text" if not text else "invalid_audio" counts[reason] += 1 rejected.append({"split": split, "id": str(source["id"]), "audio_sha256": raw_hash, "reason": reason}) continue pcm_hash = hashlib.sha256(wav.tobytes()).hexdigest() row = {"id": str(source["id"]), "path": source["path"], "text": text, "raw_text": source["raw_transcription"], "seconds": len(wav) / rate, "gender": int(source["gender"]), "audio_sha256": raw_hash, "pcm_sha256": pcm_hash, "num_samples": len(wav)} reason = exclusion_reason(row, protected, seen_audio) if split == "train" else None if reason: counts[reason] += 1 rejected.append({"split": split, "id": row["id"], "audio_sha256": raw_hash, "reason": reason}) continue audio_path = audio_dir / f"{split}-{raw_hash}.npy" np.save(audio_path, wav, allow_pickle=False) row["audio_path"] = str(audio_path) rows.append(row) seen_audio.update((raw_hash, pcm_hash)) if scanned % 100 == 0: emit("data_progress", split=split, scanned=scanned, accepted=len(rows)) if scanned != cfg["expected_split_sizes"][split] or not rows: raise ValueError(f"Unexpected {split} source count: {scanned}") data[split] = rows audit[split] = {"source_recordings": scanned, "accepted_recordings": len(rows), "hours": sum(r["seconds"] for r in rows) / 3600, "unique_transcripts": len({r["text"] for r in rows}), "min_seconds": min(r["seconds"] for r in rows), "max_seconds": max(r["seconds"] for r in rows), "rejected": dict(counts)} if split == "validation": for row in rows: protected["text"].add(row["text"]) protected["audio"].add(row["audio_sha256"]) protected["pcm"].add(row["pcm_sha256"]) emit("data_ready", split=split, **audit[split]) prepared = {"data": data, "audit": audit, "rejected": rejected} save_json(manifest, prepared) return prepared def cache_features(cfg, data, cache, device): """Extract whole recordings one at a time; only one example's states in RAM.""" feature_dir = cache / "features" feature_dir.mkdir(exist_ok=True) rows = [r for split in ("validation", "train") for r in data[split]] indexed = cache / "feature-index.json" index = json.loads(indexed.read_text()) if indexed.exists() else {} needed = [] for row in rows: key = row["audio_sha256"] path = feature_dir / f"{key}.npy" if key in index and path.exists(): if file_hash(path) != index[key]["sha256"]: raise ValueError(f"Corrupt cached features: {key}") row.update(index[key], states_path=str(path)) else: needed.append(row) emit("feature_cache_scan", total=len(rows), cached=len(rows)-len(needed), pending=len(needed)) if not needed: return # Conservative 26 fps estimate. Completed audio arrays are discarded as # their verified feature files are written, bounding peak storage. estimate = sum((r["seconds"] * 26 + 8) * 13 * 1024 * 2 for r in needed) reclaim = sum(Path(r["audio_path"]).stat().st_size for r in needed) reserve = cfg["minimum_free_disk_gib"] * 2**30 free = shutil.disk_usage(cache).free emit("storage_preflight", free_gib=free/2**30, estimated_features_gib=estimate/2**30, reclaimable_audio_gib=reclaim/2**30, reserve_gib=reserve/2**30) if free + reclaim < estimate + reserve: raise RuntimeError("Insufficient disk for complete feature cache and reserve") full = AutoModel.from_pretrained(cfg["model_id"], revision=cfg["model_revision"], dtype=torch.float32, vision_config=None) encoder = full.audio_tower del full gc.collect() encoder.requires_grad_(False).eval().to(device) extractor = AutoFeatureExtractor.from_pretrained(cfg["model_id"], revision=cfg["model_revision"]) states = [] def capture(module, args, value): states.append(value[0] if isinstance(value, tuple) else value) handles = [encoder.subsample_conv_projection.register_forward_hook(capture)] handles += [layer.register_forward_hook(capture) for layer in encoder.layers] emit("encoder_ready", parameters=sum(p.numel() for p in encoder.parameters()), trainable=0, truncation=False, device=str(device)) start = time.monotonic() for n, row in enumerate(needed, 1): key = row["audio_sha256"] if key in index: # Official validation may contain duplicate audio. Preserve its # evaluation weighting while extracting the recording only once. row.update(index[key], states_path=str(feature_dir / f"{key}.npy")) continue wav = np.load(row["audio_path"], allow_pickle=False) # The library defaults to truncating at 30 seconds: explicitly disable. inputs = extractor([wav], sampling_rate=16000, return_tensors="pt", truncation=False, max_length=None) states.clear() with torch.no_grad(): encoded = encoder(inputs["input_features"].to(device), inputs["input_features_mask"].to(device), return_dict=True) frames = int(encoded.attention_mask.sum().item()) if len(states) != 13 or frames <= 0 or frames < row["seconds"] * 24 - 2: raise ValueError("Unexpected encoder states or truncated audio") array = torch.stack([s[0, :frames].detach().cpu() for s in states]).half().numpy() if not np.isfinite(array).all(): raise ValueError("Nonfinite encoder feature") if shutil.disk_usage(cache).free < array.nbytes + reserve: raise RuntimeError("Feature cache reached reserved free-disk threshold") key = row["audio_sha256"] path = feature_dir / f"{key}.npy" temp = feature_dir / f"{key}.partial.npy" np.save(temp, array, allow_pickle=False) temp.replace(path) info = {"frames": frames, "state_shape": list(array.shape), "sha256": file_hash(path)} index[key] = info # Index before removing the reproducible temporary waveform, for restart. save_json(indexed.with_suffix(".tmp"), index) indexed.with_suffix(".tmp").replace(indexed) row.update(info, states_path=str(path)) Path(row["audio_path"]).unlink(missing_ok=True) del array, wav, inputs, encoded states.clear() if n % 25 == 0 or n == len(needed): emit("feature_progress", completed=n, pending_at_start=len(needed), elapsed_seconds=time.monotonic()-start, free_gib=shutil.disk_usage(cache).free/2**30) for handle in handles: handle.remove() del encoder gc.collect() torch.cuda.empty_cache() def prepare(cfg, tokenizer, output, device): prior_path = Path(cfg["prior_evaluation_manifest"]) prior = json.loads(prior_path.read_text()) contract = {k: cfg[k] for k in ("dataset_id", "dataset_revision", "dataset_config", "model_id", "model_revision")} contract.update(protocol="full-character-v1", prior_manifest_sha256=file_hash(prior_path)) fingerprint = hashlib.sha256(json.dumps(contract, sort_keys=True).encode()).hexdigest()[:16] cache = Path(cfg["feature_cache_root"]).expanduser() / fingerprint cache.mkdir(parents=True, exist_ok=True) save_json(output / "cache-contract.json", {**contract, "path": str(cache)}) prepared = prepare_audio(cfg, cache, prior) data = prepared["data"] train_chars = set("".join(r["text"].replace(" ", "▁") for r in data["train"])) ids = coverage_ids(tokenizer.get_vocab(), train_chars) known = set(ids) allowed = {piece: token for piece, token in tokenizer.get_vocab().items() if token in known} for split, rows in data.items(): for row in rows: row["gemma_ids"], row["target"] = encode_characters(row["text"], tokenizer, ids, allowed) save_json(output / "units.json", {"gemma_ids": ids, "blank": 0, "unk": 1, "vocabulary_uses_validation_text": False}) emit("vocabulary_ready", units=len(ids)+2, training_characters=len(train_chars), unknown_targets=0) cache_features(cfg, data, cache, device) impossible = [r for r in data["train"] if minimum_ctc_frames(r["target"]) > r["frames"]] data["train"] = [r for r in data["train"] if minimum_ctc_frames(r["target"]) <= r["frames"]] for row in impossible: prepared["rejected"].append({"split": "train", "id": row["id"], "audio_sha256": row["audio_sha256"], "reason": "ctc_infeasible"}) if not data["train"]: raise RuntimeError("No CTC-feasible training data") original_hashes = {r["audio_sha256"] for r in prior["validation"]} data["validation_24"] = [r for r in data["validation"] if r["audio_sha256"] in original_hashes] if len(data["validation_24"]) != len(original_hashes): raise ValueError("Original validation subset is not completely recoverable") audit = {"selection": prepared["audit"], "training_ctc_rejections": len(impossible), "training_recordings": len(data["train"]), "training_hours": sum(r["seconds"] for r in data["train"])/3600, "validation_recordings": len(data["validation"]), "validation_24_recordings": len(data["validation_24"]), "validation_ctc_infeasible": sum(minimum_ctc_frames(r["target"]) > r["frames"] for r in data["validation"]), "test_loaded": False, "previously_inspected_test_recordings": len(prior["previously_inspected_test"]), "vocabulary_units": len(ids)+2, "cache_path": str(cache)} save_json(output / "data-audit.json", audit) save_json(output / "rejected-recordings.json", prepared["rejected"]) save_json(output / "manifest.json", data) emit("data_audit", **audit) weights_path = hf_hub_download(cfg["model_id"], "model.safetensors", revision=cfg["model_revision"]) with safe_open(weights_path, framework="pt", device="cpu") as checkpoint: weights = checkpoint.get_slice("language_model.embed_tokens.weight") embeddings = torch.stack([weights[i:i+1].float()[0] for i in ids]) return data, embeddings, ids, audit