Download full_data.py from whiletruelearn/dulu-malayalam-ctc-778h: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/full_data.py
- Command line
-
hf download hf://whiletruelearn/dulu-malayalam-ctc-778h/full_data.py
-
curl -L -o full_data.py https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/full_data.py
13.9 kB
| """Full official train/validation selection and per-recording FP16 feature cache.""" | |
| from collections import Counter | |
| import gc | |
| import hashlib | |
| import io | |
| import json | |
| from pathlib import Path | |
| import shutil | |
| import time | |
| import datasets | |
| import numpy as np | |
| import soundfile as sf | |
| import torch | |
| from transformers import AutoFeatureExtractor, AutoModel | |
| from huggingface_hub import hf_hub_download | |
| from safetensors import safe_open | |
| from coverage_vocab import coverage_ids, segment_piece | |
| from prototype import emit, save_json | |
| from text_utils import minimum_ctc_frames, normalize | |
| def file_hash(path): | |
| with Path(path).open("rb") as stream: | |
| return hashlib.file_digest(stream, "sha256").hexdigest() | |
| def exclusion_reason(row, protected, seen_audio): | |
| if row["text"] in protected["text"]: | |
| return "heldout_transcript_overlap" | |
| if row["audio_sha256"] in protected["audio"] or row["pcm_sha256"] in protected["pcm"]: | |
| return "heldout_audio_overlap" | |
| if row["audio_sha256"] in seen_audio or row["pcm_sha256"] in seen_audio: | |
| return "duplicate_training_audio" | |
| return None | |
| def encode_characters(text, tokenizer, ids, allowed=None): | |
| if allowed is None: | |
| known = set(ids) | |
| allowed = {piece: token for piece, token in tokenizer.get_vocab().items() if token in known} | |
| tokens = [] | |
| for char in (" " + text).replace(" ", "▁"): | |
| tokens.extend(segment_piece(char, allowed)) | |
| if normalize(tokenizer.decode(tokens)) != text: | |
| raise ValueError("Character representation does not round-trip") | |
| mapping = {token: i + 2 for i, token in enumerate(ids)} | |
| return tokens, [mapping[token] for token in tokens] | |
| def prepare_audio(cfg, cache, prior): | |
| """Stream only validation and train; preserve all distinct training recordings.""" | |
| manifest = cache / "prepared.json" | |
| if manifest.exists(): | |
| prepared = json.loads(manifest.read_text()) | |
| emit("prepared_cache_reused", path=str(manifest), audit=prepared["audit"]) | |
| return prepared | |
| audio_dir = cache / "audio" | |
| audio_dir.mkdir(exist_ok=True) | |
| protected = {"text": {r["text"] for r in prior["previously_inspected_test"]}, | |
| "audio": {r["audio_sha256"] for r in prior["previously_inspected_test"]}, "pcm": set()} | |
| data, audit, rejected = {}, {}, [] | |
| for split in ("validation", "train"): | |
| emit("data_start", split=split, expected=cfg["expected_split_sizes"][split]) | |
| stream = datasets.load_dataset(cfg["dataset_id"], cfg["dataset_config"], | |
| revision=cfg["dataset_revision"], split=split, streaming=True, batch_size=16) | |
| stream = stream.cast_column("audio", datasets.Audio(decode=False)) | |
| rows, counts, seen_audio = [], Counter(), set() | |
| scanned = 0 | |
| for source in stream: | |
| scanned += 1 | |
| text = normalize(source["raw_transcription"]) | |
| blob = source["audio"]["bytes"] | |
| if blob is None: | |
| blob = Path(source["audio"]["path"]).read_bytes() | |
| raw_hash = hashlib.sha256(blob).hexdigest() | |
| try: | |
| wav, rate = sf.read(io.BytesIO(blob), dtype="float32") | |
| valid = rate == 16000 and wav.ndim == 1 and len(wav) > 0 and np.isfinite(wav).all() | |
| except (ValueError, RuntimeError): | |
| valid = False | |
| if not text or not valid: | |
| reason = "empty_normalized_text" if not text else "invalid_audio" | |
| counts[reason] += 1 | |
| rejected.append({"split": split, "id": str(source["id"]), "audio_sha256": raw_hash, "reason": reason}) | |
| continue | |
| pcm_hash = hashlib.sha256(wav.tobytes()).hexdigest() | |
| row = {"id": str(source["id"]), "path": source["path"], "text": text, | |
| "raw_text": source["raw_transcription"], "seconds": len(wav) / rate, | |
| "gender": int(source["gender"]), "audio_sha256": raw_hash, | |
| "pcm_sha256": pcm_hash, "num_samples": len(wav)} | |
| reason = exclusion_reason(row, protected, seen_audio) if split == "train" else None | |
| if reason: | |
| counts[reason] += 1 | |
| rejected.append({"split": split, "id": row["id"], "audio_sha256": raw_hash, "reason": reason}) | |
| continue | |
| audio_path = audio_dir / f"{split}-{raw_hash}.npy" | |
| np.save(audio_path, wav, allow_pickle=False) | |
| row["audio_path"] = str(audio_path) | |
| rows.append(row) | |
| seen_audio.update((raw_hash, pcm_hash)) | |
| if scanned % 100 == 0: | |
| emit("data_progress", split=split, scanned=scanned, accepted=len(rows)) | |
| if scanned != cfg["expected_split_sizes"][split] or not rows: | |
| raise ValueError(f"Unexpected {split} source count: {scanned}") | |
| data[split] = rows | |
| audit[split] = {"source_recordings": scanned, "accepted_recordings": len(rows), | |
| "hours": sum(r["seconds"] for r in rows) / 3600, | |
| "unique_transcripts": len({r["text"] for r in rows}), | |
| "min_seconds": min(r["seconds"] for r in rows), | |
| "max_seconds": max(r["seconds"] for r in rows), "rejected": dict(counts)} | |
| if split == "validation": | |
| for row in rows: | |
| protected["text"].add(row["text"]) | |
| protected["audio"].add(row["audio_sha256"]) | |
| protected["pcm"].add(row["pcm_sha256"]) | |
| emit("data_ready", split=split, **audit[split]) | |
| prepared = {"data": data, "audit": audit, "rejected": rejected} | |
| save_json(manifest, prepared) | |
| return prepared | |
| def cache_features(cfg, data, cache, device): | |
| """Extract whole recordings one at a time; only one example's states in RAM.""" | |
| feature_dir = cache / "features" | |
| feature_dir.mkdir(exist_ok=True) | |
| rows = [r for split in ("validation", "train") for r in data[split]] | |
| indexed = cache / "feature-index.json" | |
| index = json.loads(indexed.read_text()) if indexed.exists() else {} | |
| needed = [] | |
| for row in rows: | |
| key = row["audio_sha256"] | |
| path = feature_dir / f"{key}.npy" | |
| if key in index and path.exists(): | |
| if file_hash(path) != index[key]["sha256"]: | |
| raise ValueError(f"Corrupt cached features: {key}") | |
| row.update(index[key], states_path=str(path)) | |
| else: | |
| needed.append(row) | |
| emit("feature_cache_scan", total=len(rows), cached=len(rows)-len(needed), pending=len(needed)) | |
| if not needed: | |
| return | |
| # Conservative 26 fps estimate. Completed audio arrays are discarded as | |
| # their verified feature files are written, bounding peak storage. | |
| estimate = sum((r["seconds"] * 26 + 8) * 13 * 1024 * 2 for r in needed) | |
| reclaim = sum(Path(r["audio_path"]).stat().st_size for r in needed) | |
| reserve = cfg["minimum_free_disk_gib"] * 2**30 | |
| free = shutil.disk_usage(cache).free | |
| emit("storage_preflight", free_gib=free/2**30, estimated_features_gib=estimate/2**30, | |
| reclaimable_audio_gib=reclaim/2**30, reserve_gib=reserve/2**30) | |
| if free + reclaim < estimate + reserve: | |
| raise RuntimeError("Insufficient disk for complete feature cache and reserve") | |
| full = AutoModel.from_pretrained(cfg["model_id"], revision=cfg["model_revision"], | |
| dtype=torch.float32, vision_config=None) | |
| encoder = full.audio_tower | |
| del full | |
| gc.collect() | |
| encoder.requires_grad_(False).eval().to(device) | |
| extractor = AutoFeatureExtractor.from_pretrained(cfg["model_id"], revision=cfg["model_revision"]) | |
| states = [] | |
| def capture(module, args, value): | |
| states.append(value[0] if isinstance(value, tuple) else value) | |
| handles = [encoder.subsample_conv_projection.register_forward_hook(capture)] | |
| handles += [layer.register_forward_hook(capture) for layer in encoder.layers] | |
| emit("encoder_ready", parameters=sum(p.numel() for p in encoder.parameters()), | |
| trainable=0, truncation=False, device=str(device)) | |
| start = time.monotonic() | |
| for n, row in enumerate(needed, 1): | |
| key = row["audio_sha256"] | |
| if key in index: | |
| # Official validation may contain duplicate audio. Preserve its | |
| # evaluation weighting while extracting the recording only once. | |
| row.update(index[key], states_path=str(feature_dir / f"{key}.npy")) | |
| continue | |
| wav = np.load(row["audio_path"], allow_pickle=False) | |
| # The library defaults to truncating at 30 seconds: explicitly disable. | |
| inputs = extractor([wav], sampling_rate=16000, return_tensors="pt", truncation=False, max_length=None) | |
| states.clear() | |
| with torch.no_grad(): | |
| encoded = encoder(inputs["input_features"].to(device), | |
| inputs["input_features_mask"].to(device), return_dict=True) | |
| frames = int(encoded.attention_mask.sum().item()) | |
| if len(states) != 13 or frames <= 0 or frames < row["seconds"] * 24 - 2: | |
| raise ValueError("Unexpected encoder states or truncated audio") | |
| array = torch.stack([s[0, :frames].detach().cpu() for s in states]).half().numpy() | |
| if not np.isfinite(array).all(): | |
| raise ValueError("Nonfinite encoder feature") | |
| if shutil.disk_usage(cache).free < array.nbytes + reserve: | |
| raise RuntimeError("Feature cache reached reserved free-disk threshold") | |
| key = row["audio_sha256"] | |
| path = feature_dir / f"{key}.npy" | |
| temp = feature_dir / f"{key}.partial.npy" | |
| np.save(temp, array, allow_pickle=False) | |
| temp.replace(path) | |
| info = {"frames": frames, "state_shape": list(array.shape), "sha256": file_hash(path)} | |
| index[key] = info | |
| # Index before removing the reproducible temporary waveform, for restart. | |
| save_json(indexed.with_suffix(".tmp"), index) | |
| indexed.with_suffix(".tmp").replace(indexed) | |
| row.update(info, states_path=str(path)) | |
| Path(row["audio_path"]).unlink(missing_ok=True) | |
| del array, wav, inputs, encoded | |
| states.clear() | |
| if n % 25 == 0 or n == len(needed): | |
| emit("feature_progress", completed=n, pending_at_start=len(needed), | |
| elapsed_seconds=time.monotonic()-start, free_gib=shutil.disk_usage(cache).free/2**30) | |
| for handle in handles: | |
| handle.remove() | |
| del encoder | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| def prepare(cfg, tokenizer, output, device): | |
| prior_path = Path(cfg["prior_evaluation_manifest"]) | |
| prior = json.loads(prior_path.read_text()) | |
| contract = {k: cfg[k] for k in ("dataset_id", "dataset_revision", "dataset_config", "model_id", "model_revision")} | |
| contract.update(protocol="full-character-v1", prior_manifest_sha256=file_hash(prior_path)) | |
| fingerprint = hashlib.sha256(json.dumps(contract, sort_keys=True).encode()).hexdigest()[:16] | |
| cache = Path(cfg["feature_cache_root"]).expanduser() / fingerprint | |
| cache.mkdir(parents=True, exist_ok=True) | |
| save_json(output / "cache-contract.json", {**contract, "path": str(cache)}) | |
| prepared = prepare_audio(cfg, cache, prior) | |
| data = prepared["data"] | |
| train_chars = set("".join(r["text"].replace(" ", "▁") for r in data["train"])) | |
| ids = coverage_ids(tokenizer.get_vocab(), train_chars) | |
| known = set(ids) | |
| allowed = {piece: token for piece, token in tokenizer.get_vocab().items() if token in known} | |
| for split, rows in data.items(): | |
| for row in rows: | |
| row["gemma_ids"], row["target"] = encode_characters(row["text"], tokenizer, ids, allowed) | |
| save_json(output / "units.json", {"gemma_ids": ids, "blank": 0, "unk": 1, | |
| "vocabulary_uses_validation_text": False}) | |
| emit("vocabulary_ready", units=len(ids)+2, training_characters=len(train_chars), unknown_targets=0) | |
| cache_features(cfg, data, cache, device) | |
| impossible = [r for r in data["train"] if minimum_ctc_frames(r["target"]) > r["frames"]] | |
| data["train"] = [r for r in data["train"] if minimum_ctc_frames(r["target"]) <= r["frames"]] | |
| for row in impossible: | |
| prepared["rejected"].append({"split": "train", "id": row["id"], "audio_sha256": row["audio_sha256"], "reason": "ctc_infeasible"}) | |
| if not data["train"]: | |
| raise RuntimeError("No CTC-feasible training data") | |
| original_hashes = {r["audio_sha256"] for r in prior["validation"]} | |
| data["validation_24"] = [r for r in data["validation"] if r["audio_sha256"] in original_hashes] | |
| if len(data["validation_24"]) != len(original_hashes): | |
| raise ValueError("Original validation subset is not completely recoverable") | |
| audit = {"selection": prepared["audit"], "training_ctc_rejections": len(impossible), | |
| "training_recordings": len(data["train"]), "training_hours": sum(r["seconds"] for r in data["train"])/3600, | |
| "validation_recordings": len(data["validation"]), "validation_24_recordings": len(data["validation_24"]), | |
| "validation_ctc_infeasible": sum(minimum_ctc_frames(r["target"]) > r["frames"] for r in data["validation"]), | |
| "test_loaded": False, "previously_inspected_test_recordings": len(prior["previously_inspected_test"]), | |
| "vocabulary_units": len(ids)+2, "cache_path": str(cache)} | |
| save_json(output / "data-audit.json", audit) | |
| save_json(output / "rejected-recordings.json", prepared["rejected"]) | |
| save_json(output / "manifest.json", data) | |
| emit("data_audit", **audit) | |
| weights_path = hf_hub_download(cfg["model_id"], "model.safetensors", revision=cfg["model_revision"]) | |
| with safe_open(weights_path, framework="pt", device="cpu") as checkpoint: | |
| weights = checkpoint.get_slice("language_model.embed_tokens.weight") | |
| embeddings = torch.stack([weights[i:i+1].float()[0] for i in ids]) | |
| return data, embeddings, ids, audit | |