Download make_test_set_fa.py from karim23657/fa-eg2-persian: direct link, hf CLI and curl.
- Browser
- Download file 3.02 kB
-
https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/make_test_set_fa.py
- Command line
-
hf download hf://karim23657/fa-eg2-persian/make_test_set_fa.py
-
curl -L -o make_test_set_fa.py https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/make_test_set_fa.py
3.02 kB
| #!/usr/bin/env python3 | |
| """Build a held-out Persian test TSV + 16 kHz wav clips from a parquet ASR split. | |
| Reads ``<data-root>/data/<split>-*.parquet`` (Common Voice fa layout: ``audio.bytes`` + | |
| ``text``), decodes and resamples to 16 kHz mono, writes one wav per utterance to | |
| ``<out-dir>/<split>_audio/<id>.wav``, and a TSV with columns:: | |
| audio_path n_samples n_frames n_tokens text | |
| where ``text`` is Persian-normalized. ``audio_path`` is relative to the TSV's directory. | |
| Example:: | |
| python make_test_set_fa.py --data-root data/cv_fa --split test \ | |
| --out data/fa_test.tsv --out-dir data --limit 0 | |
| """ | |
| import argparse | |
| import glob | |
| import io | |
| import os | |
| import numpy as np | |
| import pyarrow.parquet as pq | |
| import soundfile as sf | |
| import persian_text as PT | |
| from cache_soft_tokens_fa import SR, decode, n_frames_for, n_tokens_for | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--data-root", required=True, help="directory holding data/<split>-*.parquet") | |
| ap.add_argument("--split", default="test") | |
| ap.add_argument("--out", required=True, help="output TSV path") | |
| ap.add_argument("--out-dir", default=None, help="directory for wav clips (default: dirname of --out)") | |
| ap.add_argument("--limit", type=int, default=0, help="max utterances (0 = all)") | |
| args = ap.parse_args() | |
| out_dir = args.out_dir or os.path.dirname(os.path.abspath(args.out)) or "." | |
| audio_dir = os.path.join(out_dir, f"{args.split}_audio") | |
| os.makedirs(audio_dir, exist_ok=True) | |
| files = sorted(glob.glob(os.path.join(args.data_root, "data", f"{args.split}-*.parquet"))) | |
| if not files: | |
| raise SystemExit(f"no parquet for split {args.split} under {args.data_root}") | |
| rows = [] | |
| for p in files: | |
| for batch in pq.ParquetFile(p).iter_batches(batch_size=32, columns=["audio", "text"]): | |
| for i, row in enumerate(batch.to_pylist()): | |
| au = row["audio"] | |
| uid = au.get("path") or f"{args.split}_{len(rows):06d}" | |
| uid = os.path.basename(str(uid)).replace(".mp3", "").replace(".flac", "").replace(".wav", "") | |
| wav = decode(au["bytes"], uid) | |
| sf.write(os.path.join(audio_dir, f"{uid}.wav"), wav, SR) | |
| text = PT.normalize(row["text"]) | |
| n = len(wav) | |
| rel = os.path.join(f"{args.split}_audio", f"{uid}.wav") | |
| rows.append((rel, n, n_frames_for(n), n_tokens_for(n_frames_for(n)), text)) | |
| if args.limit and len(rows) >= args.limit: | |
| break | |
| if args.limit and len(rows) >= args.limit: | |
| break | |
| if args.limit and len(rows) >= args.limit: | |
| break | |
| with open(args.out, "w", encoding="utf-8") as f: | |
| for r in rows: | |
| f.write("\t".join(str(x) for x in r) + "\n") | |
| print(f"wrote {args.out}: {len(rows)} utterances, wavs in {audio_dir}") | |
| if __name__ == "__main__": | |
| main() | |