#!/usr/bin/env python3 """Build a held-out Persian test TSV + 16 kHz wav clips from a parquet ASR split. Reads ``/data/-*.parquet`` (Common Voice fa layout: ``audio.bytes`` + ``text``), decodes and resamples to 16 kHz mono, writes one wav per utterance to ``/_audio/.wav``, and a TSV with columns:: audio_path n_samples n_frames n_tokens text where ``text`` is Persian-normalized. ``audio_path`` is relative to the TSV's directory. Example:: python make_test_set_fa.py --data-root data/cv_fa --split test \ --out data/fa_test.tsv --out-dir data --limit 0 """ import argparse import glob import io import os import numpy as np import pyarrow.parquet as pq import soundfile as sf import persian_text as PT from cache_soft_tokens_fa import SR, decode, n_frames_for, n_tokens_for def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--data-root", required=True, help="directory holding data/-*.parquet") ap.add_argument("--split", default="test") ap.add_argument("--out", required=True, help="output TSV path") ap.add_argument("--out-dir", default=None, help="directory for wav clips (default: dirname of --out)") ap.add_argument("--limit", type=int, default=0, help="max utterances (0 = all)") args = ap.parse_args() out_dir = args.out_dir or os.path.dirname(os.path.abspath(args.out)) or "." audio_dir = os.path.join(out_dir, f"{args.split}_audio") os.makedirs(audio_dir, exist_ok=True) files = sorted(glob.glob(os.path.join(args.data_root, "data", f"{args.split}-*.parquet"))) if not files: raise SystemExit(f"no parquet for split {args.split} under {args.data_root}") rows = [] for p in files: for batch in pq.ParquetFile(p).iter_batches(batch_size=32, columns=["audio", "text"]): for i, row in enumerate(batch.to_pylist()): au = row["audio"] uid = au.get("path") or f"{args.split}_{len(rows):06d}" uid = os.path.basename(str(uid)).replace(".mp3", "").replace(".flac", "").replace(".wav", "") wav = decode(au["bytes"], uid) sf.write(os.path.join(audio_dir, f"{uid}.wav"), wav, SR) text = PT.normalize(row["text"]) n = len(wav) rel = os.path.join(f"{args.split}_audio", f"{uid}.wav") rows.append((rel, n, n_frames_for(n), n_tokens_for(n_frames_for(n)), text)) if args.limit and len(rows) >= args.limit: break if args.limit and len(rows) >= args.limit: break if args.limit and len(rows) >= args.limit: break with open(args.out, "w", encoding="utf-8") as f: for r in rows: f.write("\t".join(str(x) for x in r) + "\n") print(f"wrote {args.out}: {len(rows)} utterances, wavs in {audio_dir}") if __name__ == "__main__": main()