fa-eg2-persian / make_test_set_fa.py
karim23657's picture
FA-EG2: model + full source + technical results
d1597be verified
Raw History Blame Contribute Delete
3.02 kB
#!/usr/bin/env python3
"""Build a held-out Persian test TSV + 16 kHz wav clips from a parquet ASR split.
Reads ``<data-root>/data/<split>-*.parquet`` (Common Voice fa layout: ``audio.bytes`` +
``text``), decodes and resamples to 16 kHz mono, writes one wav per utterance to
``<out-dir>/<split>_audio/<id>.wav``, and a TSV with columns::
audio_path n_samples n_frames n_tokens text
where ``text`` is Persian-normalized. ``audio_path`` is relative to the TSV's directory.
Example::
python make_test_set_fa.py --data-root data/cv_fa --split test \
--out data/fa_test.tsv --out-dir data --limit 0
"""
import argparse
import glob
import io
import os
import numpy as np
import pyarrow.parquet as pq
import soundfile as sf
import persian_text as PT
from cache_soft_tokens_fa import SR, decode, n_frames_for, n_tokens_for
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--data-root", required=True, help="directory holding data/<split>-*.parquet")
ap.add_argument("--split", default="test")
ap.add_argument("--out", required=True, help="output TSV path")
ap.add_argument("--out-dir", default=None, help="directory for wav clips (default: dirname of --out)")
ap.add_argument("--limit", type=int, default=0, help="max utterances (0 = all)")
args = ap.parse_args()
out_dir = args.out_dir or os.path.dirname(os.path.abspath(args.out)) or "."
audio_dir = os.path.join(out_dir, f"{args.split}_audio")
os.makedirs(audio_dir, exist_ok=True)
files = sorted(glob.glob(os.path.join(args.data_root, "data", f"{args.split}-*.parquet")))
if not files:
raise SystemExit(f"no parquet for split {args.split} under {args.data_root}")
rows = []
for p in files:
for batch in pq.ParquetFile(p).iter_batches(batch_size=32, columns=["audio", "text"]):
for i, row in enumerate(batch.to_pylist()):
au = row["audio"]
uid = au.get("path") or f"{args.split}_{len(rows):06d}"
uid = os.path.basename(str(uid)).replace(".mp3", "").replace(".flac", "").replace(".wav", "")
wav = decode(au["bytes"], uid)
sf.write(os.path.join(audio_dir, f"{uid}.wav"), wav, SR)
text = PT.normalize(row["text"])
n = len(wav)
rel = os.path.join(f"{args.split}_audio", f"{uid}.wav")
rows.append((rel, n, n_frames_for(n), n_tokens_for(n_frames_for(n)), text))
if args.limit and len(rows) >= args.limit:
break
if args.limit and len(rows) >= args.limit:
break
if args.limit and len(rows) >= args.limit:
break
with open(args.out, "w", encoding="utf-8") as f:
for r in rows:
f.write("\t".join(str(x) for x in r) + "\n")
print(f"wrote {args.out}: {len(rows)} utterances, wavs in {audio_dir}")
if __name__ == "__main__":
main()