#!/usr/bin/env python3 """Select a deterministic benchmark split from the FLEURS test sets. Requires data/data//{audio/test.tar.gz,test.tsv} (see README download step). Writes data/bench_clips.json and extracts audio to data/bench_wav//. .venv/bin/python bench_select.py --clips 100 """ import argparse, json, random, tarfile from pathlib import Path BASE = Path(__file__).resolve().parent / "data" DEFAULT_LANGS = ("en_us", "es_419", "pt_br", "de_de", "fr_fr", "cmn_hans_cn", "yue_hant_hk") def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--clips", type=int, default=100, help="utterances per language") ap.add_argument("--langs", default=",".join(DEFAULT_LANGS)) ap.add_argument("--seed", type=int, default=1234) ap.add_argument("--out", default=str(BASE / "bench_clips.json")) args = ap.parse_args() out = {} for lang in args.langs.split(","): tsv = BASE / f"data/{lang}/test.tsv" if not tsv.is_file(): raise SystemExit(f"missing {tsv} (download the FLEURS test split first)") rows = [] with open(tsv, encoding="utf-8") as f: for line in f: parts = line.rstrip("\n").split("\t") if len(parts) < 7: continue _, fn, txt, _, _, nsamp, _ = parts[:7] dur = int(nsamp) / 16000 if not (3.0 <= dur <= 25.0) or len(txt.strip()) < 10: continue rows.append({"id": fn[:-4], "file": fn, "dur": dur, "transcription": txt}) random.Random(args.seed).shuffle(rows) seen, sel = set(), [] for r in rows: key = r["transcription"].lower() if key in seen: continue seen.add(key) sel.append(r) if len(sel) == args.clips: break out[lang] = sel print(lang, "candidates", len(rows), "selected", len(sel)) for lang, rows in out.items(): dest = BASE / "bench_wav" / lang dest.mkdir(parents=True, exist_ok=True) wanted = {f"test/{r['file']}": dest / r["file"] for r in rows} with tarfile.open(BASE / f"data/{lang}/audio/test.tar.gz") as t: for m in t: if m.name in wanted: wanted[m.name].write_bytes(t.extractfile(m).read()) for r in rows: r["audio"] = f"bench_wav/{lang}/{r['file']}" r.pop("file", None) r.pop("dur", None) Path(args.out).write_text(json.dumps(out, indent=2, ensure_ascii=False)) print("wrote", args.out) if __name__ == "__main__": main()