Rumble-2-Flash-GGUF / bench_select.py
javierbits's picture
Upload folder using huggingface_hub
aff6a88 verified
Raw History Blame Contribute Delete
2.67 kB
#!/usr/bin/env python3
"""Select a deterministic benchmark split from the FLEURS test sets.
Requires data/data/<lang>/{audio/test.tar.gz,test.tsv} (see README download step).
Writes data/bench_clips.json and extracts audio to data/bench_wav/<lang>/.
.venv/bin/python bench_select.py --clips 100
"""
import argparse, json, random, tarfile
from pathlib import Path
BASE = Path(__file__).resolve().parent / "data"
DEFAULT_LANGS = ("en_us", "es_419", "pt_br", "de_de", "fr_fr", "cmn_hans_cn", "yue_hant_hk")
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--clips", type=int, default=100, help="utterances per language")
ap.add_argument("--langs", default=",".join(DEFAULT_LANGS))
ap.add_argument("--seed", type=int, default=1234)
ap.add_argument("--out", default=str(BASE / "bench_clips.json"))
args = ap.parse_args()
out = {}
for lang in args.langs.split(","):
tsv = BASE / f"data/{lang}/test.tsv"
if not tsv.is_file():
raise SystemExit(f"missing {tsv} (download the FLEURS test split first)")
rows = []
with open(tsv, encoding="utf-8") as f:
for line in f:
parts = line.rstrip("\n").split("\t")
if len(parts) < 7:
continue
_, fn, txt, _, _, nsamp, _ = parts[:7]
dur = int(nsamp) / 16000
if not (3.0 <= dur <= 25.0) or len(txt.strip()) < 10:
continue
rows.append({"id": fn[:-4], "file": fn, "dur": dur, "transcription": txt})
random.Random(args.seed).shuffle(rows)
seen, sel = set(), []
for r in rows:
key = r["transcription"].lower()
if key in seen:
continue
seen.add(key)
sel.append(r)
if len(sel) == args.clips:
break
out[lang] = sel
print(lang, "candidates", len(rows), "selected", len(sel))
for lang, rows in out.items():
dest = BASE / "bench_wav" / lang
dest.mkdir(parents=True, exist_ok=True)
wanted = {f"test/{r['file']}": dest / r["file"] for r in rows}
with tarfile.open(BASE / f"data/{lang}/audio/test.tar.gz") as t:
for m in t:
if m.name in wanted:
wanted[m.name].write_bytes(t.extractfile(m).read())
for r in rows:
r["audio"] = f"bench_wav/{lang}/{r['file']}"
r.pop("file", None)
r.pop("dur", None)
Path(args.out).write_text(json.dumps(out, indent=2, ensure_ascii=False))
print("wrote", args.out)
if __name__ == "__main__":
main()