#!/usr/bin/env python3 """Rebuild the benchmark's reference transcripts from the public datasets on Hugging Face. python make_refs.py --out refs/ [--sets fleurs tvspeech gs2 isan lotus_con123 ... tv_long gs2_long] Writes refs/.jsonl, one line per clip: {"id", "text", "cluster"}. score.py reads these files; the ids are the ones in hypotheses/*.jsonl. Every dataset is pinned to a commit, so the result never changes. No audio is downloaded: the script fetches the small transcript files (FLEURS, GigaSpeech2) or reads only the text columns of the parquet files (TVSpeech, Isan, LOTUSDIS), a few MB in all. set source id cluster fleurs google/fleurs th_th test.tsv, transcription * wav file name the clip tvspeech typhoon-ai/TVSpeech test, sentence row index YouTube video id gs2 speechcolab/gigaspeech2 data/th/test.tsv * segment id, e.g. 0-0 recording, e.g. 0 isan typhoon-ai/thai-dialect-isan-dataset test, raw dataset id speaker lotus_ nectec/LOTUSDIS test, one device, sentence chunk file name meeting, e.g. S013 (con123 conference mic, lav123 lapel, jbl speakerphone, bt3m and bt10m far channels) tv_long one TVSpeech video's clips in row order video id the file gs2_long one GS2 recording's segments in order * recording (26 left out) the file * cleaned the way our benchmark references were (see clean_text below). The long-form references join the clip references with one space (score.py removes spaces before scoring). gs2 and gs2_long need the GigaSpeech2 access terms accepted on its Hugging Face page (approval is automatic) and a Hugging Face login (`huggingface-cli login` or HF_TOKEN). Requires: huggingface_hub, pyarrow and pythainlp (5.3.5 was used for the model card). """ import argparse, json, os, re, string from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from functools import lru_cache import pyarrow.parquet as pq from huggingface_hub import HfApi, HfFileSystem, hf_hub_download from pythainlp.util import normalize as thai_normalize REV = { # dataset -> pinned commit "google/fleurs": "70bb2e84b976b7e960aa89f1c648e09c59f894dd", "typhoon-ai/TVSpeech": "b5cca17eb2933e96badabb96531545ed1f1bc318", "speechcolab/gigaspeech2": "8dc0d0e502b7e6d5649a13cd4b677cf10c0b21e3", "typhoon-ai/thai-dialect-isan-dataset": "dfc7e495b3069940880d184f0acfcc6b3cd4f364", "nectec/LOTUSDIS": "45145d7ca2286498a31958b35628ab3c1969f808", } EXPECTED = {"fleurs": 1021, "tvspeech": 570, "gs2": 8823, "isan": 500, "tv_long": 18, "gs2_long": 31, **{f"lotus_{m}": 5516 for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")}} def tsv(repo, path): p = hf_hub_download(repo, path, repo_type="dataset", revision=REV[repo]) return [line.split("\t") for line in open(p, encoding="utf-8").read().split("\n") if line] def parquet(repo, split, columns): """Rows of a split, reading only the given columns (a nested column is named like 'audio.path').""" rev, fs = REV[repo], HfFileSystem() files = sorted(f for f in HfApi().list_repo_files(repo, repo_type="dataset", revision=rev) if f.startswith(f"data/{split}-") and f.endswith(".parquet")) def read(f): # small blocks, so reading a column does not pull in the audio stored next to it with fs.open(f"datasets/{repo}@{rev}/{f}", "rb", block_size=1 << 16) as fh: return pq.ParquetFile(fh).read(columns=columns).to_pylist() with ThreadPoolExecutor(8) as ex: return [r for part in ex.map(read, files) for r in part] def clean_text(text): """How our FLEURS and GigaSpeech2 references were cleaned when the benchmark was built: ASCII punctuation except ? and ! removed, plus curly quotes, dashes and the ellipsis; lowercased; then pythainlp's normalize (removes zero-width characters, repeated vowels and misplaced marks, e.g. นำ้ -> น้ำ). Removing "." also removes decimal points (22.00 -> 2200) in 17 FLEURS references; the published FLEURS numbers were scored with them that way.""" drop = "".join(c for c in string.punctuation if c not in "?!") + "“”‘’—–…" return thai_normalize(text.translate(str.maketrans("", "", drop)).lower().strip()) def fleurs(): # columns: id, file name, raw transcription, transcription, phonemes, samples, gender return [{"id": r[1], "text": clean_text(r[3]), "cluster": r[1]} for r in tsv("google/fleurs", "data/th_th/test.tsv")] @lru_cache() def tvspeech(): rows = parquet("typhoon-ai/TVSpeech", "test", ["audio_id", "sentence"]) # audio_id is __; a YouTube id can itself contain "_" return [{"id": str(i), "text": r["sentence"], "cluster": r["audio_id"].rsplit("_", 2)[0]} for i, r in enumerate(rows)] @lru_cache() def gs2(): return [{"id": r[0], "text": clean_text(r[1]), "cluster": r[0].split("-")[0]} for r in tsv("speechcolab/gigaspeech2", "data/th/test.tsv")] def isan(): rows = parquet("typhoon-ai/thai-dialect-isan-dataset", "test", ["id", "raw", "name"]) # raw marks each dialect word as [isan spelling]; the reference keeps the Thai spelling. # (The thai_spelling column is not used: it does not line up with raw.) name is like opentyphoon;is;f_011;fin;0674.wav pair = re.compile(r"\[([^\]]*)\]<([^>]*)>") return [{"id": r["id"], "text": pair.sub(lambda m: m.group(2), r["raw"]), "cluster": (r["name"] or r["id"]).split(";")[2]} for r in rows] def lotus(mic): """LOTUSDIS test turns on one device; the same 5,516 turns exist on all five.""" rows = parquet("nectec/LOTUSDIS", "test", ["audio.path", "sentence", "mic"]) out = [] for r in rows: if r["mic"] == mic: name = os.path.basename(r["audio"]["path"])[:-4] # Hijack_S013_T014_Con123_chunk215 out.append({"id": name, "text": r["sentence"], "cluster": re.search(r"_(S\d+)_", name).group(1)}) return out def joined(clips, group_of, order_of, exclude=()): """One long file per group: its clips in recording order, references joined with one space.""" groups = defaultdict(list) for c in clips: groups[group_of(c)].append(c) return [{"id": g, "text": " ".join(c["text"].strip() for c in sorted(groups[g], key=order_of)), "cluster": g} for g in groups if g not in exclude] def tv_long(): return joined(tvspeech(), lambda c: c["cluster"], lambda c: int(c["id"])) def gs2_long(): # recording 26's references do not match its audio (every model scores ~70% CER on it), so it is left out return joined(gs2(), lambda c: c["cluster"], lambda c: int(c["id"].split("-")[1]), exclude={"26"}) SETS = {"fleurs": fleurs, "tvspeech": tvspeech, "gs2": gs2, "isan": isan, **{f"lotus_{m}": (lambda m=m: lotus(m)) for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")}, "tv_long": tv_long, "gs2_long": gs2_long} def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--out", required=True) ap.add_argument("--sets", nargs="+", default=list(SETS), choices=list(SETS)) a = ap.parse_args() os.makedirs(a.out, exist_ok=True) for name in a.sets: rows = SETS[name]() assert len({r["id"] for r in rows}) == len(rows), f"{name}: ids not unique" if len(rows) != EXPECTED[name]: raise SystemExit(f"{name}: got {len(rows)} clips, expected {EXPECTED[name]}") path = os.path.join(a.out, f"{name}.jsonl") with open(path, "w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n") print(f"{name:13s} {len(rows):5d} clips -> {path}") if __name__ == "__main__": main()