File size: 8,004 Bytes
0a9a98f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | #!/usr/bin/env python3
"""Rebuild the benchmark's reference transcripts from the public datasets on Hugging Face.
python make_refs.py --out refs/ [--sets fleurs tvspeech gs2 isan lotus_con123 ... tv_long gs2_long]
Writes refs/<set>.jsonl, one line per clip: {"id", "text", "cluster"}. score.py reads these files; the ids are the
ones in hypotheses/*.jsonl. Every dataset is pinned to a commit, so the result never changes. No audio is
downloaded: the script fetches the small transcript files (FLEURS, GigaSpeech2) or reads only the text columns of
the parquet files (TVSpeech, Isan, LOTUSDIS), a few MB in all.
set source id cluster
fleurs google/fleurs th_th test.tsv, transcription * wav file name the clip
tvspeech typhoon-ai/TVSpeech test, sentence row index YouTube video id
gs2 speechcolab/gigaspeech2 data/th/test.tsv * segment id, e.g. 0-0 recording, e.g. 0
isan typhoon-ai/thai-dialect-isan-dataset test, raw dataset id speaker
lotus_<mic> nectec/LOTUSDIS test, one device, sentence chunk file name meeting, e.g. S013
(con123 conference mic, lav123 lapel, jbl speakerphone, bt3m and bt10m far channels)
tv_long one TVSpeech video's clips in row order video id the file
gs2_long one GS2 recording's segments in order * recording (26 left out) the file
* cleaned the way our benchmark references were (see clean_text below).
The long-form references join the clip references with one space (score.py removes spaces before scoring).
gs2 and gs2_long need the GigaSpeech2 access terms accepted on its Hugging Face page (approval is automatic) and a
Hugging Face login (`huggingface-cli login` or HF_TOKEN).
Requires: huggingface_hub, pyarrow and pythainlp (5.3.5 was used for the model card).
"""
import argparse, json, os, re, string
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from functools import lru_cache
import pyarrow.parquet as pq
from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
from pythainlp.util import normalize as thai_normalize
REV = { # dataset -> pinned commit
"google/fleurs": "70bb2e84b976b7e960aa89f1c648e09c59f894dd",
"typhoon-ai/TVSpeech": "b5cca17eb2933e96badabb96531545ed1f1bc318",
"speechcolab/gigaspeech2": "8dc0d0e502b7e6d5649a13cd4b677cf10c0b21e3",
"typhoon-ai/thai-dialect-isan-dataset": "dfc7e495b3069940880d184f0acfcc6b3cd4f364",
"nectec/LOTUSDIS": "45145d7ca2286498a31958b35628ab3c1969f808",
}
EXPECTED = {"fleurs": 1021, "tvspeech": 570, "gs2": 8823, "isan": 500, "tv_long": 18, "gs2_long": 31,
**{f"lotus_{m}": 5516 for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")}}
def tsv(repo, path):
p = hf_hub_download(repo, path, repo_type="dataset", revision=REV[repo])
return [line.split("\t") for line in open(p, encoding="utf-8").read().split("\n") if line]
def parquet(repo, split, columns):
"""Rows of a split, reading only the given columns (a nested column is named like 'audio.path')."""
rev, fs = REV[repo], HfFileSystem()
files = sorted(f for f in HfApi().list_repo_files(repo, repo_type="dataset", revision=rev)
if f.startswith(f"data/{split}-") and f.endswith(".parquet"))
def read(f): # small blocks, so reading a column does not pull in the audio stored next to it
with fs.open(f"datasets/{repo}@{rev}/{f}", "rb", block_size=1 << 16) as fh:
return pq.ParquetFile(fh).read(columns=columns).to_pylist()
with ThreadPoolExecutor(8) as ex:
return [r for part in ex.map(read, files) for r in part]
def clean_text(text):
"""How our FLEURS and GigaSpeech2 references were cleaned when the benchmark was built: ASCII punctuation except
? and ! removed, plus curly quotes, dashes and the ellipsis; lowercased; then pythainlp's normalize (removes
zero-width characters, repeated vowels and misplaced marks, e.g. นำ้ -> น้ำ). Removing "." also removes decimal
points (22.00 -> 2200) in 17 FLEURS references; the published FLEURS numbers were scored with them that way."""
drop = "".join(c for c in string.punctuation if c not in "?!") + "“”‘’—–…"
return thai_normalize(text.translate(str.maketrans("", "", drop)).lower().strip())
def fleurs():
# columns: id, file name, raw transcription, transcription, phonemes, samples, gender
return [{"id": r[1], "text": clean_text(r[3]), "cluster": r[1]} for r in tsv("google/fleurs", "data/th_th/test.tsv")]
@lru_cache()
def tvspeech():
rows = parquet("typhoon-ai/TVSpeech", "test", ["audio_id", "sentence"])
# audio_id is <YouTube id>_<start>_<end>; a YouTube id can itself contain "_"
return [{"id": str(i), "text": r["sentence"], "cluster": r["audio_id"].rsplit("_", 2)[0]} for i, r in enumerate(rows)]
@lru_cache()
def gs2():
return [{"id": r[0], "text": clean_text(r[1]), "cluster": r[0].split("-")[0]}
for r in tsv("speechcolab/gigaspeech2", "data/th/test.tsv")]
def isan():
rows = parquet("typhoon-ai/thai-dialect-isan-dataset", "test", ["id", "raw", "name"])
# raw marks each dialect word as [isan spelling]<thai spelling>; the reference keeps the Thai spelling.
# (The thai_spelling column is not used: it does not line up with raw.) name is like opentyphoon;is;f_011;fin;0674.wav
pair = re.compile(r"\[([^\]]*)\]<([^>]*)>")
return [{"id": r["id"], "text": pair.sub(lambda m: m.group(2), r["raw"]), "cluster": (r["name"] or r["id"]).split(";")[2]}
for r in rows]
def lotus(mic):
"""LOTUSDIS test turns on one device; the same 5,516 turns exist on all five."""
rows = parquet("nectec/LOTUSDIS", "test", ["audio.path", "sentence", "mic"])
out = []
for r in rows:
if r["mic"] == mic:
name = os.path.basename(r["audio"]["path"])[:-4] # Hijack_S013_T014_Con123_chunk215
out.append({"id": name, "text": r["sentence"], "cluster": re.search(r"_(S\d+)_", name).group(1)})
return out
def joined(clips, group_of, order_of, exclude=()):
"""One long file per group: its clips in recording order, references joined with one space."""
groups = defaultdict(list)
for c in clips:
groups[group_of(c)].append(c)
return [{"id": g, "text": " ".join(c["text"].strip() for c in sorted(groups[g], key=order_of)), "cluster": g}
for g in groups if g not in exclude]
def tv_long():
return joined(tvspeech(), lambda c: c["cluster"], lambda c: int(c["id"]))
def gs2_long():
# recording 26's references do not match its audio (every model scores ~70% CER on it), so it is left out
return joined(gs2(), lambda c: c["cluster"], lambda c: int(c["id"].split("-")[1]), exclude={"26"})
SETS = {"fleurs": fleurs, "tvspeech": tvspeech, "gs2": gs2, "isan": isan,
**{f"lotus_{m}": (lambda m=m: lotus(m)) for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")},
"tv_long": tv_long, "gs2_long": gs2_long}
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("--out", required=True)
ap.add_argument("--sets", nargs="+", default=list(SETS), choices=list(SETS))
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
for name in a.sets:
rows = SETS[name]()
assert len({r["id"] for r in rows}) == len(rows), f"{name}: ids not unique"
if len(rows) != EXPECTED[name]:
raise SystemExit(f"{name}: got {len(rows)} clips, expected {EXPECTED[name]}")
path = os.path.join(a.out, f"{name}.jsonl")
with open(path, "w", encoding="utf-8") as f:
for r in rows:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print(f"{name:13s} {len(rows):5d} clips -> {path}")
if __name__ == "__main__":
main()
|