Model card; benchmark outputs and scorer; serve/: long recordings cut at pauses, WS /v1/live

0a9a98f verified 4 days ago

8 kB

#!/usr/bin/env python3
"""Rebuild the benchmark's reference transcripts from the public datasets on Hugging Face.
  python make_refs.py --out refs/ [--sets fleurs tvspeech gs2 isan lotus_con123 ... tv_long gs2_long]
Writes refs/<set>.jsonl, one line per clip: {"id", "text", "cluster"}. score.py reads these files; the ids are the
ones in hypotheses/*.jsonl. Every dataset is pinned to a commit, so the result never changes. No audio is
downloaded: the script fetches the small transcript files (FLEURS, GigaSpeech2) or reads only the text columns of
the parquet files (TVSpeech, Isan, LOTUSDIS), a few MB in all.
  set           source                                           id                        cluster
  fleurs        google/fleurs th_th test.tsv, transcription *    wav file name             the clip
  tvspeech      typhoon-ai/TVSpeech test, sentence               row index                 YouTube video id
  gs2           speechcolab/gigaspeech2 data/th/test.tsv *       segment id, e.g. 0-0      recording, e.g. 0
  isan          typhoon-ai/thai-dialect-isan-dataset test, raw   dataset id                speaker
  lotus_<mic>   nectec/LOTUSDIS test, one device, sentence       chunk file name           meeting, e.g. S013
                (con123 conference mic, lav123 lapel, jbl speakerphone, bt3m and bt10m far channels)
  tv_long       one TVSpeech video's clips in row order          video id                  the file
  gs2_long      one GS2 recording's segments in order *          recording (26 left out)   the file
* cleaned the way our benchmark references were (see clean_text below).
The long-form references join the clip references with one space (score.py removes spaces before scoring).
gs2 and gs2_long need the GigaSpeech2 access terms accepted on its Hugging Face page (approval is automatic) and a
Hugging Face login (`huggingface-cli login` or HF_TOKEN).
Requires: huggingface_hub, pyarrow and pythainlp (5.3.5 was used for the model card).
"""
import argparse, json, os, re, string
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from functools import lru_cache

import pyarrow.parquet as pq
from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
from pythainlp.util import normalize as thai_normalize

REV = {  # dataset -> pinned commit
    "google/fleurs": "70bb2e84b976b7e960aa89f1c648e09c59f894dd",
    "typhoon-ai/TVSpeech": "b5cca17eb2933e96badabb96531545ed1f1bc318",
    "speechcolab/gigaspeech2": "8dc0d0e502b7e6d5649a13cd4b677cf10c0b21e3",
    "typhoon-ai/thai-dialect-isan-dataset": "dfc7e495b3069940880d184f0acfcc6b3cd4f364",
    "nectec/LOTUSDIS": "45145d7ca2286498a31958b35628ab3c1969f808",
}
EXPECTED = {"fleurs": 1021, "tvspeech": 570, "gs2": 8823, "isan": 500, "tv_long": 18, "gs2_long": 31,
            **{f"lotus_{m}": 5516 for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")}}


def tsv(repo, path):
    p = hf_hub_download(repo, path, repo_type="dataset", revision=REV[repo])
    return [line.split("\t") for line in open(p, encoding="utf-8").read().split("\n") if line]


def parquet(repo, split, columns):
    """Rows of a split, reading only the given columns (a nested column is named like 'audio.path')."""
    rev, fs = REV[repo], HfFileSystem()
    files = sorted(f for f in HfApi().list_repo_files(repo, repo_type="dataset", revision=rev)
                   if f.startswith(f"data/{split}-") and f.endswith(".parquet"))

    def read(f):  # small blocks, so reading a column does not pull in the audio stored next to it
        with fs.open(f"datasets/{repo}@{rev}/{f}", "rb", block_size=1 << 16) as fh:
            return pq.ParquetFile(fh).read(columns=columns).to_pylist()
    with ThreadPoolExecutor(8) as ex:
        return [r for part in ex.map(read, files) for r in part]


def clean_text(text):
    """How our FLEURS and GigaSpeech2 references were cleaned when the benchmark was built: ASCII punctuation except
    ? and ! removed, plus curly quotes, dashes and the ellipsis; lowercased; then pythainlp's normalize (removes
    zero-width characters, repeated vowels and misplaced marks, e.g. นำ้ -> น้ำ). Removing "." also removes decimal
    points (22.00 -> 2200) in 17 FLEURS references; the published FLEURS numbers were scored with them that way."""
    drop = "".join(c for c in string.punctuation if c not in "?!") + "“”‘’—–…"
    return thai_normalize(text.translate(str.maketrans("", "", drop)).lower().strip())


def fleurs():
    # columns: id, file name, raw transcription, transcription, phonemes, samples, gender
    return [{"id": r[1], "text": clean_text(r[3]), "cluster": r[1]} for r in tsv("google/fleurs", "data/th_th/test.tsv")]


@lru_cache()
def tvspeech():
    rows = parquet("typhoon-ai/TVSpeech", "test", ["audio_id", "sentence"])
    # audio_id is <YouTube id>_<start>_<end>; a YouTube id can itself contain "_"
    return [{"id": str(i), "text": r["sentence"], "cluster": r["audio_id"].rsplit("_", 2)[0]} for i, r in enumerate(rows)]


@lru_cache()
def gs2():
    return [{"id": r[0], "text": clean_text(r[1]), "cluster": r[0].split("-")[0]}
            for r in tsv("speechcolab/gigaspeech2", "data/th/test.tsv")]


def isan():
    rows = parquet("typhoon-ai/thai-dialect-isan-dataset", "test", ["id", "raw", "name"])
    # raw marks each dialect word as [isan spelling]<thai spelling>; the reference keeps the Thai spelling.
    # (The thai_spelling column is not used: it does not line up with raw.) name is like opentyphoon;is;f_011;fin;0674.wav
    pair = re.compile(r"\[([^\]]*)\]<([^>]*)>")
    return [{"id": r["id"], "text": pair.sub(lambda m: m.group(2), r["raw"]), "cluster": (r["name"] or r["id"]).split(";")[2]}
            for r in rows]


def lotus(mic):
    """LOTUSDIS test turns on one device; the same 5,516 turns exist on all five."""
    rows = parquet("nectec/LOTUSDIS", "test", ["audio.path", "sentence", "mic"])
    out = []
    for r in rows:
        if r["mic"] == mic:
            name = os.path.basename(r["audio"]["path"])[:-4]          # Hijack_S013_T014_Con123_chunk215
            out.append({"id": name, "text": r["sentence"], "cluster": re.search(r"_(S\d+)_", name).group(1)})
    return out


def joined(clips, group_of, order_of, exclude=()):
    """One long file per group: its clips in recording order, references joined with one space."""
    groups = defaultdict(list)
    for c in clips:
        groups[group_of(c)].append(c)
    return [{"id": g, "text": " ".join(c["text"].strip() for c in sorted(groups[g], key=order_of)), "cluster": g}
            for g in groups if g not in exclude]


def tv_long():
    return joined(tvspeech(), lambda c: c["cluster"], lambda c: int(c["id"]))


def gs2_long():
    # recording 26's references do not match its audio (every model scores ~70% CER on it), so it is left out
    return joined(gs2(), lambda c: c["cluster"], lambda c: int(c["id"].split("-")[1]), exclude={"26"})


SETS = {"fleurs": fleurs, "tvspeech": tvspeech, "gs2": gs2, "isan": isan,
        **{f"lotus_{m}": (lambda m=m: lotus(m)) for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")},
        "tv_long": tv_long, "gs2_long": gs2_long}


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--out", required=True)
    ap.add_argument("--sets", nargs="+", default=list(SETS), choices=list(SETS))
    a = ap.parse_args()
    os.makedirs(a.out, exist_ok=True)
    for name in a.sets:
        rows = SETS[name]()
        assert len({r["id"] for r in rows}) == len(rows), f"{name}: ids not unique"
        if len(rows) != EXPECTED[name]:
            raise SystemExit(f"{name}: got {len(rows)} clips, expected {EXPECTED[name]}")
        path = os.path.join(a.out, f"{name}.jsonl")
        with open(path, "w", encoding="utf-8") as f:
            for r in rows:
                f.write(json.dumps(r, ensure_ascii=False) + "\n")
        print(f"{name:13s} {len(rows):5d} clips -> {path}")


if __name__ == "__main__":
    main()

	#!/usr/bin/env python3
	"""Rebuild the benchmark's reference transcripts from the public datasets on Hugging Face.

	python make_refs.py --out refs/ [--sets fleurs tvspeech gs2 isan lotus_con123 ... tv_long gs2_long]

	Writes refs/<set>.jsonl, one line per clip: {"id", "text", "cluster"}. score.py reads these files; the ids are the
	ones in hypotheses/*.jsonl. Every dataset is pinned to a commit, so the result never changes. No audio is
	downloaded: the script fetches the small transcript files (FLEURS, GigaSpeech2) or reads only the text columns of
	the parquet files (TVSpeech, Isan, LOTUSDIS), a few MB in all.

	set source id cluster
	fleurs google/fleurs th_th test.tsv, transcription * wav file name the clip
	tvspeech typhoon-ai/TVSpeech test, sentence row index YouTube video id
	gs2 speechcolab/gigaspeech2 data/th/test.tsv * segment id, e.g. 0-0 recording, e.g. 0
	isan typhoon-ai/thai-dialect-isan-dataset test, raw dataset id speaker
	lotus_<mic> nectec/LOTUSDIS test, one device, sentence chunk file name meeting, e.g. S013
	(con123 conference mic, lav123 lapel, jbl speakerphone, bt3m and bt10m far channels)
	tv_long one TVSpeech video's clips in row order video id the file
	gs2_long one GS2 recording's segments in order * recording (26 left out) the file

	* cleaned the way our benchmark references were (see clean_text below).
	The long-form references join the clip references with one space (score.py removes spaces before scoring).
	gs2 and gs2_long need the GigaSpeech2 access terms accepted on its Hugging Face page (approval is automatic) and a
	Hugging Face login (`huggingface-cli login` or HF_TOKEN).
	Requires: huggingface_hub, pyarrow and pythainlp (5.3.5 was used for the model card).
	"""
	import argparse, json, os, re, string
	from collections import defaultdict
	from concurrent.futures import ThreadPoolExecutor
	from functools import lru_cache

	import pyarrow.parquet as pq
	from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
	from pythainlp.util import normalize as thai_normalize

	REV = { # dataset -> pinned commit
	"google/fleurs": "70bb2e84b976b7e960aa89f1c648e09c59f894dd",
	"typhoon-ai/TVSpeech": "b5cca17eb2933e96badabb96531545ed1f1bc318",
	"speechcolab/gigaspeech2": "8dc0d0e502b7e6d5649a13cd4b677cf10c0b21e3",
	"typhoon-ai/thai-dialect-isan-dataset": "dfc7e495b3069940880d184f0acfcc6b3cd4f364",
	"nectec/LOTUSDIS": "45145d7ca2286498a31958b35628ab3c1969f808",
	}
	EXPECTED = {"fleurs": 1021, "tvspeech": 570, "gs2": 8823, "isan": 500, "tv_long": 18, "gs2_long": 31,
	**{f"lotus_{m}": 5516 for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")}}


	def tsv(repo, path):
	p = hf_hub_download(repo, path, repo_type="dataset", revision=REV[repo])
	return [line.split("\t") for line in open(p, encoding="utf-8").read().split("\n") if line]


	def parquet(repo, split, columns):
	"""Rows of a split, reading only the given columns (a nested column is named like 'audio.path')."""
	rev, fs = REV[repo], HfFileSystem()
	files = sorted(f for f in HfApi().list_repo_files(repo, repo_type="dataset", revision=rev)
	if f.startswith(f"data/{split}-") and f.endswith(".parquet"))

	def read(f): # small blocks, so reading a column does not pull in the audio stored next to it
	with fs.open(f"datasets/{repo}@{rev}/{f}", "rb", block_size=1 << 16) as fh:
	return pq.ParquetFile(fh).read(columns=columns).to_pylist()
	with ThreadPoolExecutor(8) as ex:
	return [r for part in ex.map(read, files) for r in part]


	def clean_text(text):
	"""How our FLEURS and GigaSpeech2 references were cleaned when the benchmark was built: ASCII punctuation except
	? and ! removed, plus curly quotes, dashes and the ellipsis; lowercased; then pythainlp's normalize (removes
	zero-width characters, repeated vowels and misplaced marks, e.g. นำ้ -> น้ำ). Removing "." also removes decimal
	points (22.00 -> 2200) in 17 FLEURS references; the published FLEURS numbers were scored with them that way."""
	drop = "".join(c for c in string.punctuation if c not in "?!") + "“”‘’—–…"
	return thai_normalize(text.translate(str.maketrans("", "", drop)).lower().strip())


	def fleurs():
	# columns: id, file name, raw transcription, transcription, phonemes, samples, gender
	return [{"id": r[1], "text": clean_text(r[3]), "cluster": r[1]} for r in tsv("google/fleurs", "data/th_th/test.tsv")]


	@lru_cache()
	def tvspeech():
	rows = parquet("typhoon-ai/TVSpeech", "test", ["audio_id", "sentence"])
	# audio_id is <YouTube id>_<start>_<end>; a YouTube id can itself contain "_"
	return [{"id": str(i), "text": r["sentence"], "cluster": r["audio_id"].rsplit("_", 2)[0]} for i, r in enumerate(rows)]


	@lru_cache()
	def gs2():
	return [{"id": r[0], "text": clean_text(r[1]), "cluster": r[0].split("-")[0]}
	for r in tsv("speechcolab/gigaspeech2", "data/th/test.tsv")]


	def isan():
	rows = parquet("typhoon-ai/thai-dialect-isan-dataset", "test", ["id", "raw", "name"])
	# raw marks each dialect word as [isan spelling]<thai spelling>; the reference keeps the Thai spelling.
	# (The thai_spelling column is not used: it does not line up with raw.) name is like opentyphoon;is;f_011;fin;0674.wav
	pair = re.compile(r"\[([^\]])\]<([^>])>")
	return [{"id": r["id"], "text": pair.sub(lambda m: m.group(2), r["raw"]), "cluster": (r["name"] or r["id"]).split(";")[2]}
	for r in rows]


	def lotus(mic):
	"""LOTUSDIS test turns on one device; the same 5,516 turns exist on all five."""
	rows = parquet("nectec/LOTUSDIS", "test", ["audio.path", "sentence", "mic"])
	out = []
	for r in rows:
	if r["mic"] == mic:
	name = os.path.basename(r["audio"]["path"])[:-4] # Hijack_S013_T014_Con123_chunk215
	out.append({"id": name, "text": r["sentence"], "cluster": re.search(r"_(S\d+)_", name).group(1)})
	return out


	def joined(clips, group_of, order_of, exclude=()):
	"""One long file per group: its clips in recording order, references joined with one space."""
	groups = defaultdict(list)
	for c in clips:
	groups[group_of(c)].append(c)
	return [{"id": g, "text": " ".join(c["text"].strip() for c in sorted(groups[g], key=order_of)), "cluster": g}
	for g in groups if g not in exclude]


	def tv_long():
	return joined(tvspeech(), lambda c: c["cluster"], lambda c: int(c["id"]))


	def gs2_long():
	# recording 26's references do not match its audio (every model scores ~70% CER on it), so it is left out
	return joined(gs2(), lambda c: c["cluster"], lambda c: int(c["id"].split("-")[1]), exclude={"26"})


	SETS = {"fleurs": fleurs, "tvspeech": tvspeech, "gs2": gs2, "isan": isan,
	**{f"lotus_{m}": (lambda m=m: lotus(m)) for m in ("con123", "lav123", "jbl", "bt3m", "bt10m")},
	"tv_long": tv_long, "gs2_long": gs2_long}


	def main():
	ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
	ap.add_argument("--out", required=True)
	ap.add_argument("--sets", nargs="+", default=list(SETS), choices=list(SETS))
	a = ap.parse_args()
	os.makedirs(a.out, exist_ok=True)
	for name in a.sets:
	rows = SETS[name]()
	assert len({r["id"] for r in rows}) == len(rows), f"{name}: ids not unique"
	if len(rows) != EXPECTED[name]:
	raise SystemExit(f"{name}: got {len(rows)} clips, expected {EXPECTED[name]}")
	path = os.path.join(a.out, f"{name}.jsonl")
	with open(path, "w", encoding="utf-8") as f:
	for r in rows:
	f.write(json.dumps(r, ensure_ascii=False) + "\n")
	print(f"{name:13s} {len(rows):5d} clips -> {path}")


	if __name__ == "__main__":
	main()