| """Paths and test sentences shared by the capu export scripts.""" |
|
|
| import csv |
| import re |
| import unicodedata |
| from pathlib import Path |
|
|
| HERE = Path(__file__).resolve().parent |
| SRC = HERE / "data" / "src" |
| FLEURS_TSV = HERE.parent / "mobile_export" / "data" / "fleurs" / "data" / "vi_vn" / "test.tsv" |
| PUNCTUATION = ".,:?" |
|
|
|
|
| def strip(text: str) -> str: |
| """Lower-cases and drops punctuation, the way ASR output looks.""" |
| text = unicodedata.normalize("NFC", text.lower()) |
| return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text)).strip() |
|
|
|
|
| def fleurs_sentences() -> list[tuple[str, str]]: |
| """(stripped, original) for each unique FLEURS Vietnamese test transcription.""" |
| seen, rows = set(), [] |
| with open(FLEURS_TSV, encoding="utf-8") as f: |
| for row in csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE): |
| truth = unicodedata.normalize("NFC", row[2].strip()) |
| if truth not in seen: |
| seen.add(truth) |
| rows.append((strip(truth), truth)) |
| return rows |
|
|
|
|
| def fleurs_paragraphs(size: int = 8) -> list[tuple[str, str]]: |
| """Consecutive FLEURS sentences joined into paragraphs of about 200 words.""" |
| rows = fleurs_sentences() |
| groups = [rows[i:i + size] for i in range(0, len(rows) - size + 1, size)] |
| return [(" ".join(s for s, _ in g), " ".join(t for _, t in g)) for g in groups] |
|
|
