File size: 1,697 Bytes
57e91f2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Paths and test texts shared by the Qwen export scripts."""
import csv
import re
import unicodedata
from pathlib import Path
HERE = Path(__file__).resolve().parent
SIZES = ("1.5B", "3B")
FLEURS_TSV = HERE.parent / "mobile_export" / "data" / "fleurs" / "data" / "vi_vn" / "test.tsv"
def src_dir(size: str) -> Path:
"""hf download Qwen/Qwen2.5-<size>-Instruct --local-dir data/Qwen2.5-<size>-Instruct"""
return HERE / "data" / f"Qwen2.5-{size}-Instruct"
def out_dir(size: str) -> Path:
"""The repo that publish.py uploads: onnx/, vocab.txt, merges.txt, LICENSE."""
return HERE / "out" / size / "repo"
def strip(text: str) -> str:
"""Lower-cases and drops punctuation, the way ASR output looks."""
text = unicodedata.normalize("NFC", text.lower())
return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text)).strip()
def fleurs_sentences() -> list[tuple[str, str]]:
"""(stripped, original) for each unique FLEURS Vietnamese test transcription."""
seen, rows = set(), []
with open(FLEURS_TSV, encoding="utf-8") as f:
for row in csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE):
truth = unicodedata.normalize("NFC", row[2].strip())
if truth not in seen:
seen.add(truth)
rows.append((strip(truth), truth))
return rows
def fleurs_paragraphs(size: int = 3) -> list[tuple[str, str]]:
"""Consecutive FLEURS sentences joined into paragraphs, about the length of a 30-second recording."""
rows = fleurs_sentences()
groups = [rows[i:i + size] for i in range(0, len(rows) - size + 1, size)]
return [(" ".join(s for s, _ in g), " ".join(t for _, t in g)) for g in groups]
|