File size: 6,421 Bytes
a777897 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | #!/usr/bin/env python3
"""
Uzbek TTS text normalizer — digits/numbers to Uzbek words + apostrophe normalization.
Usage:
from uz_normalize import normalize_uz
text = normalize_uz("1991-yil 31-avgustda soat 9 da")
# -> "bir ming to'qqiz yuz to'qson birinchi yil o'ttiz birinchi avgustda soat to'qqizda"
Rules:
1. All apostrophe-like chars (ʻ ʼ ‘ ’ `) -> ASCII '
2. Years (1000-2999 followed by -yil/-yilda etc) -> cardinal reading
3. Ordinals: digit + -inchi/-anchi suffixes -> ordinal words
4. Dates: DD-DDDD month patterns handled via ordinal day + year
5. Times: soat N da/Ni -> "soat to'qqizda"
6. Cardinals: standalone numbers -> full Uzbek reading
7. Percent, decimals covered
This normalizes for models trained on ASCII-apostrophe data (e.g. our
Chatterbox LoRA). For okina-style data flip PREFER_OKINA=True.
"""
import re
PREFER_OKINA = False
APO = "'" if not PREFER_OKINA else "\u02bb"
ONES = ["", "bir", "ikki", "uch", "to'rt", "besh", "olti", "yetti", "sakkiz", "to'qqiz"]
TENS = ["", "o'n", "yigirma", "o'ttiz", "qirq", "ellik", "oltmish", "yetmish", "sakson", "to'qson"]
def _two_digits(n: int) -> str:
"""1-99 -> uzbek words"""
t, o = divmod(n, 10)
parts = []
if t: parts.append(TENS[t])
if o: parts.append(ONES[o])
return " ".join(parts) if parts else "nol"
def _three_digits(n: int) -> str:
"""1-999"""
h, rest = divmod(n, 100)
parts = []
if h == 1: parts.append("yuz")
elif h > 1: parts.append(ONES[h] + " yuz")
if rest: parts.append(_two_digits(rest))
return " ".join(parts)
def number_to_uzbek(n: int) -> str:
"""0..999_999_999_999 -> uzbek words (long scale-ish, uzbek uses milliard/million)"""
if n == 0: return "nol"
parts = []
for divisor, name in [(10**9, "milliard"), (10**6, "million"), (1000, "ming")]:
q, n = divmod(n, divisor)
if q:
if divisor == 1000 and q == 1:
parts.append("bir ming")
else:
parts.append(number_to_uzbek(q) + " " + name)
if n: parts.append(_three_digits(n))
return " ".join(parts)
MONTHS = {
"yanvar": 1, "fevral": 2, "mart": 3, "aprel": 4, "may": 5, "iyun": 6,
"iyul": 7, "avgust": 8, "sentabr": 9, "oktabr": 10, "noyabr": 11, "dekabr": 12,
}
def _ordinal(word: str) -> str:
"""cardinal phrase -> ordinal (ayt... inchi). Uzbek: oxirgi bo'g'inga -inchi/-anchi.
Vowel harmony approximation: last vowel a/I -> -anchi? actually rule:
ends with k/t/p/q -> voice; simplified common forms used here."""
# Common convention: birinchi, ikkinchi, uchinchi are irregular; others = word + "-inchi"
IRREG = {"bir": "birinchi", "ikki": "ikkinchi", "uch": "uchinchi",
"to'rt": "to'rtinchi", "besh": "beshinchi", "olti": "oltinchi",
"yetti": "yetinchi", "sakkiz": "sakkizinchi", "to'qqiz": "to'qqizinchi",
"o'n": "o'ninchi"}
if word in IRREG: return IRREG[word]
return word + "inchi"
def _expand_year(m):
y = int(m.group(1))
suf = m.group(2) or ""
words = number_to_uzbek(y)
if suf.startswith("-yil"):
# ordinal years often read as-is; keep "yil" suffix attached
tail = suf[len("-yil"):]
return f"{words} yil{tail}"
return f"{words}{suf}"
def _expand_number_token(m):
num_str = m.group(0)
try:
return number_to_uzbek(int(num_str))
except ValueError:
return num_str
def normalize_numbers(text: str) -> str:
# protect existing words; only touch digit runs and date/time patterns
# 1. years: 4-digit number directly before yil*
text = re.sub(r"\b(\d{4})(-?\s*y[a-z']*)", _expand_year, text)
text = re.sub(r"(\bsoat\s+)?\b(\d{1,2}):(\d{2})\b", lambda m: (m.group(1) or "soat ") + f"{number_to_uzbek(int(m.group(2)))} {number_to_uzbek(int(m.group(3)))}", text)
# 2b. "soat N da/ni/dan" (without colon)
def _time(m):
h = int(m.group(1))
suf = m.group(2) or ""
return f"soat {number_to_uzbek(h)}{suf}"
text = re.sub(r"\bsoat\s+(\d{1,2})(-?[a-z']*)?", _time, text)
# 3. ordinals & day-of-date: N-avgust -> ordinal; N-inchi handled by irregular table
def _ord_date(m):
n = int(m.group(1))
rest = m.group(2) # e.g. "-avgustda" or "-inchi"
card = number_to_uzbek(n)
last = card.split()[-1]
first = " ".join(card.split()[:-1])
restword = rest.lstrip("-")
if restword.startswith("inchi") or restword.startswith("nchi"):
# already an ordinal marker: convert number only, keep suffix
ordw = _ordinal(last)
out = (first + " " + ordw if first else ordw) + (restword[5:] if restword.startswith("inchi") and len(restword) > 5 else "")
return out
ordw = _ordinal(last)
return (first + " " + ordw if first else ordw) + " " + restword
text = re.sub(r"\b(\d{1,4})-([a-z\u02bb']+)\b", _ord_date, text)
# 4. plain integers (incl. inside hyphenated like 31-avgust already partially handled;
# handle remaining digit runs)
text = re.sub(r"\b\d{1,9}\b", _expand_number_token, text)
# 5. percent
text = text.replace("%", " foiz")
return text
def normalize_apostrophes(text: str) -> str:
table = str.maketrans({"ʻ": APO, "ʼ": APO, "‘": APO, "’": APO, "`": APO,
"ʻ": APO})
return text.translate(table)
def normalize_uz(text: str) -> str:
"""Full pipeline: apostrophes + numbers. Apply BEFORE sending to TTS."""
text = normalize_apostrophes(text)
text = normalize_numbers(text)
# collapse multiple spaces
text = re.sub(r"[ \t]+", " ", text).strip()
return text
if __name__ == "__main__":
tests = [
("Assalomu alaykum, bugun havo juda ham chiroyli.", None),
("Oʻzbekiston Respublikasi 1991-yil 31-avgustda mustaqiligi eʼlon qildi.",
"O'zbekiston Respublikasi bir ming to'qqiz yuz to'qson bir yil o'ttiz birinchi avgustda mustaqiligi e'lon qildi."),
("Ertaga soat 9 da uchrashamizmi?", None),
("50 foiz odam rozi", None),
("2-inchi gʻoya", None),
("Kompyuter oʻyinlari 100 yildan beri", None),
]
for t, expected in tests:
out = normalize_uz(t)
status = ""
if expected is not None:
status = "OK" if out == expected else f"MISMATCH (want: {expected})"
print(f"IN : {t}\nOUT: {out}\n {status}\n")
|