File size: 10,304 Bytes
33abe95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | """
Badini Kurdish text normalizer — companion module for computeram/badini-w2v-bert-normalized.
Applies the same normalization used to evaluate this model (punctuation
handling, Badini number-word expansion, character/word-variant unification)
to raw CTC decoder output, so predictions match the quality reported for the
"normalized" evaluation numbers rather than the raw greedy-decode numbers.
Usage:
from badini_normalize import normalize_transcript
clean_text = normalize_transcript(raw_model_output)
"""
import re
import unicodedata as ud
USE_BADINI_FORMS = True
def number2word(text):
"""Convert numerals in `text` into spoken (Sorani) Kurdish words."""
unify = [
("٠|۰", "0"), ("١|۱", "1"), ("٢|۲", "2"), ("٣|۳", "3"), ("٤|۴", "4"),
("٥|۵", "5"), ("٦|۶", "6"), ("٧|۷", "7"), ("٨|۸", "8"), ("٩|۹", "9"),
]
for pat, rep in unify:
text = re.sub(pat, rep, text)
text = re.sub(r"([0-9]{1,3})[,،](?=[0-9]{3})", r"\1", text)
text = re.sub(r"(?<![0-9])-([0-9]+)", r"ناقس \1", text)
text = re.sub(r"(?<![0-9])% ?([0-9]+)", r"لە سەددا \1", text)
text = re.sub(r"([0-9]+) ?%", r"\1 لە سەد", text)
text = re.sub(r"\$ ?([0-9]+(\.[0-9]+)?)", r"\1 دۆلار", text)
text = re.sub(r"£ ?([0-9]+(\.[0-9]+)?)", r"\1 پاوەن", text)
text = re.sub(r"€ ?([0-9]+(\.[0-9]+)?)", r"\1 یۆرۆ", text)
text = re.sub(r"([0-9]+)\.([0-9]+)",
lambda m: _float_name(m.group(1), m.group(2)), text)
text = re.sub(r"([0-9]+)", lambda m: _integer_name(m.group(1)), text)
return text
def _float_name(integer_part, decimal_part):
leading_zeros = len(decimal_part) - len(decimal_part.lstrip("0"))
point = " پۆینت " + " سفر " * leading_zeros
return _integer_name(integer_part) + point + _integer_name(decimal_part)
def _integer_name(input_integer):
ones = ["", "یەک", "دوو", "سێ", "چوار", "پێنج", "شەش", "حەوت", "هەشت", "نۆ"]
teens = ["دە", "یازدە", "دوازدە", "سێزدە", "چواردە", "پازدە", "شازدە",
"حەڤدە", "هەژدە", "نۆزدە"]
tens = ["", "", "بیست", "سی", "چل", "پەنجا", "شەست", "هەفتا", "هەشتا", "نەوەد"]
hundreds = ["", "سەد", "دووسەد", "سێسەد", "چوارسەد", "پێنسەد", "شەشسەد",
"حەوتسەد", "هەشتسەد", "نۆسەد"]
thousands = ["", " هەزار", " ملیۆن", " ملیار", " بلیۆن", " بلیار",
" تریلیۆن", " تریلیار", " کوادرلیۆن"]
if input_integer == "0":
return "سفر"
output = ""
temp = input_integer
for i in range(0, len(input_integer), 3):
m = re.search(r"([0-9]{1,3})$", temp)
current_three = m.group(1) if m else ""
temp = temp[:len(temp) - len(current_three)]
current_three = current_three.rjust(3, "0")
C = int(current_three[0]); X = int(current_three[1]); I = int(current_three[2])
conjunction1 = " و " if (C != 0 and (X != 0 or I != 0)) else ""
conjunction2 = " و " if (X != 0 and I != 0) else ""
if X == 1:
current_three = hundreds[C] + conjunction1 + teens[I]
else:
current_three = hundreds[C] + conjunction1 + tens[X] + conjunction2 + ones[I]
magnitude = "" if current_three == "" else thousands[i // 3]
current_three += magnitude
conjunction3 = "" if output == "" else " و "
if current_three != "":
output = current_three + conjunction3 + output
output = output.replace("یەک هەزار", "هەزار")
return output
_SORANI_TO_BADINI = {
"یەک": "ئێک", "چوار": "چار", "حەوت": "هەفت", "نۆ": "نەه", "دە": "دەه",
"چواردە": "چاردە", "حەڤدە": "هەڤدە",
"سی": "سیه", "پەنجا": "پێنجی", "شەست": "شێست",
"هەفتا": "هەفتێ", "هەشتا": "هەشتێ", "نەوەد": "نۆت",
"چوارسەد": "چارسەد", "پێنسەد": "پێنجسەد",
"حەوتسەد": "هەفتسەد", "نۆسەد": "نەهسەد",
"هەزار": "هزار", "ملیۆن": "ملیون",
}
def to_badini_numbers(text):
out = []
for w in text.split():
if w in _SORANI_TO_BADINI:
out.append(_SORANI_TO_BADINI[w])
elif w[:-1] in _SORANI_TO_BADINI and w[-1:] in ('ێ', 'ی'):
out.append(_SORANI_TO_BADINI[w[:-1]] + w[-1])
elif w[:-2] in _SORANI_TO_BADINI and w[-2:] in ('یێ',):
out.append(_SORANI_TO_BADINI[w[:-2]] + w[-2:])
else:
out.append(w)
return " ".join(out)
def expand_numbers(text):
text = re.sub(r'(?<=\d)\s*/\s*(?=\d)', ' ', text)
text = number2word(text)
if USE_BADINI_FORMS:
text = re.sub(r'([،,؟!\.\-\;\:\"\'\»\«\(\)\[\]٪%؛/])', r' \1 ', text)
text = to_badini_numbers(text)
return re.sub(r'\s+', ' ', text).strip()
def normalize_chars(text):
text=text.replace('ك','ک').replace('ى','ی').replace('ي','ی').replace('ة','ە')
text=text.replace('ݒ','پ').replace('ݘ','چ').replace('ݵ','ێ')
text=text.replace('ﻩ','ه').replace('ھ','ه')
return text
def fix_ku_ko(text):
return re.sub(r'\bکو\b', 'کۆ', text)
def normalize_shared(text):
text=re.sub(r'(?<![ا-ی\w])دکەن\b','د کەن',text)
text=re.sub(r'(?<![ا-ی\w])دکەت\b','د کەت',text)
text=re.sub(r'(?<![ا-ی\w])دکەم\b','د کەم',text)
text=re.sub(r'(?<![ا-ی\w])دکەین\b','د کەین',text)
for a,b in [('دگەل','د گەل'),('لسەر','ل سەر'),('لپێش','ل پێش'),('لگەل','ل گەل'),
('بخۆڤە','ب خۆڤە'),('دناڤ','د ناڤ'),('لدویڤدا','ل دویڤدا'),('لدور','ل دور')]:
text=text.replace(a,b)
text=re.sub(r'\bب ڕێکا\b','برێکا',text)
for a,b in [('نەخۆشییا','نەخۆشیا'),('هاتییە','هاتیە'),('بووییە','بوویە'),
('گرێداییی','گرێدایی'),('دەریاییی','دەریایی'),('ئاساییی','ئاسای'),
('ئاسایی','ئاسای'),('پترییا','پتریا')]:
text=text.replace(a,b)
for a,b in [('دور','دۆر'),('توز','تۆز'),('تسونامی','تسۆنامی'),('گورەی','گۆرەی'),
('ڕوژێ','ڕۆژێ'),('تومەتبارکرن','تۆمەتبارکرن')]:
text=text.replace(a,b)
for a,b in [('ڤەگوهازتن','ڤەگوهاستن'),('پێشکەڤتی','پێشکەفتی'),('بەرەڤ','بەرەف'),
('چاڤدێڕیکڕنا','چاڤدێریکرنا'),('بەرامبەر','بەرانبەر')]:
text=text.replace(a,b)
for a,b in [('بچویک','بچیک'),('نموونە','نمونە'),('دووبارە','دوبارە'),
('کۆمپانییێن','کۆمپانیێن'),('نڤێسین','نڤیسین'),('نێزیکی','نیزیکی'),
('زارۆکێن','زاروکێن'),('چاوا','چەوا'),('تەمووزێ','تەموزێ'),
('شێوەیەکێ','شێوەکێ'),('گووت','گوت'),('قۆتابیان','قوتابیان'),
('نیڤدەولەتی','نێڤدەولەتی'),('ئیتاڵی','ئیتالی'),('ئاگاهداریێن','ئاگەهداریێن'),
('هاریکاریا','هاریکارییا'),('گەشتیێن','گەشتییێن'),('ساڵ','سال')]:
text=text.replace(a,b)
return re.sub(r'\s+', ' ', text).strip()
WORD_VARIANTS = {
'خوە':'خۆ', 'دەستپێکێ':'دەسپێکێ', 'دوماهیک':'دویماهیک',
'دویڤ':'دیف', 'گەهە':'گهە',
'بو':'بۆ','خو':'خۆ','کو':'کۆ','کوێ':'کۆێ','کویێ':'کۆیێ','کوژمێ':'کۆژمێ',
'زور':'زۆر','تومارکرن':'تۆمارکرن','نەخوشیا':'نەخۆشیا','مروڤ':'مرۆڤ','مروڤان':'مرۆڤان',
'کومەلێن':'کۆمەلێن','سەروک':'سەرۆک','سەروکێ':'سەرۆکێ','سەروکا':'سەرۆکا',
'فروکە':'فرۆکە','فروکەیەک':'فرۆکەیەک','ڕوژا':'ڕۆژا','ڕوژان':'ڕۆژان',
'توشی':'تووشی','ساڵا':'سالا','ساڵ':'سال','داکو':'دا کو',
'هەروەسا':'هەر وەسا','هەروەسان':'هەر وەسان','هەرچەندە':'هەر چەندە','هەردوو':'هەر دوو',
'بتنێ':'ب تنێ','ژبو':'ژ بۆ','ژبۆ':'ژ بۆ','شورەشا':'شۆڕەشا','پشتراست':'پشتڕاست',
'کەلتور':'کەلتوور','کەلتورێ':'کەلتوورێ','کەلتوری':'کەلتووری','برێكا':'برێکا',
'دەریای':'دەریایی','ساال':'سال','شێوەیەکێ':'شێوەکێ',
}
def strip_invisibles(text):
return re.sub(r'[\u200c\u200d\ufeff\u0640]', '', text)
def collapse_doubles(text):
return text.replace('وو','و').replace('یی','ی')
_DIGIT_MAP = str.maketrans('٠١٢٣٤٥٦٧٨٩', '0123456789')
def _front(text):
text = strip_invisibles(text)
text = ud.normalize('NFKC', text)
text = text.translate(_DIGIT_MAP)
text = normalize_chars(text)
text = expand_numbers(text)
return text
def _back(text):
text = ' '.join(WORD_VARIANTS.get(w, w) for w in text.split())
text = re.sub(r'\bر', 'ڕ', text)
text = fix_ku_ko(text)
text = normalize_shared(text)
text = collapse_doubles(text)
return text
def normalize(text):
text = _front(text)
text = re.sub(r'[،,؟!\.\-\;\:\"\'»«\(\)\[\]٪%؛/]', ' ', text)
return _back(text)
def normalize_punct_ext(text):
text = _front(text)
text = text.replace('?', '؟').replace(',', '،')
text = re.sub(r'([؟،\.\!\(\)«»":؛])', r' \1 ', text)
return _back(text)
def normalize_transcript(text):
"""Recommended: punctuation stripped, word-level normalization.
This is the configuration the model's reported WER/CER are measured on."""
return normalize(text)
def normalize_transcript_keep_punct(text):
"""Alternative: keeps the 11 punctuation marks the model was trained on."""
return normalize_punct_ext(text)
|