Spaces:
Running on Zero
Running on Zero
| """One canonical spelling for every dataset, shared by training and scoring. | |
| The sources disagree: the Bible text is truecase with punctuation, the radio transcripts are lowercase | |
| with none, `history-data` is lowercase with glued clitics. A model trained on the mix has to guess which | |
| convention a given recording wants, and every wrong guess counts as an error. So the training targets and | |
| the metric both go through `normalize`, and the model only ever has to learn one form: lowercase, no | |
| punctuation, digits spelled out, modern spelling without apostrophes. | |
| Casing and punctuation are a separate job for a text model on top of the transcript. | |
| """ | |
| import re | |
| import unicodedata | |
| # Kreyòl numerals. 0-19 are irregular; 20-99 is built below; the forms follow common usage | |
| # (25 vennsenk, 75 swasannkenz, 98 katrevendizuit, 2026 de mil vennsis). | |
| UNITS = ["zewo", "en", "de", "twa", "kat", "senk", "sis", "sèt", "uit", "nèf", | |
| "dis", "onz", "douz", "trèz", "katòz", "kenz", "sèz", "disèt", "dizuit", "diznèf"] | |
| TENS = {20: "ven", 30: "trant", 40: "karant", 50: "senkant", 60: "swasant"} | |
| NASAL = {20: "venn", 30: "trann", 40: "karann", 50: "senkann", 60: "swasann"} | |
| HARD = {20: "vent", 30: "trant", 40: "karant", 50: "senkant", 60: "swasant"} | |
| HUNDREDS = [None, "san", "desan", "twasan", "katsan", "senksan", "sisan", "sètsan", "uisan", "nèfsan"] | |
| LETTER = re.compile(r"[^\W\d_]", re.UNICODE) | |
| # Tokens that appear in French but not in Kreyòl, used to flag French-language rows | |
| FRENCH = {"les", "des", "est", "sont", "nous", "vous", "dans", "avec", "qui", "que", "qu", "une", | |
| "cette", "ces", "il", "elle", "ils", "elles", "leur", "notre", "votre", "etait", "était", | |
| "peut", "faire", "aussi", "donc", "alors", "tres", "très", "beaucoup", "toujours", "jamais", | |
| "français", "francais", "haïti", "sur", "plus", "cet", "vers", "chez", "quelque", "le", | |
| "du", "au", "aux", "ce", "vient", "suis", "sais", "veux", "avez", "avons", "ont", "été", | |
| "comment", "pourquoi", "aujourd", "hui", "parce", "puis", "ensuite", "toute", "autre", | |
| "meme", "même", "c", "j", "quand", "notamment", "ainsi"} | |
| # Kreyòl writes only à, è and ò; any other accent is a French (or English) spelling | |
| NON_KREYOL = re.compile(r"[éêëçùûîïôœæáíúñ]") | |
| def spell_int(n): | |
| """Spell a non-negative integer below one million.""" | |
| if n < 20: | |
| return UNITS[n] | |
| if n < 100: | |
| ten, unit = n // 10 * 10, n % 10 | |
| if ten == 70: | |
| return "swasann" + UNITS[10 + unit] | |
| if ten == 80: | |
| return "katreven" if unit == 0 else f"katreven {UNITS[unit]}" | |
| if ten == 90: | |
| return "katreven" + UNITS[10 + unit] | |
| if unit == 0: | |
| return TENS[ten] | |
| if unit == 1: | |
| return HARD[ten] + "eyen" | |
| if unit in (8, 9): | |
| return HARD[ten] + UNITS[unit] | |
| return NASAL[ten] + UNITS[unit] | |
| if n < 1000: | |
| hund, rest = n // 100, n % 100 | |
| return HUNDREDS[hund] + ("" if rest == 0 else " " + spell_int(rest)) | |
| thou, rest = n // 1000, n % 1000 | |
| head = "mil" if thou == 1 else f"{spell_int(thou)} mil" | |
| return head + ("" if rest == 0 else " " + spell_int(rest)) | |
| def spell_digits(text): | |
| """Replace every run of digits with its Kreyòl words, plus the symbols that read as words.""" | |
| text = re.sub(r"(\d)\s*%", r"\1 pousan", text) | |
| text = re.sub(r"(?<=\d)[ ,](?=\d{3}\b)", "", text) # 1 000 / 1,000 -> 1000 | |
| def one(m): | |
| whole, frac = m.group(1), m.group(2) | |
| n = int(whole) | |
| out = spell_int(n) if n < 1_000_000 else " ".join(spell_int(int(c)) for c in whole) | |
| if frac: | |
| out += " pwen " + " ".join(spell_int(int(c)) for c in frac) | |
| return out | |
| return re.sub(r"(\d+)(?:[.,](\d+))?", one, text) | |
| def modern_spelling(text): | |
| """Drop the apostrophes the older orthography used: `pa t' gen` -> `pa t gen`, `t'ap` -> `t ap`.""" | |
| t = text.replace("’", "'").replace("‘", "'").replace("ʼ", "'") | |
| t = re.sub(r"(\w)'(?=[\s,.;:!?\"»”)\]]|$)", r"\1", t) | |
| t = re.sub(r"(\w)'(?=\w)", r"\1 ", t) | |
| return t | |
| def normalize(text, lower=True, punct=False): | |
| """The canonical form used for both training targets and error rates.""" | |
| if not text: | |
| return "" | |
| t = unicodedata.normalize("NFC", text) | |
| t = t.replace("", " ").replace("\xa0", " ") | |
| t = re.sub(r"[“”«»]", '"', t).replace("–", "-").replace("—", "-").replace("…", " ") | |
| t = modern_spelling(t) | |
| t = spell_digits(t) | |
| if lower: | |
| t = t.lower() | |
| if not punct: | |
| t = re.sub(r"[^\w\s'-]", " ", t, flags=re.UNICODE) | |
| t = re.sub(r"(?<!\w)[-']|[-'](?!\w)", " ", t) | |
| return re.sub(r"\s+", " ", t).strip() | |
| def metric_form(text): | |
| """Strip accents on top of `normalize`, so è/e and ò/o never decide a word error.""" | |
| t = unicodedata.normalize("NFKD", normalize(text)) | |
| return "".join(c for c in t if not unicodedata.combining(c)) | |
| def french_score(text): | |
| """Share of tokens that are French-only words; radio broadcasts switch into French mid-sentence.""" | |
| toks = [p for t in normalize(text).split() for p in t.split("-") if p] | |
| if not toks: | |
| return 0.0 | |
| return sum(t in FRENCH for t in toks) / len(toks) | |
| def is_french(text, min_share=0.18): | |
| """A row written in French rather than Kreyòl. | |
| Kreyòl prose about Haiti is full of French-spelled proper nouns (Éstimé, François, Déjoie), so the | |
| accent test skips capitalised words and only judges ordinary ones. | |
| Catches French *text*. It cannot catch French *speech* that a Kreyòl-only ASR wrote down as | |
| Kreyòl-looking words, which is how French appears in the radio transcripts. | |
| """ | |
| ordinary = [] | |
| for word in text.split(): | |
| first = next((c for c in word if c.isalpha()), "") | |
| if not first or first.islower(): | |
| ordinary.append(word) | |
| return french_score(text) >= min_share or len(NON_KREYOL.findall(" ".join(ordinary))) >= 2 | |
| def suspect(text, duration=None): | |
| """Reasons a row is bad training data. Empty list means the row looks usable.""" | |
| why, norm = [], normalize(text) | |
| if len(norm) < 2: | |
| why.append("empty") | |
| if re.search(r"\d", norm): | |
| why.append("digits_left") | |
| if is_french(text): | |
| why.append("french") | |
| letters = len(LETTER.findall(norm)) | |
| if letters and len(norm) and letters / len(norm) < 0.7: | |
| why.append("low_letter_ratio") | |
| if norm and max((len(w) for w in norm.split()), default=0) > 24: | |
| why.append("long_token") | |
| if duration: | |
| rate = letters / duration | |
| if rate < 5 or rate > 25: | |
| why.append(f"rate_{rate:.0f}") | |
| return why | |
| if __name__ == "__main__": | |
| for n in (0, 1, 8, 15, 21, 25, 31, 48, 70, 75, 80, 81, 90, 98, 100, 101, 250, 1000, 1804, 2026, 15000): | |
| print(f"{n:>6} {spell_int(n)}") | |
| for s in ["Jenèz chapit pemye. Istwa kreyasyon an.", | |
| "Nan lane 1804, 25 moun te peye 50% nan 2,500 goud.", | |
| "pa t' gen anyen, epi t'ap vini", | |
| "Qu'est-ce que le vaudou, d'où vient-il?"]: | |
| print(f"\n{s}\n-> {normalize(s)}\n french={french_score(s):.2f} suspect={suspect(s)}") | |