nida-tts / src /nida /text /normalize_data.py
Bilal7Dev's picture
Nida v0.5 (research preview): weights, 16 voices, code, model card
c0bce59 verified
Raw History Blame Contribute Delete
12.8 kB
"""Lexicons of the TTS text normaliser (:mod:`nida.text.normalize`).
Everything here is plain data so it can be extended without touching the rules: add an
:class:`Abbreviation` to :data:`ABBREVIATIONS`, a word to :data:`READ_AS_WORD`, a trigger to
:data:`ID_TRIGGER_WORDS`. Every spoken form must be lower-case Turkish text without digits or
symbols (``tests/test_text_normalize.py`` checks that every one of them encodes).
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
__all__ = [
"ABBREVIATIONS",
"ASCII_SPELLINGS",
"EXTRA_UNITS",
"ID_BLOCKER_PREFIXES",
"ID_TRIGGER_PREFIXES",
"ID_TRIGGER_WORDS",
"LETTER_NAMES",
"ORDINAL_HEADS",
"RATE_WORD_PREFIXES",
"READ_AS_WORD",
"ROMAN_RULER_TITLES",
"ROMAN_TITLE_WORDS",
"SCORE_WORD_PREFIXES",
"SECTION_WORDS",
"SENTENCE_STARTERS",
"SPELLED_ACRONYMS",
"Abbreviation",
]
@dataclass(frozen=True)
class Abbreviation:
"""A written abbreviation and how it is spoken.
Attributes:
written: Exact spelling including its dot or colon (``"Prof."``, ``"No:"``).
spoken: Lower-case spoken form (``"profesör"``).
ignore_case: Also accept the all-lower, all-upper (Turkish rules) and
capitalised spellings. Off for abbreviations that are also a word in another
case (``Av.`` / ``av.`` "hunt", ``Sok.`` / ``sok.`` "insert").
ends_sentence: The abbreviation may close a sentence (``vs.``, ``A.Ş.``): its dot is
kept when the text ends there or a capitalised word follows. Not set for titles
(``Dr.``) and for address parts that normally continue (``Cad.``).
needs_digit_after: Expand only before a digit (``No: 12``), so a plain word
``no.`` is left alone.
possessive: The spoken form ends in the possessive ``-(s)I`` (``caddesi``): a case
suffix written after the abbreviation gets the pronominal ``n``
(``Cad.'de`` -> ``caddesinde``).
soften_final_k: A final ``k`` becomes ``ğ`` before a vowel-initial suffix
(``Sok.'a`` -> ``sokağa``).
not_after_number: Do not expand right after a number (``144 Hz.`` is a unit).
after_capitalised: Expand only right after a capitalised word that is not the first
of its sentence (``Gül Sok.``, never the imperative ``Sok.``).
year_context: Expand only before a three or four digit number after a year word
(``MS`` in ``yıl MS 500``); otherwise the abbreviation is reported.
"""
written: str
spoken: str
ignore_case: bool = False
ends_sentence: bool = False
needs_digit_after: bool = False
possessive: bool = False
soften_final_k: bool = False
not_after_number: bool = False
after_capitalised: bool = False
year_context: bool = False
#: Abbreviation lexicon (word-initial, case rules per entry, see :class:`Abbreviation`).
ABBREVIATIONS: tuple[Abbreviation, ...] = (
Abbreviation("Prof.", "profesör", ignore_case=True),
Abbreviation("Doç.", "doçent", ignore_case=True),
Abbreviation("Yrd.", "yardımcı", ignore_case=True),
Abbreviation("Dr.", "doktor", ignore_case=True),
Abbreviation("Av.", "avukat"),
Abbreviation("Müh.", "mühendis", ignore_case=True),
Abbreviation("Hz.", "hazreti", ignore_case=True, not_after_number=True),
Abbreviation("Cad.", "caddesi", ignore_case=True, possessive=True),
Abbreviation("Sok.", "sokak", soften_final_k=True, after_capitalised=True),
Abbreviation("Bul.", "bulvarı", possessive=True, after_capitalised=True),
Abbreviation("Mah.", "mahallesi", ignore_case=True, possessive=True),
Abbreviation("Apt.", "apartmanı", ignore_case=True, possessive=True),
Abbreviation("No:", "numara", needs_digit_after=True),
Abbreviation("No.", "numara", needs_digit_after=True),
Abbreviation("no:", "numara", needs_digit_after=True),
Abbreviation("no.", "numara", needs_digit_after=True),
Abbreviation("Tel:", "telefon", ignore_case=True),
Abbreviation("Tel.", "telefon", ignore_case=True),
Abbreviation("San.", "sanayi"),
Abbreviation("Tic.", "ticaret"),
Abbreviation("A.Ş.", "anonim şirketi", possessive=True, ends_sentence=True),
Abbreviation("Ltd.", "limited"),
Abbreviation("Şti.", "şirketi", possessive=True, ends_sentence=True),
Abbreviation("vs.", "vesaire", ignore_case=True, ends_sentence=True),
Abbreviation("vb.", "ve benzeri", ignore_case=True, ends_sentence=True),
Abbreviation("vd.", "ve diğerleri", ignore_case=True, ends_sentence=True),
Abbreviation("bkz.", "bakınız", ignore_case=True),
Abbreviation("örn.", "örneğin", ignore_case=True),
Abbreviation("Sn.", "sayın"),
Abbreviation("yy.", "yüzyıl"),
Abbreviation("T.C.", "te ce"),
Abbreviation("(s.a.v.)", "sallallahu aleyhi ve sellem", ignore_case=True),
Abbreviation("MÖ", "milattan önce", needs_digit_after=True),
Abbreviation("MS", "milattan sonra", needs_digit_after=True, year_context=True),
)
#: Capital-letter acronyms that are pronounced as a word, not spelled: written form -> lower-case
#: spoken word. Matched exactly, diacritics included (``YÖK`` is the council, ``YOK`` the word
#: "none"); an ASCII spelling of a name is accepted only through :data:`ASCII_SPELLINGS`. The
#: spoken form is explicit because ``tr_lower`` would turn the ``I`` of a foreign word into
#: ``ı`` (``BIST`` -> ``bıst``).
READ_AS_WORD: Mapping[str, str] = MappingProxyType(
{
"AFAD": "afad",
"AK": "ak",
"ASELSAN": "aselsan",
"ASKİ": "aski",
"BİM": "bim",
"BIST": "bist",
"BOTAŞ": "botaş",
"COVID": "covid",
"DASK": "dask",
"FIFA": "fifa",
"HAVELSAN": "havelsan",
"IBAN": "iban",
"İSKİ": "iski",
"KOSGEB": "kosgeb",
"MEB": "meb",
"NASA": "nasa",
"NATO": "nato",
"ODTÜ": "odtü",
"OPEC": "opec",
"RAM": "ram",
"ROKETSAN": "roketsan",
"TEİAŞ": "teiaş",
"TEKNOFEST": "teknofest",
"TOGG": "togg",
"TÜBİTAK": "tübitak",
"TÜİK": "tüik",
"TÜPRAŞ": "tüpraş",
"TÜRKSAT": "türksat",
"TÜRSAB": "türsab",
"TÜVTÜRK": "tüvtürk",
"UEFA": "uefa",
"UNESCO": "unesco",
"UNICEF": "unicef",
"VIP": "vip",
"YÖK": "yök",
"ŞOK": "şok",
}
)
#: ASCII-only spellings of the names above that are unambiguous (no ordinary Turkish word
#: looks like them): written form -> key of :data:`READ_AS_WORD`.
ASCII_SPELLINGS: Mapping[str, str] = MappingProxyType(
{
"ODTU": "ODTÜ",
"TUBITAK": "TÜBİTAK",
"TUIK": "TÜİK",
"TUPRAS": "TÜPRAŞ",
"TURKSAT": "TÜRKSAT",
"TURSAB": "TÜRSAB",
"TUVTURK": "TÜVTÜRK",
}
)
#: Acronyms that are spelled although they could be pronounced as a word (``AB`` reads like
#: "ab"). Unpronounceable acronyms (``ABD``, ``TBMM``) need no entry.
SPELLED_ACRONYMS: frozenset[str] = frozenset({"AB", "ÇED", "ÖSS", "ÖSYM", "TEM"})
#: Spoken name of each capital letter when an acronym is spelled. Turkish convention:
#: ``K`` = ``ka``, ``H`` = ``ha`` (not the English-style ``ke`` / ``he``). Q, W and X are not
#: Turkish letters but occur in acronyms (QR, WWF, XL).
LETTER_NAMES: Mapping[str, str] = MappingProxyType(
{
"A": "a",
"B": "be",
"C": "ce",
"Ç": "çe",
"D": "de",
"E": "e",
"F": "fe",
"G": "ge",
"Ğ": "ğe",
"H": "ha",
"I": "ı",
"İ": "i",
"J": "je",
"K": "ka",
"L": "le",
"M": "me",
"N": "ne",
"O": "o",
"Ö": "ö",
"P": "pe",
"Q": "kü",
"R": "re",
"S": "se",
"Ş": "şe",
"T": "te",
"U": "u",
"Ü": "ü",
"V": "ve",
"W": "çift ve",
"X": "iks",
"Y": "ye",
"Z": "ze",
}
)
#: Words (lower-case, exact) that mark the digit string after them as an identifier that is
#: read digit by digit (``sipariş numaranız 584213``).
ID_TRIGGER_WORDS: frozenset[str] = frozenset({"no", "tc", "pin", "otp"})
#: Prefixes of identifier nouns too (``numara`` covers numarası, numaranız, ...). The noun must
#: be an identifier noun: ``sipariş`` or ``fatura`` alone is not (``Fatura tutarı 12500 TL``).
ID_TRIGGER_PREFIXES: tuple[str, ...] = ("numara", "kod", "şifre")
#: A word with one of these prefixes before the digits makes them a quantity, not an identifier
#: (``kod sayısı 12500``).
ID_BLOCKER_PREFIXES: tuple[str, ...] = (
"sayı",
"tutar",
"adet",
"miktar",
"bedel",
"fiyat",
"toplam",
)
#: Title / section words after which a bare Roman numeral is an ordinal
#: (``Sultan II Abdülhamid``, ``Bölüm IV``), lower-case.
ROMAN_TITLE_WORDS: frozenset[str] = frozenset(
{
"sultan",
"kral",
"kraliçe",
"papa",
"çar",
"imparator",
"imparatoriçe",
"şah",
"halife",
"hakan",
"bölüm",
"kısım",
"cilt",
"madde",
"fasıl",
"perde",
"kitap",
}
)
#: The titles among :data:`ROMAN_TITLE_WORDS` that belong to a person: ``Papa II. Jean Paul``
#: stays one sentence whatever name follows the dotted numeral (``Bölüm IV. Sonra`` does not).
ROMAN_RULER_TITLES: frozenset[str] = frozenset(
{
"sultan",
"kral",
"kraliçe",
"papa",
"çar",
"imparator",
"imparatoriçe",
"şah",
"halife",
"hakan",
}
)
#: Units after a number that ``turkificate`` does not know (case-sensitive), spoken form.
#: (``mAh``, the squared/cubed units and ``km/sa`` are handled by dedicated rules.)
EXTRA_UNITS: Mapping[str, str] = MappingProxyType(
{
"kcal": "kilokalori",
"dB": "desibel",
"Wh": "watt saat",
"W": "watt",
"V": "volt",
}
)
#: Capitalised words that a digit ordinal can precede (``2. Dünya Savaşı``, ``3. Murat``,
#: ``5. Lig``), lower-case. After ``N.`` any other capitalised word starts a new sentence, so
#: ``N`` stays a cardinal and its dot a sentence end (``Fiyat 50. Sonra ...``). A lower-case
#: word after ``N.`` is an ordinal (``5. kat``) and needs no entry.
ORDINAL_HEADS: frozenset[str] = frozenset(
{
"abdülhamid", "abdülhamit", "ahmet", "asır", "ay", "baskı", "bayezid", "beyazıt",
"bölüm", "cadde", "cilt", "cumhurbaşkanı", "dönem", "dünya", "gün", "hafta", "kat",
"kısım", "kolordu", "lig", "madde", "mahmut", "mehmet", "murat", "ordu", "orhan",
"osman", "sayfa", "selim", "sınıf", "sokak", "süleyman", "tümen", "uluslararası",
"yıl", "yüzyıl",
}
) # fmt: skip
#: Words that typically open a sentence. After ``N.`` a capitalised one confirms that the dot
#: ends a sentence; any other unlisted capitalised word is reported as ``ambiguous_ordinal``.
#: A lower-case one after ``N.`` is still read as an ordinal but reported too (``25. sonra``).
SENTENCE_STARTERS: frozenset[str] = frozenset(
{
"ama", "ancak", "ardından", "ayrıca", "böylece", "bu", "bugün", "bunlar", "bunun",
"bir", "çünkü", "daha", "dün", "eğer", "evet", "fakat", "hayır", "her", "ise",
"lütfen", "o", "ona", "onun", "peki", "sonra", "sonuç", "sonuçta", "şimdi", "şu",
"tamam", "ve", "yarın", "yeni",
}
) # fmt: skip
#: Word prefixes (lower-case) that make a number pair a score: ``Skor 1-2`` is read "bir iki".
SCORE_WORD_PREFIXES: tuple[str, ...] = (
"maç",
"skor",
"gol",
"berabere",
"galibiyet",
"mağlubiyet",
"yenil",
"yendi",
"yendik",
)
#: Words (lower-case) after which ``3.2`` is a section number, read "üç nokta iki".
SECTION_WORDS: frozenset[str] = frozenset(
{"madde", "bölüm", "kısım", "bent", "fıkra", "başlık", "paragraf", "adım"}
)
#: Word prefixes (lower-case) after which ``8.50`` is an amount or a rate, not a clock time.
RATE_WORD_PREFIXES: tuple[str, ...] = (
"faiz",
"oran",
"fiyat",
"ücret",
"maaş",
"kur",
"tutar",
"bedel",
"puan",
"zam",
)