Blame for src/nida/text/normalize_data.py

"""Lexicons of the TTS text normaliser (:mod:`nida.text.normalize`).



Everything here is plain data so it can be extended without touching the rules: add an

:class:`Abbreviation` to :data:`ABBREVIATIONS`, a word to :data:`READ_AS_WORD`, a trigger to

:data:`ID_TRIGGER_WORDS`. Every spoken form must be lower-case Turkish text without digits or

symbols (``tests/test_text_normalize.py`` checks that every one of them encodes).

"""

from __future__ import annotations

from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType

__all__ = [
    "ABBREVIATIONS",
    "ASCII_SPELLINGS",
    "EXTRA_UNITS",
    "ID_BLOCKER_PREFIXES",
    "ID_TRIGGER_PREFIXES",
    "ID_TRIGGER_WORDS",
    "LETTER_NAMES",
    "ORDINAL_HEADS",
    "RATE_WORD_PREFIXES",
    "READ_AS_WORD",
    "ROMAN_RULER_TITLES",
    "ROMAN_TITLE_WORDS",
    "SCORE_WORD_PREFIXES",
    "SECTION_WORDS",
    "SENTENCE_STARTERS",
    "SPELLED_ACRONYMS",
    "Abbreviation",
]


@dataclass(frozen=True)
class Abbreviation:
    """A written abbreviation and how it is spoken.



    Attributes:

        written: Exact spelling including its dot or colon (``"Prof."``, ``"No:"``).

        spoken: Lower-case spoken form (``"profesör"``).

        ignore_case: Also accept the all-lower, all-upper (Turkish rules) and

            capitalised spellings. Off for abbreviations that are also a word in another

            case (``Av.`` / ``av.`` "hunt", ``Sok.`` / ``sok.`` "insert").

        ends_sentence: The abbreviation may close a sentence (``vs.``, ``A.Ş.``): its dot is

            kept when the text ends there or a capitalised word follows. Not set for titles

            (``Dr.``) and for address parts that normally continue (``Cad.``).

        needs_digit_after: Expand only before a digit (``No: 12``), so a plain word

            ``no.`` is left alone.

        possessive: The spoken form ends in the possessive ``-(s)I`` (``caddesi``): a case

            suffix written after the abbreviation gets the pronominal ``n``

            (``Cad.'de`` -> ``caddesinde``).

        soften_final_k: A final ``k`` becomes ``ğ`` before a vowel-initial suffix

            (``Sok.'a`` -> ``sokağa``).

        not_after_number: Do not expand right after a number (``144 Hz.`` is a unit).

        after_capitalised: Expand only right after a capitalised word that is not the first

            of its sentence (``Gül Sok.``, never the imperative ``Sok.``).

        year_context: Expand only before a three or four digit number after a year word

            (``MS`` in ``yıl MS 500``); otherwise the abbreviation is reported.

    """

    written: str
    spoken: str
    ignore_case: bool = False
    ends_sentence: bool = False
    needs_digit_after: bool = False
    possessive: bool = False
    soften_final_k: bool = False
    not_after_number: bool = False
    after_capitalised: bool = False
    year_context: bool = False


#: Abbreviation lexicon (word-initial, case rules per entry, see :class:`Abbreviation`).
ABBREVIATIONS: tuple[Abbreviation, ...] = (
    Abbreviation("Prof.", "profesör", ignore_case=True),
    Abbreviation("Doç.", "doçent", ignore_case=True),
    Abbreviation("Yrd.", "yardımcı", ignore_case=True),
    Abbreviation("Dr.", "doktor", ignore_case=True),
    Abbreviation("Av.", "avukat"),
    Abbreviation("Müh.", "mühendis", ignore_case=True),
    Abbreviation("Hz.", "hazreti", ignore_case=True, not_after_number=True),
    Abbreviation("Cad.", "caddesi", ignore_case=True, possessive=True),
    Abbreviation("Sok.", "sokak", soften_final_k=True, after_capitalised=True),
    Abbreviation("Bul.", "bulvarı", possessive=True, after_capitalised=True),
    Abbreviation("Mah.", "mahallesi", ignore_case=True, possessive=True),
    Abbreviation("Apt.", "apartmanı", ignore_case=True, possessive=True),
    Abbreviation("No:", "numara", needs_digit_after=True),
    Abbreviation("No.", "numara", needs_digit_after=True),
    Abbreviation("no:", "numara", needs_digit_after=True),
    Abbreviation("no.", "numara", needs_digit_after=True),
    Abbreviation("Tel:", "telefon", ignore_case=True),
    Abbreviation("Tel.", "telefon", ignore_case=True),
    Abbreviation("San.", "sanayi"),
    Abbreviation("Tic.", "ticaret"),
    Abbreviation("A.Ş.", "anonim şirketi", possessive=True, ends_sentence=True),
    Abbreviation("Ltd.", "limited"),
    Abbreviation("Şti.", "şirketi", possessive=True, ends_sentence=True),
    Abbreviation("vs.", "vesaire", ignore_case=True, ends_sentence=True),
    Abbreviation("vb.", "ve benzeri", ignore_case=True, ends_sentence=True),
    Abbreviation("vd.", "ve diğerleri", ignore_case=True, ends_sentence=True),
    Abbreviation("bkz.", "bakınız", ignore_case=True),
    Abbreviation("örn.", "örneğin", ignore_case=True),
    Abbreviation("Sn.", "sayın"),
    Abbreviation("yy.", "yüzyıl"),
    Abbreviation("T.C.", "te ce"),
    Abbreviation("(s.a.v.)", "sallallahu aleyhi ve sellem", ignore_case=True),
    Abbreviation("MÖ", "milattan önce", needs_digit_after=True),
    Abbreviation("MS", "milattan sonra", needs_digit_after=True, year_context=True),
)

#: Capital-letter acronyms that are pronounced as a word, not spelled: written form -> lower-case
#: spoken word. Matched exactly, diacritics included (``YÖK`` is the council, ``YOK`` the word
#: "none"); an ASCII spelling of a name is accepted only through :data:`ASCII_SPELLINGS`. The
#: spoken form is explicit because ``tr_lower`` would turn the ``I`` of a foreign word into
#: ``ı`` (``BIST`` -> ``bıst``).
READ_AS_WORD: Mapping[str, str] = MappingProxyType(
    {
        "AFAD": "afad",
        "AK": "ak",
        "ASELSAN": "aselsan",
        "ASKİ": "aski",
        "BİM": "bim",
        "BIST": "bist",
        "BOTAŞ": "botaş",
        "COVID": "covid",
        "DASK": "dask",
        "FIFA": "fifa",
        "HAVELSAN": "havelsan",
        "IBAN": "iban",
        "İSKİ": "iski",
        "KOSGEB": "kosgeb",
        "MEB": "meb",
        "NASA": "nasa",
        "NATO": "nato",
        "ODTÜ": "odtü",
        "OPEC": "opec",
        "RAM": "ram",
        "ROKETSAN": "roketsan",
        "TEİAŞ": "teiaş",
        "TEKNOFEST": "teknofest",
        "TOGG": "togg",
        "TÜBİTAK": "tübitak",
        "TÜİK": "tüik",
        "TÜPRAŞ": "tüpraş",
        "TÜRKSAT": "türksat",
        "TÜRSAB": "türsab",
        "TÜVTÜRK": "tüvtürk",
        "UEFA": "uefa",
        "UNESCO": "unesco",
        "UNICEF": "unicef",
        "VIP": "vip",
        "YÖK": "yök",
        "ŞOK": "şok",
    }
)

#: ASCII-only spellings of the names above that are unambiguous (no ordinary Turkish word
#: looks like them): written form -> key of :data:`READ_AS_WORD`.
ASCII_SPELLINGS: Mapping[str, str] = MappingProxyType(
    {
        "ODTU": "ODTÜ",
        "TUBITAK": "TÜBİTAK",
        "TUIK": "TÜİK",
        "TUPRAS": "TÜPRAŞ",
        "TURKSAT": "TÜRKSAT",
        "TURSAB": "TÜRSAB",
        "TUVTURK": "TÜVTÜRK",
    }
)

#: Acronyms that are spelled although they could be pronounced as a word (``AB`` reads like
#: "ab"). Unpronounceable acronyms (``ABD``, ``TBMM``) need no entry.
SPELLED_ACRONYMS: frozenset[str] = frozenset({"AB", "ÇED", "ÖSS", "ÖSYM", "TEM"})

#: Spoken name of each capital letter when an acronym is spelled. Turkish convention:
#: ``K`` = ``ka``, ``H`` = ``ha`` (not the English-style ``ke`` / ``he``). Q, W and X are not
#: Turkish letters but occur in acronyms (QR, WWF, XL).
LETTER_NAMES: Mapping[str, str] = MappingProxyType(
    {
        "A": "a",
        "B": "be",
        "C": "ce",
        "Ç": "çe",
        "D": "de",
        "E": "e",
        "F": "fe",
        "G": "ge",
        "Ğ": "ğe",
        "H": "ha",
        "I": "ı",
        "İ": "i",
        "J": "je",
        "K": "ka",
        "L": "le",
        "M": "me",
        "N": "ne",
        "O": "o",
        "Ö": "ö",
        "P": "pe",
        "Q": "kü",
        "R": "re",
        "S": "se",
        "Ş": "şe",
        "T": "te",
        "U": "u",
        "Ü": "ü",
        "V": "ve",
        "W": "çift ve",
        "X": "iks",
        "Y": "ye",
        "Z": "ze",
    }
)

#: Words (lower-case, exact) that mark the digit string after them as an identifier that is
#: read digit by digit (``sipariş numaranız 584213``).
ID_TRIGGER_WORDS: frozenset[str] = frozenset({"no", "tc", "pin", "otp"})

#: Prefixes of identifier nouns too (``numara`` covers numarası, numaranız, ...). The noun must
#: be an identifier noun: ``sipariş`` or ``fatura`` alone is not (``Fatura tutarı 12500 TL``).
ID_TRIGGER_PREFIXES: tuple[str, ...] = ("numara", "kod", "şifre")

#: A word with one of these prefixes before the digits makes them a quantity, not an identifier
#: (``kod sayısı 12500``).
ID_BLOCKER_PREFIXES: tuple[str, ...] = (
    "sayı",
    "tutar",
    "adet",
    "miktar",
    "bedel",
    "fiyat",
    "toplam",
)

#: Title / section words after which a bare Roman numeral is an ordinal
#: (``Sultan II Abdülhamid``, ``Bölüm IV``), lower-case.
ROMAN_TITLE_WORDS: frozenset[str] = frozenset(
    {
        "sultan",
        "kral",
        "kraliçe",
        "papa",
        "çar",
        "imparator",
        "imparatoriçe",
        "şah",
        "halife",
        "hakan",
        "bölüm",
        "kısım",
        "cilt",
        "madde",
        "fasıl",
        "perde",
        "kitap",
    }
)

#: The titles among :data:`ROMAN_TITLE_WORDS` that belong to a person: ``Papa II. Jean Paul``
#: stays one sentence whatever name follows the dotted numeral (``Bölüm IV. Sonra`` does not).
ROMAN_RULER_TITLES: frozenset[str] = frozenset(
    {
        "sultan",
        "kral",
        "kraliçe",
        "papa",
        "çar",
        "imparator",
        "imparatoriçe",
        "şah",
        "halife",
        "hakan",
    }
)

#: Units after a number that ``turkificate`` does not know (case-sensitive), spoken form.
#: (``mAh``, the squared/cubed units and ``km/sa`` are handled by dedicated rules.)
EXTRA_UNITS: Mapping[str, str] = MappingProxyType(
    {
        "kcal": "kilokalori",
        "dB": "desibel",
        "Wh": "watt saat",
        "W": "watt",
        "V": "volt",
    }
)

#: Capitalised words that a digit ordinal can precede (``2. Dünya Savaşı``, ``3. Murat``,
#: ``5. Lig``), lower-case. After ``N.`` any other capitalised word starts a new sentence, so
#: ``N`` stays a cardinal and its dot a sentence end (``Fiyat 50. Sonra ...``). A lower-case
#: word after ``N.`` is an ordinal (``5. kat``) and needs no entry.
ORDINAL_HEADS: frozenset[str] = frozenset(
    {
        "abdülhamid", "abdülhamit", "ahmet", "asır", "ay", "baskı", "bayezid", "beyazıt",
        "bölüm", "cadde", "cilt", "cumhurbaşkanı", "dönem", "dünya", "gün", "hafta", "kat",
        "kısım", "kolordu", "lig", "madde", "mahmut", "mehmet", "murat", "ordu", "orhan",
        "osman", "sayfa", "selim", "sınıf", "sokak", "süleyman", "tümen", "uluslararası",
        "yıl", "yüzyıl",
    }
)  # fmt: skip

#: Words that typically open a sentence. After ``N.`` a capitalised one confirms that the dot
#: ends a sentence; any other unlisted capitalised word is reported as ``ambiguous_ordinal``.
#: A lower-case one after ``N.`` is still read as an ordinal but reported too (``25. sonra``).
SENTENCE_STARTERS: frozenset[str] = frozenset(
    {
        "ama", "ancak", "ardından", "ayrıca", "böylece", "bu", "bugün", "bunlar", "bunun",
        "bir", "çünkü", "daha", "dün", "eğer", "evet", "fakat", "hayır", "her", "ise",
        "lütfen", "o", "ona", "onun", "peki", "sonra", "sonuç", "sonuçta", "şimdi", "şu",
        "tamam", "ve", "yarın", "yeni",
    }
)  # fmt: skip

#: Word prefixes (lower-case) that make a number pair a score: ``Skor 1-2`` is read "bir iki".
SCORE_WORD_PREFIXES: tuple[str, ...] = (
    "maç",
    "skor",
    "gol",
    "berabere",
    "galibiyet",
    "mağlubiyet",
    "yenil",
    "yendi",
    "yendik",
)

#: Words (lower-case) after which ``3.2`` is a section number, read "üç nokta iki".
SECTION_WORDS: frozenset[str] = frozenset(
    {"madde", "bölüm", "kısım", "bent", "fıkra", "başlık", "paragraf", "adım"}
)

#: Word prefixes (lower-case) after which ``8.50`` is an amount or a rate, not a clock time.
RATE_WORD_PREFIXES: tuple[str, ...] = (
    "faiz",
    "oran",
    "fiyat",
    "ücret",
    "maaş",
    "kur",
    "tutar",
    "bedel",
    "puan",
    "zam",
)