Download src/nida/text/normalize_data.py from Bilal7Dev/nida-tts: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/Bilal7Dev/nida-tts/resolve/main/src/nida/text/normalize_data.py
- Command line
-
hf download hf://Bilal7Dev/nida-tts/src/nida/text/normalize_data.py
-
curl -L -o normalize_data.py https://huggingface.co/Bilal7Dev/nida-tts/resolve/main/src/nida/text/normalize_data.py
12.8 kB
| """Lexicons of the TTS text normaliser (:mod:`nida.text.normalize`). | |
| Everything here is plain data so it can be extended without touching the rules: add an | |
| :class:`Abbreviation` to :data:`ABBREVIATIONS`, a word to :data:`READ_AS_WORD`, a trigger to | |
| :data:`ID_TRIGGER_WORDS`. Every spoken form must be lower-case Turkish text without digits or | |
| symbols (``tests/test_text_normalize.py`` checks that every one of them encodes). | |
| """ | |
| from __future__ import annotations | |
| from collections.abc import Mapping | |
| from dataclasses import dataclass | |
| from types import MappingProxyType | |
| __all__ = [ | |
| "ABBREVIATIONS", | |
| "ASCII_SPELLINGS", | |
| "EXTRA_UNITS", | |
| "ID_BLOCKER_PREFIXES", | |
| "ID_TRIGGER_PREFIXES", | |
| "ID_TRIGGER_WORDS", | |
| "LETTER_NAMES", | |
| "ORDINAL_HEADS", | |
| "RATE_WORD_PREFIXES", | |
| "READ_AS_WORD", | |
| "ROMAN_RULER_TITLES", | |
| "ROMAN_TITLE_WORDS", | |
| "SCORE_WORD_PREFIXES", | |
| "SECTION_WORDS", | |
| "SENTENCE_STARTERS", | |
| "SPELLED_ACRONYMS", | |
| "Abbreviation", | |
| ] | |
| class Abbreviation: | |
| """A written abbreviation and how it is spoken. | |
| Attributes: | |
| written: Exact spelling including its dot or colon (``"Prof."``, ``"No:"``). | |
| spoken: Lower-case spoken form (``"profesör"``). | |
| ignore_case: Also accept the all-lower, all-upper (Turkish rules) and | |
| capitalised spellings. Off for abbreviations that are also a word in another | |
| case (``Av.`` / ``av.`` "hunt", ``Sok.`` / ``sok.`` "insert"). | |
| ends_sentence: The abbreviation may close a sentence (``vs.``, ``A.Ş.``): its dot is | |
| kept when the text ends there or a capitalised word follows. Not set for titles | |
| (``Dr.``) and for address parts that normally continue (``Cad.``). | |
| needs_digit_after: Expand only before a digit (``No: 12``), so a plain word | |
| ``no.`` is left alone. | |
| possessive: The spoken form ends in the possessive ``-(s)I`` (``caddesi``): a case | |
| suffix written after the abbreviation gets the pronominal ``n`` | |
| (``Cad.'de`` -> ``caddesinde``). | |
| soften_final_k: A final ``k`` becomes ``ğ`` before a vowel-initial suffix | |
| (``Sok.'a`` -> ``sokağa``). | |
| not_after_number: Do not expand right after a number (``144 Hz.`` is a unit). | |
| after_capitalised: Expand only right after a capitalised word that is not the first | |
| of its sentence (``Gül Sok.``, never the imperative ``Sok.``). | |
| year_context: Expand only before a three or four digit number after a year word | |
| (``MS`` in ``yıl MS 500``); otherwise the abbreviation is reported. | |
| """ | |
| written: str | |
| spoken: str | |
| ignore_case: bool = False | |
| ends_sentence: bool = False | |
| needs_digit_after: bool = False | |
| possessive: bool = False | |
| soften_final_k: bool = False | |
| not_after_number: bool = False | |
| after_capitalised: bool = False | |
| year_context: bool = False | |
| #: Abbreviation lexicon (word-initial, case rules per entry, see :class:`Abbreviation`). | |
| ABBREVIATIONS: tuple[Abbreviation, ...] = ( | |
| Abbreviation("Prof.", "profesör", ignore_case=True), | |
| Abbreviation("Doç.", "doçent", ignore_case=True), | |
| Abbreviation("Yrd.", "yardımcı", ignore_case=True), | |
| Abbreviation("Dr.", "doktor", ignore_case=True), | |
| Abbreviation("Av.", "avukat"), | |
| Abbreviation("Müh.", "mühendis", ignore_case=True), | |
| Abbreviation("Hz.", "hazreti", ignore_case=True, not_after_number=True), | |
| Abbreviation("Cad.", "caddesi", ignore_case=True, possessive=True), | |
| Abbreviation("Sok.", "sokak", soften_final_k=True, after_capitalised=True), | |
| Abbreviation("Bul.", "bulvarı", possessive=True, after_capitalised=True), | |
| Abbreviation("Mah.", "mahallesi", ignore_case=True, possessive=True), | |
| Abbreviation("Apt.", "apartmanı", ignore_case=True, possessive=True), | |
| Abbreviation("No:", "numara", needs_digit_after=True), | |
| Abbreviation("No.", "numara", needs_digit_after=True), | |
| Abbreviation("no:", "numara", needs_digit_after=True), | |
| Abbreviation("no.", "numara", needs_digit_after=True), | |
| Abbreviation("Tel:", "telefon", ignore_case=True), | |
| Abbreviation("Tel.", "telefon", ignore_case=True), | |
| Abbreviation("San.", "sanayi"), | |
| Abbreviation("Tic.", "ticaret"), | |
| Abbreviation("A.Ş.", "anonim şirketi", possessive=True, ends_sentence=True), | |
| Abbreviation("Ltd.", "limited"), | |
| Abbreviation("Şti.", "şirketi", possessive=True, ends_sentence=True), | |
| Abbreviation("vs.", "vesaire", ignore_case=True, ends_sentence=True), | |
| Abbreviation("vb.", "ve benzeri", ignore_case=True, ends_sentence=True), | |
| Abbreviation("vd.", "ve diğerleri", ignore_case=True, ends_sentence=True), | |
| Abbreviation("bkz.", "bakınız", ignore_case=True), | |
| Abbreviation("örn.", "örneğin", ignore_case=True), | |
| Abbreviation("Sn.", "sayın"), | |
| Abbreviation("yy.", "yüzyıl"), | |
| Abbreviation("T.C.", "te ce"), | |
| Abbreviation("(s.a.v.)", "sallallahu aleyhi ve sellem", ignore_case=True), | |
| Abbreviation("MÖ", "milattan önce", needs_digit_after=True), | |
| Abbreviation("MS", "milattan sonra", needs_digit_after=True, year_context=True), | |
| ) | |
| #: Capital-letter acronyms that are pronounced as a word, not spelled: written form -> lower-case | |
| #: spoken word. Matched exactly, diacritics included (``YÖK`` is the council, ``YOK`` the word | |
| #: "none"); an ASCII spelling of a name is accepted only through :data:`ASCII_SPELLINGS`. The | |
| #: spoken form is explicit because ``tr_lower`` would turn the ``I`` of a foreign word into | |
| #: ``ı`` (``BIST`` -> ``bıst``). | |
| READ_AS_WORD: Mapping[str, str] = MappingProxyType( | |
| { | |
| "AFAD": "afad", | |
| "AK": "ak", | |
| "ASELSAN": "aselsan", | |
| "ASKİ": "aski", | |
| "BİM": "bim", | |
| "BIST": "bist", | |
| "BOTAŞ": "botaş", | |
| "COVID": "covid", | |
| "DASK": "dask", | |
| "FIFA": "fifa", | |
| "HAVELSAN": "havelsan", | |
| "IBAN": "iban", | |
| "İSKİ": "iski", | |
| "KOSGEB": "kosgeb", | |
| "MEB": "meb", | |
| "NASA": "nasa", | |
| "NATO": "nato", | |
| "ODTÜ": "odtü", | |
| "OPEC": "opec", | |
| "RAM": "ram", | |
| "ROKETSAN": "roketsan", | |
| "TEİAŞ": "teiaş", | |
| "TEKNOFEST": "teknofest", | |
| "TOGG": "togg", | |
| "TÜBİTAK": "tübitak", | |
| "TÜİK": "tüik", | |
| "TÜPRAŞ": "tüpraş", | |
| "TÜRKSAT": "türksat", | |
| "TÜRSAB": "türsab", | |
| "TÜVTÜRK": "tüvtürk", | |
| "UEFA": "uefa", | |
| "UNESCO": "unesco", | |
| "UNICEF": "unicef", | |
| "VIP": "vip", | |
| "YÖK": "yök", | |
| "ŞOK": "şok", | |
| } | |
| ) | |
| #: ASCII-only spellings of the names above that are unambiguous (no ordinary Turkish word | |
| #: looks like them): written form -> key of :data:`READ_AS_WORD`. | |
| ASCII_SPELLINGS: Mapping[str, str] = MappingProxyType( | |
| { | |
| "ODTU": "ODTÜ", | |
| "TUBITAK": "TÜBİTAK", | |
| "TUIK": "TÜİK", | |
| "TUPRAS": "TÜPRAŞ", | |
| "TURKSAT": "TÜRKSAT", | |
| "TURSAB": "TÜRSAB", | |
| "TUVTURK": "TÜVTÜRK", | |
| } | |
| ) | |
| #: Acronyms that are spelled although they could be pronounced as a word (``AB`` reads like | |
| #: "ab"). Unpronounceable acronyms (``ABD``, ``TBMM``) need no entry. | |
| SPELLED_ACRONYMS: frozenset[str] = frozenset({"AB", "ÇED", "ÖSS", "ÖSYM", "TEM"}) | |
| #: Spoken name of each capital letter when an acronym is spelled. Turkish convention: | |
| #: ``K`` = ``ka``, ``H`` = ``ha`` (not the English-style ``ke`` / ``he``). Q, W and X are not | |
| #: Turkish letters but occur in acronyms (QR, WWF, XL). | |
| LETTER_NAMES: Mapping[str, str] = MappingProxyType( | |
| { | |
| "A": "a", | |
| "B": "be", | |
| "C": "ce", | |
| "Ç": "çe", | |
| "D": "de", | |
| "E": "e", | |
| "F": "fe", | |
| "G": "ge", | |
| "Ğ": "ğe", | |
| "H": "ha", | |
| "I": "ı", | |
| "İ": "i", | |
| "J": "je", | |
| "K": "ka", | |
| "L": "le", | |
| "M": "me", | |
| "N": "ne", | |
| "O": "o", | |
| "Ö": "ö", | |
| "P": "pe", | |
| "Q": "kü", | |
| "R": "re", | |
| "S": "se", | |
| "Ş": "şe", | |
| "T": "te", | |
| "U": "u", | |
| "Ü": "ü", | |
| "V": "ve", | |
| "W": "çift ve", | |
| "X": "iks", | |
| "Y": "ye", | |
| "Z": "ze", | |
| } | |
| ) | |
| #: Words (lower-case, exact) that mark the digit string after them as an identifier that is | |
| #: read digit by digit (``sipariş numaranız 584213``). | |
| ID_TRIGGER_WORDS: frozenset[str] = frozenset({"no", "tc", "pin", "otp"}) | |
| #: Prefixes of identifier nouns too (``numara`` covers numarası, numaranız, ...). The noun must | |
| #: be an identifier noun: ``sipariş`` or ``fatura`` alone is not (``Fatura tutarı 12500 TL``). | |
| ID_TRIGGER_PREFIXES: tuple[str, ...] = ("numara", "kod", "şifre") | |
| #: A word with one of these prefixes before the digits makes them a quantity, not an identifier | |
| #: (``kod sayısı 12500``). | |
| ID_BLOCKER_PREFIXES: tuple[str, ...] = ( | |
| "sayı", | |
| "tutar", | |
| "adet", | |
| "miktar", | |
| "bedel", | |
| "fiyat", | |
| "toplam", | |
| ) | |
| #: Title / section words after which a bare Roman numeral is an ordinal | |
| #: (``Sultan II Abdülhamid``, ``Bölüm IV``), lower-case. | |
| ROMAN_TITLE_WORDS: frozenset[str] = frozenset( | |
| { | |
| "sultan", | |
| "kral", | |
| "kraliçe", | |
| "papa", | |
| "çar", | |
| "imparator", | |
| "imparatoriçe", | |
| "şah", | |
| "halife", | |
| "hakan", | |
| "bölüm", | |
| "kısım", | |
| "cilt", | |
| "madde", | |
| "fasıl", | |
| "perde", | |
| "kitap", | |
| } | |
| ) | |
| #: The titles among :data:`ROMAN_TITLE_WORDS` that belong to a person: ``Papa II. Jean Paul`` | |
| #: stays one sentence whatever name follows the dotted numeral (``Bölüm IV. Sonra`` does not). | |
| ROMAN_RULER_TITLES: frozenset[str] = frozenset( | |
| { | |
| "sultan", | |
| "kral", | |
| "kraliçe", | |
| "papa", | |
| "çar", | |
| "imparator", | |
| "imparatoriçe", | |
| "şah", | |
| "halife", | |
| "hakan", | |
| } | |
| ) | |
| #: Units after a number that ``turkificate`` does not know (case-sensitive), spoken form. | |
| #: (``mAh``, the squared/cubed units and ``km/sa`` are handled by dedicated rules.) | |
| EXTRA_UNITS: Mapping[str, str] = MappingProxyType( | |
| { | |
| "kcal": "kilokalori", | |
| "dB": "desibel", | |
| "Wh": "watt saat", | |
| "W": "watt", | |
| "V": "volt", | |
| } | |
| ) | |
| #: Capitalised words that a digit ordinal can precede (``2. Dünya Savaşı``, ``3. Murat``, | |
| #: ``5. Lig``), lower-case. After ``N.`` any other capitalised word starts a new sentence, so | |
| #: ``N`` stays a cardinal and its dot a sentence end (``Fiyat 50. Sonra ...``). A lower-case | |
| #: word after ``N.`` is an ordinal (``5. kat``) and needs no entry. | |
| ORDINAL_HEADS: frozenset[str] = frozenset( | |
| { | |
| "abdülhamid", "abdülhamit", "ahmet", "asır", "ay", "baskı", "bayezid", "beyazıt", | |
| "bölüm", "cadde", "cilt", "cumhurbaşkanı", "dönem", "dünya", "gün", "hafta", "kat", | |
| "kısım", "kolordu", "lig", "madde", "mahmut", "mehmet", "murat", "ordu", "orhan", | |
| "osman", "sayfa", "selim", "sınıf", "sokak", "süleyman", "tümen", "uluslararası", | |
| "yıl", "yüzyıl", | |
| } | |
| ) # fmt: skip | |
| #: Words that typically open a sentence. After ``N.`` a capitalised one confirms that the dot | |
| #: ends a sentence; any other unlisted capitalised word is reported as ``ambiguous_ordinal``. | |
| #: A lower-case one after ``N.`` is still read as an ordinal but reported too (``25. sonra``). | |
| SENTENCE_STARTERS: frozenset[str] = frozenset( | |
| { | |
| "ama", "ancak", "ardından", "ayrıca", "böylece", "bu", "bugün", "bunlar", "bunun", | |
| "bir", "çünkü", "daha", "dün", "eğer", "evet", "fakat", "hayır", "her", "ise", | |
| "lütfen", "o", "ona", "onun", "peki", "sonra", "sonuç", "sonuçta", "şimdi", "şu", | |
| "tamam", "ve", "yarın", "yeni", | |
| } | |
| ) # fmt: skip | |
| #: Word prefixes (lower-case) that make a number pair a score: ``Skor 1-2`` is read "bir iki". | |
| SCORE_WORD_PREFIXES: tuple[str, ...] = ( | |
| "maç", | |
| "skor", | |
| "gol", | |
| "berabere", | |
| "galibiyet", | |
| "mağlubiyet", | |
| "yenil", | |
| "yendi", | |
| "yendik", | |
| ) | |
| #: Words (lower-case) after which ``3.2`` is a section number, read "üç nokta iki". | |
| SECTION_WORDS: frozenset[str] = frozenset( | |
| {"madde", "bölüm", "kısım", "bent", "fıkra", "başlık", "paragraf", "adım"} | |
| ) | |
| #: Word prefixes (lower-case) after which ``8.50`` is an amount or a rate, not a clock time. | |
| RATE_WORD_PREFIXES: tuple[str, ...] = ( | |
| "faiz", | |
| "oran", | |
| "fiyat", | |
| "ücret", | |
| "maaş", | |
| "kur", | |
| "tutar", | |
| "bedel", | |
| "puan", | |
| "zam", | |
| ) | |