Blame for src/nida/text/normalize.py

"""Turkish TTS text normaliser: raw text to fully spoken text the model's encoder accepts.



``normalize_for_tts`` turns written Turkish (digits, symbols, abbreviations, acronyms, clock

times, phone numbers, ...) into the words a news reader would say, so that

:func:`nida.model.vocab.encode_text` never meets a digit or a symbol. Conventions are

standard spoken Turkish; every choice that is a convention is listed here.



Pipeline (per plain-text segment; tags ``[...]``, ``<...>`` and ``@emotion`` directives pass

through untouched)::



    NFC -> typographic quotes -> our pre-rules -> turkificate -> our post-rules -> validation



Pre-rules (in this order, each owns the shapes the base library gets wrong):



1. abbreviation lexicon (``Dr.`` -> doktor, ``A.Ş.``, ``T.C.``, ``No: 12``, ...), see

   :mod:`nida.text.normalize_data`;

2. Turkish phone numbers, read in groups without commas (commas are clause marks for the

   sentence splitter): ``0212 444 55 66`` -> ``sıfır iki yüz on iki dört yüz kırk dört elli beş

   altmış altı``; ``+90`` is read ``artı doksan``;

3. plates ``34 TBC 512``;

4. clock times ``09.00'da`` -> ``dokuzda`` (``:00`` minutes dropped, ``06:05`` -> ``altı sıfır

   beş``, ``09.00-17.00`` -> ``dokuz ila on yedi``), dot form only in time context;

5. identifier numbers (5+ digits after numara / no / kod / sipariş / ...) digit by digit;

6. ``1 000 000`` (space-grouped thousands) and ``007`` (three or more digits with a leading

   zero: a code, read digit by digit);

7. Roman-numeral ordinals (``XIX. yüzyıl``; bare ones only after a title or a name);

8. English-style decimals ``3.5`` -> ``3,5`` (a dot followed by exactly three digits stays a

   thousands separator);

9. numeric ranges ``a-b`` / ``a–b`` (ascending) -> ``a ila b``;

10. units the base library misses (``km/sa`` -> ``saatte <n> kilometre``, ``mAh``, ``m²``,

    ``°C`` -> ``derece``, ``kcal``, ...);

11. words joined by a slash (``ve/veya``) are two words; ``TL`` / ``₺`` not preceded by an

    amount (``milyon TL``, ``TL'den``) -> ``lira``;

12. capital acronyms: read-as-word lexicon (``NATO`` -> ``nato``), otherwise spelled with the

    Turkish letter names (``TBMM'de`` -> ``te be me mede``); alphanumeric codes ``TK1985``,

    ``F-16``, ``3D``.



The base library runs without its ``abbreviations``, ``phones`` and ``roman_numerals``

concepts (ours replace them) and without its brand and technology-term lists; see

:data:`_BASE_FEATURES`.



Post-rules: a suffix written after a digit (``40'ı``) is held out of the base library and

re-attached with :func:`nida.text.suffix.attach_suffix`, so it agrees with the *spoken* last

word (``kırkı``, not the base library's ``kırğı``); an apostrophe left after a spoken number,

unit or currency word is resolved the same way (``lira'dır`` -> ``liradır``); ``X virgül beş``

that came from ``X,5`` becomes ``X buçuk`` (``0,5`` stays ``sıfır virgül beş``).



Conventions chosen here (spec: "standard spoken Turkish, as in news reading"):



* Dot clock times are times only in time context: after ``saat`` / ``at``, with an apostrophe

  suffix, with a zero-padded or 13+ hour (``09.00``, ``21.45``) or at the start of a clause;

  never before a currency / unit / ``%`` (``3.50 TL``). A colon time is always a time. A

  reading taken on weak evidence (clause start, or a plain ``9.30`` read as a decimal) is

  reported as ``ambiguous_time``.

* A range is ``a ila b`` only when ``a < b``; ``3-0`` (a score) and ``10-5`` are left to the base

  library (``üç tire sıfır``) and reported.

* Capital tokens are spelled only if they are in the lexicon (``SPELLED_ACRONYMS``) or cannot

  be a Turkish word (no vowel, an impossible consonant cluster: ``ABD``, ``ÖSYM``). Any other

  (``DUR``, ``BORUSAN``) stays a word, lower-cased, and is reported as ``acronym_guess``; an

  all-capitals clause (``ONU GÖR``) is left alone. Lexicon words match exactly (``YOK`` is not

  ``YÖK``).

* A read-as-word acronym keeps its written suffix and apostrophe (``NATO'nun`` ->

  ``nato'nun``), like any proper name; a spelled one gets the suffix re-harmonised and attached

  to the last letter name (``SGK'ya`` -> ``se ge kaya``).

* ``.5`` as the fraction is ``buçuk`` only with a non-zero integer part and no suffix.



Anything with digits, symbols or characters outside the model vocabulary left after the

pipeline is reported in :attr:`TtsNormalization.issues`; nothing is dropped silently.

Text that needs no change is returned byte-identical (after NFC).

"""

from __future__ import annotations

import functools
import re
import unicodedata
from collections.abc import Sequence
from dataclasses import dataclass, field
from typing import Protocol

import turkificate
from turkificate.data import CURRENCY, MONTHS, UNITS

from nida.model.vocab import VOCAB, encode_text
from nida.text.normalize_data import (
    ABBREVIATIONS,
    ASCII_SPELLINGS,
    EXTRA_UNITS,
    ID_BLOCKER_PREFIXES,
    ID_TRIGGER_PREFIXES,
    ID_TRIGGER_WORDS,
    LETTER_NAMES,
    ORDINAL_HEADS,
    RATE_WORD_PREFIXES,
    READ_AS_WORD,
    ROMAN_RULER_TITLES,
    ROMAN_TITLE_WORDS,
    SCORE_WORD_PREFIXES,
    SECTION_WORDS,
    SENTENCE_STARTERS,
    SPELLED_ACRONYMS,
    Abbreviation,
)
from nida.text.numbers import number_words, ordinal_words
from nida.text.suffix import attach_suffix
from nida.text.turkish import nfc, tr_lower, tr_upper

__all__ = [
    "DANGEROUS_ISSUE_KINDS",
    "TtsNormalization",
    "dangerous_issues",
    "issue_kind",
    "normalize_for_tts",
    "normalize_or_raise",
]


@dataclass(frozen=True)
class TtsNormalization:
    """Result of :func:`normalize_for_tts`.



    Attributes:

        text: The spoken text (NFC).

        issues: Machine-readable reasons for everything left unresolved, as ``kind:detail``

            (``digits:3D``, ``symbol:%``, ``char:ä``, ``roman_numeral_ambiguous:XI``,

            ``range_not_ascending:3-0``, ``invalid_time:25:70``, ``encode:...``), in order of

            appearance, without duplicates. Empty when ``text`` is fully spoken.

    """

    text: str
    issues: tuple[str, ...] = ()

    @property
    def ok(self) -> bool:
        """True when nothing was left unresolved."""
        return not self.issues


@dataclass
class _Ctx:
    """Mutable issue collector shared by the rules of one call."""

    issues: list[str] = field(default_factory=list)


# --------------------------------------------------------------------------------------
# Shared pieces
# --------------------------------------------------------------------------------------

#: Typographic quotes / apostrophes outside the model vocabulary -> the ASCII ones it knows.
_TYPOGRAPHIC = str.maketrans(
    {
        chr(0x2018): "'",  # left single quotation mark
        chr(0x2019): "'",  # right single quotation mark
        chr(0x02BC): "'",  # modifier letter apostrophe
        chr(0x201C): '"',  # left double quotation mark
        chr(0x201D): '"',  # right double quotation mark
        chr(0x201E): '"',  # double low-9 quotation mark
    }
)
#: Private-use characters mark held-out suffixes below; they must not occur in the input.
_SENTINEL_OPEN = chr(0xE000)
_SENTINEL_CLOSE = chr(0xE001)
#: Stands for the dot of a cardinal that ends a sentence (``%65.``) while the base library runs,
#: so that it does not take the dot for an ordinal marker.
_KEPT_DOT = chr(0xE002)

_LETTER = r"[^\W\d_]"
#: Letters a written suffix is made of (Latin, Turkish; not digits, superscripts or accents of
#: other languages).
_SUFFIX_LETTER = "[A-Za-zÇĞİÖŞÜçğıöşüÂÎÛâîû]"
#: Optional written suffix after an apostrophe (named group ``sfx``).
_SFX = rf"(?:'(?P<sfx>{_SUFFIX_LETTER}+))?"
#: Capital letters of the Turkish alphabet plus Q W X (inside regex character classes).
_CAPITAL = "A-ZÇĞİÖŞÜ"
#: Turkish-formatted number: thousands ``.``, decimal ``,``.
_NUM = r"(?:\d{1,3}(?:\.\d{3})+(?:,\d+)?|\d+(?:,\d+)?)"

#: Tags, spans and ``@emotion`` directives are never touched.
_PROTECTED = re.compile(r"\[[^\[\]<>]*\]|</?[^\[\]<>]*>|(?<!\S)@\S+")
#: ``[duraksama:0.3]``: a pause marker with a value, consumed by ``nida say`` before encoding.
_VALUED_MARKER = re.compile(r"\[\s*[^\[\]:<>]+?\s*:[^\]]*\]")
_DIRECTIVE = re.compile(r"(?<!\S)@\S+")

#: Characters the vocabulary accepts as punctuation (everything but letters), minus ``/`` which
#: is reported as a symbol (nothing in running text should still carry one).
_ALLOWED_PUNCT = frozenset(token for token in VOCAB if len(token) == 1 and not token.isalpha())
_ALLOWED_PUNCT -= {"/"}


def _digit_word_list(digits: str) -> list[str]:
    return [number_words(int(char)) for char in digits]


def _suffixed(spoken: str, suffix: str | None) -> str:
    """``spoken`` with the written ``suffix`` re-attached (no-op without a suffix)."""
    return spoken if not suffix else attach_suffix(spoken, suffix)


def _group_words(group: str) -> str:
    """Words for a digit group with its leading zeros spoken (``05`` -> ``sıfır beş``)."""
    stripped = group.lstrip("0")
    words = ["sıfır"] * (len(group) - len(stripped))
    if stripped:
        words.append(number_words(int(stripped)))
    return " ".join(words)


def _previous_words(text: str, pos: int, count: int) -> list[str]:
    """The last ``count`` whitespace-separated words before ``pos``, lower-cased, stripped of

    surrounding punctuation and of an apostrophe suffix."""
    words: list[str] = []
    for token in text[:pos].split()[-count:]:
        core = token.strip(".,:;!?()\"'«»-–—…")
        words.append(tr_lower(core.split("'")[0]))
    return words


@functools.cache
def _unit_words() -> tuple[frozenset[str], frozenset[str]]:
    """(codes, spoken words) of units and currencies that follow a number.



    ``codes`` are the lower-cased written forms (``kg``, ``tl``, ``w``); ``words`` the spoken

    ones (``kilogram``, ``lira``). Both serve to recognise "a quantity word comes next".

    """
    codes = {tr_lower(key) for key in UNITS} | {tr_lower(key) for key in CURRENCY}
    codes |= {tr_lower(key) for key in EXTRA_UNITS} | {"mah"}
    spoken: set[str] = set()
    for value in [*UNITS.values(), *CURRENCY.values(), *EXTRA_UNITS.values()]:
        spoken.update(str(value).split())
    spoken |= {"kuruş", "sent", "peni", "miliamper", "yüzde"}
    return frozenset(codes), frozenset(spoken)


def _is_quantity_word(word: str) -> bool:
    codes, spoken = _unit_words()
    return word in codes or word in spoken


# --------------------------------------------------------------------------------------
# 1. Abbreviations
# --------------------------------------------------------------------------------------


@functools.cache
def _abbreviation_table() -> tuple[re.Pattern[str], dict[str, Abbreviation]]:
    table: dict[str, Abbreviation] = {}
    for entry in ABBREVIATIONS:
        spellings = {entry.written}
        if entry.ignore_case:
            spellings |= {
                tr_lower(entry.written),
                tr_upper(entry.written),
                tr_upper(entry.written[0]) + tr_lower(entry.written[1:]),
            }
        for spelling in spellings:
            table.setdefault(spelling, entry)
    keys = sorted(table, key=len, reverse=True)
    pattern = re.compile(rf"(?<!\w)(?P<key>{'|'.join(re.escape(key) for key in keys)}){_SFX}")
    return pattern, table


#: Case suffixes written after an abbreviation whose spoken form ends in the possessive -(s)I
#: (``Cad.'de``): they need the pronominal n (``caddesinde``). Written form -> with the n.
_CASE_AFTER_POSSESSIVE = (
    (re.compile("[dt][ae]"), "nde"),
    (re.compile("[dt][ae]n"), "nden"),
    (re.compile("y?[ae]"), "ne"),
    (re.compile("y?[ıiuü]"), "ni"),
)
_VOWEL_LETTERS = "aeıioöuü"


def _abbreviation_spoken(entry: Abbreviation, suffix: str | None) -> str:
    """Spoken form of an abbreviation with its written case suffix (``Cad.'de`` ->

    ``caddesinde``, ``Sok.'a`` -> ``sokağa``)."""
    if not suffix:
        return entry.spoken
    written = tr_lower(suffix)
    if entry.possessive:
        for family, with_n in _CASE_AFTER_POSSESSIVE:
            if family.fullmatch(written):
                written = with_n
                break
    joined = attach_suffix(entry.spoken, written)
    tail = joined[len(entry.spoken) :]
    if entry.soften_final_k and tail[:1] in tuple(_VOWEL_LETTERS) and entry.spoken.endswith("k"):
        joined = entry.spoken[:-1] + "ğ" + tail
    return joined


def _after_capitalised_word(before: str) -> bool:
    """Whether the text before ends with a capitalised word that does not end a sentence."""
    tokens = before.split()
    if not tokens or tokens[-1][-1] in ".!?…":
        return False
    word = tokens[-1].strip("\"'()«»")
    return bool(word) and word[0].isalpha() and word[0] != tr_lower(word[0])


def _capital_or_end_follows(rest: str) -> bool:
    """Whether the text ends here or the next word is capitalised."""
    stripped = rest.lstrip()
    return not stripped or (stripped[0].isalpha() and stripped[0] != tr_lower(stripped[0]))


def _rule_abbreviations(text: str, ctx: _Ctx) -> str:
    pattern, table = _abbreviation_table()

    def expand(match: re.Match[str]) -> str:
        entry = table[match.group("key")]
        suffix = match.group("sfx")
        before, rest = text[: match.start()], text[match.end() :]
        if entry.needs_digit_after and not re.match(r"\s*\d", rest):
            return match.group()
        if entry.not_after_number and re.search(r"\d\s*$", before):
            return match.group()  # "144 Hz.": a unit
        if entry.after_capitalised and not _after_capitalised_word(before):
            return match.group()  # "Eline sok.": a verb
        if entry.year_context:
            previous = _previous_words(text, match.start(), 1)
            if not (
                re.match(r"\s*\d{3,4}(?!\d)", rest)
                and previous[0:1]
                and previous[0].startswith(("yıl", "tarih", "sene"))
            ):
                if re.match(r"\s*\d", rest):
                    ctx.issues.append(f"ambiguous_abbreviation:{match.group('key')}")
                return match.group()
        spoken = _abbreviation_spoken(entry, suffix)
        if rest[:1].isalnum():
            return spoken + " "
        if entry.ends_sentence and suffix is None and _capital_or_end_follows(rest):
            return spoken + "."  # the dot also ended the sentence
        return spoken

    return pattern.sub(expand, text)


# --------------------------------------------------------------------------------------
# 2. Phone numbers
# --------------------------------------------------------------------------------------

_PHONE_SEP = r"[\s.-]*"
_PHONE_TAIL = rf"{_PHONE_SEP}(\d{{3}}){_PHONE_SEP}(\d{{2}}){_PHONE_SEP}(\d{{2}})(?!\d){_SFX}"
_PHONE_INTL = re.compile(rf"(?<![\w+])(?:\+|00)90{_PHONE_SEP}\(?0?(\d{{3}})\)?{_PHONE_TAIL}")
_PHONE_LOCAL = re.compile(rf"(?<![\w.,+])\(?0(\d{{3}})\)?{_PHONE_TAIL}")
_PHONE_BARE = re.compile(
    rf"(?<![\w.,+])([2-58]\d{{2}})[\s.-](\d{{3}})[\s.-](\d{{2}})[\s.-](\d{{2}})(?!\d){_SFX}"
)
_PHONE_SHORT = re.compile(rf"(?<![\w.,+])(444)\s(\d)\s(\d{{3}})(?!\d){_SFX}")


def _rule_phones(text: str, ctx: _Ctx) -> str:
    def read(lead: str, groups: tuple[str, ...], suffix: str | None) -> str:
        words = [_group_words(group) for group in groups]
        if lead == "sıfır":
            words[0] = f"sıfır {words[0]}"
        elif lead:
            words.insert(0, lead)
        return _suffixed(" ".join(words), suffix)

    def intl(match: re.Match[str]) -> str:
        return read("artı doksan", match.group(1, 2, 3, 4), match.group("sfx"))

    def local(match: re.Match[str]) -> str:
        return read("sıfır", match.group(1, 2, 3, 4), match.group("sfx"))

    def bare(match: re.Match[str]) -> str:
        return read("", match.group(1, 2, 3, 4), match.group("sfx"))

    def short(match: re.Match[str]) -> str:
        return read("", match.group(1, 2, 3), match.group("sfx"))

    text = _PHONE_INTL.sub(intl, text)
    text = _PHONE_LOCAL.sub(local, text)
    text = _PHONE_BARE.sub(bare, text)
    return _PHONE_SHORT.sub(short, text)


# --------------------------------------------------------------------------------------
# 3. Plates
# --------------------------------------------------------------------------------------

_PLATE = re.compile(
    rf"(?<![\w.,-])(0[1-9]|[1-7]\d|8[01])\s([A-Z]{{1,3}})\s(\d{{2,4}})(?!\w)(?![.,:/]\d){_SFX}"
)


def _rule_plates(text: str, ctx: _Ctx) -> str:
    def read(match: re.Match[str]) -> str:
        city, letters, number = match.group(1, 2, 3)
        if _is_quantity_word(tr_lower(letters)):
            return match.group()  # "50 TL 100", "12 KG 500": not a plate
        spelled = " ".join(LETTER_NAMES[char] for char in letters)
        spoken = f"{number_words(int(city))} {spelled} {number_words(int(number))}"
        return _suffixed(spoken, match.group("sfx"))

    return _PLATE.sub(read, text)


# --------------------------------------------------------------------------------------
# 4. Clock times
# --------------------------------------------------------------------------------------

_TIME = re.compile(
    r"(?<![\w.:,/+-])(?P<h>\d{1,2})(?P<sep>[.:])(?P<m>\d{2})(?::(?P<s>\d{2}))?(?!\d)(?![.:]\d)"
)
_TIME_TAIL = re.compile(
    r"\s*[-–]\s*(?P<h>\d{1,2})(?P<sep>[.:])(?P<m>\d{2})(?::(?P<s>\d{2}))?(?!\d)(?![.:]\d)"
)
_SUFFIX_AT = re.compile(rf"'(?P<sfx>{_SUFFIX_LETTER}+)")
_TIME_WORDS_BEFORE = frozenset({"saat", "at", "sa"})
_CLAUSE_END = ".!?…;:(\"'«"


def _time_words(hour: int, minute: int, second: int | None) -> str:
    """Spoken clock time: ``:00`` minutes are dropped, other minutes keep a spoken zero

    (``6:05`` -> ``altı sıfır beş``); with seconds every field is read."""
    words = [number_words(hour)]
    if second is None:
        if minute:
            words.append(_group_words(f"{minute:02d}"))
    else:
        words.append(_group_words(f"{minute:02d}"))
        words.append(_group_words(f"{second:02d}"))
    return " ".join(words)


def _valid_time(hour: str, minute: str, second: str | None) -> bool:
    if int(minute) > 59 or (second is not None and int(second) > 59):
        return False
    # 24:00 is midnight; no other minute of hour 24
    return int(hour) <= 23 or (int(hour) == 24 and int(minute) == 0 and second in (None, "00"))


def _follows_time_word(text: str, match: re.Match[str]) -> bool:
    """Whether the match comes right after ``saat`` / ``at`` / ``sa``."""
    previous = re.search(r"(\w+)[:\s]*$", text[: match.start()])
    return previous is not None and tr_lower(previous.group(1)) in _TIME_WORDS_BEFORE


def _dot_time_reading(text: str, match: re.Match[str], has_suffix: bool) -> tuple[bool, bool]:
    """Whether a dot-separated ``H.MM`` is a clock time, and whether that is a guess.



    Returns ``(is_time, ambiguous)``: ``ambiguous`` marks a reading chosen on weak evidence

    (start of a clause, or nothing at all) that is reported as an issue.

    """
    before = text[: match.start()]
    if _follows_time_word(text, match):
        return True, False
    after = text[match.end() :]
    following = re.match(r"\s*(\S+)", after)
    if after[:1] == "%" or (
        following is not None
        and _is_quantity_word(tr_lower(following.group(1).split("'")[0].strip('.,;:!?()"')))
    ):
        return False, False  # "3.50 TL", "2.30 kg": an amount
    previous = _previous_words(text, match.start(), 1)
    if previous and previous[0].startswith(RATE_WORD_PREFIXES):
        return False, True  # "Faiz 8.50'ye", "Fiyat 3.50": an amount or a rate
    if has_suffix:
        return True, False
    hour = match.group("h")
    if len(hour) == 2 and (hour.startswith("0") or int(hour) >= 13):
        return True, False  # clock notation; a decimal is written with a comma
    stripped = before.rstrip()
    clause_start = not stripped or stripped[-1] in _CLAUSE_END
    return clause_start, True


_SECTION_NUMBER = re.compile(r"(?<![\w.,])\d+(?:\.\d+)+(?!\d)(?![.,]\d)")


def _rule_section_numbers(text: str, ctx: _Ctx) -> str:
    """``Madde 3.2`` is a section number: read "üç nokta iki"."""

    def read(match: re.Match[str]) -> str:
        previous = _previous_words(text, match.start(), 1)
        if previous and previous[0] in SECTION_WORDS:
            return match.group().replace(".", " nokta ")
        return match.group()

    return _SECTION_NUMBER.sub(read, text)


def _rule_times(text: str, ctx: _Ctx) -> str:
    out: list[str] = []
    pos = 0
    for match in _TIME.finditer(text):
        if match.start() < pos:
            continue  # the second half of a range consumed below
        hour, sep, minute, second = match.group("h", "sep", "m", "s")
        suffix_match = _SUFFIX_AT.match(text, match.end())
        if not _valid_time(hour, minute, second):
            # "25:70" is no time; a dot form is a decimal unless "saat" says it is a time.
            if sep == ":" or _follows_time_word(text, match):
                ctx.issues.append(f"invalid_time:{match.group()}")
            continue
        if sep == ".":
            is_time, ambiguous = _dot_time_reading(text, match, suffix_match is not None)
            if ambiguous:
                ctx.issues.append(f"ambiguous_time:{match.group()}")
            if not is_time:
                continue
        words = _time_words(int(hour), int(minute), None if second is None else int(second))
        end = match.end()
        tail = _TIME_TAIL.match(text, end)
        if tail is not None and _valid_time(*tail.group("h", "m", "s")):
            tail_second = tail.group("s")
            words += " ila " + _time_words(
                int(tail.group("h")),
                int(tail.group("m")),
                None if tail_second is None else int(tail_second),
            )
            end = tail.end()
            suffix_match = _SUFFIX_AT.match(text, end)
        if suffix_match is not None:
            end = suffix_match.end()
        out.append(text[pos : match.start()])
        out.append(_suffixed(words, None if suffix_match is None else suffix_match.group("sfx")))
        pos = end
    out.append(text[pos:])
    return "".join(out)


# --------------------------------------------------------------------------------------
# 5. Identifier numbers
# --------------------------------------------------------------------------------------

_ID = re.compile(rf"(?<![\w.,])(\d{{5,}})(?!\w)(?![.,]\d){_SFX}")


def _is_id_trigger(word: str) -> bool:
    return word in ID_TRIGGER_WORDS or word.startswith(ID_TRIGGER_PREFIXES)


def _rule_ids(text: str, ctx: _Ctx) -> str:
    def read(match: re.Match[str]) -> str:
        window = _previous_words(text, match.start(), 3)
        if any(w.startswith(ID_BLOCKER_PREFIXES) for w in window):
            return match.group()  # "kod sayısı 12500": a quantity
        if not any(_is_id_trigger(w) for w in window):
            return match.group()
        following = re.match(r"\s*(\S+)", text[match.end() :])
        if following is not None and _is_quantity_word(
            tr_lower(following.group(1).split("'")[0].strip('.,;:!?()"'))
        ):
            return match.group()  # "numara 12500 TL": an amount
        spoken = " ".join(_digit_word_list(match.group(1)))
        return _suffixed(spoken, match.group("sfx"))

    return _ID.sub(read, text)


# --------------------------------------------------------------------------------------
# 6. Roman numerals
# --------------------------------------------------------------------------------------

_ROMAN_DIGITS = {"I": 1, "V": 5, "X": 10}
_ROMAN_ONES = ("", "I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX")


def _roman_value(token: str) -> int | None:
    """Value of a canonical Roman numeral in ``I V X`` between 1 and 39, else ``None``.



    L, C, D and M are left out on purpose: as single letters they are initials (``M. Kemal``)

    and in groups acronyms (``CD``, ``MC``).

    """
    if not token or any(char not in _ROMAN_DIGITS for char in token):
        return None
    total = 0
    for index, char in enumerate(token):
        value = _ROMAN_DIGITS[char]
        following = _ROMAN_DIGITS[token[index + 1]] if index + 1 < len(token) else 0
        total += -value if value < following else value
    if not 1 <= total <= 39:
        return None
    canonical = "X" * (total // 10) + _ROMAN_ONES[total % 10]
    return total if canonical == token else None


_ROMAN_DOT = re.compile(r"(?<![\w.])(?P<r>[IVX]{1,6})\.(?=\s*[^\s.,;:!?)\]\"'])")
_ROMAN_BARE = re.compile(rf"(?<![\w.])(?P<r>[IVX]{{2,6}})(?!\w)(?!\.\w){_SFX}")


def _rule_roman(text: str, ctx: _Ctx) -> str:
    def dotted(match: re.Match[str]) -> str:
        value = _roman_value(match.group("r"))
        if value is None:
            return match.group()
        spoken = ordinal_words(value)
        previous = _previous_words(text, match.start(), 1)
        after_title = bool(previous) and previous[0] in ROMAN_RULER_TITLES
        if not after_title and not _dot_is_ordinal(text[match.end() :], ctx, match.group()):
            spoken += "."  # "Bölüm IV. Sonra ...": the dot also ended the sentence
        return spoken + (" " if text[match.end() : match.end() + 1].isalnum() else "")

    text = _ROMAN_DOT.sub(dotted, text)

    def bare(match: re.Match[str]) -> str:
        value = _roman_value(match.group("r"))
        previous = text[: match.start()].split()
        if value is None or not previous:
            return match.group()
        word = previous[-1].strip(".,:;!?()\"'«»")
        spoken = _suffixed(ordinal_words(value), match.group("sfx"))
        if tr_lower(word) in ROMAN_TITLE_WORDS:
            return spoken
        # A capitalised name counts only inside a sentence: "Kral Louis XIV", not "Bu XI ne".
        capitalised = word[:1] != tr_lower(word[:1])
        sentence_initial = len(previous) == 1 or previous[-2][-1:] in ".!?…:"
        if capitalised and not sentence_initial:
            return spoken
        return match.group()

    return _ROMAN_BARE.sub(bare, text)


# --------------------------------------------------------------------------------------
# 7. Decimals, 8. ranges
# --------------------------------------------------------------------------------------

#: ``3.5`` / ``0.25`` / ``2.5000``: one dot, 1-2 or 4+ fraction digits (``1.250`` is thousands).
_DOT_DECIMAL = re.compile(r"(?<![\w.,:+-])(\d+)\.(\d{1,2}|\d{4,})(?!\d)(?![.,:]\d)")
_RANGE = re.compile(
    rf"(?<![\w.,/:+-])(?P<lo>{_NUM})(?:\s*–\s*|-)(?P<hi>{_NUM})(?![\d-])(?![.,/:]\d)"
)


def _as_float(number: str) -> float:
    return float(number.replace(".", "").replace(",", "."))


def _rule_dot_decimals(text: str, ctx: _Ctx) -> str:
    return _DOT_DECIMAL.sub(lambda m: f"{m.group(1)},{m.group(2)}", text)


def _is_score_context(text: str, match: re.Match[str]) -> bool:
    """Whether a score word (maç, skor, yenildi, ...) is near the number pair."""
    words = _previous_words(text, match.start(), 3)
    words += [
        tr_lower(token.strip('.,:;!?()"').split("'")[0])
        for token in text[match.end() :].split()[:3]
    ]
    return any(word.startswith(SCORE_WORD_PREFIXES) for word in words)


def _rule_ranges(text: str, ctx: _Ctx) -> str:
    def read(match: re.Match[str]) -> str:
        low, high = match.group("lo"), match.group("hi")
        if low.isdigit() and high.isdigit() and _is_score_context(text, match):
            return f"{low} {high}"  # "Skor 1-2": a score is read "bir iki"
        if not _as_float(low) < _as_float(high):
            # "3-0" (a score), "10-5": no range. The digits stay so a unit after them is still
            # found; the base library reads the dash as "tire".
            ctx.issues.append(f"range_not_ascending:{match.group()}")
            return f"{low} tire {high}"
        return f"{low} ila {high}"

    return _RANGE.sub(read, text)


#: Space, no-break space and narrow no-break space as thousands separators.
_THOUSANDS_SPACES = f" {chr(0x00A0)}{chr(0x202F)}"
_SPACE_GROUPED = re.compile(
    rf"(?<![\w.,'])(?P<head>[1-9]\d{{0,2}})(?P<rest>(?:[{_THOUSANDS_SPACES}]\d{{3}})+)(?!\d)(?![.,]\d)"
)
_LEADING_ZEROS = re.compile(r"(?<![\w.,])0\d{2,}(?!\w)(?![.,:/]\d)")
_DIGIT_DOT = re.compile(r"(?<!\d)\d+\.(?=\s|$)")


def _rule_grouped_thousands(text: str, ctx: _Ctx) -> str:
    """``1 000 000`` (space or no-break space as the thousands separator) -> ``1000000``."""
    return _SPACE_GROUPED.sub(lambda m: m.group("head") + re.sub(r"\D", "", m.group("rest")), text)


def _dot_is_ordinal(rest: str, ctx: _Ctx, shown: str) -> bool:
    """Whether the dot after a number / Roman numeral marks an ordinal (``rest`` follows it).



    A lower-case word after it means an ordinal (``5. kat``; reported when that word is a

    typical sentence opener: ``25. sonra``). A capitalised word of :data:`ORDINAL_HEADS` too

    (``2. Dünya Savaşı``). The end of the text, an opening quote or bracket, a digit and any

    other capitalised word mean the dot ends a sentence; a capitalised word that is not a

    known sentence opener is reported as ``ambiguous_ordinal``.

    """
    follower = re.match(r"\s*(\S+)", rest)
    if follower is None or not follower.group(1)[0].isalpha():
        return False
    word = follower.group(1)
    core = tr_lower(word.split("'")[0].strip('.,;:!?()"«»'))
    if word[0] == tr_lower(word[0]):
        if core in SENTENCE_STARTERS:
            ctx.issues.append(f"ambiguous_ordinal:{shown}")
        return True
    if core in ORDINAL_HEADS:
        return True
    if core not in SENTENCE_STARTERS:
        ctx.issues.append(f"ambiguous_ordinal:{shown}")
    return False


def _rule_digit_dots(text: str, ctx: _Ctx) -> str:
    """Decide between an ordinal marker and a sentence end for ``N.`` before whitespace

    (see :func:`_dot_is_ordinal`). A dot that ends a sentence is held back from the base

    library, which would take it for an ordinal marker (``%30. Yeni ...``)."""

    def decide(match: re.Match[str]) -> str:
        # "2,5." / "1.250.": the end of a decimal or a thousands group is never an ordinal
        after_decimal = re.search(r"\d[,.]$", text[: match.start()]) is not None
        if not after_decimal and _dot_is_ordinal(text[match.end() :], ctx, match.group()):
            return match.group()
        return match.group()[:-1] + _KEPT_DOT

    return _DIGIT_DOT.sub(decide, text)


def _rule_leading_zeros(text: str, ctx: _Ctx) -> str:
    """``007`` is a code, not 7: three or more digits with a leading zero are read digit by

    digit (two-digit ``05`` is an ordinary number for the base library)."""
    return _LEADING_ZEROS.sub(lambda m: " ".join(_digit_word_list(m.group())), text)


# --------------------------------------------------------------------------------------
# 9. Units, 10. standalone currency
# --------------------------------------------------------------------------------------

_KM_PER_HOUR = re.compile(r"(?P<num>\d+(?:[.,]\d+)*)\s*km\s*/\s*(?:sa|saat|h)(?!\w)")
_MAH = re.compile(rf"(?<=\d)\s*mAh{_SFX}(?!\w)")
_POWER_UNITS = {"km": "kilometre", "cm": "santimetre", "mm": "milimetre", "m": "metre"}
_SQUARED = re.compile(rf"(?<=\d)\s*(?P<u>km|cm|mm|m)(?P<e>[²³23])(?![\d\w]){_SFX}")
_HERTZ = re.compile(rf"(?<=\d)\s*(?P<u>[kKmMgG]?[hH][zZ]){_SFX}(?!\w)")
_HERTZ_WORDS = {"hz": "hertz", "khz": "kilohertz", "mhz": "megahertz", "ghz": "gigahertz"}
_DEGREE = re.compile(rf"\s*°\s?(?P<scale>[CF])(?!\w){_SFX}")
_EXTRA_UNIT = re.compile(
    rf"(?<=\d)\s*(?P<u>{'|'.join(sorted(map(re.escape, EXTRA_UNITS), key=len, reverse=True))})"
    rf"(?!\w){_SFX}"
)
_STANDALONE_CURRENCY = re.compile(rf"(?<!\w)(?:TL|₺){_SFX}(?!\w)")
#: A plain amount right before a currency: the base library reads "750 TL" itself (a date,
#: a percentage or a fraction before it is not an amount).
_AMOUNT_BEFORE = re.compile(rf"(?<![\w.,/:%'-])(?<!% ){_NUM}\s*$")


def _rule_units(text: str, ctx: _Ctx) -> str:
    text = _KM_PER_HOUR.sub(lambda m: f"saatte {m.group('num')} kilometre", text)
    text = _MAH.sub(lambda m: " " + _suffixed("miliamper saat", m.group("sfx")), text)
    text = _SQUARED.sub(
        lambda m: (
            " "
            + _suffixed(
                f"{_POWER_UNITS[m.group('u')]} {'kare' if m.group('e') in '²2' else 'küp'}",
                m.group("sfx"),
            )
        ),
        text,
    )
    text = _DEGREE.sub(
        lambda m: (
            " " + _suffixed("derece" if m.group("scale") == "C" else "fahrenheit", m.group("sfx"))
        ),
        text,
    )
    text = _HERTZ.sub(
        lambda m: " " + _suffixed(_HERTZ_WORDS[tr_lower(m.group("u"))], m.group("sfx")), text
    )
    return _EXTRA_UNIT.sub(
        lambda m: " " + _suffixed(EXTRA_UNITS[m.group("u")], m.group("sfx")), text
    )


#: ``ve/veya``, ``Dolar/TL``: words joined by a slash (no digit, dot, colon or ``@`` around:
#: those are fractions, units, dates, URLs and addresses).
_LETTER_SLASH = re.compile(r"(?<![\w/.:@-])[^\W\d_]+(?:/[^\W\d_]+)+(?![\w/.:@-])")


def _rule_letter_slashes(text: str, ctx: _Ctx) -> str:
    """Words joined by a slash are two words: ``ve/veya`` -> ``ve veya``."""

    def split(match: re.Match[str]) -> str:
        if re.search(r"\d\s*$", text[: match.start()]):
            return match.group()  # "100 m/s": a unit, left to the base library
        return match.group().replace("/", " ")

    return _LETTER_SLASH.sub(split, text)


def _rule_standalone_currency(text: str, ctx: _Ctx) -> str:
    def read(match: re.Match[str]) -> str:
        if _AMOUNT_BEFORE.search(text[: match.start()]):
            return match.group()  # "750 TL": the base library reads amount and currency
        if match.group().startswith("₺") and re.match(r"\s*\d", text[match.end() :]):
            return match.group()  # "₺750"
        return _suffixed("lira", match.group("sfx"))

    return _STANDALONE_CURRENCY.sub(read, text)


# --------------------------------------------------------------------------------------
# 11. Acronyms and alphanumeric codes
# --------------------------------------------------------------------------------------

_UPPER_VOWELS = "AEIİOÖUÜ"
_DOTTED_ACRONYM = re.compile(rf"(?<!\w)(?P<letters>(?:[{_CAPITAL}]\.){{2,}}){_SFX}")
_CODE = re.compile(
    rf"(?<![\w'])(?P<letters>[{_CAPITAL}]{{1,6}})['-]?(?P<digits>\d{{1,6}})"
    rf"(?P<tail>[{_CAPITAL}]{{1,3}})?(?!\w){_SFX}"
)
_DIGITS_LETTERS = re.compile(
    rf"(?<![\w.,])(?P<digits>\d+)(?P<letters>[{_CAPITAL}]{{1,3}})(?!\w){_SFX}"
)
#: ``Grup C'de``: one capital letter with a written suffix is a letter name.
_SINGLE_CAPITAL = re.compile(rf"(?<![\w'])(?P<tok>[{_CAPITAL}])'(?P<sfx>[a-zçğıöşüâîû]+)(?!\w)")
#: Consonant clusters a Turkish word may start with (loan onsets) or end with. Inside a word a
#: run of up to three consonants is fine (``lütfen``, ``korkma``, ``şampiyon``), so only the
#: edges tell an acronym (``ABD``: coda ``bd``; ``DHA``: onset ``dh``) from a word.
_ONSETS = frozenset({"tr", "pr", "kr", "br", "gr", "dr", "fr", "pl", "bl", "kl", "gl", "fl",
                     "st", "sp", "sk"})  # fmt: skip
_CODAS = frozenset(
    {
        "rk", "rt", "rs", "rl", "rm", "rp", "rf", "rş", "rz", "rn", "rç", "rd", "rb", "rg",
        "lk", "lt", "lp", "lm", "ls", "lf", "lç", "ld", "nk", "nt", "nç", "nd", "ns", "ng",
        "st", "şt", "ft", "kt", "pt", "ks", "ps", "sk", "sp",
    }
)  # fmt: skip
_CAPS_WORD = re.compile(rf"(?<![\w'])(?P<tok>[{_CAPITAL}]{{2,}}){_SFX}(?!\w)")


def _read_as_word(token: str) -> str | None:
    """The spoken word of a lexicon acronym (exact match; see ``ASCII_SPELLINGS``)."""
    return READ_AS_WORD.get(ASCII_SPELLINGS.get(token, token))


def _spell(letters: str, suffix: str | None) -> str:
    """Letter names of ``letters``; the suffix goes on the last name."""
    names = [LETTER_NAMES[char] for char in letters]
    names[-1] = _suffixed(names[-1], suffix)
    return " ".join(names)


def _pronounceable(token: str) -> bool:
    """Whether a capital token could be a Turkish word.



    It needs a vowel; at most one consonant (or a loan onset such as ``tr``) before the first

    vowel; at most two consonants, a known coda, after the last; and no run of more than three

    consonants between vowels. ``ABD``, ``AKP``, ``DHA``, ``IMF`` are acronyms; ``İMDAT``,

    ``LÜTFEN``, ``KORKMA``, ``ŞİMDİ`` are words.

    """
    lowered = tr_lower(token)
    parts = re.split(f"[{_VOWEL_LETTERS}]+", lowered)
    if len(parts) < 2:
        return False  # no vowel
    onset, *inner, coda = parts
    if len(onset) > 1 and onset not in _ONSETS:
        return False
    if len(coda) > 1 and coda not in _CODAS:
        return False
    return all(len(run) <= 3 for run in inner)


_SPELLED_NAME = re.compile(
    "(?:" + "|".join(sorted({n for n in LETTER_NAMES.values() if len(n) > 1 and " " not in n},
                            key=len, reverse=True)) + ")[a-zçğıöşüâîû]{0,4}"
)  # fmt: skip
_SINGLE_LETTER_NAMES = frozenset(name for name in LETTER_NAMES.values() if len(name) == 1)


def _is_spelled_word(word: str) -> bool:
    """A letter name, or one with a suffix (``be``, ``mede``): what spelling an acronym leaves

    behind. It says nothing about the case of the line (so a second pass reads it alike)."""
    return word in _SINGLE_LETTER_NAMES or (
        len(word) > 1 and _SPELLED_NAME.fullmatch(word) is not None
    )


def _clause_is_shouted(clause: str) -> bool:
    """An all-capitals line (``ONU GÖR``): its capital words are text, not acronyms.



    At least two capital words that could be Turkish words (:func:`_pronounceable`; ``ABD``,

    ``AB`` and the like do not count) making up 80% or more of the line's words. A suffix after

    an apostrophe, numbers and letter names do not count as words. Such a line is left alone,

    except for its acronyms, which are spelled.

    """
    words = [
        w
        for w in re.findall(r"(?<!')[^\W\d_]+", clause)
        if len(w) >= 2
        and w not in _spoken_number_words()
        and not _ORDINAL_WORD.fullmatch(w)
        and not _is_spelled_word(w)
    ]
    caps = [w for w in words if w == tr_upper(w)]
    wordy = [w for w in caps if w not in SPELLED_ACRONYMS and _pronounceable(w)]
    return len(wordy) >= 2 and len(caps) >= 0.8 * len(words)


def _rule_acronyms(text: str, ctx: _Ctx) -> str:
    def dotted(match: re.Match[str]) -> str:
        letters = match.group("letters").replace(".", "")
        spoken = _spell(letters, match.group("sfx"))
        after = text[match.end() :]
        keep_dot = match.group("sfx") is None and not after.strip()
        return spoken + ("." if keep_dot else "")

    def code(match: re.Match[str]) -> str:
        letters, digits = match.group("letters", "digits")
        word = _read_as_word(letters)
        if word is not None:
            head = word
        elif len(letters) <= 4:
            head = _spell(letters, None)
        else:
            ctx.issues.append(f"ambiguous_code:{match.group()}")
            return match.group()
        if len(digits) <= 4 and not (len(digits) > 1 and digits.startswith("0")):
            number = number_words(int(digits))
        else:
            number = " ".join(_digit_word_list(digits))
        trailing = match.group("tail")
        spoken = f"{head} {number}" + (f" {_spell(trailing, None)}" if trailing else "")
        return _suffixed(spoken, match.group("sfx"))

    def digits_letters(match: re.Match[str]) -> str:
        digits, letters = match.group("digits", "letters")
        glued_unit = (len(letters) >= 2 and _is_quantity_word(tr_lower(letters))) or (
            letters in EXTRA_UNITS
        )
        if glued_unit:
            return match.group()  # "16GB", "12V": a unit, left to the base library
        return f"{digits} {_spell(letters, match.group('sfx'))}"  # "3D", "5G", "2XL"

    text = _CODE.sub(code, _DOTTED_ACRONYM.sub(dotted, text))
    text = _DIGITS_LETTERS.sub(digits_letters, text)
    text = _SINGLE_CAPITAL.sub(lambda m: _spell(m.group("tok"), m.group("sfx")), text)

    def caps(match: re.Match[str], *, shouted: bool) -> str:
        token, suffix = match.group("tok"), match.group("sfx")
        word = _read_as_word(token)
        if word is not None:
            return match.group() if shouted else word + (f"'{suffix}" if suffix else "")
        if _roman_value(token) is not None:
            if not shouted:
                ctx.issues.append(f"roman_numeral_ambiguous:{token}")
            return match.group()
        if _AMOUNT_BEFORE.search(text[: match.start()]) and _is_quantity_word(tr_lower(token)):
            return match.group()  # "16 GB", "100 USD": left to the base library
        if token in SPELLED_ACRONYMS or not _pronounceable(token):
            return _spell(token, suffix)  # an acronym, even inside an all-capitals line
        if shouted:
            return match.group()  # an all-capitals line is text
        # A short capital word (DUR! GEL) could be emphasis or an acronym we do not know:
        # keep it as a word and say so.
        ctx.issues.append(f"acronym_guess:{token}")
        return tr_lower(token) + (f"'{suffix}" if suffix else "")

    # The whole line decides, not a sentence: where a sentence ends (a digit ordinal "3." loses
    # its dot when it becomes a word) changes with the very rewriting done here.
    shouted = _clause_is_shouted(text)
    return _CAPS_WORD.sub(lambda m: caps(m, shouted=shouted), text)


# --------------------------------------------------------------------------------------
# The base library and the post-rules
# --------------------------------------------------------------------------------------


class _Normalizer(Protocol):
    def normalize(self, text: str) -> str: ...


#: The concepts of the base library we use. Left out, with the reason: ``abbreviations``
#: (ours is case-aware and extendable; its ``av.`` -> avukat rewrites the common word),
#: ``phones`` (ours reads the groups; its pattern also fires inside ``12.345.678 55 10``),
#: ``roman_numerals`` (it reads the initial in ``M. Kemal`` as 1000), ``companies`` and
#: ``technology_terms`` (they rewrite words by their spelling, e.g. the capitals ``ANI`` /
#: ``ASI`` of a shouted sentence, and invent English-style readings), ``custom_lexicon``.
_BASE_FEATURES = (
    "emails",
    "urls",
    "ibans",
    "turkish_ids",
    "dates",
    "times",
    "percent",
    "currency",
    "fractions",
    "ranges",
    "ordinals",
    "units",
    "numbers",
    "symbols",
)


@functools.cache
def _base_normalizer() -> _Normalizer:
    """turkificate restricted to :data:`_BASE_FEATURES`."""
    normalizer: _Normalizer = turkificate.TurkishNormalizer(features=list(_BASE_FEATURES))
    return normalizer


_SUFFIX_AFTER_DIGIT = re.compile(rf"(?<=\d)'({_SUFFIX_LETTER}+)")
_HELD_SUFFIX = re.compile(rf"(\S+?){_SENTINEL_OPEN}([^{_SENTINEL_CLOSE}]*){_SENTINEL_CLOSE}")
_HALF = re.compile(rf"(?<![\w.,])(?:\d{{1,3}}(?:\.\d{{3}})+|\d+),5(?!\d)(?!'{_SUFFIX_LETTER})")
_VIRGUL_BES = re.compile(r"(?P<head>[^\s]+) virgül beş(?!\w)(?!\s+yüz\b)")


_NUMERALS = frozenset(
    {
        "sıfır", "bir", "iki", "üç", "dört", "beş", "altı", "yedi", "sekiz", "dokuz", "on",
        "yirmi", "otuz", "kırk", "elli", "altmış", "yetmiş", "seksen", "doksan", "yüz", "bin",
        "milyon", "milyar", "trilyon", "katrilyon",
    }
)  # fmt: skip


@functools.cache
def _spoken_number_words() -> frozenset[str]:
    """Words the number, unit and currency rules produce from digits and symbols; they say

    nothing about the case of the text around them."""
    _, spoken = _unit_words()
    months = {str(name) for name in MONTHS.values()}
    return _NUMERALS | spoken | months | {"ila", "buçuk", "virgül", "tire", "eksi", "artı"}


_ORDINAL_WORD = re.compile(r"\w*(?:ıncı|inci|uncu|üncü|ncı|nci|ncu|ncü)")


@functools.cache
def _spoken_stem_words() -> re.Pattern[str]:
    """Words after which an apostrophe suffix is part of a verbalised number or quantity."""
    _, spoken = _unit_words()
    words = sorted(_NUMERALS | set(spoken), key=len, reverse=True)
    return re.compile(rf"(?<![\w'])({'|'.join(map(re.escape, words))})'({_SUFFIX_LETTER}+)")


def _reattach_held_suffix(match: re.Match[str]) -> str:
    stem, suffix = match.group(1), match.group(2)
    if not stem[-1].isalpha():
        return f"{stem}'{suffix}"  # digits survived: keep the written form; the scan reports it
    return attach_suffix(stem, suffix)


def _normalize_core(text: str, ctx: _Ctx) -> str:
    """Run the whole pipeline on one plain-text segment (no tags, no surrounding spaces)."""
    work = text.translate(_TYPOGRAPHIC)
    for rule in (
        _rule_abbreviations,
        _rule_phones,
        _rule_plates,
        _rule_section_numbers,
        _rule_times,
        _rule_ids,
        _rule_grouped_thousands,
        _rule_leading_zeros,
        _rule_roman,
        _rule_dot_decimals,
        _rule_ranges,
        _rule_units,
        _rule_letter_slashes,
        _rule_standalone_currency,
        _rule_acronyms,
        _rule_digit_dots,
    ):
        work = rule(work, ctx)
    halves = len(_HALF.findall(work))
    work = _SUFFIX_AFTER_DIGIT.sub(lambda m: f"{_SENTINEL_OPEN}{m.group(1)}{_SENTINEL_CLOSE}", work)
    try:
        out = _base_normalizer().normalize(work)
    except ValueError as exc:  # e.g. a number beyond the base library's largest scale
        ctx.issues.append(f"base_normalizer_error:{exc}")
        out = work
    out = _HELD_SUFFIX.sub(_reattach_held_suffix, out).replace(_KEPT_DOT, ".")
    out = _spoken_stem_words().sub(lambda m: attach_suffix(m.group(1), m.group(2)), out)
    if halves:

        def half(match: re.Match[str]) -> str:
            head = match.group("head")
            return match.group() if head == "sıfır" else f"{head} buçuk"

        out = _VIRGUL_BES.sub(half, out, count=halves)
    return out


def _canonical_spaces(text: str) -> str:
    """``text`` as the whitespace step of the base library would leave it."""
    return re.sub(r"\s+", " ", re.sub(r"\s+([,.!?;:])", r"\1", text)).strip()


_LINE_BREAK = re.compile(r"(\r\n|\n|\r)")


def _normalize_line(piece: str, ctx: _Ctx) -> str:
    """Normalise one line of plain text, keeping its edge whitespace; unchanged text is

    returned as is (the base library collapses inner whitespace, which is not a change)."""
    core = piece.strip()
    if not core:
        return piece
    lead = piece[: len(piece) - len(piece.lstrip())]
    trail = piece[len(piece.rstrip()) :]
    out = _normalize_core(core, ctx)
    if _canonical_spaces(out) == _canonical_spaces(core):
        return piece
    return f"{lead}{out}{trail}"


def _normalize_segment(piece: str, ctx: _Ctx) -> str:
    """Normalise one plain-text piece line by line: a line break is a hard boundary, no rule

    reads across it (``%65.`` at the end of a line is a sentence end, not an ordinal)."""
    parts = _LINE_BREAK.split(piece)
    return "".join(
        part if index % 2 else _normalize_line(part, ctx) for index, part in enumerate(parts)
    )


# --------------------------------------------------------------------------------------
# Validation and the public API
# --------------------------------------------------------------------------------------


def _segments(text: str) -> list[tuple[bool, str]]:
    """Split into ``(is_plain_text, piece)``; tags and directives are the protected pieces."""
    pieces: list[tuple[bool, str]] = []
    pos = 0
    for match in _PROTECTED.finditer(text):
        if match.start() > pos:
            pieces.append((True, text[pos : match.start()]))
        pieces.append((False, match.group()))
        pos = match.end()
    if pos < len(text):
        pieces.append((True, text[pos:]))
    return pieces


#: Letter, dot or comma, letter with no space (``mümkün.Ama``, ``ve,veya``): usually a missing
#: space; initials (``M.K.``, ``e.g.``) are fine.
_GLUED_PUNCT = re.compile(r"[^\W\d_][.,][^\W\d_]")
_INITIALS = re.compile(r"(?:[^\W\d_]\.)+[^\W\d_]?\.?")

#: Issue kinds whose text is misread or unusable even though it still encodes: ``nida say``
#: stops on them (every other kind is a warning unless ``strict_normalize`` is set).
DANGEROUS_ISSUE_KINDS: frozenset[str] = frozenset(
    {"roman_numeral_ambiguous", "digits", "symbol", "glued_punct", "char", "encode"}
)


def issue_kind(issue: str) -> str:
    """The kind of an issue string (``"digits:3D"`` -> ``"digits"``)."""
    return issue.partition(":")[0]


def dangerous_issues(issues: Sequence[str]) -> tuple[str, ...]:
    """The issues of kind :data:`DANGEROUS_ISSUE_KINDS`, in order."""
    return tuple(i for i in issues if issue_kind(i) in DANGEROUS_ISSUE_KINDS)


def _scan_plain(piece: str) -> list[str]:
    """Issues of one plain-text piece: digits, symbols and characters the vocabulary lacks."""
    issues: list[str] = []
    for token in piece.split():
        if any(unicodedata.category(char).startswith("N") for char in token):
            shown = token.strip('.,;:!?()"')
            issues.append(f"digits:{shown}")
            continue
        if _GLUED_PUNCT.search(token) and not _INITIALS.fullmatch(token.strip(',;:!?)"')):
            shown = token.strip('.,;:!?()"')
            issues.append(f"glued_punct:{shown}")
        for char in token:
            if char.isalpha():
                if tr_lower(char) not in VOCAB:
                    issues.append(f"char:{char}")
            elif char not in _ALLOWED_PUNCT:
                issues.append(f"symbol:{char}")
    return issues


def _validate(text: str) -> list[str]:
    issues: list[str] = []
    for is_plain, piece in _segments(text):
        if is_plain:
            issues.extend(_scan_plain(piece))
    checked = _DIRECTIVE.sub("", _VALUED_MARKER.sub("", text))
    try:
        encode_text(checked)
    except ValueError as exc:
        # A digit, symbol or foreign letter makes the encoder fail too and is already listed
        # precisely above; the encoder's own message is added when it is the only evidence
        # (a bad or crossing tag, a stray "<").
        if not issues:
            issues.append(f"encode:{exc}")
    return issues


def normalize_for_tts(text: str) -> TtsNormalization:
    """Normalise raw Turkish ``text`` into fully spoken text for the model's encoder.



    Plain-text parts are normalised; event tags ``[gülme]``, span tags ``<fısıltı>``,

    pause markers ``[duraksama:0.3]`` and ``@emotion`` directives are left byte-identical.

    Text that needs no change is returned unchanged (after NFC). Never raises on a ``str``:

    whatever cannot be resolved stays in the text and is listed in

    :attr:`TtsNormalization.issues`.



    Args:

        text: Raw text, any Unicode normalisation form.



    Returns:

        The normalised text and the machine-readable issues (empty when fully spoken).



    Raises:

        TypeError: ``text`` is not a ``str``.



    Examples:

        >>> normalize_for_tts("Toplantı saat 09.00'da başlıyor.").text

        'Toplantı saat dokuzda başlıyor.'

        >>> normalize_for_tts("TBMM'de %50 indirim [gülme]").text

        'te be me mede yüzde elli indirim [gülme]'

    """
    if not isinstance(text, str):
        raise TypeError(f"normalize_for_tts: expected str, got {type(text).__name__}")
    ctx = _Ctx()
    source = nfc(text)
    if any(mark in source for mark in (_SENTINEL_OPEN, _SENTINEL_CLOSE, _KEPT_DOT)):
        for mark in (_SENTINEL_OPEN, _SENTINEL_CLOSE, _KEPT_DOT):
            source = source.replace(mark, "")
        ctx.issues.append("private_use_character_removed")
    pieces = [
        _normalize_segment(piece, ctx) if is_plain else piece
        for is_plain, piece in _segments(source)
    ]
    result = nfc("".join(pieces))
    issues = [*ctx.issues, *_validate(result)]
    return TtsNormalization(result, tuple(dict.fromkeys(issues)))


def normalize_or_raise(text: str) -> str:
    """:func:`normalize_for_tts`, but raise if anything is left unresolved.



    Raises:

        ValueError: with the issue list, when ``text`` cannot be fully spoken.

    """
    result = normalize_for_tts(text)
    if result.issues:
        raise ValueError(
            f"text not fully normalised ({len(result.issues)} issue(s): "
            f"{', '.join(result.issues)}): {result.text!r}"
        )
    return result.text