File size: 5,011 Bytes
2a7158b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
"""Modern Standard Arabic front end: text -> phoneme tokens with a stress feature.

Same interface as the English front end it replaces (text_english.py), so the model, annotation
and training code are unchanged. The pipeline is in ``arabic_frontend``:

  undiacritized text -> phrases (split at punctuation) -> CATT diacritizer -> MSA rules -> phonemes

* A token is one phoneme; geminates are two identical consonant tokens.
* Stress (0/1) is a parallel feature from MSA syllable weight, not a token.
* Punctuation, word boundaries and the utterance edges are tokens that may last zero frames.
* ``Phonemized.optional`` marks case endings inside a phrase. The aligner keeps one only when
  the recording has it; inference always says them (formal MSA).

Diacritizations can be served from a cache (``TINY_VOCA_DIACRITIZED`` = jsonl of {"p", "d"}) so
annotation and training use exactly the vowels that were aligned.
"""

from dataclasses import dataclass, field
import hashlib
import json
import os

from . import arabic_frontend as af

FRONTEND_VERSION = "ar-1"
FRONTENDS = ("arabic",)
FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND"
_frontend = "arabic"

PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>"
SPECIALS = (PAD, BOS, EOS, SP, UNK)
PUNCTUATION = (",", ".", "?", "!", ";", ":", "…")
_PUNCT_MAP = {"،": ",", ",": ",", ".": ".", "؟": "?", "?": "?", "!": "!", "؛": ";", ";": ";", ":": ":", "…": "…"}
CONSONANTS = tuple(af.CONSONANTS)
VOWELS = tuple(af.VOWELS)
PHONES = CONSONANTS + VOWELS
# Append-only: changing the order or removing a symbol invalidates checkpoints.
SYMBOLS = SPECIALS + PUNCTUATION + PHONES
SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)}
PHONE_SET = frozenset(PHONES)
PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]}
VOICELESS = frozenset(af.VOICELESS)
VOWEL_SET = frozenset(VOWELS)


class NothingToPronounce(ValueError):
    """The text has no Arabic word to say."""


@dataclass(frozen=True)
class Phonemized:
    symbols: tuple
    ids: tuple
    stress: tuple
    unknown: tuple
    optional: tuple = field(default=())
    diacritized: tuple = field(default=())

    def __len__(self):
        return len(self.ids)


def frontend():
    return _frontend


def set_frontend(name):
    if name not in FRONTENDS:
        raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}")


def frontend_identity():
    return {"frontend_version": FRONTEND_VERSION, "phonemizer": "tiny_voca.arabic_frontend",
            "diacritizer": "catt-tashkeel ed_model v2", "symbols": list(SYMBOLS)}


def frontend_runtime():
    return {"phonemizer": hashlib.sha256(open(af.__file__, "rb").read()).hexdigest()}


def frontend_fingerprint():
    return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest()


_cache = None


def _diacritized(phrases):
    global _cache
    if _cache is None:
        _cache = {}
        path = os.environ.get("TINY_VOCA_DIACRITIZED")
        if path and os.path.exists(path):
            with open(path, encoding="utf-8") as f:
                for line in f:
                    r = json.loads(line); _cache[r["p"]] = r["d"]
    missing = [p for p in phrases if p not in _cache]
    if missing:
        for p, d in zip(missing, af.diacritize(missing)):
            _cache[p] = d
    return [_cache[p] for p in phrases]


def phonemize(text):
    if not isinstance(text, str) or not text.strip():
        raise ValueError("text must not be empty")
    parts = af.split_phrases(text)
    if not parts:
        raise NothingToPronounce("the text has nothing to pronounce")
    res = af.phonemize(text, _diacritized([p for p, _ in parts]))
    symbols, stress, optional = [], [], []
    for x in res.phones:
        if x.p == "^": s = BOS
        elif x.p == "$": s = EOS
        elif x.p == " ": s = SP
        elif x.word < 0: s = _PUNCT_MAP.get(x.p, ",")
        else: s = x.p
        if s in PUNCTUATION and symbols and symbols[-1] in PUNCTUATION:
            continue
        if s in PUNCTUATION and symbols and symbols[-1] == SP:
            symbols.pop(); stress.pop(); optional.pop()
        symbols.append(s); stress.append(x.stress); optional.append(int(x.optional))
    if not any(s in PHONE_SET for s in symbols):
        raise NothingToPronounce("the text has nothing to pronounce")
    return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), (),
                      tuple(optional), tuple(res.phrases))


def drop(ph, remove):
    """Copy of ``ph`` without the token positions in ``remove`` (dropped optional case endings)."""
    remove = set(remove)
    keep = [i for i in range(len(ph.symbols)) if i not in remove]
    pick = lambda seq: tuple(seq[i] for i in keep) if seq else seq
    return Phonemized(pick(ph.symbols), pick(ph.ids), pick(ph.stress), ph.unknown, pick(ph.optional), ph.diacritized)


def is_phone(token_ids):
    return [int(i) in PHONE_IDS for i in token_ids]