File size: 5,011 Bytes
2a7158b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """Modern Standard Arabic front end: text -> phoneme tokens with a stress feature.
Same interface as the English front end it replaces (text_english.py), so the model, annotation
and training code are unchanged. The pipeline is in ``arabic_frontend``:
undiacritized text -> phrases (split at punctuation) -> CATT diacritizer -> MSA rules -> phonemes
* A token is one phoneme; geminates are two identical consonant tokens.
* Stress (0/1) is a parallel feature from MSA syllable weight, not a token.
* Punctuation, word boundaries and the utterance edges are tokens that may last zero frames.
* ``Phonemized.optional`` marks case endings inside a phrase. The aligner keeps one only when
the recording has it; inference always says them (formal MSA).
Diacritizations can be served from a cache (``TINY_VOCA_DIACRITIZED`` = jsonl of {"p", "d"}) so
annotation and training use exactly the vowels that were aligned.
"""
from dataclasses import dataclass, field
import hashlib
import json
import os
from . import arabic_frontend as af
FRONTEND_VERSION = "ar-1"
FRONTENDS = ("arabic",)
FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND"
_frontend = "arabic"
PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>"
SPECIALS = (PAD, BOS, EOS, SP, UNK)
PUNCTUATION = (",", ".", "?", "!", ";", ":", "…")
_PUNCT_MAP = {"،": ",", ",": ",", ".": ".", "؟": "?", "?": "?", "!": "!", "؛": ";", ";": ";", ":": ":", "…": "…"}
CONSONANTS = tuple(af.CONSONANTS)
VOWELS = tuple(af.VOWELS)
PHONES = CONSONANTS + VOWELS
# Append-only: changing the order or removing a symbol invalidates checkpoints.
SYMBOLS = SPECIALS + PUNCTUATION + PHONES
SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)}
PHONE_SET = frozenset(PHONES)
PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]}
VOICELESS = frozenset(af.VOICELESS)
VOWEL_SET = frozenset(VOWELS)
class NothingToPronounce(ValueError):
"""The text has no Arabic word to say."""
@dataclass(frozen=True)
class Phonemized:
symbols: tuple
ids: tuple
stress: tuple
unknown: tuple
optional: tuple = field(default=())
diacritized: tuple = field(default=())
def __len__(self):
return len(self.ids)
def frontend():
return _frontend
def set_frontend(name):
if name not in FRONTENDS:
raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}")
def frontend_identity():
return {"frontend_version": FRONTEND_VERSION, "phonemizer": "tiny_voca.arabic_frontend",
"diacritizer": "catt-tashkeel ed_model v2", "symbols": list(SYMBOLS)}
def frontend_runtime():
return {"phonemizer": hashlib.sha256(open(af.__file__, "rb").read()).hexdigest()}
def frontend_fingerprint():
return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest()
_cache = None
def _diacritized(phrases):
global _cache
if _cache is None:
_cache = {}
path = os.environ.get("TINY_VOCA_DIACRITIZED")
if path and os.path.exists(path):
with open(path, encoding="utf-8") as f:
for line in f:
r = json.loads(line); _cache[r["p"]] = r["d"]
missing = [p for p in phrases if p not in _cache]
if missing:
for p, d in zip(missing, af.diacritize(missing)):
_cache[p] = d
return [_cache[p] for p in phrases]
def phonemize(text):
if not isinstance(text, str) or not text.strip():
raise ValueError("text must not be empty")
parts = af.split_phrases(text)
if not parts:
raise NothingToPronounce("the text has nothing to pronounce")
res = af.phonemize(text, _diacritized([p for p, _ in parts]))
symbols, stress, optional = [], [], []
for x in res.phones:
if x.p == "^": s = BOS
elif x.p == "$": s = EOS
elif x.p == " ": s = SP
elif x.word < 0: s = _PUNCT_MAP.get(x.p, ",")
else: s = x.p
if s in PUNCTUATION and symbols and symbols[-1] in PUNCTUATION:
continue
if s in PUNCTUATION and symbols and symbols[-1] == SP:
symbols.pop(); stress.pop(); optional.pop()
symbols.append(s); stress.append(x.stress); optional.append(int(x.optional))
if not any(s in PHONE_SET for s in symbols):
raise NothingToPronounce("the text has nothing to pronounce")
return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), (),
tuple(optional), tuple(res.phrases))
def drop(ph, remove):
"""Copy of ``ph`` without the token positions in ``remove`` (dropped optional case endings)."""
remove = set(remove)
keep = [i for i in range(len(ph.symbols)) if i not in remove]
pick = lambda seq: tuple(seq[i] for i in keep) if seq else seq
return Phonemized(pick(ph.symbols), pick(ph.ids), pick(ph.stress), ph.unknown, pick(ph.optional), ph.diacritized)
def is_phone(token_ids):
return [int(i) in PHONE_IDS for i in token_ids]
|