Download tiny_voca/text.py from Xeven7/tiny-swara: direct link, hf CLI and curl.
- Browser
- Download file 14 kB
-
https://huggingface.co/Xeven7/tiny-swara/resolve/main/tiny_voca/text.py
- Command line
-
hf download hf://Xeven7/tiny-swara/tiny_voca/text.py
-
curl -L -o text.py https://huggingface.co/Xeven7/tiny-swara/resolve/main/tiny_voca/text.py
14 kB
| """English front end: text -> phoneme tokens with a stress feature. | |
| The same code runs during annotation, training-set preparation and inference, | |
| so tokens can never drift between them. Two programs can produce the phonemes: | |
| * ``espeak`` (front-end version 3, the default): eSpeak NG, as described below. | |
| The current checkpoints and annotations were made with it. | |
| * ``own`` (front-end version 4): the separate ``dizione`` package, a dictionary | |
| built from CMUdict, a small model for unknown words and rules. No eSpeak, so | |
| the whole system can be shipped under a permissive licence. Same 76 symbols. | |
| Choose with ``set_frontend("own")`` or ``TINY_VOCA_FRONTEND=own``. Everything | |
| below about tokens holds for both; the rest of this note is the eSpeak path. | |
| * eSpeak NG (``en-us``) is the only pronunciation source. It is an external | |
| rule-based program: no neural G2P and no downloaded weights. | |
| * A token is one eSpeak phoneme exactly as eSpeak segments it (``--sep``), for | |
| example ``t``, ``aɪ``, ``ɑːɹ`` or ``n̩``. This is also the label inventory of | |
| the phoneme recogniser used for offline forced alignment, so one aligned | |
| label corresponds to exactly one token. | |
| * Lexical stress is a parallel feature (0 none, 1 primary, 2 secondary), not a | |
| token, so it never has to be given audio frames. | |
| * Punctuation, word boundaries and the utterance edges are explicit tokens that | |
| may last zero frames. They carry pauses when the recording has them. | |
| Versions 1 and 2 encoded IPA as UTF-8 bytes and piped text to eSpeak without a | |
| final newline; eSpeak NG 1.51 then drops the last character of the input, so | |
| the final phonemes of every clause were wrong. Version 3 terminates the input | |
| and has a regression test for it. | |
| """ | |
| from dataclasses import dataclass | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import subprocess | |
| import unicodedata | |
| FRONTEND_VERSION = 3 | |
| ESPEAK_VOICE = "en-us" | |
| SEPARATOR = "_" | |
| FRONTENDS = ("espeak", "own") | |
| FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND" | |
| _frontend = os.environ.get(FRONTEND_VARIABLE, "espeak") | |
| if _frontend not in FRONTENDS: | |
| raise ValueError(f"{FRONTEND_VARIABLE} must be one of {FRONTENDS}, not {_frontend!r}") | |
| PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>" | |
| SPECIALS = (PAD, BOS, EOS, SP, UNK) | |
| PUNCTUATION = (",", ".", "?", "!", ";", ":", '"', "'", "(", ")", "—", "…") | |
| CONSONANTS = ("p", "b", "t", "d", "k", "ɡ", "f", "v", "θ", "ð", "s", "z", "ʃ", "ʒ", "h", | |
| "m", "n", "ŋ", "l", "ɹ", "j", "w", "tʃ", "dʒ", "ɾ", "ʔ") | |
| VOWELS = ("ə", "ɐ", "ɚ", "ᵻ", "ɪ", "i", "iː", "ɛ", "æ", "ʌ", "ɑː", "ɔ", "ɔː", "ʊ", "uː", "ɜː", | |
| "eɪ", "aɪ", "ɔɪ", "oʊ", "aʊ", "iə", "oː", | |
| "ɑːɹ", "ɔːɹ", "oːɹ", "ɪɹ", "ɛɹ", "ʊɹ", "aɪɚ", "aɪə", "əl", "n̩") | |
| PHONES = CONSONANTS + VOWELS | |
| # Append-only: changing the order or removing a symbol invalidates checkpoints. | |
| SYMBOLS = SPECIALS + PUNCTUATION + PHONES | |
| SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)} | |
| PHONE_SET = frozenset(PHONES) | |
| PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]} | |
| VOICELESS = frozenset(("p", "t", "k", "f", "θ", "s", "ʃ", "h", "tʃ", "ʔ")) | |
| VOWEL_SET = frozenset(VOWELS) | |
| STRESS_MARKS = {"ˈ": 1, "ˌ": 2} | |
| # eSpeak phonemes outside the inventory (foreign words, rare allophones) map to | |
| # the closest inventory phone(s). Anything else becomes <unk> and is reported. | |
| FALLBACK = { | |
| "r": ("ɹ",), "ɻ": ("ɹ",), "ʁ": ("ɹ",), "x": ("k",), "ç": ("h",), "ɬ": ("l",), "ɫ": ("l",), "ɲ": ("n", "j"), | |
| "ts": ("t", "s"), "dz": ("d", "z"), "β": ("v",), "ɣ": ("ɡ",), "ʎ": ("l", "j"), | |
| "a": ("æ",), "aː": ("ɑː",), "ɑ": ("ɑː",), "ɒ": ("ɑː",), "ɑ̃": ("ɑː",), "ɔ̃": ("ɔː",), "ɛ̃": ("æ",), | |
| "e": ("eɪ",), "eː": ("eɪ",), "o": ("oʊ",), "u": ("uː",), "y": ("uː",), "yː": ("uː",), | |
| "ø": ("ɜː",), "øː": ("ɜː",), "œ": ("ɜː",), "ɜ": ("ɜː",), "ɛː": ("ɛ",), "ɪː": ("ɪ",), | |
| "əʊ": ("oʊ",), "eə": ("ɛɹ",), "ɪə": ("iə",), "ʊə": ("ʊɹ",), "aʊɚ": ("aʊ", "ɚ"), "l̩": ("əl",), "m̩": ("ə", "m"), | |
| } | |
| # --- bengali-patch --- | |
| INVENTORY_VARIABLE = "TINY_VOCA_INVENTORY" | |
| ALIGNER_LABELS = {} # inventory phone -> label of the aligner recogniser (when they differ) | |
| if os.environ.get(INVENTORY_VARIABLE): | |
| _inv = json.load(open(os.environ[INVENTORY_VARIABLE], encoding="utf-8")) | |
| ESPEAK_VOICE = _inv["voice"] | |
| CONSONANTS = tuple(_inv["consonants"]) | |
| VOWELS = tuple(_inv["vowels"]) | |
| VOICELESS = frozenset(_inv["voiceless"]) | |
| FALLBACK = {k: tuple(v) for k, v in _inv["fallback"].items()} | |
| ALIGNER_LABELS = dict(_inv.get("aligner_labels", {})) | |
| PHONES = CONSONANTS + VOWELS | |
| SYMBOLS = SPECIALS + PUNCTUATION + PHONES | |
| SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)} | |
| PHONE_SET = frozenset(PHONES) | |
| PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]} | |
| VOWEL_SET = frozenset(VOWELS) | |
| _QUOTES = str.maketrans({"‘": "'", "’": "'", "‚": "'", "‛": "'", "`": "'", "´": "'", | |
| "“": '"', "”": '"', "„": '"', "‟": '"', "«": '"', "»": '"', | |
| "–": "—", "―": "—", "।": ".", "॥": ".", "[": "(", "{": "(", "]": ")", "}": ")"}) | |
| _PUNCTUATION_CHARS = frozenset(PUNCTUATION) | |
| ABBREVIATIONS = {"mr", "mrs", "ms", "dr", "prof", "st", "jr", "sr", "vs", "etc"} | |
| _LANGUAGE_TAG = re.compile(r"\((?:_?[a-z0-9]_?){2,3}(?:-(?:_?[a-z0-9]_?)+)*\)") | |
| class NothingToPronounce(ValueError): | |
| """The text has no word, number or letter to say: only marks, emoji and the like.""" | |
| class Phonemized: | |
| symbols: tuple | |
| ids: tuple | |
| stress: tuple | |
| unknown: tuple # eSpeak phonemes that had no inventory mapping | |
| def __len__(self): | |
| return len(self.ids) | |
| def frontend(): | |
| """Name of the program that produces the phonemes: ``"espeak"`` or ``"own"``.""" | |
| return _frontend | |
| def set_frontend(name): | |
| """Select the phonemizer for this process. Tokens of the two front ends | |
| must not be mixed: annotations and checkpoints record which one made them.""" | |
| global _frontend | |
| if name not in FRONTENDS: | |
| raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}") | |
| _frontend = name | |
| def _dizione(): | |
| try: | |
| import dizione | |
| except ImportError as error: | |
| raise RuntimeError('the "own" front end needs the dizione package: pip install dizione') from error | |
| if tuple(dizione.symbols.SYMBOLS) != SYMBOLS: | |
| raise RuntimeError("dizione's symbol inventory differs from this model's") | |
| return dizione | |
| def frontend_identity(): | |
| if _frontend == "own": | |
| phonemizer = _dizione() | |
| return {"frontend_version": phonemizer.VERSION, "phonemizer": "dizione", "symbols": list(SYMBOLS)} | |
| return {"frontend_version": FRONTEND_VERSION, "voice": ESPEAK_VOICE, "separator": SEPARATOR, | |
| "symbols": list(SYMBOLS), "fallback": {key: list(value) for key, value in sorted(FALLBACK.items())}} | |
| def frontend_runtime(): | |
| """What an annotation run records about the exact program behind its | |
| tokens: the eSpeak version, or the fingerprint of the phonemizer build.""" | |
| if _frontend == "own": | |
| return {"phonemizer": _dizione().identity()["fingerprint"]} | |
| return {"espeak_version": espeak_version()} | |
| def frontend_fingerprint(): | |
| return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest() | |
| def normalize_text(text): | |
| text = unicodedata.normalize("NFKC", text).translate(_QUOTES) | |
| text = re.sub(r"\.{3,}", "…", text) | |
| text = re.sub(r"-{2,}|\s-\s", "—", text) | |
| return re.sub(r"\s+", " ", text).strip() | |
| def chunks(text): | |
| """Yield ``(is_punctuation, piece)``. Decimals, clock times, digit groups, | |
| contractions, title abbreviations and initialisms stay inside the text piece | |
| for eSpeak; every other punctuation mark becomes its own token.""" | |
| start = 0 | |
| for i, char in enumerate(text): | |
| if char not in _PUNCTUATION_CHARS: | |
| continue | |
| previous = text[i-1] if i else "" | |
| following = text[i+1] if i+1 < len(text) else "" | |
| if char == "'" and previous.isalnum() and following.isalnum(): | |
| continue | |
| if char in ".,:" and previous.isdigit() and following.isdigit(): | |
| continue | |
| if char == ".": | |
| word = re.search(r"([A-Za-z]+)$", text[:i]) | |
| if word and word[1].lower() in ABBREVIATIONS: | |
| continue | |
| if previous.isalpha() and following.isalpha(): | |
| continue | |
| if text[start:i].strip(): | |
| yield False, text[start:i].strip() | |
| yield True, char | |
| start = i+1 | |
| if text[start:].strip(): | |
| yield False, text[start:].strip() | |
| def espeak_executable(): | |
| executable = shutil.which("espeak-ng") | |
| if executable is None: | |
| raise RuntimeError("Install espeak-ng on the cloud machine before preprocessing/inference") | |
| return executable | |
| def espeak_version(): | |
| return subprocess.run([espeak_executable(), "--version"], capture_output=True, text=True, check=True, timeout=10).stdout.strip() | |
| def espeak_words(piece, voice=None): | |
| """Phonemize one text piece. Returns words as lists of ``(phone, stress)``. | |
| The input must end with a newline: eSpeak NG 1.51 silently drops the last | |
| character of unterminated standard input. | |
| """ | |
| result = subprocess.run( | |
| [espeak_executable(), "-q", "--ipa", "--sep=" + SEPARATOR, "-v", voice or ESPEAK_VOICE, "--stdin"], | |
| input=piece.rstrip("\n") + "\n", text=True, capture_output=True, check=True, timeout=30) | |
| return parse_espeak(result.stdout) | |
| def parse_espeak(output): | |
| words = [] | |
| for word in _LANGUAGE_TAG.sub("", output).split(): | |
| units = [] | |
| for unit in word.split(SEPARATOR): | |
| stress = 0 | |
| for mark, level in STRESS_MARKS.items(): | |
| if mark in unit: | |
| stress = stress or level | |
| unit = unit.replace(mark, "") | |
| unit = "".join(c for c in unit if unicodedata.category(c) != "Cf" and c not in "()") | |
| if unit: | |
| units.append((unit, stress)) | |
| if units: | |
| words.append(units) | |
| return words | |
| def canonical(phone): | |
| """Map an eSpeak phoneme to inventory phones; ``None`` when unknown.""" | |
| if phone in PHONE_SET: | |
| return (phone,) | |
| if phone in FALLBACK: | |
| return FALLBACK[phone] | |
| stripped = "".join(c for c in unicodedata.normalize("NFD", phone) if not unicodedata.combining(c)) | |
| for candidate in (stripped, stripped.replace("ː", ""), stripped + "ː"): | |
| if candidate != phone and candidate in PHONE_SET: | |
| return (candidate,) | |
| if candidate != phone and candidate in FALLBACK: | |
| return FALLBACK[candidate] | |
| return None | |
| def tokenize(pieces): | |
| """Build the token sequence from ``(is_punctuation, value)`` pieces where a | |
| text piece's value is the parsed eSpeak word list. Pure function (testable | |
| without eSpeak).""" | |
| symbols, stress, unknown = [BOS], [0], [] | |
| for punctuation, value in pieces: | |
| if punctuation: | |
| if symbols[-1] != value: # collapse repeats such as "!!" | |
| symbols.append(value) | |
| stress.append(0) | |
| continue | |
| for word in value: | |
| if symbols[-1] in PHONE_SET or symbols[-1] == UNK: | |
| symbols.append(SP) | |
| stress.append(0) | |
| for phone, level in word: | |
| mapped = canonical(phone) | |
| if mapped is None: | |
| unknown.append(phone) | |
| mapped = (UNK,) | |
| # A fallback may expand to several phones: stress goes to its first vowel. | |
| carrier = next((i for i, item in enumerate(mapped) if item in VOWEL_SET), 0) | |
| for index, item in enumerate(mapped): | |
| symbols.append(item) | |
| stress.append(level if index == carrier else 0) | |
| symbols.append(EOS) | |
| stress.append(0) | |
| if not any(s in PHONE_SET or s == UNK for s in symbols): | |
| raise NothingToPronounce("the text has nothing to pronounce") | |
| return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), tuple(unknown)) | |
| LATIN_RUN = re.compile(r"[A-Za-z][A-Za-z']*") | |
| def load_lexicon(): | |
| path = os.environ.get("TINY_VOCA_LEXICON") | |
| if not path: | |
| return {} | |
| with open(path, encoding="utf-8") as f: | |
| data = json.load(f) | |
| return {str(k).lower(): str(v) for k, v in data.items()} | |
| LEXICON = load_lexicon() | |
| def espeak_mixed(piece): | |
| 'Phonemize Bengali text and route known Latin words through the Bengali lexicon.' | |
| words = [] | |
| position = 0 | |
| for match in LATIN_RUN.finditer(piece): | |
| prefix = piece[position:match.start()] | |
| if prefix.strip(): | |
| words += espeak_words(prefix) | |
| latin_word = match.group() | |
| bengali_spelling = LEXICON.get(latin_word.casefold()) | |
| if bengali_spelling: | |
| words += espeak_words(bengali_spelling, voice="bn") | |
| else: | |
| words += espeak_words(latin_word, voice="en-us") | |
| position = match.end() | |
| suffix = piece[position:] | |
| if suffix.strip(): | |
| words += espeak_words(suffix) | |
| return words | |
| def phonemize(text): | |
| if not isinstance(text, str) or not text.strip(): | |
| raise ValueError("text must not be empty") | |
| if _frontend == "own": | |
| return tokenize(_dizione().pieces(text)) | |
| pieces = [] | |
| for punctuation, piece in chunks(normalize_text(text)): | |
| pieces.append((True, piece) if punctuation else (False, espeak_mixed(piece))) | |
| return tokenize(pieces) | |
| def is_phone(token_ids): | |
| """Boolean list/tensor helper: tokens that must be realised as audio.""" | |
| return [int(i) in PHONE_IDS for i in token_ids] | |