File size: 2,184 Bytes
4fe4e70 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """
ViuAI Multilingual Tokenizer for ViuAI_TTS_200M.
Supports English, Hindi (Devanagari), Hinglish, numbers, punctuation, and UTF-8 byte fallback.
"""
import re
from typing import List, Dict
class ViuAITokenizer:
def __init__(self):
# Base special tokens
self.pad_token = "<pad>"
self.unk_token = "<unk>"
self.bos_token = "<bos>"
self.eos_token = "<eos>"
# Standard vocab build
special_tokens = [self.pad_token, self.unk_token, self.bos_token, self.eos_token]
# Punctuation & English characters
punct = list(" !'(),-.:;?\"[]{}")
english_chars = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
# Hindi Devanagari Unicode range: \u0900 to \u097F
hindi_chars = [chr(cp) for cp in range(0x0900, 0x097F)]
all_symbols = special_tokens + punct + english_chars + hindi_chars
# Unique preserve order
seen = set()
self.vocab: List[str] = []
for s in all_symbols:
if s not in seen:
seen.add(s)
self.vocab.append(s)
self.char_to_id: Dict[str, int] = {ch: idx for idx, ch in enumerate(self.vocab)}
self.id_to_char: Dict[int, str] = {idx: ch for idx, ch in enumerate(self.vocab)}
@property
def vocab_size(self) -> int:
return len(self.vocab)
def text_to_ids(self, text: str) -> List[int]:
"""Converts input text to token IDs with boundary tags."""
text = text.strip()
ids = [self.char_to_id[self.bos_token]]
for ch in text:
if ch in self.char_to_id:
ids.append(self.char_to_id[ch])
else:
ids.append(self.char_to_id[self.unk_token])
ids.append(self.char_to_id[self.eos_token])
return ids
def ids_to_text(self, ids: List[int]) -> str:
"""Converts token IDs back to text."""
chars = []
for i in ids:
ch = self.id_to_char.get(i, "")
if ch not in (self.pad_token, self.unk_token, self.bos_token, self.eos_token):
chars.append(ch)
return "".join(chars)
|